Merge eb18a6f6d3 into 4feb467dab

README.md

@@ -43,6 +43,7 @@ found in .editorconfig files.
 * **--outdent_attributes**: See **dfmt_outdent_attributes** below
 * **--space_after_cast**: See **dfmt_space_after_cast** below
 * **--space_before_function_parameters**: See **dfmt_space_before_function_parameters** below
+* **--sort_imports**: See **sort_imports** below
 * **--split_operator_at_line_end**: See **dfmt_split_operator_at_line_end** below
 * **--tab_width**: See **tab_width** below
 * **--selective_import_space**: See **dfmt_selective_import_space** below
@@ -102,6 +103,7 @@ dfmt_split_operator_at_line_end | `true`, `false` | `false` | Place operators on
 dfmt_space_after_cast | `true`, `false` | `true` | Insert space after the closing paren of a `cast` expression.
 dfmt_space_after_keywords (Not yet implemented) | `true`, `false` | `true` | Insert space after `if`, `while`, `foreach`, etc, and before the `(`.
 dfmt_space_before_function_parameters | `true`, `false` | `false` | Insert space before the opening paren of a function parameter list.
+dfmt_sort_imports | `true`, `false` | `false` | Sort Imports alphabetically group by root-package
 dfmt_selective_import_space | `true`, `false` | `true` | Insert space after the module name and before the `:` for selective imports.
 dfmt_compact_labeled_statements | `true`, `false` | `true` | Place labels on the same line as the labeled `switch`, `for`, `foreach`, or `while` statement.
 dfmt_template_constraint_style | `conditional_newline_indent` `conditional_newline` `always_newline` `always_newline_indent` | `conditional_newline_indent` | Control the formatting of template constraints.

makefile

@@ -3,7 +3,7 @@ SRC := $(shell find src -name "*.d") \
 INCLUDE_PATHS := -Ilibdparse/src -Isrc
 DMD_COMMON_FLAGS := -dip25 -w $(INCLUDE_PATHS) -Jviews
 DMD_DEBUG_FLAGS := -debug -g $(DMD_COMMON_FLAGS)
-DMD_FLAGS := -O -inline $(DMD_COMMON_FLAGS)
+DMD_FLAGS :=  $(DMD_COMMON_FLAGS) -g
 DMD_TEST_FLAGS := -unittest -g $(DMD_COMMON_FLAGS)
 LDC_FLAGS := -g -w -oq $(INCLUDE_PATHS)
 GDC_FLAGS := -g -w -oq $(INCLUDE_PATHS)

src/dfmt/ast_info.d

@@ -8,12 +8,30 @@ module dfmt.ast_info;
 import dparse.lexer;
 import dparse.ast;
 
+struct Import
+{
+    /// the identifier chain of the import
+    string[] importStrings;
+    /// the lhs of renamed imports 
+    string renamedAs;
+    /// attribs for the import
+	string attribString;
+}
+
 /// AST information that is needed by the formatter.
 struct ASTInformation
 {
+    struct LocationRange
+    {
+        size_t startLocation;
+        size_t endLocation;
+    }
+
     /// Sorts the arrays so that binary search will work on them
     void cleanup()
     {
+        finished = true;
+
         import std.algorithm : sort;
 
         sort(doubleNewlineLocations);
@@ -30,6 +48,8 @@ struct ASTInformation
         sort(arrayStartLocations);
         sort(contractLocations);
         sort(constraintLocations);
+        sort(skipTokenLocations);
+
     }
 
     /// Locations of end braces for struct bodies
@@ -38,12 +58,145 @@ struct ASTInformation
     /// Locations of tokens where a space is needed (such as the '*' in a type)
     size_t[] spaceAfterLocations;
 
+    /// Location-Ranges where scopes begin and end
+    LocationRange[] scopeLocationRanges;
+
+    /// zero is module scope
+    size_t scopeOrdinalOfLocation(const size_t location) const
+    {
+        size_t bestOrdinal = 0;
+
+        LocationRange bestRange = scopeLocationRanges[bestOrdinal];
+
+        foreach (i; 1 .. scopeLocationRanges.length)
+        {
+            LocationRange nextRange = scopeLocationRanges[i];
+
+            if (nextRange.startLocation > location)
+                break;
+
+            if (nextRange.endLocation > location)
+            {
+                bestRange = nextRange;
+                bestOrdinal = i;
+            }
+
+        }
+
+        return bestOrdinal;
+    }
+
+    /// true if a is closer to t then b
+    /// with bias towards true
+    static bool isCloserTo(const string[] a, const string[] b, const string[] t)
+    {
+        import std.algorithm : min;
+
+        foreach(i;0 .. min(a.length, b.length, t.length))
+        {
+            
+        }
+
+        return false;
+    }
+
+    bool importStringLess(const Import a, const Import b) const
+    {
+        bool result;
+        const (string)[] sortKeyA = a.importStrings;
+        const (string)[] sortKeyB = b.importStrings;
+
+        result = sortKeyA < sortKeyB;
+
+        //TODO take the module-name of the module we are formatting into account
+
+        return result;
+    }
+
+    static struct ImportLine
+    {
+        string importString;
+        string attribString;
+        string renamedAs;
+    }
+
+    /// returns an array of indecies into the token array
+    /// which are the indecies of the imports to be written
+    /// in sorted order
+    /// newlines for grouping are enoded as a null entry
+    ImportLine[] importLinesFor(const size_t scopeOrdinal) const
+    {
+        import std.algorithm;
+        import std.range;
+
+        uint idx = 0;
+        ImportLine[] result;
+
+        auto imports = importScopes[scopeOrdinal];
+
+        if (imports.length)
+        {
+            const max_sorted_imports_length = imports.length * 2;
+            // account for newlines
+            result.length = max_sorted_imports_length;
+            auto sortedImports =
+                (cast(Import[])imports).sort!((a, b) => importStringLess(a, b))
+                .release;
+
+            foreach(i, imp;sortedImports)
+            {
+                if (i > 0)
+                {
+                    const prev = sortedImports[i-1];
+                    static if (false)
+                    {
+                        if (prev.importStrings.length < 2
+                            || imp.importStrings[0 .. $-1] 
+                                != prev.importStrings[0 .. $-1]
+                        )
+                        {
+                            result[idx++].importString = null;
+                            // a null importString means a blank line is inserted
+                        }
+                    }
+                    else
+                    {
+                        if (imp.importStrings[0] != prev.importStrings[0])
+                        {
+                            result[idx++].importString = null;
+                            // a null importString means a blank line is inserted
+                        }
+                    }
+                }
+
+                result[idx].importString = imp.importStrings.join(".");
+                result[idx].renamedAs = imp.renamedAs;
+                result[idx].attribString = imp.attribString;
+                idx++;
+            }
+
+            result = result[0 .. idx];
+        }
+        else
+        {
+            result = null;
+        }
+
+        return result;
+    }
+
     /// Locations of unary operators
     size_t[] unaryLocations;
 
+    /// Locations of tokens to be skipped
+    size_t[] skipTokenLocations;
+
     /// Lines containing attribute declarations
     size_t[] attributeDeclarationLines;
 
+    /// lines in which imports end
+    size_t[] importEndLines;
+
     /// Case statement colon locations
     size_t[] caseEndLocations;
 
@@ -73,6 +226,15 @@ struct ASTInformation
 
     /// Locations of template constraint "if" tokens
     size_t[] constraintLocations;
+
+    /// cleanup run;
+    bool finished;
+
+    /// contains all imports inside scope
+    Import[][] importScopes;
+
+    ///contain the current fqn of the module
+    string[] moduleNameStrings;
 }
 
 /// Collects information from the AST that is useful for the formatter
@@ -82,9 +244,22 @@ final class FormatVisitor : ASTVisitor
      * Params:
      *     astInformation = the AST information that will be filled in
      */
+
     this(ASTInformation* astInformation)
     {
         this.astInformation = astInformation;
+        if (this.astInformation.scopeLocationRanges.length != 0)
+            assert(0, "astinformation seems to be dirty");
+
+        this.astInformation.scopeLocationRanges ~= ASTInformation.LocationRange(0, size_t.max);
+        this.astInformation.importScopes.length = 1;
+    }
+
+    void addScope(const size_t startLocation, const size_t endLocation)
+    {
+        astInformation.scopeLocationRanges ~= ASTInformation.LocationRange(startLocation,
+                endLocation);
+        astInformation.importScopes.length += 1;
     }
 
     override void visit(const ArrayInitializer arrayInitializer)
@@ -93,6 +268,42 @@ final class FormatVisitor : ASTVisitor
         arrayInitializer.accept(this);
     }
  
+    void addImport(size_t scopeId, string[] importString, string renamedAs, string importAttribString)
+    {
+        astInformation.importScopes[scopeId] ~= Import(importString, renamedAs, importAttribString);
+    }
+
+    override void visit(const SingleImport singleImport)
+    {
+        auto scopeOrdinal = size_t.max;
+
+        if (singleImport.identifierChain)
+        {
+            string[] importString;
+            string renamedAs = null;
+
+            auto ic = singleImport.identifierChain;
+            foreach (ident; ic.identifiers)
+            {
+                importString ~= ident.text;
+            }
+
+            scopeOrdinal = astInformation.scopeOrdinalOfLocation(ic.identifiers[0].index);
+
+            if (singleImport.rename.text && singleImport.rename.text.length)
+                renamedAs = singleImport.rename.text;
+
+            addImport(scopeOrdinal, importString, renamedAs, importAttribString);
+
+        }
+        else
+        {
+            assert (0, "singleImport without identifierChain");
+        }
+
+        singleImport.accept(this);
+    }
+
     override void visit(const ConditionalDeclaration dec)
     {
         if (dec.hasElse)
@@ -137,7 +348,8 @@ final class FormatVisitor : ASTVisitor
     {
         if (funcLit.functionBody !is null)
         {
-            astInformation.funLitStartLocations ~= funcLit.functionBody.blockStatement.startLocation;
+            astInformation.funLitStartLocations ~= funcLit.functionBody
+                .blockStatement.startLocation;
             astInformation.funLitEndLocations ~= funcLit.functionBody.blockStatement.endLocation;
         }
         funcLit.accept(this);
@@ -149,6 +361,49 @@ final class FormatVisitor : ASTVisitor
         defaultStatement.accept(this);
     }
 
+    /// this is the very limited usecase of printing attribs which may be
+    /// attached to imports (therefore it's not compleate at all)
+    /// HACK this method also adds the original token to the ignore_tokens
+
+    private string toImportAttribString (const (Attribute)[] attributes)
+    {
+        string result;
+
+        foreach(attrib;attributes)
+        {
+
+            if (attrib.attribute.type == tok!"public")
+            {
+                result ~= "public ";
+                astInformation.skipTokenLocations ~= attrib.attribute.index;
+            }
+            else if (attrib.attribute.type == tok!"private")
+            {
+                result ~= "private ";
+                astInformation.skipTokenLocations ~= attrib.attribute.index;
+            }
+            else if (attrib.attribute.type == tok!"static")
+            {
+                result ~= "static ";
+                astInformation.skipTokenLocations ~= attrib.attribute.index;
+            }
+        }
+
+        return result;
+    }
+
+    override void visit(const Declaration declaration)
+    {
+        if (declaration.importDeclaration)
+        {
+            importAttribString = toImportAttribString(declaration.attributes);
+        }
+
+        declaration.accept(this);
+
+        importAttribString = null;
+    }
+
     override void visit(const CaseStatement caseStatement)
     {
         astInformation.caseEndLocations ~= caseStatement.colonLocation;
@@ -164,11 +419,22 @@ final class FormatVisitor : ASTVisitor
     override void visit(const FunctionBody functionBody)
     {
         if (functionBody.blockStatement !is null)
-            astInformation.doubleNewlineLocations ~= functionBody.blockStatement.endLocation;
+        {
+            auto bs = functionBody.blockStatement;
+
+            addScope(bs.startLocation, bs.endLocation);
+            astInformation.doubleNewlineLocations ~= bs.endLocation;
+        }
+
         if (functionBody.bodyStatement !is null && functionBody.bodyStatement
                 .blockStatement !is null)
-            astInformation.doubleNewlineLocations
+        {
-                ~= functionBody.bodyStatement.blockStatement.endLocation;
+            auto bs = functionBody.bodyStatement.blockStatement;
+
+            addScope(bs.startLocation, bs.endLocation);
+            astInformation.doubleNewlineLocations ~= bs.endLocation;
+        }
+
         functionBody.accept(this);
     }
 
@@ -199,6 +465,7 @@ final class FormatVisitor : ASTVisitor
 
     override void visit(const StructBody structBody)
     {
+        addScope(structBody.startLocation, structBody.endLocation);
         astInformation.doubleNewlineLocations ~= structBody.endLocation;
         structBody.accept(this);
     }
@@ -247,5 +514,7 @@ final class FormatVisitor : ASTVisitor
 
 private:
     ASTInformation* astInformation;
+    string importAttribString;
+
     alias visit = ASTVisitor.visit;
 }

src/dfmt/config.d

@@ -50,6 +50,8 @@ struct Config
     ///
     OptionalBoolean dfmt_selective_import_space;
     ///
+    OptionalBoolean dfmt_sort_imports;
+    ///
     OptionalBoolean dfmt_compact_labeled_statements;
     ///
     TemplateConstraintStyle dfmt_template_constraint_style;
@@ -77,6 +79,7 @@ struct Config
         dfmt_space_before_function_parameters = OptionalBoolean.f;
         dfmt_split_operator_at_line_end = OptionalBoolean.f;
         dfmt_selective_import_space = OptionalBoolean.t;
+        dfmt_sort_imports = OptionalBoolean.f;
         dfmt_compact_labeled_statements = OptionalBoolean.t;
         dfmt_template_constraint_style = TemplateConstraintStyle.conditional_newline_indent;
     }

src/dfmt/formatter.d

@@ -4,7 +4,6 @@
 //          http://www.boost.org/LICENSE_1_0.txt)
 
 module dfmt.formatter;
-
 import dparse.lexer;
 import dparse.parser;
 import dparse.rollback_allocator;
@@ -148,7 +147,7 @@ private:
     /// Configuration
     const Config* config;
 
-    /// chached end of line string
+    /// cached end of line string
     const string eolString;
 
     /// Keep track of whether or not an extra newline was just added because of
@@ -170,6 +169,9 @@ private:
     /// True if we're in an ASM block
     bool inAsm;
 
+    /// formarts a variable number of tokes per call
+    /// called until no tokens remain
+    /// maybe called recursively
     void formatStep()
     {
         import std.range : assumeSorted;
@@ -191,7 +193,7 @@ private:
                     write(" ");
             }
         }
-        else if (currentIs(tok!"module") || currentIs(tok!"import"))
+        else if (currentIs(tok!"import") || currentIs(tok!"module"))
         {
             formatModuleOrImport();
         }
@@ -281,11 +283,12 @@ private:
         {
             writeToken();
             if (index < tokens.length && (currentIs(tok!"identifier")
-                    || ( index < 1 && ( isBasicType(peekBack(2).type) || peekBack2Is(tok!"identifier") ) &&
+                    || (index < 1 && (isBasicType(peekBack(2).type)
-                         currentIs(tok!("(")) && config.dfmt_space_before_function_parameters )
+                    || peekBack2Is(tok!"identifier")) && currentIs(tok!("("))
-                    || isBasicType(current.type) || currentIs(tok!"@") || currentIs(tok!"if")
+                    && config.dfmt_space_before_function_parameters)
-                    || isNumberLiteral(tokens[index].type) || (inAsm
+                    || isBasicType(current.type) || currentIs(tok!"@")
-                    && peekBack2Is(tok!";") && currentIs(tok!"["))))
+                    || currentIs(tok!"if") || isNumberLiteral(tokens[index].type)
+                    || (inAsm && peekBack2Is(tok!";") && currentIs(tok!"["))))
             {
                 write(" ");
             }
@@ -433,7 +436,9 @@ private:
 
     void formatModuleOrImport()
     {
-        immutable t = current.type;
+        immutable isImport = (current.type == tok!"import");
+        if (!config.dfmt_sort_imports || peekIs(tok!("(")) || !isImport)
+        {
             writeToken();
             if (currentIs(tok!"("))
             {
@@ -441,6 +446,7 @@ private:
                 return;
             }
             write(" ");
+
             while (index < tokens.length)
             {
                 if (currentIs(tok!";"))
@@ -457,7 +463,7 @@ private:
                     }
                     else if (currentIs(tok!"{") && config.dfmt_brace_style == BraceStyle.allman)
                         break;
-                else if (t == tok!"import" && !currentIs(tok!"import")
+                    else if (isImport && !currentIs(tok!"import")
                         && !currentIs(tok!"}")
                         && !((currentIs(tok!"public")
                             || currentIs(tok!"private")
@@ -495,7 +501,8 @@ private:
                     }
                     assert(lengthOfNextChunk > 0);
                     writeToken();
-                if (currentLineLength + 1 + lengthOfNextChunk >= config.dfmt_soft_max_line_length)
+                    if (currentLineLength + 1 + lengthOfNextChunk >= config
+                            .dfmt_soft_max_line_length)
                     {
                         pushWrapIndent(tok!",");
                         newline();
@@ -506,6 +513,21 @@ private:
                 else
                       formatStep();
               }
+
+              if (config.dfmt_sort_imports && !isImport)
+                  writeImportLinesFor(0);
+        }
+        else
+        {
+            while(currentIs(tok!"import"))
+            {
+                // skip to the ending ; of the import statement
+                while(!currentIs(tok!";")) 
+                    index++;
+                // skip past the ;
+                index++;
+            }
+        }
     }
 
     void formatLeftParenOrBracket()
@@ -603,6 +625,41 @@ private:
             write(" ");
     }
 
+    void writeImportLinesFor(size_t scopeOrdinal)
+    {
+        foreach(importLine;astInformation.importLinesFor(scopeOrdinal))
+        {
+            if (importLine.importString !is null)
+            {
+                // for some reason newline() creates double 
+                // newlines in module-scope
+                scopeOrdinal ? newline() : simpleNewline();
+
+                write(importLine.attribString);
+                write("import ");
+                if (importLine.renamedAs)
+                {
+                    write(importLine.renamedAs);
+                    write(" = ");
+                }
+                /+ TODO deal with selective imports
+                if (importLine.selctiveImports)
+                {
+                }
+                +/
+                write(importLine.importString);
+                write(";");
+            }
+            else
+            {
+                simpleNewline();
+            }
+        }
+
+        simpleNewline();
+        simpleNewline();
+    }
+
     void formatColon()
     {
         import dfmt.editorconfig : OptionalBoolean;
@@ -749,6 +806,19 @@ private:
         }
         else
         {
+            bool writeImports = false;
+            size_t scopeOrdinal = void;
+            if (config.dfmt_sort_imports)
+            {
+                scopeOrdinal = astInformation.scopeOrdinalOfLocation(current.index);
+                if (scopeOrdinal)
+                {
+                    auto range = astInformation.scopeLocationRanges[scopeOrdinal];
+                    if (range.startLocation == current.index)
+                        writeImports = true;
+                }
+            }
+
             if (peekBackIsSlashSlash())
             {
                 if (peekBack2Is(tok!";"))
@@ -773,9 +843,15 @@ private:
             }
 
             indents.push(tok!"{");
+
             if (!currentIs(tok!"{"))
                 newline();
             linebreakHints = [];
+
+            if (writeImports && astInformation.importScopes[scopeOrdinal].length)
+            {
+                writeImportLinesFor(scopeOrdinal);
+            }
         }
     }
 
@@ -1051,8 +1127,7 @@ private:
 
     bool currentIsIndentedTemplateConstraint()
     {
-        return index < tokens.length
+        return index < tokens.length && astInformation.constraintLocations.canFindIndex(current.index)
-            && astInformation.constraintLocations.canFindIndex(current.index)
             && (config.dfmt_template_constraint_style == TemplateConstraintStyle.always_newline
                     || currentLineLength >= config.dfmt_soft_max_line_length);
     }
@@ -1438,6 +1513,12 @@ private:
 
     void writeToken()
     {
+        if (config.dfmt_sort_imports && astInformation.skipTokenLocations.canFindIndex(current.index))
+        {
+            index++;
+            return ;
+        }
+
         import std.range : retro;
         import std.algorithm.searching : countUntil;
 

src/dfmt/main.d

@@ -87,7 +87,7 @@ else
             import dfmt.editorconfig : OptionalBoolean;
             import std.exception : enforceEx;
 
-            enforceEx!GetOptException(value == "true" || value == "false", "Invalid argument");
+            enforceEx!GetOptException(value == "true" || value == "false", "Invalid argument '" ~ value ~ "' for " ~ option ~ " should be 'true' or 'false'");
             immutable OptionalBoolean optVal = value == "true" ? OptionalBoolean.t
                 : OptionalBoolean.f;
             switch (option)
@@ -110,6 +110,9 @@ else
             case "selective_import_space":
                 optConfig.dfmt_selective_import_space = optVal;
                 break;
+            case "sort_imports":
+                optConfig.dfmt_sort_imports = optVal;
+                break;
             case "compact_labeled_statements":
                 optConfig.dfmt_compact_labeled_statements = optVal;
                 break;
@@ -136,6 +139,7 @@ else
                 "outdent_attributes", &handleBooleans,
                 "space_after_cast", &handleBooleans,
                 "selective_import_space", &handleBooleans,
+                "sort_imports", &handleBooleans,
                 "space_before_function_parameters", &handleBooleans,
                 "split_operator_at_line_end", &handleBooleans,
                 "compact_labeled_statements", &handleBooleans,