feat: implement visitors and writers

Signed-off-by: Prajwal S N <prajwalnadig21@gmail.com>
This commit is contained in:
Prajwal S N 2023-10-19 21:27:17 +05:30
parent 6c1c543194
commit 3ac8e5afa6
No known key found for this signature in database
GPG Key ID: 60701A603988FAC2
10 changed files with 3870 additions and 3907 deletions

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "libdparse"]
path = libdparse
url = https://github.com/dlang-community/libdparse.git
[submodule "stdx-allocator"]
path = stdx-allocator
url = https://github.com/dlang-community/stdx-allocator

View File

@ -4,18 +4,17 @@
"targetType": "autodetect",
"license": "BSL-1.0",
"dependencies": {
"libdparse": ">=0.19.2 <1.0.0",
"dmd": "~>2.105.2"
"dmd": "~>2.106.0-beta.1"
},
"targetPath" : "bin/",
"targetName" : "dfmt",
"stringImportPaths" : [
"bin"
"targetPath": "bin/",
"targetName": "dfmt",
"stringImportPaths": [
"bin"
],
"versions" : [
"versions": [
"built_with_dub"
],
"preBuildCommands" : [
"$DC -run \"$PACKAGE_DIR/dubhash.d\""
"preBuildCommands": [
"$DC -run \"$PACKAGE_DIR/dubhash.d\""
]
}

@ -1 +0,0 @@
Subproject commit fe6d1e38fb4fc04323170389cfec67ed7fd4e24a

3656
src/dfmt/ast.d Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,492 +0,0 @@
// Copyright Brian Schott 2015.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dfmt.ast_info;
import dmd.transitivevisitor;
import dmd.tokens;
enum BraceIndentInfoFlags
{
tempIndent = 1 << 0,
}
struct BraceIndentInfo
{
size_t startLocation;
size_t endLocation;
uint flags;
uint beginIndentLevel;
}
struct StructInitializerInfo
{
size_t startLocation;
size_t endLocation;
}
/// AST information that is needed by the formatter.
struct ASTInformation
{
/// Sorts the arrays so that binary search will work on them
void cleanup()
{
import std.algorithm : sort, uniq;
import std.array : array;
sort(doubleNewlineLocations);
sort(spaceAfterLocations);
sort(unaryLocations);
sort(attributeDeclarationLines);
sort(atAttributeStartLocations);
sort(caseEndLocations);
sort(structInitStartLocations);
sort(structInitEndLocations);
sort(funLitStartLocations);
sort(funLitEndLocations);
sort(conditionalWithElseLocations);
sort(conditionalStatementLocations);
sort(arrayStartLocations);
sort(assocArrayStartLocations);
sort(contractLocations);
sort(constraintLocations);
sort(constructorDestructorLocations);
sort(staticConstructorDestructorLocations);
sort(sharedStaticConstructorDestructorLocations);
sort!((a, b) => a.endLocation < b.endLocation)(indentInfoSortedByEndLocation);
sort!((a, b) => a.endLocation < b.endLocation)(structInfoSortedByEndLocation);
sort(ufcsHintLocations);
ufcsHintLocations = ufcsHintLocations.uniq().array();
sort(ternaryColonLocations);
sort(namedArgumentColonLocations);
}
/// Locations of end braces for struct bodies
size_t[] doubleNewlineLocations;
/// Locations of tokens where a space is needed (such as the '*' in a type)
size_t[] spaceAfterLocations;
/// Locations of unary operators
size_t[] unaryLocations;
/// Lines containing attribute declarations
size_t[] attributeDeclarationLines;
/// Lines containing attribute declarations that can be followed by a new line
size_t[] atAttributeStartLocations;
/// Case statement colon locations
size_t[] caseEndLocations;
/// Opening braces of struct initializers
size_t[] structInitStartLocations;
/// Closing braces of struct initializers
size_t[] structInitEndLocations;
/// Opening braces of function literals
size_t[] funLitStartLocations;
/// Closing braces of function literals
size_t[] funLitEndLocations;
/// Locations of aggregate bodies (struct, class, union)
size_t[] aggregateBodyLocations;
/// Locations of function bodies
size_t[] funBodyLocations;
/// Conditional statements that have matching "else" statements
size_t[] conditionalWithElseLocations;
/// Conditional statement locations
size_t[] conditionalStatementLocations;
/// Locations of start locations of array initializers
size_t[] arrayStartLocations;
/// Locations of start locations of associative array initializers
size_t[] assocArrayStartLocations;
/// Locations of "in" and "out" tokens that begin contracts
size_t[] contractLocations;
/// Locations of template constraint "if" tokens
size_t[] constraintLocations;
/// Locations of constructor/destructor "shared" tokens ?
size_t[] sharedStaticConstructorDestructorLocations;
/// Locations of constructor/destructor "static" tokens ?
size_t[] staticConstructorDestructorLocations;
/// Locations of constructor/destructor "this" tokens ?
size_t[] constructorDestructorLocations;
/// Locations of '.' characters that might be UFCS chains.
size_t[] ufcsHintLocations;
BraceIndentInfo[] indentInfoSortedByEndLocation;
/// Opening & closing braces of struct initializers
StructInitializerInfo[] structInfoSortedByEndLocation;
/// Locations ternary expression colons.
size_t[] ternaryColonLocations;
/// Locations of named arguments of function call or struct constructor.
size_t[] namedArgumentColonLocations;
}
/// Collects information from the AST that is useful for the formatter
extern (C++) class FormatVisitor(AST) : ParseTimeTransitiveVisitor!AST
{
alias visit = ParseTimeTransitiveVisitor!AST.visit;
/**
* Params:
* astInformation = the AST information that will be filled in
*/
this(ASTInformation* astInformation)
{
this.astInformation = astInformation;
}
/* override void visit(const ArrayInitializer arrayInitializer) */
/* { */
/* astInformation.arrayStartLocations ~= arrayInitializer.startLocation; */
/* arrayInitializer.accept(this); */
/* } */
/* override void visit(const ArrayLiteral arrayLiteral) */
/* { */
/* astInformation.arrayStartLocations ~= arrayLiteral.tokens[0].index; */
/* arrayLiteral.accept(this); */
/* } */
/* override void visit(const AssocArrayLiteral assocArrayLiteral) */
/* { */
/* astInformation.arrayStartLocations ~= assocArrayLiteral.tokens[0].index; */
/* astInformation.assocArrayStartLocations ~= assocArrayLiteral.tokens[0].index; */
/* assocArrayLiteral.accept(this); */
/* } */
/* override void visit(const SharedStaticConstructor sharedStaticConstructor) */
/* { */
/* astInformation.sharedStaticConstructorDestructorLocations ~= sharedStaticConstructor */
/* .location; */
/* sharedStaticConstructor.accept(this); */
/* } */
/* override void visit(const SharedStaticDestructor sharedStaticDestructor) */
/* { */
/* astInformation.sharedStaticConstructorDestructorLocations ~= sharedStaticDestructor */
/* .location; */
/* sharedStaticDestructor.accept(this); */
/* } */
/* override void visit(const StaticConstructor staticConstructor) */
/* { */
/* astInformation.staticConstructorDestructorLocations ~= staticConstructor.location; */
/* staticConstructor.accept(this); */
/* } */
/* override void visit(const StaticDestructor staticDestructor) */
/* { */
/* astInformation.staticConstructorDestructorLocations ~= staticDestructor.location; */
/* staticDestructor.accept(this); */
/* } */
/* override void visit(const Constructor constructor) */
/* { */
/* astInformation.constructorDestructorLocations ~= constructor.location; */
/* constructor.accept(this); */
/* } */
/* override void visit(const Destructor destructor) */
/* { */
/* astInformation.constructorDestructorLocations ~= destructor.index; */
/* destructor.accept(this); */
/* } */
/* override void visit(const FunctionBody functionBody) */
/* { */
/* if (auto bd = functionBody.specifiedFunctionBody) */
/* { */
/* if (bd.blockStatement) */
/* { */
/* astInformation.funBodyLocations ~= bd.blockStatement.startLocation; */
/* } */
/* } */
/* functionBody.accept(this); */
/* } */
/* override void visit(const ConditionalDeclaration dec) */
/* { */
/* if (dec.hasElse) */
/* { */
/* auto condition = dec.compileCondition; */
/* if (condition.versionCondition !is null) */
/* { */
/* astInformation.conditionalWithElseLocations */
/* ~= condition.versionCondition.versionIndex; */
/* } */
/* else if (condition.debugCondition !is null) */
/* { */
/* astInformation.conditionalWithElseLocations ~= condition.debugCondition.debugIndex; */
/* } */
/* // Skip "static if" because the formatting for normal "if" handles */
/* // it properly */
/* } */
/* dec.accept(this); */
/* } */
/* override void visit(const Constraint constraint) */
/* { */
/* astInformation.constraintLocations ~= constraint.location; */
/* constraint.accept(this); */
/* } */
/* override void visit(const ConditionalStatement statement) */
/* { */
/* auto condition = statement.compileCondition; */
/* if (condition.versionCondition !is null) */
/* { */
/* astInformation.conditionalStatementLocations ~= condition.versionCondition.versionIndex; */
/* } */
/* else if (condition.debugCondition !is null) */
/* { */
/* astInformation.conditionalStatementLocations ~= condition.debugCondition.debugIndex; */
/* } */
/* statement.accept(this); */
/* } */
/* override void visit(const FunctionLiteralExpression funcLit) */
/* { */
/* if (funcLit.specifiedFunctionBody !is null) */
/* { */
/* const bs = funcLit.specifiedFunctionBody.blockStatement; */
/* astInformation.funLitStartLocations ~= bs.startLocation; */
/* astInformation.funLitEndLocations ~= bs.endLocation; */
/* astInformation.indentInfoSortedByEndLocation ~= */
/* BraceIndentInfo(bs.startLocation, bs.endLocation); */
/* } */
/* funcLit.accept(this); */
/* } */
/* override void visit(const DefaultStatement defaultStatement) */
/* { */
/* astInformation.caseEndLocations ~= defaultStatement.colonLocation; */
/* defaultStatement.accept(this); */
/* } */
/* override void visit(const CaseStatement caseStatement) */
/* { */
/* astInformation.caseEndLocations ~= caseStatement.colonLocation; */
/* caseStatement.accept(this); */
/* } */
/* override void visit(const CaseRangeStatement caseRangeStatement) */
/* { */
/* astInformation.caseEndLocations ~= caseRangeStatement.colonLocation; */
/* caseRangeStatement.accept(this); */
/* } */
/* override void visit(const SpecifiedFunctionBody specifiedFunctionBody) */
/* { */
/* if (specifiedFunctionBody.blockStatement !is null) */
/* astInformation.doubleNewlineLocations ~= specifiedFunctionBody */
/* .blockStatement.endLocation; */
/* specifiedFunctionBody.accept(this); */
/* } */
/* override void visit(const StructInitializer structInitializer) */
/* { */
/* astInformation.structInitStartLocations ~= structInitializer.startLocation; */
/* astInformation.structInitEndLocations ~= structInitializer.endLocation; */
/* astInformation.structInfoSortedByEndLocation ~= */
/* StructInitializerInfo( */
/* structInitializer.startLocation, structInitializer.endLocation); */
/* astInformation.indentInfoSortedByEndLocation ~= */
/* BraceIndentInfo( */
/* structInitializer.startLocation, structInitializer.endLocation); */
/* structInitializer.accept(this); */
/* } */
/* override void visit(const EnumBody enumBody) */
/* { */
/* astInformation.doubleNewlineLocations ~= enumBody.endLocation; */
/* enumBody.accept(this); */
/* } */
/* override void visit(const Unittest unittest_) */
/* { */
/* astInformation.doubleNewlineLocations ~= unittest_.blockStatement.endLocation; */
/* unittest_.accept(this); */
/* } */
/* override void visit(const Invariant invariant_) */
/* { */
/* if (invariant_.blockStatement !is null) */
/* astInformation.doubleNewlineLocations ~= invariant_.blockStatement.endLocation; */
/* invariant_.accept(this); */
/* } */
/* override void visit(const StructBody structBody) */
/* { */
/* astInformation.aggregateBodyLocations ~= structBody.startLocation; */
/* astInformation.doubleNewlineLocations ~= structBody.endLocation; */
/* structBody.accept(this); */
/* } */
/* override void visit(const TemplateDeclaration templateDeclaration) */
/* { */
/* astInformation.doubleNewlineLocations ~= templateDeclaration.endLocation; */
/* templateDeclaration.accept(this); */
/* } */
/* override void visit(const TypeSuffix typeSuffix) */
/* { */
/* if (typeSuffix.star.type != TOK.error) */
/* astInformation.spaceAfterLocations ~= typeSuffix.star.index; */
/* typeSuffix.accept(this); */
/* } */
/* override void visit(const UnaryExpression unary) */
/* { */
/* import std.typecons : rebindable; */
/* int chainLength; */
/* auto u = rebindable(unary); */
/* while (u !is null) */
/* { */
/* if (u.identifierOrTemplateInstance !is null */
/* && u.identifierOrTemplateInstance.templateInstance !is null) */
/* chainLength++; */
/* u = u.unaryExpression; */
/* } */
/* if (chainLength > 1) */
/* { */
/* u = unary; */
/* while (u.unaryExpression !is null) */
/* { */
/* astInformation.ufcsHintLocations ~= u.dotLocation; */
/* u = u.unaryExpression; */
/* } */
/* } */
/* if (unary.prefix.type == TOK.tilde || unary.prefix.type == TOK.and */
/* || unary.prefix.type == TOK.mul */
/* || unary.prefix.type == TOK.add || unary.prefix.type == TOK.min) */
/* { */
/* astInformation.unaryLocations ~= unary.prefix.index; */
/* } */
/* unary.accept(this); */
/* } */
/* override void visit(const AttributeDeclaration attributeDeclaration) */
/* { */
/* astInformation.attributeDeclarationLines ~= attributeDeclaration.line; */
/* attributeDeclaration.accept(this); */
/* } */
/* override void visit(const FunctionAttribute functionAttribute) */
/* { */
/* if (functionAttribute.atAttribute !is null) */
/* astInformation.atAttributeStartLocations ~= functionAttribute.atAttribute.startLocation; */
/* functionAttribute.accept(this); */
/* } */
/* override void visit(const MemberFunctionAttribute memberFunctionAttribute) */
/* { */
/* if (memberFunctionAttribute.atAttribute !is null) */
/* astInformation.atAttributeStartLocations ~= memberFunctionAttribute */
/* .atAttribute.startLocation; */
/* memberFunctionAttribute.accept(this); */
/* } */
/* override void visit(const Attribute attribute) */
/* { */
/* if (attribute.atAttribute !is null) */
/* astInformation.atAttributeStartLocations ~= attribute.atAttribute.startLocation; */
/* attribute.accept(this); */
/* } */
/* override void visit(const StorageClass storageClass) */
/* { */
/* if (storageClass.atAttribute !is null) */
/* astInformation.atAttributeStartLocations ~= storageClass.atAttribute.startLocation; */
/* storageClass.accept(this); */
/* } */
/* override void visit(const InContractExpression inContractExpression) */
/* { */
/* astInformation.contractLocations ~= inContractExpression.inTokenLocation; */
/* inContractExpression.accept(this); */
/* } */
/* override void visit(const InStatement inStatement) */
/* { */
/* astInformation.contractLocations ~= inStatement.inTokenLocation; */
/* inStatement.accept(this); */
/* } */
/* override void visit(const OutContractExpression outContractExpression) */
/* { */
/* astInformation.contractLocations ~= outContractExpression.outTokenLocation; */
/* outContractExpression.accept(this); */
/* } */
/* override void visit(const OutStatement outStatement) */
/* { */
/* astInformation.contractLocations ~= outStatement.outTokenLocation; */
/* outStatement.accept(this); */
/* } */
override void visit(const TernaryExpression ternaryExpression)
{
astInformation.ternaryColonLocations ~= ternaryExpression.colon.index;
ternaryExpression.accept(this);
}
override void visit(const FunctionCallExpression functionCall)
{
// Check if function has any arguments.
if (functionCall.arguments.namedArgumentList is null)
{
functionCall.accept(this);
return;
}
foreach (item; functionCall.arguments.namedArgumentList.items)
{
// Do nothing if not a named argument.
if (item.name == tok!"")
{
continue;
}
// Find first colon if named argument.
foreach (t; item.tokens)
{
if (t.type == tok!":")
{
astInformation.namedArgumentColonLocations ~= t.index;
break;
}
}
}
functionCall.accept(this);
}
private:
ASTInformation* astInformation;
}

File diff suppressed because it is too large Load Diff

View File

@ -1,392 +0,0 @@
// Copyright Brian Schott 2015.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dfmt.indentation;
import dfmt.config;
import dfmt.editorconfig;
import dmd.tokens;
import std.bitmanip : bitfields;
/**
* Returns: true if the given token type is a wrap indent type
*/
bool isWrapIndent(TOK type) pure nothrow @nogc @safe
{
switch (type)
{
case TOK.leftCurly:
case TOK.case_:
case TOK.at:
case TOK.rightBracket:
case TOK.leftParenthesis:
case TOK.rightParenthesis:
return false;
// Operators
case TOK.lessThan:
case TOK.greaterThan:
case TOK.lessOrEqual:
case TOK.greaterOrEqual:
case TOK.equal:
case TOK.notEqual:
case TOK.identity:
case TOK.notIdentity:
case TOK.is_:
case TOK.leftShift:
case TOK.rightShift:
case TOK.leftShiftAssign:
case TOK.rightShiftAssign:
case TOK.unsignedRightShift:
case TOK.unsignedRightShiftAssign:
case TOK.concatenateAssign:
case TOK.add:
case TOK.min:
case TOK.addAssign:
case TOK.minAssign:
case TOK.mul:
case TOK.div:
case TOK.mod:
case TOK.mulAssign:
case TOK.divAssign:
case TOK.modAssign:
case TOK.and:
case TOK.or:
case TOK.xor:
case TOK.andAssign:
case TOK.orAssign:
case TOK.xorAssign:
case TOK.assign:
case TOK.not:
case TOK.tilde:
case TOK.plusPlus:
case TOK.minusMinus:
case TOK.dot:
case TOK.comma:
case TOK.question:
case TOK.andAnd:
case TOK.orOr:
return true;
default:
return false;
}
}
/**
* Returns: true if the given token type is a temporary indent type
*/
bool isTempIndent(TOK type) pure nothrow @nogc @safe
{
return type != TOK.rightParenthesis && type != TOK.leftCurly && type != TOK.case_ && type != TOK
.at;
}
/**
* Stack for managing indent levels.
*/
struct IndentStack
{
/// Configuration
private const Config* config;
this(const Config* config)
{
this.config = config;
}
static struct Details
{
mixin(bitfields!(
// generally true for all operators except {, case, @, ], (, )
bool, "wrap", 1, // temporary indentation which get's reverted when a block starts
// generally true for all tokens except ), {, case, @
bool, "temp", 1, // emit minimal newlines
bool, "mini", 1, // for associative arrays or arrays containing them, break after every item
bool, "breakEveryItem", 1, // when an item inside an array would break mid-item, definitely break at the comma first
bool, "preferLongBreaking", 1,
uint, "", 27));
}
/**
* Get the indent size at the most recent occurrence of the given indent type
*/
int indentToMostRecent(TOK item) const
{
if (index == 0)
return -1;
size_t i = index - 1;
while (true)
{
if (arr[i] == item)
return indentSize(i);
if (i > 0)
i--;
else
return -1;
}
}
int wrapIndents() const pure nothrow @property
{
if (index == 0)
return 0;
int tempIndentCount = 0;
for (size_t i = index; i > 0; i--)
{
if (!details[i - 1].wrap && arr[i - 1] != TOK.rightBracket)
break;
tempIndentCount++;
}
return tempIndentCount;
}
/**
* Pushes the given indent type on to the stack.
*/
void push(TOK item) pure nothrow
{
Details detail;
detail.wrap = isWrapIndent(item);
detail.temp = isTempIndent(item);
push(item, detail);
}
/**
* Pushes the given indent type on to the stack.
*/
void push(TOK item, Details detail) pure nothrow
{
arr[index] = item;
details[index] = detail;
//FIXME this is actually a bad thing to do,
//we should not just override when the stack is
//at it's limit
if (index < arr.length)
{
index++;
}
}
/**
* Pops the top indent from the stack.
*/
void pop() pure nothrow
{
if (index)
index--;
}
/**
* Pops all wrapping indents from the top of the stack.
*/
void popWrapIndents() pure nothrow @safe @nogc
{
while (index > 0 && details[index - 1].wrap)
index--;
}
/**
* Pops all temporary indents from the top of the stack.
*/
void popTempIndents() pure nothrow @safe @nogc
{
while (index > 0 && details[index - 1].temp)
index--;
}
bool topAre(TOK[] types...)
{
if (types.length > index)
return false;
return arr[index - types.length .. index] == types;
}
/**
* Returns: `true` if the top of the indent stack is the given indent type.
*/
bool topIs(TOK type) const pure nothrow @safe @nogc
{
return index > 0 && index <= arr.length && arr[index - 1] == type;
}
/**
* Returns: `true` if the top of the indent stack is a temporary indent
*/
bool topIsTemp()
{
return index > 0 && index <= arr.length && details[index - 1].temp;
}
/**
* Returns: `true` if the top of the indent stack is a temporary indent with the specified token
*/
bool topIsTemp(TOK item)
{
return index > 0 && index <= arr.length && arr[index - 1] == item && details[index - 1]
.temp;
}
/**
* Returns: `true` if the top of the indent stack is a wrapping indent
*/
bool topIsWrap()
{
return index > 0 && index <= arr.length && details[index - 1].wrap;
}
/**
* Returns: `true` if the top of the indent stack is a temporary indent with the specified token
*/
bool topIsWrap(TOK item)
{
return index > 0 && index <= arr.length && arr[index - 1] == item && details[index - 1]
.wrap;
}
/**
* Returns: `true` if the top of the indent stack is one of the given token
* types.
*/
bool topIsOneOf(TOK[] types...) const pure nothrow @safe @nogc
{
if (index == 0)
return false;
immutable topType = arr[index - 1];
foreach (t; types)
if (t == topType)
return true;
return false;
}
TOK top() const pure nothrow @property @safe @nogc
{
return arr[index - 1];
}
Details topDetails() const pure nothrow @property @safe @nogc
{
return details[index - 1];
}
int indentLevel() const pure nothrow @property @safe @nogc
{
return indentSize();
}
int length() const pure nothrow @property @safe @nogc
{
return cast(int) index;
}
/**
* Dumps the current state of the indentation stack to `stderr`. Used for debugging.
*/
void dump(size_t pos = size_t.max, string file = __FILE__, uint line = __LINE__) const
{
import std.algorithm.iteration : map;
import std.stdio : stderr;
if (pos == size_t.max)
stderr.writefln("\033[31m%s:%d %(%s %)\033[0m", file, line, arr[0 .. index].map!(
a => Token.toString(a)));
else
stderr.writefln("\033[31m%s:%d at %d %(%s %)\033[0m", file, line, pos, arr[0 .. index].map!(
a => Token.toString(a)));
}
private:
size_t index;
TOK[256] arr;
Details[arr.length] details;
int indentSize(const size_t k = size_t.max) const pure nothrow @safe @nogc
{
import std.algorithm : among;
if (index == 0 || k == 0)
return 0;
immutable size_t j = k == size_t.max ? index : k;
int size = 0;
int parenCount;
foreach (i; 0 .. j)
{
immutable int pc = (arr[i] == TOK.not || arr[i] == TOK.leftParenthesis || arr[i] == TOK
.rightParenthesis) ? parenCount + 1 : parenCount;
if ((details[i].wrap || arr[i] == TOK.leftParenthesis) && parenCount > 1)
{
parenCount = pc;
continue;
}
if (i + 1 < index)
{
if (config.dfmt_single_indent == OptionalBoolean.t && skipDoubleIndent(i, parenCount))
{
parenCount = pc;
continue;
}
immutable currentIsNonWrapTemp = !details[i].wrap
&& details[i].temp && arr[i] != TOK.rightParenthesis && arr[i] != TOK.not;
if (currentIsNonWrapTemp && arr[i + 1] == TOK.rightBracket)
{
parenCount = pc;
continue;
}
if (arr[i] == TOK.static_
&& arr[i + 1].among!(TOK.if_, TOK.else_, TOK.foreach_, TOK.foreach_reverse_)
&& (i + 2 >= index || arr[i + 2] != TOK.leftCurly))
{
parenCount = pc;
continue;
}
if (currentIsNonWrapTemp && (arr[i + 1] == TOK.switch_
|| arr[i + 1] == TOK.leftCurly || arr[i + 1] == TOK.rightParenthesis))
{
parenCount = pc;
continue;
}
}
else if (parenCount == 0 && arr[i] == TOK.leftParenthesis && config.dfmt_single_indent == OptionalBoolean
.f)
size++;
if (arr[i] == TOK.not)
size++;
parenCount = pc;
size++;
}
return size;
}
bool skipDoubleIndent(size_t i, int parenCount) const pure nothrow @safe @nogc
{
return (details[i + 1].wrap && arr[i] == TOK.rightParenthesis)
|| (parenCount == 0 && arr[i + 1] == TOK.comma && arr[i] == TOK.leftParenthesis);
}
}
unittest
{
IndentStack stack;
stack.push(TOK.leftCurly);
assert(stack.length == 1);
assert(stack.indentLevel == 1);
stack.pop();
assert(stack.length == 0);
assert(stack.indentLevel == 0);
stack.push(TOK.if_);
assert(stack.topIsTemp());
stack.popTempIndents();
assert(stack.length == 0);
}

View File

@ -30,91 +30,85 @@ static immutable VERSION = () {
}
return DFMT_VERSION ~ DEBUG_SUFFIX;
} ();
}();
import dfmt.config : Config;
import dfmt.editorconfig : getConfigFor;
import dfmt.formatter : format;
import std.array : appender, front, popFront;
import std.getopt : getopt, GetOptException;
import std.path : buildPath, dirName, expandTilde;
import std.stdio : File, stderr, stdin, stdout, writeln;
version (NoMain)
int main(string[] args)
{
}
else
{
import dfmt.config : Config;
import dfmt.editorconfig : getConfigFor;
import dfmt.formatter : format;
import std.array : appender, front, popFront;
import std.getopt : getopt, GetOptException;
import std.path : buildPath, dirName, expandTilde;
import std.stdio : File, stderr, stdin, stdout, writeln;
bool inplace = false;
Config optConfig;
optConfig.pattern = "*.d";
bool showHelp;
bool showVersion;
string explicitConfigDir;
int main(string[] args)
void handleBooleans(string option, string value)
{
bool inplace = false;
Config optConfig;
optConfig.pattern = "*.d";
bool showHelp;
bool showVersion;
string explicitConfigDir;
import dfmt.editorconfig : OptionalBoolean;
import std.exception : enforce;
void handleBooleans(string option, string value)
enforce!GetOptException(value == "true" || value == "false", "Invalid argument");
immutable OptionalBoolean optVal = value == "true" ? OptionalBoolean.t
: OptionalBoolean.f;
switch (option)
{
import dfmt.editorconfig : OptionalBoolean;
import std.exception : enforce;
enforce!GetOptException(value == "true" || value == "false", "Invalid argument");
immutable OptionalBoolean optVal = value == "true" ? OptionalBoolean.t
: OptionalBoolean.f;
switch (option)
{
case "align_switch_statements":
optConfig.dfmt_align_switch_statements = optVal;
break;
case "outdent_attributes":
optConfig.dfmt_outdent_attributes = optVal;
break;
case "space_after_cast":
optConfig.dfmt_space_after_cast = optVal;
break;
case "space_before_function_parameters":
optConfig.dfmt_space_before_function_parameters = optVal;
break;
case "split_operator_at_line_end":
optConfig.dfmt_split_operator_at_line_end = optVal;
break;
case "selective_import_space":
optConfig.dfmt_selective_import_space = optVal;
break;
case "compact_labeled_statements":
optConfig.dfmt_compact_labeled_statements = optVal;
break;
case "single_template_constraint_indent":
optConfig.dfmt_single_template_constraint_indent = optVal;
break;
case "space_before_aa_colon":
optConfig.dfmt_space_before_aa_colon = optVal;
break;
case "space_before_named_arg_colon":
optConfig.dfmt_space_before_named_arg_colon = optVal;
break;
case "keep_line_breaks":
optConfig.dfmt_keep_line_breaks = optVal;
break;
case "single_indent":
optConfig.dfmt_single_indent = optVal;
break;
case "reflow_property_chains":
optConfig.dfmt_reflow_property_chains = optVal;
break;
case "space_after_keywords":
optConfig.dfmt_space_after_keywords = optVal;
break;
default:
assert(false, "Invalid command-line switch");
}
case "align_switch_statements":
optConfig.dfmt_align_switch_statements = optVal;
break;
case "outdent_attributes":
optConfig.dfmt_outdent_attributes = optVal;
break;
case "space_after_cast":
optConfig.dfmt_space_after_cast = optVal;
break;
case "space_before_function_parameters":
optConfig.dfmt_space_before_function_parameters = optVal;
break;
case "split_operator_at_line_end":
optConfig.dfmt_split_operator_at_line_end = optVal;
break;
case "selective_import_space":
optConfig.dfmt_selective_import_space = optVal;
break;
case "compact_labeled_statements":
optConfig.dfmt_compact_labeled_statements = optVal;
break;
case "single_template_constraint_indent":
optConfig.dfmt_single_template_constraint_indent = optVal;
break;
case "space_before_aa_colon":
optConfig.dfmt_space_before_aa_colon = optVal;
break;
case "space_before_named_arg_colon":
optConfig.dfmt_space_before_named_arg_colon = optVal;
break;
case "keep_line_breaks":
optConfig.dfmt_keep_line_breaks = optVal;
break;
case "single_indent":
optConfig.dfmt_single_indent = optVal;
break;
case "reflow_property_chains":
optConfig.dfmt_reflow_property_chains = optVal;
break;
case "space_after_keywords":
optConfig.dfmt_space_after_keywords = optVal;
break;
default:
assert(false, "Invalid command-line switch");
}
}
try
{
// dfmt off
try
{
// dfmt off
getopt(args,
"version", &showVersion,
"align_switch_statements", &handleBooleans,
@ -143,68 +137,116 @@ else
"single_indent", &handleBooleans,
"reflow_property_chains", &handleBooleans);
// dfmt on
}
catch (GetOptException e)
}
catch (GetOptException e)
{
stderr.writeln(e.msg);
return 1;
}
if (showVersion)
{
writeln(VERSION);
return 0;
}
if (showHelp)
{
printHelp();
return 0;
}
args.popFront();
immutable bool readFromStdin = args.length == 0;
version (Windows)
{
// On Windows, set stdout to binary mode (needed for correct EOL writing)
// See Phobos' stdio.File.rawWrite
{
stderr.writeln(e.msg);
import std.stdio : _O_BINARY;
immutable fd = stdout.fileno;
_setmode(fd, _O_BINARY);
version (CRuntime_DigitalMars)
{
import core.atomic : atomicOp;
import core.stdc.stdio : __fhnd_info, FHND_TEXT;
atomicOp!"&="(__fhnd_info[fd], ~FHND_TEXT);
}
}
}
ubyte[] buffer;
Config explicitConfig;
if (explicitConfigDir)
{
import std.file : exists, isDir;
if (!exists(explicitConfigDir) || !isDir(explicitConfigDir))
{
stderr.writeln("--config|c must specify existing directory path");
return 1;
}
explicitConfig = getConfigFor!Config(explicitConfigDir);
explicitConfig.pattern = "*.d";
}
if (showVersion)
if (readFromStdin)
{
import std.file : getcwd;
auto cwdDummyPath = buildPath(getcwd(), "dummy.d");
Config config;
config.initializeWithDefaults();
if (explicitConfigDir != "")
{
writeln(VERSION);
return 0;
config.merge(explicitConfig, buildPath(explicitConfigDir, "dummy.d"));
}
if (showHelp)
else
{
printHelp();
return 0;
Config fileConfig = getConfigFor!Config(getcwd());
fileConfig.pattern = "*.d";
config.merge(fileConfig, cwdDummyPath);
}
args.popFront();
immutable bool readFromStdin = args.length == 0;
version (Windows)
config.merge(optConfig, cwdDummyPath);
if (!config.isValid())
return 1;
ubyte[4096] inputBuffer;
ubyte[] b;
while (true)
{
// On Windows, set stdout to binary mode (needed for correct EOL writing)
// See Phobos' stdio.File.rawWrite
b = stdin.rawRead(inputBuffer);
if (b.length)
buffer ~= b;
else
break;
}
immutable bool formatSuccess = format("stdin", buffer,
stdout.lockingTextWriter(), &config);
return formatSuccess ? 0 : 1;
}
else
{
import std.file : dirEntries, isDir, SpanMode;
if (args.length >= 2)
inplace = true;
int retVal;
while (args.length > 0)
{
const path = args.front;
args.popFront();
if (isDir(path))
{
import std.stdio : _O_BINARY;
immutable fd = stdout.fileno;
_setmode(fd, _O_BINARY);
version (CRuntime_DigitalMars)
{
import core.atomic : atomicOp;
import core.stdc.stdio : __fhnd_info, FHND_TEXT;
atomicOp!"&="(__fhnd_info[fd], ~FHND_TEXT);
}
inplace = true;
foreach (string name; dirEntries(path, "*.d", SpanMode.depth))
args ~= name;
continue;
}
}
ubyte[] buffer;
Config explicitConfig;
if (explicitConfigDir)
{
import std.file : exists, isDir;
if (!exists(explicitConfigDir) || !isDir(explicitConfigDir))
{
stderr.writeln("--config|c must specify existing directory path");
return 1;
}
explicitConfig = getConfigFor!Config(explicitConfigDir);
explicitConfig.pattern = "*.d";
}
if (readFromStdin)
{
import std.file : getcwd;
auto cwdDummyPath = buildPath(getcwd(), "dummy.d");
Config config;
config.initializeWithDefaults();
if (explicitConfigDir != "")
@ -213,94 +255,42 @@ else
}
else
{
Config fileConfig = getConfigFor!Config(getcwd());
Config fileConfig = getConfigFor!Config(path);
fileConfig.pattern = "*.d";
config.merge(fileConfig, cwdDummyPath);
config.merge(fileConfig, path);
}
config.merge(optConfig, cwdDummyPath);
config.merge(optConfig, path);
if (!config.isValid())
return 1;
ubyte[4096] inputBuffer;
ubyte[] b;
while (true)
File f = File(path);
// ignore empty files
if (f.size)
{
b = stdin.rawRead(inputBuffer);
if (b.length)
buffer ~= b;
buffer = new ubyte[](cast(size_t) f.size);
f.rawRead(buffer);
File.LockingTextWriter output;
if (inplace)
output = File(path, "wb").lockingTextWriter();
else
break;
output = stdout.lockingTextWriter();
immutable bool formatSuccess = format(path, buffer, output, &config);
retVal = formatSuccess ? 0 : 1;
}
immutable bool formatSuccess = format("stdin", buffer,
stdout.lockingTextWriter(), &config);
return formatSuccess ? 0 : 1;
}
else
{
import std.file : dirEntries, isDir, SpanMode;
if (args.length >= 2)
inplace = true;
int retVal;
while (args.length > 0)
{
const path = args.front;
args.popFront();
if (isDir(path))
{
inplace = true;
foreach (string name; dirEntries(path, "*.d", SpanMode.depth))
args ~= name;
continue;
}
Config config;
config.initializeWithDefaults();
if (explicitConfigDir != "")
{
config.merge(explicitConfig, buildPath(explicitConfigDir, "dummy.d"));
}
else
{
Config fileConfig = getConfigFor!Config(path);
fileConfig.pattern = "*.d";
config.merge(fileConfig, path);
}
config.merge(optConfig, path);
if (!config.isValid())
return 1;
File f = File(path);
// ignore empty files
if (f.size)
{
buffer = new ubyte[](cast(size_t) f.size);
f.rawRead(buffer);
auto output = appender!string;
immutable bool formatSuccess = format(path, buffer, output, &config);
if (formatSuccess)
{
if (inplace)
File(path, "wb").rawWrite(output.data);
else
stdout.rawWrite(output.data);
}
else
retVal = 1;
}
}
return retVal;
}
return retVal;
}
}
private version (Windows)
{
version(CRuntime_DigitalMars)
version (CRuntime_DigitalMars)
{
extern(C) int setmode(int, int) nothrow @nogc;
extern (C) int setmode(int, int) nothrow @nogc;
alias _setmode = setmode;
}
else version(CRuntime_Microsoft)
else version (CRuntime_Microsoft)
{
extern(C) int _setmode(int, int) nothrow @nogc;
extern (C) int _setmode(int, int) nothrow @nogc;
}
}
@ -318,11 +308,14 @@ template optionsToString(E) if (is(E == enum))
}
result = result[0 .. $ - 1] ~ ")";
return result;
} ();
}();
}
private void printHelp()
{
import std.stdio : writeln;
import dfmt.config;
writeln(`dfmt `, VERSION, `
https://github.com/dlang-community/dfmt
@ -335,11 +328,11 @@ Options:
Formatting Options:
--align_switch_statements
--brace_style `, optionsToString!(typeof(Config.dfmt_brace_style)),
`
`
--end_of_line `, optionsToString!(typeof(Config.end_of_line)), `
--indent_size
--indent_style, -t `,
optionsToString!(typeof(Config.indent_style)), `
optionsToString!(typeof(Config.indent_style)), `
--keep_line_breaks
--soft_max_line_length
--max_line_length
@ -357,13 +350,13 @@ Formatting Options:
--single_indent
--reflow_property_chains
`,
optionsToString!(typeof(Config.dfmt_template_constraint_style)));
optionsToString!(typeof(Config.dfmt_template_constraint_style)));
}
private string createFilePath(bool readFromStdin, string fileName)
{
import std.file : getcwd;
import std.path : isRooted;
import std.path : isRooted, buildPath;
immutable string cwd = getcwd();
if (readFromStdin)

View File

@ -1,247 +0,0 @@
// Copyright Brian Schott 2015.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dfmt.tokens;
import dmd.tokens;
/// Length of an invalid token
enum int INVALID_TOKEN_LENGTH = -1;
uint betweenParenLength(const Token[] tokens) @safe
in
{
assert(tokens[0].value == TOK.leftParenthesis);
}
do
{
uint length = 0;
size_t i = 1;
int depth = 1;
while (i < tokens.length && depth > 0)
{
if (tokens[i].value == TOK.leftParenthesis)
depth++;
else if (tokens[i].value == TOK.rightParenthesis)
depth--;
length += tokenLength(tokens[i]);
i++;
}
return length;
}
int tokenLength(ref const Token t) @safe
{
import std.algorithm : countUntil;
if (t.isKeyword())
return cast(int) Token.toString(t.value).length;
int c;
switch (t.value)
{
// Numeric literals
case TOK.int32Literal:
case TOK.uns32Literal:
case TOK.int64Literal:
case TOK.uns64Literal:
case TOK.int128Literal:
case TOK.uns128Literal:
case TOK.float32Literal:
case TOK.float64Literal:
case TOK.float80Literal:
case TOK.imaginary32Literal:
case TOK.imaginary64Literal:
case TOK.imaginary80Literal:
// Char constants
case TOK.charLiteral:
case TOK.wcharLiteral:
case TOK.dcharLiteral:
// Identifiers
case TOK.identifier:
return cast(int) Token.toString(t.value).length;
// Spaced operators
case TOK.add:
case TOK.addAssign:
case TOK.and:
case TOK.andAnd:
case TOK.andAssign:
case TOK.arrow:
case TOK.assign:
case TOK.colon:
case TOK.colonColon:
case TOK.comma:
case TOK.concatenateAssign:
case TOK.div:
case TOK.divAssign:
case TOK.dot:
case TOK.dotDotDot:
case TOK.equal:
case TOK.goesTo:
case TOK.greaterOrEqual:
case TOK.greaterThan:
case TOK.identity:
case TOK.is_:
case TOK.leftShift:
case TOK.leftShiftAssign:
case TOK.lessOrEqual:
case TOK.lessThan:
case TOK.min:
case TOK.minAssign:
case TOK.minusMinus:
case TOK.mod:
case TOK.modAssign:
case TOK.mul:
case TOK.mulAssign:
case TOK.not:
case TOK.notEqual:
case TOK.notIdentity:
case TOK.or:
case TOK.orAssign:
case TOK.orOr:
case TOK.plusPlus:
case TOK.pound:
case TOK.pow:
case TOK.powAssign:
case TOK.question:
case TOK.rightShift:
case TOK.rightShiftAssign:
case TOK.semicolon:
case TOK.slice:
case TOK.tilde:
case TOK.unsignedRightShift:
case TOK.unsignedRightShiftAssign:
case TOK.xor:
case TOK.xorAssign:
return cast(int) Token.toString(t.value).length + 1;
case TOK.string_:
// TODO: Unicode line breaks and old-Mac line endings
c = cast(int) Token.toString(t.value).countUntil('\n');
if (c == -1)
return cast(int) Token.toString(t.value).length;
else
return c;
default:
return INVALID_TOKEN_LENGTH;
}
}
bool isBreakToken(TOK t) pure nothrow @safe @nogc
{
switch (t)
{
case TOK.orOr:
case TOK.andAnd:
case TOK.leftParenthesis:
case TOK.leftBracket:
case TOK.comma:
case TOK.colon:
case TOK.semicolon:
case TOK.pow:
case TOK.powAssign:
case TOK.xor:
case TOK.concatenateAssign:
case TOK.leftShiftAssign:
case TOK.leftShift:
case TOK.lessOrEqual:
case TOK.lessThan:
case TOK.equal:
case TOK.goesTo:
case TOK.assign:
case TOK.greaterOrEqual:
case TOK.rightShiftAssign:
case TOK.unsignedRightShift:
case TOK.unsignedRightShiftAssign:
case TOK.rightShift:
case TOK.greaterThan:
case TOK.orAssign:
case TOK.or:
case TOK.minAssign:
case TOK.notEqual:
case TOK.question:
case TOK.divAssign:
case TOK.div:
case TOK.slice:
case TOK.mulAssign:
case TOK.mul:
case TOK.andAssign:
case TOK.modAssign:
case TOK.mod:
case TOK.addAssign:
case TOK.dot:
case TOK.tilde:
case TOK.add:
case TOK.min:
return true;
default:
return false;
}
}
int breakCost(TOK p, TOK c) pure nothrow @safe @nogc
{
switch (c)
{
case TOK.orOr:
case TOK.andAnd:
case TOK.comma:
case TOK.question:
return 0;
case TOK.leftParenthesis:
return 60;
case TOK.leftBracket:
return 300;
case TOK.semicolon:
case TOK.pow:
case TOK.xorAssign:
case TOK.xor:
case TOK.concatenateAssign:
case TOK.leftShiftAssign:
case TOK.leftShift:
case TOK.lessOrEqual:
case TOK.lessThan:
case TOK.equal:
case TOK.goesTo:
case TOK.assign:
case TOK.greaterOrEqual:
case TOK.rightShiftAssign:
case TOK.unsignedRightShiftAssign:
case TOK.unsignedRightShift:
case TOK.rightShift:
case TOK.greaterThan:
case TOK.orAssign:
case TOK.or:
case TOK.minAssign:
case TOK.divAssign:
case TOK.div:
case TOK.slice:
case TOK.mulAssign:
case TOK.mul:
case TOK.andAssign:
case TOK.modAssign:
case TOK.mod:
case TOK.add:
case TOK.min:
case TOK.tilde:
case TOK.addAssign:
return 200;
case TOK.colon:
// colon could be after a label or an import, where it should normally wrap like before
// for everything else (associative arrays) try not breaking around colons
return p == TOK.identifier ? 0 : 300;
case TOK.dot:
return p == TOK.rightParenthesis ? 0 : 300;
default:
return 1000;
}
}
pure nothrow @safe @nogc unittest
{
foreach (ubyte u; 0 .. ubyte.max)
if (isBreakToken(u))
assert(breakCost(TOK.dot, u) != 1000);
}

View File

@ -1,182 +0,0 @@
// Copyright Brian Schott 2015.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dfmt.wrapping;
import dmd.tokens;
import dfmt.tokens;
import dfmt.config;
struct State
{
this(uint breaks, const Token[] tokens, immutable short[] depths,
const Config* config, int currentLineLength, int indentLevel) @safe
{
import std.math : abs;
import core.bitop : popcnt, bsf;
import std.algorithm : min, map, sum;
immutable int remainingCharsMultiplier = 25;
immutable int newlinePenalty = 480;
this.breaks = breaks;
this._cost = 0;
this._solved = true;
if (breaks == 0)
{
immutable int l = currentLineLength + tokens.map!(a => tokenLength(a)).sum();
if (l > config.dfmt_soft_max_line_length)
{
immutable int longPenalty = (l - config.dfmt_soft_max_line_length)
* remainingCharsMultiplier;
this._cost += longPenalty;
this._solved = longPenalty < newlinePenalty;
}
}
else
{
foreach (size_t i; 0 .. (uint.sizeof * 8))
{
if (((1 << i) & breaks) == 0)
continue;
immutable prevType = i > 0 ? tokens[i - 1].value : TOK.error;
immutable currentType = tokens[i].value;
immutable p = abs(depths[i]);
immutable bc = breakCost(prevType, currentType) * (p == 0 ? 1 : p * 2);
this._cost += bc + newlinePenalty;
}
int ll = currentLineLength;
size_t i = 0;
foreach (_; 0 .. uint.sizeof * 8)
{
immutable uint k = breaks >>> i;
immutable bool b = k == 0;
immutable uint bits = b ? ALGORITHMIC_COMPLEXITY_SUCKS : bsf(k);
immutable size_t j = min(i + bits + 1, tokens.length);
ll += tokens[i .. j].map!(a => tokenLength(a)).sum();
if (ll > config.dfmt_soft_max_line_length)
{
immutable int longPenalty = (ll - config.dfmt_soft_max_line_length)
* remainingCharsMultiplier;
this._cost += longPenalty;
}
if (ll > config.max_line_length)
{
this._solved = false;
break;
}
i = j;
if (indentLevel < 0)
ll = (abs(indentLevel) + 1) * config.indent_size;
else
ll = (indentLevel + (i == 0 ? 0 : 1)) * config.indent_size;
if (b)
break;
}
}
}
int cost() const pure nothrow @safe @property
{
return _cost;
}
int solved() const pure nothrow @safe @property
{
return _solved;
}
int opCmp(ref const State other) const pure nothrow @safe
{
import core.bitop : bsf, popcnt;
if (_cost < other._cost)
return -1;
if (_cost == other._cost && (breaks != 0 && other.breaks != 0
&& bsf(breaks) > bsf(other.breaks)))
return -1;
return _cost > other._cost;
}
bool opEquals(ref const State other) const pure nothrow @safe
{
return other.breaks == breaks;
}
size_t toHash() const pure nothrow @safe
{
return breaks;
}
uint breaks;
private:
int _cost;
bool _solved;
}
private enum ALGORITHMIC_COMPLEXITY_SUCKS = uint.sizeof * 8;
/**
* Note: Negative values for `indentLevel` are treated specially: costs for
* continuation indents are reduced. This is used for array literals.
*/
size_t[] chooseLineBreakTokens(size_t index, const Token[] tokens,
immutable short[] depths, const Config* config, int currentLineLength, int indentLevel)
{
import std.container.rbtree : RedBlackTree;
import std.algorithm : filter, min;
import core.bitop : popcnt;
static size_t[] genRetVal(uint breaks, size_t index) pure nothrow @safe
{
auto retVal = new size_t[](popcnt(breaks));
size_t j = 0;
foreach (uint i; 0 .. uint.sizeof * 8)
if ((1 << i) & breaks)
retVal[j++] = index + i;
return retVal;
}
immutable size_t tokensEnd = min(tokens.length, ALGORITHMIC_COMPLEXITY_SUCKS);
auto open = new RedBlackTree!State;
open.insert(State(0, tokens[0 .. tokensEnd], depths[0 .. tokensEnd], config,
currentLineLength, indentLevel));
State lowest;
lowest._solved = false;
int tries = 0;
while (!open.empty && tries < 10_00)
{
tries++;
State current = open.front();
open.removeFront();
if (current.solved)
return genRetVal(current.breaks, index);
if (current < lowest)
lowest = current;
validMoves!(typeof(open))(open, tokens[0 .. tokensEnd], depths[0 .. tokensEnd],
current.breaks, config, currentLineLength, indentLevel);
}
foreach (r; open[].filter!(a => a.solved))
return genRetVal(r.breaks, index);
if (open[].front < lowest)
return genRetVal(open[].front.breaks, index);
else
return genRetVal(lowest.breaks, index);
}
void validMoves(OR)(auto ref OR output, const Token[] tokens, immutable short[] depths,
uint current, const Config* config, int currentLineLength, int indentLevel)
{
foreach (i, token; tokens)
{
if (!isBreakToken(token.value) || (((1 << i) & current) != 0))
continue;
immutable uint breaks = current | (1 << i);
output.insert(State(breaks, tokens, depths, config, currentLineLength, indentLevel));
}
}