whitespace
This commit is contained in:
Hackerpilot
parent
c1fcef1873
commit
dc81410008
178
std/d/lexer.d
178
std/d/lexer.d
|
|
@ -4,7 +4,7 @@
|
|||
* This module contains a range-based _lexer for the D programming language.
|
||||
*
|
||||
* For performance reasons the _lexer contained in this module operates only on
|
||||
* ASCII and UTF-8 encoded source code. If the use of other encodings is
|
||||
* ASCII or UTF-8 encoded source code. If the use of other encodings is
|
||||
* desired, the source code must be converted to UTF-8 before passing it to this
|
||||
* _lexer.
|
||||
*
|
||||
|
|
@ -125,8 +125,8 @@ version (unittest) import std.stdio;
|
|||
public:
|
||||
|
||||
/**
|
||||
* Represents a D token
|
||||
*/
|
||||
* Represents a D token
|
||||
*/
|
||||
struct Token
|
||||
{
|
||||
/**
|
||||
|
|
@ -188,9 +188,9 @@ struct Token
|
|||
}
|
||||
|
||||
/**
|
||||
* Configure the behavior of the byToken() function. These flags may be
|
||||
* combined using a bitwise or.
|
||||
*/
|
||||
* Configure the behavior of the byToken() function. These flags may be
|
||||
* combined using a bitwise or.
|
||||
*/
|
||||
enum IterationStyle
|
||||
{
|
||||
/// Only include code, not whitespace or comments
|
||||
|
|
@ -208,9 +208,9 @@ enum IterationStyle
|
|||
}
|
||||
|
||||
/**
|
||||
* Configuration of the token lexing style. These flags may be combined with a
|
||||
* bitwise or.
|
||||
*/
|
||||
* Configuration of the token lexing style. These flags may be combined with a
|
||||
* bitwise or.
|
||||
*/
|
||||
enum TokenStyle : uint
|
||||
{
|
||||
/**
|
||||
|
|
@ -251,8 +251,8 @@ enum TokenStyle : uint
|
|||
}
|
||||
|
||||
/**
|
||||
* Lexer configuration
|
||||
*/
|
||||
* Lexer configuration
|
||||
*/
|
||||
struct LexerConfig
|
||||
{
|
||||
/**
|
||||
|
|
@ -292,14 +292,14 @@ struct LexerConfig
|
|||
}
|
||||
|
||||
/**
|
||||
* Iterate over the given range of characters by D tokens.
|
||||
* Params:
|
||||
* range = the range of characters
|
||||
* config = the lexer configuration
|
||||
* bufferSize = initial size of internal circular buffer
|
||||
* Returns:
|
||||
* an input range of tokens
|
||||
*/
|
||||
* Iterate over the given range of characters by D tokens.
|
||||
* Params:
|
||||
* range = the range of characters
|
||||
* config = the lexer configuration
|
||||
* bufferSize = initial size of internal circular buffer
|
||||
* Returns:
|
||||
* an input range of tokens
|
||||
*/
|
||||
auto byToken(R)(R range, LexerConfig config, size_t bufferSize = 4*1024)
|
||||
if (isForwardRange!(R) && !isRandomAccessRange!(R)
|
||||
&& is(ElementType!R : const(ubyte)))
|
||||
|
|
@ -326,8 +326,8 @@ auto byToken(R)(R range, LexerConfig config)
|
|||
}
|
||||
|
||||
/**
|
||||
* Range of tokens. Use byToken$(LPAREN)$(RPAREN) to instantiate.
|
||||
*/
|
||||
* Range of tokens. Use byToken$(LPAREN)$(RPAREN) to instantiate.
|
||||
*/
|
||||
struct TokenRange(LexSrc)
|
||||
//if ( is(LexSrc : LexSource!(U...), U...)) //check for LexSource
|
||||
{
|
||||
|
|
@ -431,15 +431,15 @@ L_advance:
|
|||
"=", "TokenType.assign",
|
||||
"@", "TokenType.at",
|
||||
"&", "TokenType.bitAnd",
|
||||
"&=", "TokenType.bitAndEquals",
|
||||
"&=", "TokenType.bitAndEqual",
|
||||
"|", "TokenType.bitOr",
|
||||
"|=", "TokenType.bitOrEquals",
|
||||
"~=", "TokenType.catEquals",
|
||||
"|=", "TokenType.bitOrEqual",
|
||||
"~=", "TokenType.catEqual",
|
||||
":", "TokenType.colon",
|
||||
",", "TokenType.comma",
|
||||
"--", "TokenType.decrement",
|
||||
"$", "TokenType.dollar",
|
||||
"==", "TokenType.equals",
|
||||
"==", "TokenType.equal",
|
||||
"=>", "TokenType.goesTo",
|
||||
">", "TokenType.greater",
|
||||
">=", "TokenType.greaterEqual",
|
||||
|
|
@ -454,21 +454,21 @@ L_advance:
|
|||
"||", "TokenType.logicOr",
|
||||
"(", "TokenType.lParen",
|
||||
"-", "TokenType.minus",
|
||||
"-=", "TokenType.minusEquals",
|
||||
"-=", "TokenType.minusEqual",
|
||||
"%", "TokenType.mod",
|
||||
"%=", "TokenType.modEquals",
|
||||
"*=", "TokenType.mulEquals",
|
||||
"%=", "TokenType.modEqual",
|
||||
"*=", "TokenType.mulEqual",
|
||||
"!", "TokenType.not",
|
||||
"!=", "TokenType.notEquals",
|
||||
"!=", "TokenType.notEqual",
|
||||
"!>", "TokenType.notGreater",
|
||||
"!>=", "TokenType.notGreaterEqual",
|
||||
"!<", "TokenType.notLess",
|
||||
"!<=", "TokenType.notLessEqual",
|
||||
"!<>", "TokenType.notLessEqualGreater",
|
||||
"+", "TokenType.plus",
|
||||
"+=", "TokenType.plusEquals",
|
||||
"+=", "TokenType.plusEqual",
|
||||
"^^", "TokenType.pow",
|
||||
"^^=", "TokenType.powEquals",
|
||||
"^^=", "TokenType.powEqual",
|
||||
"}", "TokenType.rBrace",
|
||||
"]", "TokenType.rBracket",
|
||||
")", "TokenType.rParen",
|
||||
|
|
@ -484,7 +484,7 @@ L_advance:
|
|||
">>>", "TokenType.unsignedShiftRight",
|
||||
">>>=", "TokenType.unsignedShiftRightEqual",
|
||||
"^", "TokenType.xor",
|
||||
"^=", "TokenType.xorEquals",
|
||||
"^=", "TokenType.xorEqual",
|
||||
));
|
||||
case '/':
|
||||
nextCharNonLF();
|
||||
|
|
@ -505,7 +505,7 @@ L_advance:
|
|||
goto L_advance; // tail-recursion
|
||||
|
||||
case '=':
|
||||
current.type = TokenType.divEquals;
|
||||
current.type = TokenType.divEqual;
|
||||
current.value = "/=";
|
||||
src.popFront();
|
||||
return;
|
||||
|
|
@ -1877,186 +1877,186 @@ L_advance:
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is an operator
|
||||
*/
|
||||
* Returns: true if the token is an operator
|
||||
*/
|
||||
pure nothrow bool isOperator(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.assign && t <= TokenType.xorEquals;
|
||||
return t >= TokenType.assign && t <= TokenType.xorEqual;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isOperator(ref const Token t)
|
||||
{
|
||||
return isOperator(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a keyword
|
||||
*/
|
||||
* Returns: true if the token is a keyword
|
||||
*/
|
||||
pure nothrow bool isKeyword(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.bool_ && t <= TokenType.with_;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isKeyword(ref const Token t)
|
||||
{
|
||||
return isKeyword(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a built-in type
|
||||
*/
|
||||
* Returns: true if the token is a built-in type
|
||||
*/
|
||||
pure nothrow bool isType(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.bool_ && t <= TokenType.wchar_;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isType(ref const Token t)
|
||||
{
|
||||
return isType(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is an attribute
|
||||
*/
|
||||
* Returns: true if the token is an attribute
|
||||
*/
|
||||
pure nothrow bool isAttribute(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.align_ && t <= TokenType.static_;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isAttribute(ref const Token t)
|
||||
{
|
||||
return isAttribute(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a protection attribute
|
||||
*/
|
||||
* Returns: true if the token is a protection attribute
|
||||
*/
|
||||
pure nothrow bool isProtection(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.export_ && t <= TokenType.public_;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isProtection(ref const Token t)
|
||||
{
|
||||
return isProtection(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a compile-time constant such as ___DATE__
|
||||
*/
|
||||
* Returns: true if the token is a compile-time constant such as ___DATE__
|
||||
*/
|
||||
pure nothrow bool isConstant(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.date && t <= TokenType.traits;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isConstant(ref const Token t)
|
||||
{
|
||||
return isConstant(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a string or number literal
|
||||
*/
|
||||
* Returns: true if the token is a string or number literal
|
||||
*/
|
||||
pure nothrow bool isLiteral(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.doubleLiteral && t <= TokenType.wstringLiteral;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isLiteral(ref const Token t)
|
||||
{
|
||||
return isLiteral(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a number literal
|
||||
*/
|
||||
* Returns: true if the token is a number literal
|
||||
*/
|
||||
pure nothrow bool isNumberLiteral(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.doubleLiteral && t <= TokenType.ulongLiteral;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isNumberLiteral(ref const Token t)
|
||||
{
|
||||
return isNumberLiteral(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is a string literal
|
||||
*/
|
||||
* Returns: true if the token is a string literal
|
||||
*/
|
||||
pure nothrow bool isStringLiteral(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.dstringLiteral && t <= TokenType.wstringLiteral;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isStringLiteral(ref const Token t)
|
||||
{
|
||||
return isStringLiteral(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns: true if the token is whitespace, a commemnt, a special token
|
||||
* sequence, or an identifier
|
||||
*/
|
||||
* Returns: true if the token is whitespace, a commemnt, a special token
|
||||
* sequence, or an identifier
|
||||
*/
|
||||
pure nothrow bool isMisc(const TokenType t)
|
||||
{
|
||||
return t >= TokenType.comment && t <= TokenType.specialTokenSequence;
|
||||
}
|
||||
|
||||
/**
|
||||
* ditto
|
||||
*/
|
||||
* ditto
|
||||
*/
|
||||
pure nothrow bool isMisc(ref const Token t)
|
||||
{
|
||||
return isMisc(t.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Listing of all the tokens in the D language.
|
||||
*/
|
||||
* Listing of all the tokens in the D language.
|
||||
*/
|
||||
enum TokenType: ushort
|
||||
{
|
||||
assign, /// =
|
||||
at, /// @
|
||||
bitAnd, /// &
|
||||
bitAndEquals, /// &=
|
||||
bitAndEqual, /// &=
|
||||
bitOr, /// |
|
||||
bitOrEquals, /// |=
|
||||
catEquals, /// ~=
|
||||
bitOrEqual, /// |=
|
||||
catEqual, /// ~=
|
||||
colon, /// :
|
||||
comma, /// ,
|
||||
decrement, /// --
|
||||
div, /// /
|
||||
divEquals, /// /=
|
||||
divEqual, /// /=
|
||||
dollar, /// $
|
||||
dot, /// .
|
||||
equals, /// ==
|
||||
equal, /// ==
|
||||
goesTo, /// =>
|
||||
greater, /// >
|
||||
greaterEqual, /// >=
|
||||
|
|
@ -2072,21 +2072,21 @@ enum TokenType: ushort
|
|||
logicOr, /// ||
|
||||
lParen, /// $(LPAREN)
|
||||
minus, /// -
|
||||
minusEquals, /// -=
|
||||
minusEqual, /// -=
|
||||
mod, /// %
|
||||
modEquals, /// %=
|
||||
mulEquals, /// *=
|
||||
modEqual, /// %=
|
||||
mulEqual, /// *=
|
||||
not, /// !
|
||||
notEquals, /// !=
|
||||
notEqual, /// !=
|
||||
notGreater, /// !>
|
||||
notGreaterEqual, /// !>=
|
||||
notLess, /// !<
|
||||
notLessEqual, /// !<=
|
||||
notLessEqualGreater, /// !<>
|
||||
plus, /// +
|
||||
plusEquals, /// +=
|
||||
plusEqual, /// +=
|
||||
pow, /// ^^
|
||||
powEquals, /// ^^=
|
||||
powEqual, /// ^^=
|
||||
rBrace, /// }
|
||||
rBracket, /// ]
|
||||
rParen, /// $(RPAREN)
|
||||
|
|
@ -2104,7 +2104,7 @@ enum TokenType: ushort
|
|||
unsignedShiftRightEqual, /// >>>=
|
||||
vararg, /// ...
|
||||
xor, /// ^
|
||||
xorEquals, /// ^=
|
||||
xorEqual, /// ^=
|
||||
|
||||
bool_, /// $(D_KEYWORD bool)
|
||||
byte_, /// $(D_KEYWORD byte)
|
||||
|
|
|
|||
Loading…
Reference in New Issue