Merged changes from @blackwhale. Added and fixed unit tests.

std/d/lexer.d

@@ -337,7 +337,7 @@ auto byToken(R)(R range, LexerConfig config)
 // ATM it is byte-oriented
 private struct LexSource(R)
 	if(isForwardRange!R && !isRandomAccessRange!R)
-{
+    {
 	bool empty() const { return _empty; }
 
 	auto ref front() const
@@ -2158,169 +2158,169 @@ private:
 }
 
 /**
-* Returns: true if the token is an operator
+ * Returns: true if the token is an operator
-*/
+ */
 pure nothrow bool isOperator(const TokenType t)
 {
     return t >= TokenType.assign && t <= TokenType.xorEquals;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isOperator(ref const Token t)
 {
     return isOperator(t.type);
 }
 
 /**
-* Returns: true if the token is a keyword
+ * Returns: true if the token is a keyword
-*/
+ */
 pure nothrow bool isKeyword(const TokenType t)
 {
     return t >= TokenType.bool_ && t <= TokenType.with_;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isKeyword(ref const Token t)
 {
     return isKeyword(t.type);
 }
 
 /**
-* Returns: true if the token is a built-in type
+ * Returns: true if the token is a built-in type
-*/
+ */
 pure nothrow bool isType(const TokenType t)
 {
     return t >= TokenType.bool_ && t <= TokenType.wchar_;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isType(ref const Token t)
 {
     return isType(t.type);
 }
 
 /**
-* Returns: true if the token is an attribute
+ * Returns: true if the token is an attribute
-*/
+ */
 pure nothrow bool isAttribute(const TokenType t)
 {
     return t >= TokenType.align_ && t <= TokenType.static_;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isAttribute(ref const Token t)
 {
     return isAttribute(t.type);
 }
 
 /**
-* Returns: true if the token is a protection attribute
+ * Returns: true if the token is a protection attribute
-*/
+ */
 pure nothrow bool isProtection(const TokenType t)
 {
     return t >= TokenType.export_ && t <= TokenType.public_;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isProtection(ref const Token t)
 {
     return isProtection(t.type);
 }
 
 /**
-* Returns: true if the token is a compile-time constant such as ___DATE__
+ * Returns: true if the token is a compile-time constant such as ___DATE__
-*/
+ */
 pure nothrow bool isConstant(const TokenType t)
 {
     return t >= TokenType.date && t <= TokenType.traits;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isConstant(ref const Token t)
 {
     return isConstant(t.type);
 }
 
 /**
-* Returns: true if the token is a string or number literal
+ * Returns: true if the token is a string or number literal
-*/
+ */
 pure nothrow bool isLiteral(const TokenType t)
 {
     return t >= TokenType.doubleLiteral && t <= TokenType.wstringLiteral;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isLiteral(ref const Token t)
 {
     return isLiteral(t.type);
 }
 
 /**
-* Returns: true if the token is a number literal
+ * Returns: true if the token is a number literal
-*/
+ */
 pure nothrow bool isNumberLiteral(const TokenType t)
 {
     return t >= TokenType.doubleLiteral && t <= TokenType.ulongLiteral;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isNumberLiteral(ref const Token t)
 {
     return isNumberLiteral(t.type);
 }
 
 /**
-* Returns: true if the token is a string literal
+ * Returns: true if the token is a string literal
-*/
+ */
 pure nothrow bool isStringLiteral(const TokenType t)
 {
     return t >= TokenType.dstringLiteral && t <= TokenType.wstringLiteral;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isStringLiteral(ref const Token t)
 {
     return isStringLiteral(t.type);
 }
 
 /**
-* Returns: true if the token is whitespace, a commemnt, a special token
+ * Returns: true if the token is whitespace, a commemnt, a special token
-*     sequence, or an identifier
+ *     sequence, or an identifier
-*/
+ */
 pure nothrow bool isMisc(const TokenType t)
 {
     return t >= TokenType.comment && t <= TokenType.specialTokenSequence;
 }
 
 /**
-* ditto
+ * ditto
-*/
+ */
 pure nothrow bool isMisc(ref const Token t)
 {
     return isMisc(t.type);
 }
 
 /**
-* Listing of all the tokens in the D language.
+ * Listing of all the tokens in the D language.
-*/
+ */
 enum TokenType: ushort
 {
     assign, /// =
@@ -3344,7 +3344,9 @@ unittest
     LexerConfig config;
     auto tokens = byToken(source, config);
     Token a = tokens.moveFront();
+    assert (a.type == TokenType.import_);
     Token b = tokens.moveFront();
+    assert (b.type == TokenType.identifier);
     assert (a != b);
     assert (a != "foo");
     assert (a < b);
@@ -3368,6 +3370,4 @@ unittest
 }
 
 
-//void main(string[] args)
+//void main(string[] args){}
-//{
-//}