forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

compile errors

This commit is contained in:
Hackerpilot 2013-01-27 14:21:04 -08:00
parent 31f59384b9 9a6e765335
commit 31ffb3f470
3 changed files with 363 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
highlighter.d
View File

@ -15,6 +15,8 @@ void writeSpan(string cssClass, string value)
stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`); stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
} }
// http://ethanschoonover.com/solarized
void highlight(R)(R tokens) void highlight(R)(R tokens)
{ {
stdout.writeln(q"[<!DOCTYPE html> stdout.writeln(q"[<!DOCTYPE html>
@ -23,13 +25,14 @@ void highlight(R)(R tokens)
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/> <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<body> <body>
<style type="text/css"> <style type="text/css">
html { background-color: #fff; color: #222; } html { background-color: #fdf6e3; color: #002b36; }
.kwrd { font-weight: bold; color: blue; } .kwrd { color: #b58900; font-weight: bold; }
.com { color: green; font-style: italic;} .com { color: #93a1a1; font-style: italic; }
.num { color: orangered; font-weigth: bold; } .num { color: #dc322f; font-weigth: bold; }
.str { color: red; font-style: italic; } .str { color: #2aa198; font-style: italic; }
.op { color: 333; font-weight: bold; } .op { color: #586e75; font-weight: bold; }
.type { color: magenta; font-weight: bold; } .type { color: #268bd2; font-weight: bold; }
.cons { color: #859900; font-weight: bold; }
</style> </style>
<pre>]"); <pre>]");
@ -47,6 +50,8 @@ html { background-color: #fff; color: #222; }
writeSpan("num", t.value); writeSpan("num", t.value);
else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END) else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END)
writeSpan("op", t.value); writeSpan("op", t.value);
else if (t.type > TokenType.CONSTANTS_BEGIN && t.type < TokenType.CONSTANTS_END)
writeSpan("cons", t.value);
else else
stdout.write(t.value.replace("<", "&lt;")); stdout.write(t.value.replace("<", "&lt;"));
} }

4
main.d
View File

@ -143,7 +143,7 @@ int main(string[] args)
} }
else else
{ {
writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken())() writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken(a))()
.joiner().count!(a => isLineOfCode(a.type))()); .joiner().count!(a => isLineOfCode(a.type))());
} }
return 0; return 0;
@ -153,7 +153,7 @@ int main(string[] args)
{ {
File f = args.length == 1 ? stdin : File(args[1]); File f = args.length == 1 ? stdin : File(args[1]);
highlighter.highlight(f.byLine(KeepTerminator.yes).join().byToken( highlighter.highlight(f.byLine(KeepTerminator.yes).join().byToken(
IterationStyle.Everything, StringStyle.Source)); "", IterationStyle.Everything, TokenStyle.Source));
return 0; return 0;
} }

580
std/d/lexer.d
View File

@ -1,71 +1,93 @@
// Written in the D programming language // Written in the D programming language
/** /**
* This module contains a range-based lexer for the D programming language. * This module contains a range-based _lexer for the D programming language.
* *
* Examples: * Examples:
* *
* Generate HTML markup of D code. * Generate HTML markup of D code.
* --- * ---
* import std.stdio; * import std.stdio;
* import std.array; * import std.array;
* import std.file; * import std.file;
* import std.d.lexer; * import std.d.lexer;
* *
* void writeSpan(string cssClass, string value) * void writeSpan(string cssClass, string value)
* { * {
* stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`); * stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
* } * }
* *
* void highlight(R)(R tokens) * // http://ethanschoonover.com/solarized
* { * void highlight(R)(R tokens)
* stdout.writeln(q"[<!DOCTYPE html> * {
* <html> * stdout.writeln(q"[<!DOCTYPE html>
* <head> * <html>
* <meta http-equiv="content-type" content="text/html; charset=UTF-8"/> * <head>
* <body> * <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
* <style type="text/css"> * <body>
* html { background-color: #fff; color: #222; } * <style type="text/css">
* .kwrd { font-weight: bold; color: blue; } * html { background-color: #fdf6e3; color: #002b36; }
* .com { color: green; font-style: italic;} * .kwrd { color: #b58900; font-weight: bold; }
* .num { color: orangered; font-weigth: bold; } * .com { color: #93a1a1; font-style: italic; }
* .str { color: red; font-style: italic; } * .num { color: #dc322f; font-weigth: bold; }
* .op { color: 333; font-weight: bold; } * .str { color: #2aa198; font-style: italic; }
* .type { color: magenta; font-weight: bold; } * .op { color: #586e75; font-weight: bold; }
* </style> * .type { color: #268bd2; font-weight: bold; }
* <pre>]"); * .cons { color: #859900; font-weight: bold; }
* * </style>
* foreach (Token t; tokens) * <pre>]");
* { *
* if (t.type > TokenType.TYPES_BEGIN && t.type < TokenType.TYPES_END) * foreach (Token t; tokens)
* writeSpan("type", t.value); * {
* else if (t.type > TokenType.KEYWORDS_BEGIN && t.type < TokenType.KEYWORDS_END) * if (t.type > TokenType.TYPES_BEGIN && t.type < TokenType.TYPES_END)
* writeSpan("kwrd", t.value); * writeSpan("type", t.value);
* else if (t.type == TokenType.Comment) * else if (t.type > TokenType.KEYWORDS_BEGIN && t.type < TokenType.KEYWORDS_END)
* writeSpan("com", t.value); * writeSpan("kwrd", t.value);
* else if (t.type > TokenType.STRINGS_BEGIN && t.type < TokenType.STRINGS_END) * else if (t.type == TokenType.Comment)
* writeSpan("str", t.value); * writeSpan("com", t.value);
* else if (t.type > TokenType.NUMBERS_BEGIN && t.type < TokenType.NUMBERS_END) * else if (t.type > TokenType.STRINGS_BEGIN && t.type < TokenType.STRINGS_END)
* writeSpan("num", t.value); * writeSpan("str", t.value);
* else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END) * else if (t.type > TokenType.NUMBERS_BEGIN && t.type < TokenType.NUMBERS_END)
* writeSpan("op", t.value); * writeSpan("num", t.value);
* else * else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END)
* stdout.write(t.value.replace("<", "&lt;")); * writeSpan("op", t.value);
* } * else
* stdout.writeln("</pre>\n</body></html>"); * stdout.write(t.value.replace("<", "&lt;"));
* } * }
* * stdout.writeln("</pre>\n</body></html>");
* void main(string[] args) * }
* { *
* args[1].readText().byToken(IterationStyle.Everything, StringStyle.Source).highlight(); * void main(string[] args)
* } * {
* --- * args[1].readText().byToken(args[1], IterationStyle.Everything, TokenStyle.Source).highlight();
* * }
* Copyright: Brian Schott 2013 * ---
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0) * Iterate by tokens that would be significant to a parser
* Authors: Brian Schott * ---
* Source: $(PHOBOSSRC std/d/_lexer.d) * import std.range;
*/ * import std.d.lexer;
*
* // ...
*
* string s = "import std.stdio; // comment";
* auto tokens = byToken(s);
* // The comment and whitespace are not included
* assert (walkLength(tokens) == 5);
* ---
* Replace special tokens
* ---
* string s = "#line 5\n__VERSION__";
* auto tokens = byToken(s, "example.d", IterationStyle.CodeOnly, TokenStyle.Default, "foo", "1.0");
* assert (tokens.front.type == TokenType.IntLiteral);
* assert (tokens.front.value == "1.0")
* assert (tokens.front.lineNumber == 5);
* ---
*
* Copyright: Brian Schott 2013
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
* Authors: Brian Schott
* Source: $(PHOBOSSRC std/d/_lexer.d)
*/
module std.d.lexer; module std.d.lexer;
@ -76,6 +98,8 @@ import std.conv;
import std.uni; import std.uni;
import std.ascii; import std.ascii;
import std.exception; import std.exception;
import std.datetime;
import std.string;
import std.d.entities; import std.d.entities;
public: public:
@ -129,8 +153,9 @@ struct Token
} }
/** /**
* Configure the behavior of the byToken() function * Configure the behavior of the byToken() function. These flags may be
*/ * combined using a bitwise or.
*/
enum IterationStyle enum IterationStyle
{ {
/// Only include code, not whitespace or comments /// Only include code, not whitespace or comments
@ -141,20 +166,24 @@ enum IterationStyle
IncludeWhitespace = 0b0010, IncludeWhitespace = 0b0010,
/// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens) /// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens)
IncludeSpecialTokens = 0b0100, IncludeSpecialTokens = 0b0100,
/// Do not stop iteration on reaching the ___EOF__ token
IgnoreEOF = 0b1000,
/// Include everything /// Include everything
Everything = IncludeComments | IncludeWhitespace Everything = IncludeComments | IncludeWhitespace | IgnoreEOF
} }
/** /**
* Configuration of the string lexing style * Configuration of the token lexing style. These flags may be combined with a
*/ * bitwise or.
enum StringStyle : uint */
enum TokenStyle : uint
{ {
/** /**
* Escape sequences will be replaced with their equivalent characters, * Escape sequences will be replaced with their equivalent characters,
* enclosing quote characters will not be included. Useful for creating a * enclosing quote characters will not be included. Special tokens such as
* compiler or interpreter. * __VENDOR__ will be replaced with their equivalent strings. Useful for
*/ * creating a compiler or interpreter.
*/
Default = 0b0000, Default = 0b0000,
/** /**
@ -165,33 +194,48 @@ enum StringStyle : uint
NotEscaped = 0b0001, NotEscaped = 0b0001,
/** /**
* Strings will include their opening and closing quote characters as well * Strings will include their opening and closing quote characters as well
* as any prefixes or suffixes $(LPAREN)e.g.: $(D_STRING "abcde"w) will * as any prefixes or suffixes $(LPAREN)e.g.: $(D_STRING "abcde"w) will
* include the $(D_STRING 'w') character as well as the opening and closing * include the $(D_STRING 'w') character as well as the opening and closing
* quotes$(RPAREN) * quotes$(RPAREN)
*/ */
IncludeQuotes = 0x0010, IncludeQuotes = 0b0010,
/** /**
* Strings will be read exactly as they appeared in the source, including * Do not replace the value field of the special tokens such as ___DATE__
* their opening and closing quote characters. Useful for syntax * with their string equivalents.
* highlighting. */
*/ DoNotReplaceSpecial = 0b0100,
Source = NotEscaped | IncludeQuotes,
/**
* Strings will be read exactly as they appeared in the source, including
* their opening and closing quote characters. Useful for syntax
* highlighting.
*/
Source = NotEscaped | IncludeQuotes | DoNotReplaceSpecial,
} }
/// Default replacement for the ___VERSION__ special token
immutable string VERSION = "1.0";
/// Default replacement for the ___VENDOR__ special token
immutable string VENDOR = "std.d.lexer";
/** /**
* Iterate over the given range of characters by D tokens. * Iterate over the given range of characters by D tokens.
* Params: * Params:
* range = the range of characters * range = the range of characters
* iterationStyle = See IterationStyle * iterationStyle = See IterationStyle
* stringStyle = see StringStyle * stringStyle = see TokenStyle
* Returns: * vendor = the string literal that should replace the ___VENDOR__ special token
* an input range of tokens * ver = the string literal that should replace the ___VERSION__ special token
*/ * Returns:
TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly, * an input range of tokens
const StringStyle stringStyle = StringStyle.Default) */
if (isForwardRange!(R) && (isSomeChar!(ElementType!(R)) || is (ElementType!(R) == ubyte))) TokenRange!(R) byToken(R)(R range, string fileName = "",
const IterationStyle iterationStyle = IterationStyle.CodeOnly,
const TokenStyle stringStyle = TokenStyle.Default, string vendor = VENDOR,
string ver = VERSION) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{ {
auto r = new TokenRange!(R)(range); auto r = new TokenRange!(R)(range);
r.stringStyle = stringStyle; r.stringStyle = stringStyle;
@ -202,19 +246,13 @@ TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = Iterati
} }
/** /**
* Range of tokens. Avoid creating instances of this manually. Use * Range of tokens. Use byToken$(LPAREN)$(RPAREN) to instantiate.
* $(DDOC_PSYMBOL byToken$(LPAREN)$(RPAREN)) instead, as it does some initialization work. */
*/
class TokenRange(R) : InputRange!(Token) class TokenRange(R) : InputRange!(Token)
{ {
this(ref R range)
{
this.range = range;
}
/** /**
* Returns: true if the range is empty * Returns: true if the range is empty
*/ */
override bool empty() const @property override bool empty() const @property
{ {
return _empty; return _empty;
@ -295,9 +333,14 @@ class TokenRange(R) : InputRange!(Token)
private: private:
this(ref R range)
{
this.range = range;
}
/* /*
* Advances the range to the next token * Advances the range to the next token
*/ */
void advance() void advance()
{ {
if (range.empty) if (range.empty)
@ -483,6 +526,55 @@ private:
} }
current.value = to!string(app.data); current.value = to!string(app.data);
current.type = lookupTokenType(current.value); current.type = lookupTokenType(current.value);
if (!(iterStyle & IterationStyle.IgnoreEOF) && current.type == TokenType.EOF)
{
_empty = true;
return;
}
if (!(iterStyle & TokenStyle.DoNotReplaceSpecial))
break;
switch (current.type)
{
case TokenType.Date:
current.type = TokenType.StringLiteral;
auto time = Clock.currTime();
current.value = format("%s %02d %04d", time.month, time.day, time.year);
break;
case TokenType.Time:
auto time = Clock.currTime();
current.type = TokenType.StringLiteral;
current.value = (cast(TimeOfDay)(time)).toISOExtString();
break;
case TokenType.Timestamp:
auto time = Clock.currTime();
auto dt = cast(DateTime) time;
current.type = TokenType.StringLiteral;
current.value = format("%s %s %02d %02d:%02d:%02d %04d",
dt.dayOfWeek, dt.month, dt.day, dt.hour, dt.minute,
dt.second, dt.year);
break;
case TokenType.Vendor:
current.type = TokenType.StringLiteral;
current.value = vendor;
break;
case TokenType.CompilerVersion:
current.type = TokenType.StringLiteral;
current.value = ver;
break;
case TokenType.Line:
current.type = TokenType.IntLiteral;
current.value = format("%d", current.lineNumber);
break;
case TokenType.File:
current.type = TokenType.StringLiteral;
current.value = fileName;
break;
default:
break;
}
break; break;
} }
} }
@ -493,7 +585,10 @@ private:
R range; R range;
bool _empty; bool _empty;
IterationStyle iterStyle; IterationStyle iterStyle;
StringStyle stringStyle; TokenStyle stringStyle;
string ver;
string vendor;
string fileName;
} }
unittest unittest
@ -505,98 +600,98 @@ unittest
} }
/** /**
* Listing of all the tokens in the D language. * Listing of all the tokens in the D language.
* *
* Token types are arranged so that it is easy to group tokens while iterating * Token types are arranged so that it is easy to group tokens while iterating
* over them. For example: * over them. For example:
* --- * ---
* assert(TokenType.Increment < TokenType.OPERATORS_END); * assert(TokenType.Increment < TokenType.OPERATORS_END);
* assert(TokenType.Increment > TokenType.OPERATORS_BEGIN); * assert(TokenType.Increment > TokenType.OPERATORS_BEGIN);
* --- * ---
* The non-token values are documented below: * The non-token values are documented below:
* *
* $(BOOKTABLE , * $(BOOKTABLE ,
* $(TR $(TH Begin) $(TH End) $(TH Content) $(TH Examples)) * $(TR $(TH Begin) $(TH End) $(TH Content) $(TH Examples))
* $(TR $(TD OPERATORS_BEGIN) $(TD OPERATORS_END) $(TD operatiors) $(TD +, -, <<=)) * $(TR $(TD OPERATORS_BEGIN) $(TD OPERATORS_END) $(TD operatiors) $(TD +, -, <<=))
* $(TR $(TD TYPES_BEGIN) $(TD TYPES_END) $(TD types) $(TD bool, char, double)) * $(TR $(TD TYPES_BEGIN) $(TD TYPES_END) $(TD types) $(TD bool, char, double))
* $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert)) * $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert))
* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared)) * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared))
* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected)) * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected))
* $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD __FILE__, __TIME__)) * $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD ___FILE__, ___TIME__))
* $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123)) * $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123))
* $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110)) * $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110))
* $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde")) * $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde"))
* $(TR $(TD MISC_BEGIN) $(TD MISC_END) $(TD anything else) $(TD whitespace, comments, identifiers)) * $(TR $(TD MISC_BEGIN) $(TD MISC_END) $(TD anything else) $(TD whitespace, comments, identifiers))
* ) * )
* Note that several of the above ranges overlap. * Note that several of the above ranges overlap.
*/ */
enum TokenType: uint enum TokenType: uint
{ {
// Operators // Operators
OPERATORS_BEGIN, /// OPERATORS_BEGIN, ///
Assign, /// $(D_KEYWORD =) Assign, /// =
At, /// $(D_KEYWORD @) At, /// @
BitAnd, /// $(D_KEYWORD &) BitAnd, /// &
BitAndEquals, /// $(D_KEYWORD &=) BitAndEquals, /// &=
BitOr, /// $(D_KEYWORD |) BitOr, /// |
BitOrEquals, /// $(D_KEYWORD |=) BitOrEquals, /// |=
CatEquals, /// $(D_KEYWORD ~=) CatEquals, /// ~=
Colon, /// $(D_KEYWORD :) Colon, /// :
Comma, /// $(D_KEYWORD ,) Comma, /// ,
Decrement, /// $(D_KEYWORD --) Decrement, /// --
Div, /// $(D_KEYWORD /) Div, /// /
DivEquals, /// $(D_KEYWORD /=) DivEquals, /// /=
Dollar, /// $(D_KEYWORD $) Dollar, /// $
Dot, /// $(D_KEYWORD .) Dot, /// .
Equals, /// $(D_KEYWORD ==) Equals, /// ==
GoesTo, // => GoesTo, // =>
Greater, /// $(D_KEYWORD >) Greater, /// >
GreaterEqual, /// $(D_KEYWORD >=) GreaterEqual, /// >=
Hash, // $(D_KEYWORD #) Hash, // #
Increment, /// $(D_KEYWORD ++) Increment, /// ++
LBrace, /// $(D_KEYWORD {) LBrace, /// {
LBracket, /// $(D_KEYWORD [) LBracket, /// [
Less, /// $(D_KEYWORD <) Less, /// <
LessEqual, /// $(D_KEYWORD <=) LessEqual, /// <=
LessEqualGreater, // $(D_KEYWORD <>=) LessEqualGreater, // <>=
LessOrGreater, /// $(D_KEYWORD <>) LessOrGreater, /// <>
LogicAnd, /// $(D_KEYWORD &&) LogicAnd, /// &&
LogicOr, /// $(D_KEYWORD ||) LogicOr, /// ||
LParen, /// $(D_KEYWORD $(LPAREN)) LParen, /// $(LPAREN)
Minus, /// $(D_KEYWORD -) Minus, /// -
MinusEquals, /// $(D_KEYWORD -=) MinusEquals, /// -=
Mod, /// $(D_KEYWORD %) Mod, /// %
ModEquals, /// $(D_KEYWORD %=) ModEquals, /// %=
MulEquals, /// $(D_KEYWORD *=) MulEquals, /// *=
Not, /// $(D_KEYWORD !) Not, /// !
NotEquals, /// $(D_KEYWORD !=) NotEquals, /// !=
NotGreater, /// $(D_KEYWORD !>) NotGreater, /// !>
NotGreaterEqual, /// $(D_KEYWORD !>=) NotGreaterEqual, /// !>=
NotLess, /// $(D_KEYWORD !<) NotLess, /// !<
NotLessEqual, /// $(D_KEYWORD !<=) NotLessEqual, /// !<=
NotLessEqualGreater, /// $(D_KEYWORD !<>) NotLessEqualGreater, /// !<>
Plus, /// $(D_KEYWORD +) Plus, /// +
PlusEquals, /// $(D_KEYWORD +=) PlusEquals, /// +=
Pow, /// $(D_KEYWORD ^^) Pow, /// ^^
PowEquals, /// $(D_KEYWORD ^^=) PowEquals, /// ^^=
RBrace, /// $(D_KEYWORD }) RBrace, /// }
RBracket, /// $(D_KEYWORD ]) RBracket, /// ]
RParen, /// $(D_KEYWORD $(RPAREN)) RParen, /// $(RPAREN)
Semicolon, /// $(D_KEYWORD ;) Semicolon, /// ;
ShiftLeft, /// $(D_KEYWORD <<) ShiftLeft, /// <<
ShiftLeftEqual, /// $(D_KEYWORD <<=) ShiftLeftEqual, /// <<=
ShiftRight, /// $(D_KEYWORD >>) ShiftRight, /// >>
ShiftRightEqual, /// $(D_KEYWORD >>=) ShiftRightEqual, /// >>=
Slice, // .. Slice, // ..
Star, /// $(D_KEYWORD *) Star, /// *
Ternary, /// $(D_KEYWORD ?) Ternary, /// ?
Tilde, /// $(D_KEYWORD ~) Tilde, /// ~
Unordered, /// $(D_KEYWORD !<>=) Unordered, /// !<>=
UnsignedShiftRight, /// $(D_KEYWORD >>>) UnsignedShiftRight, /// >>>
UnsignedShiftRightEqual, /// $(D_KEYWORD >>>=) UnsignedShiftRightEqual, /// >>>=
Vararg, /// $(D_KEYWORD ...) Vararg, /// ...
Xor, /// $(D_KEYWORD ^) Xor, /// ^
XorEquals, /// $(D_KEYWORD ^=) XorEquals, /// ^=
OPERATORS_END, /// OPERATORS_END, ///
@ -718,10 +813,16 @@ enum TokenType: uint
// Constants // Constants
CONSTANTS_BEGIN, /// CONSTANTS_BEGIN, ///
File, /// $(D_KEYWORD __FILE__) Date, /// ___DATE__
Line, /// $(D_KEYWORD __LINE__) EOF, /// ___EOF__
Thread, /// $(D_KEYWORD __thread) Time, /// ___TIME__
Traits, /// $(D_KEYWORD __traits) Timestamp, /// ___TIMESTAMP__
Vendor, /// ___VENDOR__
CompilerVersion, /// ___VERSION__
File, /// ___FILE__
Line, /// ___LINE__
Thread, /// ___thread
Traits, /// ___traits
CONSTANTS_END, /// CONSTANTS_END, ///
// Misc // Misc
@ -1120,7 +1221,7 @@ unittest
} }
Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber, Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Default) const TokenStyle style = TokenStyle.Default)
in in
{ {
assert (input.front == 'x'); assert (input.front == 'x');
@ -1132,7 +1233,7 @@ body
t.startIndex = index; t.startIndex = index;
t.type = TokenType.StringLiteral; t.type = TokenType.StringLiteral;
auto app = appender!(C[])(); auto app = appender!(C[])();
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put("x\""); app.put("x\"");
input.popFront(); input.popFront();
input.popFront(); input.popFront();
@ -1150,7 +1251,7 @@ body
input.popFront(); input.popFront();
++index; ++index;
} }
else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped)) else if (std.uni.isWhite(input.front) && (style & TokenStyle.NotEscaped))
{ {
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
@ -1158,7 +1259,7 @@ body
} }
else if (input.front == '"') else if (input.front == '"')
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put('"'); app.put('"');
input.popFront(); input.popFront();
++index; ++index;
@ -1180,7 +1281,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1189,7 +1290,7 @@ body
break; break;
} }
} }
if (style & StringStyle.NotEscaped) if (style & TokenStyle.NotEscaped)
t.value = to!string(app.data); t.value = to!string(app.data);
else else
{ {
@ -1219,17 +1320,17 @@ unittest
assert (br == TokenType.WStringLiteral); assert (br == TokenType.WStringLiteral);
auto c = `x"6d"`; auto c = `x"6d"`;
auto cr = lexHexString(c, i, l, StringStyle.NotEscaped); auto cr = lexHexString(c, i, l, TokenStyle.NotEscaped);
assert (cr == "6d"); assert (cr == "6d");
auto d = `x"5e5f"d`; auto d = `x"5e5f"d`;
auto dr = lexHexString(d, i, l, StringStyle.NotEscaped | StringStyle.IncludeQuotes); auto dr = lexHexString(d, i, l, TokenStyle.NotEscaped | TokenStyle.IncludeQuotes);
assert (dr == `x"5e5f"d`); assert (dr == `x"5e5f"d`);
assert (dr == TokenType.DStringLiteral); assert (dr == TokenType.DStringLiteral);
} }
Token lexString(R)(ref R input, ref uint index, ref uint lineNumber, Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Default) const TokenStyle style = TokenStyle.Default)
in in
{ {
assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r'); assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r');
@ -1244,7 +1345,7 @@ body
bool isWysiwyg = input.front == 'r' || input.front == '`'; bool isWysiwyg = input.front == 'r' || input.front == '`';
if (input.front == 'r') if (input.front == 'r')
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put('r'); app.put('r');
input.popFront(); input.popFront();
} }
@ -1252,7 +1353,7 @@ body
input.popFront(); input.popFront();
++index; ++index;
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(quote); app.put(quote);
while (!isEoF(input)) while (!isEoF(input))
{ {
@ -1263,7 +1364,7 @@ body
} }
else if (input.front == '\\') else if (input.front == '\\')
{ {
if (style & StringStyle.NotEscaped) if (style & TokenStyle.NotEscaped)
{ {
auto r = input.save(); auto r = input.save();
r.popFront(); r.popFront();
@ -1295,7 +1396,7 @@ body
} }
else if (input.front == quote) else if (input.front == quote)
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(quote); app.put(quote);
input.popFront(); input.popFront();
++index; ++index;
@ -1319,7 +1420,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1341,7 +1442,7 @@ unittest
auto b = "\"ab\\ncd\""; auto b = "\"ab\\ncd\"";
assert (lexString(b, i, l) == "ab\ncd"); assert (lexString(b, i, l) == "ab\ncd");
auto c = "`abc\\ndef`"; auto c = "`abc\\ndef`";
assert (lexString(c, i, l, StringStyle.NotEscaped) == "abc\\ndef"); assert (lexString(c, i, l, TokenStyle.NotEscaped) == "abc\\ndef");
auto d = `"12345"w`; auto d = `"12345"w`;
assert (lexString(d, i, l).type == TokenType.WStringLiteral); assert (lexString(d, i, l).type == TokenType.WStringLiteral);
auto e = `"abc"c`; auto e = `"abc"c`;
@ -1353,7 +1454,7 @@ unittest
} }
Token lexDelimitedString(R)(ref R input, ref uint index, Token lexDelimitedString(R)(ref R input, ref uint index,
ref uint lineNumber, const StringStyle stringStyle = StringStyle.Default) ref uint lineNumber, const TokenStyle stringStyle = TokenStyle.Default)
in in
{ {
assert(input.front == 'q'); assert(input.front == 'q');
@ -1369,7 +1470,7 @@ body
input.popFront(); // q input.popFront(); // q
input.popFront(); // " input.popFront(); // "
index += 2; index += 2;
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put('q'); app.put('q');
app.put('"'); app.put('"');
@ -1415,7 +1516,7 @@ body
app.put('"'); app.put('"');
++index; ++index;
input.popFront(); input.popFront();
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
t.value = to!string(app.data); t.value = to!string(app.data);
else else
t.value = to!string(app.data[0 .. app.data.length - hereOpen.data.length - 1]); t.value = to!string(app.data[0 .. app.data.length - hereOpen.data.length - 1]);
@ -1431,7 +1532,7 @@ body
} }
else else
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
int depth = 1; int depth = 1;
@ -1446,7 +1547,7 @@ body
--depth; --depth;
if (depth == 0) if (depth == 0)
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put(close); app.put(close);
app.put('"'); app.put('"');
@ -1475,7 +1576,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1504,13 +1605,13 @@ unittest
assert (br == TokenType.WStringLiteral); assert (br == TokenType.WStringLiteral);
auto c = `q"[<xml></xml>]");`; auto c = `q"[<xml></xml>]");`;
auto cr = lexDelimitedString(c, i, l, StringStyle.Source); auto cr = lexDelimitedString(c, i, l, TokenStyle.Source);
assert (cr == `q"[<xml></xml>]"`); assert (cr == `q"[<xml></xml>]"`);
assert (cr == TokenType.StringLiteral); assert (cr == TokenType.StringLiteral);
} }
Token lexTokenString(R)(ref R input, ref uint index, ref uint lineNumber, Token lexTokenString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle stringStyle = StringStyle.Default) const TokenStyle stringStyle = TokenStyle.Default)
in in
{ {
assert (input.front == 'q'); assert (input.front == 'q');
@ -1525,12 +1626,12 @@ body
input.popFront(); // q input.popFront(); // q
input.popFront(); // { input.popFront(); // {
index += 2; index += 2;
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put('q'); app.put('q');
app.put('{'); app.put('{');
} }
auto r = byToken(input, IterationStyle.Everything, StringStyle.Source); auto r = byToken(input, "", IterationStyle.Everything, TokenStyle.Source);
r.index = index; r.index = index;
int depth = 1; int depth = 1;
while (!r.empty) while (!r.empty)
@ -1544,7 +1645,7 @@ body
--depth; --depth;
if (depth <= 0) if (depth <= 0)
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put('}'); app.put('}');
r.popFront(); r.popFront();
break; break;
@ -1554,7 +1655,7 @@ body
r.popFront(); r.popFront();
} }
auto n = app.data.length - (stringStyle & StringStyle.IncludeQuotes ? 2 : 0); auto n = app.data.length - (stringStyle & TokenStyle.IncludeQuotes ? 2 : 0);
input.popFrontN(n); input.popFrontN(n);
if (!input.isEoF()) if (!input.isEoF())
{ {
@ -1567,7 +1668,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1592,7 +1693,7 @@ unittest
assert (ar == "import std.stdio;"); assert (ar == "import std.stdio;");
auto b = `q{writeln("hello world");}`; auto b = `q{writeln("hello world");}`;
auto br = lexTokenString(b, i, l, StringStyle.Source); auto br = lexTokenString(b, i, l, TokenStyle.Source);
assert (br == TokenType.StringLiteral); assert (br == TokenType.StringLiteral);
assert (br == `q{writeln("hello world");}`); assert (br == `q{writeln("hello world");}`);
} }
@ -2414,6 +2515,7 @@ pure nothrow TokenType lookupTokenType(const string input)
case 7: case 7:
switch (input) switch (input)
{ {
case "__EOF__": return TokenType.EOF;
case "cdouble": return TokenType.Cdouble; case "cdouble": return TokenType.Cdouble;
case "default": return TokenType.Default; case "default": return TokenType.Default;
case "dstring": return TokenType.DString; case "dstring": return TokenType.DString;
@ -2444,6 +2546,8 @@ pure nothrow TokenType lookupTokenType(const string input)
case "function": return TokenType.Function; case "function": return TokenType.Function;
case "unittest": return TokenType.Unittest; case "unittest": return TokenType.Unittest;
case "__FILE__": return TokenType.File; case "__FILE__": return TokenType.File;
case "__DATE__": return TokenType.Date;
case "__TIME__": return TokenType.Date;
default: break; default: break;
} }
break; break;
@ -2459,14 +2563,26 @@ pure nothrow TokenType lookupTokenType(const string input)
} }
break; break;
case 10: case 10:
if (input == "deprecated") switch (input)
return TokenType.Deprecated; {
case "deprecated": return TokenType.Deprecated;
case "__VENDOR__": return TokenType.Vendor;
default: break;
}
break; break;
case 11: case 11:
if (input == "__VERSION__")
return TokenType.CompilerVersion;
break;
case 12:
if (input == "synchronized") if (input == "synchronized")
return TokenType.Synchronized; return TokenType.Synchronized;
break; break;
case 13: case 13:
if (input == "__TIMESTAMP__")
return TokenType.Timestamp;
break;
case 15:
if (input == "foreach_reverse") if (input == "foreach_reverse")
return TokenType.Foreach_reverse; return TokenType.Foreach_reverse;
break; break;
@ -2574,3 +2690,5 @@ string generateCaseTrie(string[] args ...)
} }
return printCaseStatements(t, ""); return printCaseStatements(t, "");
} }
//void main(string[] args) {}