forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Special tokens, ddoc, better highlighter colors

This commit is contained in:
Hackerpilot 2013-01-27 01:09:02 +00:00
parent c7b84ca0cc
commit 9a6e765335
3 changed files with 505 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
highlighter.d
View File

@ -17,6 +17,8 @@ void writeSpan(string cssClass, string value)
stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`); stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
} }
// http://ethanschoonover.com/solarized
void highlight(R)(R tokens) void highlight(R)(R tokens)
{ {
stdout.writeln(q"[<!DOCTYPE html> stdout.writeln(q"[<!DOCTYPE html>
@ -25,13 +27,14 @@ void highlight(R)(R tokens)
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/> <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<body> <body>
<style type="text/css"> <style type="text/css">
html { background-color: #fff; color: #222; } html { background-color: #fdf6e3; color: #002b36; }
.kwrd { font-weight: bold; color: blue; } .kwrd { color: #b58900; font-weight: bold; }
.com { color: green; font-style: italic;} .com { color: #93a1a1; font-style: italic; }
.num { color: orangered; font-weigth: bold; } .num { color: #dc322f; font-weigth: bold; }
.str { color: red; font-style: italic; } .str { color: #2aa198; font-style: italic; }
.op { color: 333; font-weight: bold; } .op { color: #586e75; font-weight: bold; }
.type { color: magenta; font-weight: bold; } .type { color: #268bd2; font-weight: bold; }
.cons { color: #859900; font-weight: bold; }
</style> </style>
<pre>]"); <pre>]");
@ -49,6 +52,8 @@ html { background-color: #fff; color: #222; }
writeSpan("num", t.value); writeSpan("num", t.value);
else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END) else if (t.type > TokenType.OPERATORS_BEGIN && t.type < TokenType.OPERATORS_END)
writeSpan("op", t.value); writeSpan("op", t.value);
else if (t.type > TokenType.CONSTANTS_BEGIN && t.type < TokenType.CONSTANTS_END)
writeSpan("cons", t.value);
else else
stdout.write(t.value.replace("<", "&lt;")); stdout.write(t.value.replace("<", "&lt;"));
} }

8
main.d
View File

@ -160,13 +160,13 @@ int main(string[] args)
char[] buf; char[] buf;
while (stdin.readln(buf)) while (stdin.readln(buf))
f.put(buf); f.put(buf);
highlighter.highlight(f.data.byToken(IterationStyle.Everything, highlighter.highlight(f.data.byToken("stdin", IterationStyle.Everything,
StringStyle.Source)); TokenStyle.Source));
} }
else else
{ {
highlighter.highlight(args[1].readText().byToken( highlighter.highlight(args[1].readText().byToken(args[1],
IterationStyle.Everything, StringStyle.Source)); IterationStyle.Everything, TokenStyle.Source));
} }
return 0; return 0;
} }

374
std/d/lexer.d
View File

@ -1,7 +1,7 @@
// Written in the D programming language // Written in the D programming language
/** /**
* This module contains a range-based lexer for the D programming language. * This module contains a range-based _lexer for the D programming language.
* *
* Examples: * Examples:
* *
@ -17,6 +17,7 @@
* stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`); * stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
* } * }
* *
* // http://ethanschoonover.com/solarized
* void highlight(R)(R tokens) * void highlight(R)(R tokens)
* { * {
* stdout.writeln(q"[<!DOCTYPE html> * stdout.writeln(q"[<!DOCTYPE html>
@ -25,13 +26,14 @@
* <meta http-equiv="content-type" content="text/html; charset=UTF-8"/> * <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
* <body> * <body>
* <style type="text/css"> * <style type="text/css">
* html { background-color: #fff; color: #222; } * html { background-color: #fdf6e3; color: #002b36; }
* .kwrd { font-weight: bold; color: blue; } * .kwrd { color: #b58900; font-weight: bold; }
* .com { color: green; font-style: italic;} * .com { color: #93a1a1; font-style: italic; }
* .num { color: orangered; font-weigth: bold; } * .num { color: #dc322f; font-weigth: bold; }
* .str { color: red; font-style: italic; } * .str { color: #2aa198; font-style: italic; }
* .op { color: 333; font-weight: bold; } * .op { color: #586e75; font-weight: bold; }
* .type { color: magenta; font-weight: bold; } * .type { color: #268bd2; font-weight: bold; }
* .cons { color: #859900; font-weight: bold; }
* </style> * </style>
* <pre>]"); * <pre>]");
* *
@ -57,9 +59,29 @@
* *
* void main(string[] args) * void main(string[] args)
* { * {
* args[1].readText().byToken(IterationStyle.Everything, StringStyle.Source).highlight(); * args[1].readText().byToken(args[1], IterationStyle.Everything, TokenStyle.Source).highlight();
* } * }
* --- * ---
* Iterate by tokens that would be significant to a parser
* ---
* import std.range;
* import std.d.lexer;
*
* // ...
*
* string s = "import std.stdio; // comment";
* auto tokens = byToken(s);
* // The comment and whitespace are not included
* assert (walkLength(tokens) == 5);
* ---
* Replace special tokens
* ---
* string s = "#line 5\n__VERSION__";
* auto tokens = byToken(s, "example.d", IterationStyle.CodeOnly, TokenStyle.Default, "foo", "1.0");
* assert (tokens.front.type == TokenType.IntLiteral);
* assert (tokens.front.value == "1.0")
* assert (tokens.front.lineNumber == 5);
* ---
* *
* Copyright: Brian Schott 2013 * Copyright: Brian Schott 2013
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0) * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
@ -76,6 +98,8 @@ import std.conv;
import std.uni; import std.uni;
import std.ascii; import std.ascii;
import std.exception; import std.exception;
import std.datetime;
import std.string;
import std.d.entities; import std.d.entities;
public: public:
@ -129,7 +153,8 @@ struct Token
} }
/** /**
* Configure the behavior of the byToken() function * Configure the behavior of the byToken() function. These flags may be
* combined using a bitwise or.
*/ */
enum IterationStyle enum IterationStyle
{ {
@ -139,21 +164,25 @@ enum IterationStyle
IncludeComments = 0b0001, IncludeComments = 0b0001,
/// Includes whitespace /// Includes whitespace
IncludeWhitespace = 0b0010, IncludeWhitespace = 0b0010,
/// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens) /// Include $(LINK2 http://dlang.org/lex.html#Special%20Tokens%20Sequence, special token sequences)
IncludeSpecialTokens = 0b0100, IncludeSpecialTokens = 0b0100,
/// Include everything /// Do not terminate iteration upon reaching the ___EOF__ token
Everything = IncludeComments | IncludeWhitespace IgnoreEOF = 0b1000,
/// Include everything, including the __EOF__ token.
Everything = IncludeComments | IncludeWhitespace | IgnoreEOF
} }
/** /**
* Configuration of the string lexing style * Configuration of the string lexing style. These flags may be combined with a
* bitwise or.
*/ */
enum StringStyle : uint enum TokenStyle : uint
{ {
/** /**
* Escape sequences will be replaced with their equivalent characters, * Escape sequences will be replaced with their equivalent characters,
* enclosing quote characters will not be included. Useful for creating a * enclosing quote characters will not be included. Special tokens such as
* compiler or interpreter. * __VENDOR__ will be replaced with their equivalent strings. Useful for
* creating a compiler or interpreter.
*/ */
Default = 0b0000, Default = 0b0000,
@ -170,27 +199,43 @@ enum StringStyle : uint
* include the $(D_STRING 'w') character as well as the opening and closing * include the $(D_STRING 'w') character as well as the opening and closing
* quotes$(RPAREN) * quotes$(RPAREN)
*/ */
IncludeQuotes = 0x0010, IncludeQuotes = 0b0010,
/**
* Do not replace the value field of the special tokens such as ___DATE__
* with their string equivalents.
*/
DoNotReplaceSpecial = 0b0100,
/** /**
* Strings will be read exactly as they appeared in the source, including * Strings will be read exactly as they appeared in the source, including
* their opening and closing quote characters. Useful for syntax * their opening and closing quote characters. Useful for syntax
* highlighting. * highlighting.
*/ */
Source = NotEscaped | IncludeQuotes, Source = NotEscaped | IncludeQuotes | DoNotReplaceSpecial,
} }
/// Default replacement for the ___VERSION__ special token
immutable string VERSION = "1.0";
/// Default replacement for the ___VENDOR__ special token
immutable string VENDOR = "std.d.lexer";
/** /**
* Iterate over the given range of characters by D tokens. * Iterate over the given range of characters by D tokens.
* Params: * Params:
* range = the range of characters * range = the range of characters
* iterationStyle = See IterationStyle * iterationStyle = See IterationStyle
* stringStyle = see StringStyle * stringStyle = see TokenStyle
* vendor = the string literal that should replace the ___VENDOR__ special token
* ver = the string literal that should replace the ___VERSION__ special token
* Returns: * Returns:
* an input range of tokens * an input range of tokens
*/ */
TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly, TokenRange!(R) byToken(R)(R range, string fileName = "",
const StringStyle stringStyle = StringStyle.Default) if (isForwardRange!(R) && isSomeChar!(ElementType!(R))) const IterationStyle iterationStyle = IterationStyle.CodeOnly,
const TokenStyle stringStyle = TokenStyle.Default, string vendor = VENDOR,
string ver = VERSION) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{ {
auto r = new TokenRange!(R)(range); auto r = new TokenRange!(R)(range);
r.stringStyle = stringStyle; r.stringStyle = stringStyle;
@ -201,16 +246,10 @@ TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = Iterati
} }
/** /**
* Range of tokens. Avoid creating instances of this manually. Use * Range of tokens. Use byToken$(LPAREN)$(RPAREN) to instantiate.
* $(DDOC_PSYMBOL byToken$(LPAREN)$(RPAREN)) instead, as it does some initialization work.
*/ */
class TokenRange(R) : InputRange!(Token) class TokenRange(R) : InputRange!(Token)
{ {
this(ref R range)
{
this.range = range;
}
/** /**
* Returns: true if the range is empty * Returns: true if the range is empty
*/ */
@ -294,6 +333,11 @@ class TokenRange(R) : InputRange!(Token)
private: private:
this(ref R range)
{
this.range = range;
}
/* /*
* Advances the range to the next token * Advances the range to the next token
*/ */
@ -482,6 +526,55 @@ private:
} }
current.value = to!string(app.data); current.value = to!string(app.data);
current.type = lookupTokenType(current.value); current.type = lookupTokenType(current.value);
if (!(iterStyle & IterationStyle.IgnoreEOF) && current.type == TokenType.EOF)
{
_empty = true;
return;
}
if (!(iterStyle & TokenStyle.DoNotReplaceSpecial))
break;
switch (current.type)
{
case TokenType.Date:
current.type = TokenType.StringLiteral;
auto time = Clock.currTime();
current.value = format("%s %02d %04d", time.month, time.day, time.year);
break;
case TokenType.Time:
auto time = Clock.currTime();
current.type = TokenType.StringLiteral;
current.value = (cast(TimeOfDay)(time)).toISOExtString();
break;
case TokenType.Timestamp:
auto time = Clock.currTime();
auto dt = cast(DateTime) time;
current.type = TokenType.StringLiteral;
current.value = format("%s %s %02d %02d:%02d:%02d %04d",
dt.dayOfWeek, dt.month, dt.day, dt.hour, dt.minute,
dt.second, dt.year);
break;
case TokenType.Vendor:
current.type = TokenType.StringLiteral;
current.value = vendor;
break;
case TokenType.CompilerVersion:
current.type = TokenType.StringLiteral;
current.value = ver;
break;
case TokenType.Line:
current.type = TokenType.IntLiteral;
current.value = format("%d", current.lineNumber);
break;
case TokenType.File:
current.type = TokenType.StringLiteral;
current.value = fileName;
break;
default:
break;
}
break; break;
} }
} }
@ -492,7 +585,10 @@ private:
R range; R range;
bool _empty; bool _empty;
IterationStyle iterStyle; IterationStyle iterStyle;
StringStyle stringStyle; TokenStyle stringStyle;
string ver;
string vendor;
string fileName;
} }
unittest unittest
@ -521,7 +617,7 @@ unittest
* $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert)) * $(TR $(TD KEYWORDS_BEGIN) $(TD KEYWORDS) $(TD keywords) $(TD class, if, assert))
* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared)) * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD attributes) $(TD override synchronized, __gshared))
* $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected)) * $(TR $(TD ATTRIBUTES_BEGIN) $(TD ATTRIBUTES_END) $(TD protection) $(TD public, protected))
* $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD __FILE__, __TIME__)) * $(TR $(TD CONSTANTS_BEGIN) $(TD CONSTANTS_END) $(TD compile-time constants) $(TD ___FILE__, ___TIME__))
* $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123)) * $(TR $(TD LITERALS_BEGIN) $(TD LITERALS_END) $(TD string and numeric literals) $(TD "str", 123))
* $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110)) * $(TR $(TD NUMBERS_BEGIN) $(TD NUMBERS_END) $(TD numeric literals) $(TD 0x123p+9, 0b0110))
* $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde")) * $(TR $(TD STRINGS_BEGIN) $(TD STRINGS_END) $(TD string literals) $(TD `123`c, q{tokens;}, "abcde"))
@ -533,69 +629,69 @@ enum TokenType: uint
{ {
// Operators // Operators
OPERATORS_BEGIN, /// OPERATORS_BEGIN, ///
Assign, /// $(D_KEYWORD =) Assign, /// =
At, /// $(D_KEYWORD @) At, /// @
BitAnd, /// $(D_KEYWORD &) BitAnd, /// &
BitAndEquals, /// $(D_KEYWORD &=) BitAndEquals, /// &=
BitOr, /// $(D_KEYWORD |) BitOr, /// |
BitOrEquals, /// $(D_KEYWORD |=) BitOrEquals, /// |=
CatEquals, /// $(D_KEYWORD ~=) CatEquals, /// ~=
Colon, /// $(D_KEYWORD :) Colon, /// :
Comma, /// $(D_KEYWORD ,) Comma, /// ,
Decrement, /// $(D_KEYWORD --) Decrement, /// --
Div, /// $(D_KEYWORD /) Div, /// /
DivEquals, /// $(D_KEYWORD /=) DivEquals, /// /=
Dollar, /// $(D_KEYWORD $) Dollar, /// $
Dot, /// $(D_KEYWORD .) Dot, /// .
Equals, /// $(D_KEYWORD ==) Equals, /// ==
GoesTo, // => GoesTo, // =>
Greater, /// $(D_KEYWORD >) Greater, /// >
GreaterEqual, /// $(D_KEYWORD >=) GreaterEqual, /// >=
Hash, // $(D_KEYWORD #) Hash, // #
Increment, /// $(D_KEYWORD ++) Increment, /// ++
LBrace, /// $(D_KEYWORD {) LBrace, /// {
LBracket, /// $(D_KEYWORD [) LBracket, /// [
Less, /// $(D_KEYWORD <) Less, /// <
LessEqual, /// $(D_KEYWORD <=) LessEqual, /// <=
LessEqualGreater, // $(D_KEYWORD <>=) LessEqualGreater, // <>=
LessOrGreater, /// $(D_KEYWORD <>) LessOrGreater, /// <>
LogicAnd, /// $(D_KEYWORD &&) LogicAnd, /// &&
LogicOr, /// $(D_KEYWORD ||) LogicOr, /// ||
LParen, /// $(D_KEYWORD $(LPAREN)) LParen, /// $(LPAREN)
Minus, /// $(D_KEYWORD -) Minus, /// -
MinusEquals, /// $(D_KEYWORD -=) MinusEquals, /// -=
Mod, /// $(D_KEYWORD %) Mod, /// %
ModEquals, /// $(D_KEYWORD %=) ModEquals, /// %=
MulEquals, /// $(D_KEYWORD *=) MulEquals, /// *=
Not, /// $(D_KEYWORD !) Not, /// !
NotEquals, /// $(D_KEYWORD !=) NotEquals, /// !=
NotGreater, /// $(D_KEYWORD !>) NotGreater, /// !>
NotGreaterEqual, /// $(D_KEYWORD !>=) NotGreaterEqual, /// !>=
NotLess, /// $(D_KEYWORD !<) NotLess, /// !<
NotLessEqual, /// $(D_KEYWORD !<=) NotLessEqual, /// !<=
NotLessEqualGreater, /// $(D_KEYWORD !<>) NotLessEqualGreater, /// !<>
Plus, /// $(D_KEYWORD +) Plus, /// +
PlusEquals, /// $(D_KEYWORD +=) PlusEquals, /// +=
Pow, /// $(D_KEYWORD ^^) Pow, /// ^^
PowEquals, /// $(D_KEYWORD ^^=) PowEquals, /// ^^=
RBrace, /// $(D_KEYWORD }) RBrace, /// }
RBracket, /// $(D_KEYWORD ]) RBracket, /// ]
RParen, /// $(D_KEYWORD $(RPAREN)) RParen, /// $(RPAREN)
Semicolon, /// $(D_KEYWORD ;) Semicolon, /// ;
ShiftLeft, /// $(D_KEYWORD <<) ShiftLeft, /// <<
ShiftLeftEqual, /// $(D_KEYWORD <<=) ShiftLeftEqual, /// <<=
ShiftRight, /// $(D_KEYWORD >>) ShiftRight, /// >>
ShiftRightEqual, /// $(D_KEYWORD >>=) ShiftRightEqual, /// >>=
Slice, // .. Slice, // ..
Star, /// $(D_KEYWORD *) Star, /// *
Ternary, /// $(D_KEYWORD ?) Ternary, /// ?
Tilde, /// $(D_KEYWORD ~) Tilde, /// ~
Unordered, /// $(D_KEYWORD !<>=) Unordered, /// !<>=
UnsignedShiftRight, /// $(D_KEYWORD >>>) UnsignedShiftRight, /// >>>
UnsignedShiftRightEqual, /// $(D_KEYWORD >>>=) UnsignedShiftRightEqual, /// >>>=
Vararg, /// $(D_KEYWORD ...) Vararg, /// ...
Xor, /// $(D_KEYWORD ^) Xor, /// ^
XorEquals, /// $(D_KEYWORD ^=) XorEquals, /// ^=
OPERATORS_END, /// OPERATORS_END, ///
@ -717,10 +813,16 @@ enum TokenType: uint
// Constants // Constants
CONSTANTS_BEGIN, /// CONSTANTS_BEGIN, ///
File, /// $(D_KEYWORD __FILE__) Date, /// ___DATE__
Line, /// $(D_KEYWORD __LINE__) EOF, /// ___EOF__
Thread, /// $(D_KEYWORD __thread) Time, /// ___TIME__
Traits, /// $(D_KEYWORD __traits) Timestamp, /// ___TIMESTAMP__
Vendor, /// ___VENDOR__
CompilerVersion, /// ___VERSION__
File, /// ___FILE__
Line, /// ___LINE__
Thread, /// ___thread
Traits, /// ___traits
CONSTANTS_END, /// CONSTANTS_END, ///
// Misc // Misc
@ -1119,7 +1221,7 @@ unittest
} }
Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber, Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Default) const TokenStyle style = TokenStyle.Default)
in in
{ {
assert (input.front == 'x'); assert (input.front == 'x');
@ -1131,7 +1233,7 @@ body
t.startIndex = index; t.startIndex = index;
t.type = TokenType.StringLiteral; t.type = TokenType.StringLiteral;
auto app = appender!(C[])(); auto app = appender!(C[])();
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put("x\""); app.put("x\"");
input.popFront(); input.popFront();
input.popFront(); input.popFront();
@ -1149,7 +1251,7 @@ body
input.popFront(); input.popFront();
++index; ++index;
} }
else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped)) else if (std.uni.isWhite(input.front) && (style & TokenStyle.NotEscaped))
{ {
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
@ -1157,7 +1259,7 @@ body
} }
else if (input.front == '"') else if (input.front == '"')
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put('"'); app.put('"');
input.popFront(); input.popFront();
++index; ++index;
@ -1179,7 +1281,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1188,7 +1290,7 @@ body
break; break;
} }
} }
if (style & StringStyle.NotEscaped) if (style & TokenStyle.NotEscaped)
t.value = to!string(app.data); t.value = to!string(app.data);
else else
{ {
@ -1218,17 +1320,17 @@ unittest
assert (br == TokenType.WStringLiteral); assert (br == TokenType.WStringLiteral);
auto c = `x"6d"`; auto c = `x"6d"`;
auto cr = lexHexString(c, i, l, StringStyle.NotEscaped); auto cr = lexHexString(c, i, l, TokenStyle.NotEscaped);
assert (cr == "6d"); assert (cr == "6d");
auto d = `x"5e5f"d`; auto d = `x"5e5f"d`;
auto dr = lexHexString(d, i, l, StringStyle.NotEscaped | StringStyle.IncludeQuotes); auto dr = lexHexString(d, i, l, TokenStyle.NotEscaped | TokenStyle.IncludeQuotes);
assert (dr == `x"5e5f"d`); assert (dr == `x"5e5f"d`);
assert (dr == TokenType.DStringLiteral); assert (dr == TokenType.DStringLiteral);
} }
Token lexString(R)(ref R input, ref uint index, ref uint lineNumber, Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Default) const TokenStyle style = TokenStyle.Default)
in in
{ {
assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r'); assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r');
@ -1243,7 +1345,7 @@ body
bool isWysiwyg = input.front == 'r' || input.front == '`'; bool isWysiwyg = input.front == 'r' || input.front == '`';
if (input.front == 'r') if (input.front == 'r')
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put('r'); app.put('r');
input.popFront(); input.popFront();
} }
@ -1251,7 +1353,7 @@ body
input.popFront(); input.popFront();
++index; ++index;
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(quote); app.put(quote);
while (!isEoF(input)) while (!isEoF(input))
{ {
@ -1262,7 +1364,7 @@ body
} }
else if (input.front == '\\') else if (input.front == '\\')
{ {
if (style & StringStyle.NotEscaped) if (style & TokenStyle.NotEscaped)
{ {
auto r = input.save(); auto r = input.save();
r.popFront(); r.popFront();
@ -1294,7 +1396,7 @@ body
} }
else if (input.front == quote) else if (input.front == quote)
{ {
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(quote); app.put(quote);
input.popFront(); input.popFront();
++index; ++index;
@ -1318,7 +1420,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (style & StringStyle.IncludeQuotes) if (style & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1340,7 +1442,7 @@ unittest
auto b = "\"ab\\ncd\""; auto b = "\"ab\\ncd\"";
assert (lexString(b, i, l) == "ab\ncd"); assert (lexString(b, i, l) == "ab\ncd");
auto c = "`abc\\ndef`"; auto c = "`abc\\ndef`";
assert (lexString(c, i, l, StringStyle.NotEscaped) == "abc\\ndef"); assert (lexString(c, i, l, TokenStyle.NotEscaped) == "abc\\ndef");
auto d = `"12345"w`; auto d = `"12345"w`;
assert (lexString(d, i, l).type == TokenType.WStringLiteral); assert (lexString(d, i, l).type == TokenType.WStringLiteral);
auto e = `"abc"c`; auto e = `"abc"c`;
@ -1352,7 +1454,7 @@ unittest
} }
Token lexDelimitedString(R)(ref R input, ref uint index, Token lexDelimitedString(R)(ref R input, ref uint index,
ref uint lineNumber, const StringStyle stringStyle = StringStyle.Default) ref uint lineNumber, const TokenStyle stringStyle = TokenStyle.Default)
in in
{ {
assert(input.front == 'q'); assert(input.front == 'q');
@ -1368,7 +1470,7 @@ body
input.popFront(); // q input.popFront(); // q
input.popFront(); // " input.popFront(); // "
index += 2; index += 2;
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put('q'); app.put('q');
app.put('"'); app.put('"');
@ -1414,7 +1516,7 @@ body
app.put('"'); app.put('"');
++index; ++index;
input.popFront(); input.popFront();
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
t.value = to!string(app.data); t.value = to!string(app.data);
else else
t.value = to!string(app.data[0 .. app.data.length - hereOpen.data.length - 1]); t.value = to!string(app.data[0 .. app.data.length - hereOpen.data.length - 1]);
@ -1430,7 +1532,7 @@ body
} }
else else
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
int depth = 1; int depth = 1;
@ -1445,7 +1547,7 @@ body
--depth; --depth;
if (depth == 0) if (depth == 0)
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put(close); app.put(close);
app.put('"'); app.put('"');
@ -1474,7 +1576,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1503,13 +1605,13 @@ unittest
assert (br == TokenType.WStringLiteral); assert (br == TokenType.WStringLiteral);
auto c = `q"[<xml></xml>]");`; auto c = `q"[<xml></xml>]");`;
auto cr = lexDelimitedString(c, i, l, StringStyle.Source); auto cr = lexDelimitedString(c, i, l, TokenStyle.Source);
assert (cr == `q"[<xml></xml>]"`); assert (cr == `q"[<xml></xml>]"`);
assert (cr == TokenType.StringLiteral); assert (cr == TokenType.StringLiteral);
} }
Token lexTokenString(R)(ref R input, ref uint index, ref uint lineNumber, Token lexTokenString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle stringStyle = StringStyle.Default) const TokenStyle stringStyle = TokenStyle.Default)
in in
{ {
assert (input.front == 'q'); assert (input.front == 'q');
@ -1524,12 +1626,12 @@ body
input.popFront(); // q input.popFront(); // q
input.popFront(); // { input.popFront(); // {
index += 2; index += 2;
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
{ {
app.put('q'); app.put('q');
app.put('{'); app.put('{');
} }
auto r = byToken(input, IterationStyle.Everything, StringStyle.Source); auto r = byToken(input, "", IterationStyle.Everything, TokenStyle.Source);
r.index = index; r.index = index;
int depth = 1; int depth = 1;
while (!r.empty) while (!r.empty)
@ -1543,7 +1645,7 @@ body
--depth; --depth;
if (depth <= 0) if (depth <= 0)
{ {
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put('}'); app.put('}');
r.popFront(); r.popFront();
break; break;
@ -1553,7 +1655,7 @@ body
r.popFront(); r.popFront();
} }
auto n = app.data.length - (stringStyle & StringStyle.IncludeQuotes ? 2 : 0); auto n = app.data.length - (stringStyle & TokenStyle.IncludeQuotes ? 2 : 0);
input.popFrontN(n); input.popFrontN(n);
if (!input.isEoF()) if (!input.isEoF())
{ {
@ -1566,7 +1668,7 @@ body
t.type = TokenType.DStringLiteral; t.type = TokenType.DStringLiteral;
goto case 'c'; goto case 'c';
case 'c': case 'c':
if (stringStyle & StringStyle.IncludeQuotes) if (stringStyle & TokenStyle.IncludeQuotes)
app.put(input.front); app.put(input.front);
input.popFront(); input.popFront();
++index; ++index;
@ -1591,7 +1693,7 @@ unittest
assert (ar == "import std.stdio;"); assert (ar == "import std.stdio;");
auto b = `q{writeln("hello world");}`; auto b = `q{writeln("hello world");}`;
auto br = lexTokenString(b, i, l, StringStyle.Source); auto br = lexTokenString(b, i, l, TokenStyle.Source);
assert (br == TokenType.StringLiteral); assert (br == TokenType.StringLiteral);
assert (br == `q{writeln("hello world");}`); assert (br == `q{writeln("hello world");}`);
} }
@ -2384,6 +2486,7 @@ pure nothrow TokenType lookupTokenType(const string input)
default: break; default: break;
} }
break; break;
case 6: case 6:
switch (input) switch (input)
{ {
@ -2413,6 +2516,7 @@ pure nothrow TokenType lookupTokenType(const string input)
case 7: case 7:
switch (input) switch (input)
{ {
case "__EOF__": return TokenType.EOF;
case "cdouble": return TokenType.Cdouble; case "cdouble": return TokenType.Cdouble;
case "default": return TokenType.Default; case "default": return TokenType.Default;
case "dstring": return TokenType.DString; case "dstring": return TokenType.DString;
@ -2443,6 +2547,8 @@ pure nothrow TokenType lookupTokenType(const string input)
case "function": return TokenType.Function; case "function": return TokenType.Function;
case "unittest": return TokenType.Unittest; case "unittest": return TokenType.Unittest;
case "__FILE__": return TokenType.File; case "__FILE__": return TokenType.File;
case "__DATE__": return TokenType.Date;
case "__TIME__": return TokenType.Date;
default: break; default: break;
} }
break; break;
@ -2458,14 +2564,26 @@ pure nothrow TokenType lookupTokenType(const string input)
} }
break; break;
case 10: case 10:
if (input == "deprecated") switch (input)
return TokenType.Deprecated; {
case "deprecated": return TokenType.Deprecated;
case "__VENDOR__": return TokenType.Vendor;
default: break;
}
break; break;
case 11: case 11:
if (input == "__VERSION__")
return TokenType.CompilerVersion;
break;
case 12:
if (input == "synchronized") if (input == "synchronized")
return TokenType.Synchronized; return TokenType.Synchronized;
break; break;
case 13: case 13:
if (input == "__TIMESTAMP__")
return TokenType.Timestamp;
break;
case 15:
if (input == "foreach_reverse") if (input == "foreach_reverse")
return TokenType.Foreach_reverse; return TokenType.Foreach_reverse;
break; break;
@ -2573,3 +2691,5 @@ string generateCaseTrie(string[] args ...)
} }
return printCaseStatements(t, ""); return printCaseStatements(t, "");
} }
//void main(string[] args) {}