forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

This revision has a code gen bug.

This commit is contained in:
Hackerpilot 2013-12-27 23:05:03 +00:00
parent d5a3918dd0
commit 1febda6fc7
4 changed files with 159 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.sh
View File

@ -1,5 +1,5 @@
#dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd #dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -g -m64 -wi -ofdscanner dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -g -m64 -wi -ofdscanner
#ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -O3 -of=dscanner-ldc -release -m64 -oq #ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq
#ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g #ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g
#/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d #/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d

3
highlighter.d
View File

@ -51,7 +51,8 @@ html { background-color: #fdf6e3; color: #002b36; }
{ {
version(Windows) version(Windows)
{ {
// Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong. // Stupid Windows automatically does a LF → CRLF, so
// CRLF → CRCRLF, which is obviously wrong.
// Strip out the CR characters here to avoid this. // Strip out the CR characters here to avoid this.
stdout.write(t.text.replace("<", "&lt;").replace("\r", "")); stdout.write(t.text.replace("<", "&lt;").replace("\r", ""));
} }

239
stdx/d/lexer.d
View File

@ -3,7 +3,6 @@ module stdx.d.lexer;
import std.typecons; import std.typecons;
import std.typetuple; import std.typetuple;
import std.array; import std.array;
import std.stdio;
import std.algorithm; import std.algorithm;
import std.range; import std.range;
import stdx.lexer; import stdx.lexer;
@ -322,45 +321,46 @@ public bool isStringLiteral(IdType type) pure nothrow @safe
public struct DLexer(R) public struct DLexer(R)
{ {
import std.conv; import std.conv;
import core.vararg;
mixin Lexer!(R, IdType, Token, isSeparating, lexIdentifier, staticTokens, dynamicTokens, mixin Lexer!(R, IdType, Token, isSeparating, lexIdentifier, staticTokens,
pseudoTokens, possibleDefaultTokens); dynamicTokens, pseudoTokens, possibleDefaultTokens);
this(R range) this(R range)
{ {
registerPostProcess!"\""(&lexStringLiteral!RangeType); registerPostProcess!"\""(&lexStringLiteral);
registerPostProcess!"`"(&lexWysiwygString!RangeType); registerPostProcess!"`"(&lexWysiwygString);
registerPostProcess!"//"(&lexSlashSlashComment!RangeType); registerPostProcess!"//"(&lexSlashSlashComment);
registerPostProcess!"/*"(&lexSlashStarComment!RangeType); registerPostProcess!"/*"(&lexSlashStarComment);
registerPostProcess!"/+"(&lexSlashPlusComment!RangeType); registerPostProcess!"/+"(&lexSlashPlusComment);
registerPostProcess!"."(&lexDot!RangeType); registerPostProcess!"."(&lexDot);
registerPostProcess!"'"(&lexCharacterLiteral!RangeType); registerPostProcess!"'"(&lexCharacterLiteral);
registerPostProcess!"0"(&lexNumber!RangeType); registerPostProcess!"0"(&lexNumber);
registerPostProcess!"1"(&lexNumber!RangeType); registerPostProcess!"1"(&lexNumber);
registerPostProcess!"2"(&lexNumber!RangeType); registerPostProcess!"2"(&lexNumber);
registerPostProcess!"3"(&lexNumber!RangeType); registerPostProcess!"3"(&lexNumber);
registerPostProcess!"4"(&lexNumber!RangeType); registerPostProcess!"4"(&lexNumber);
registerPostProcess!"5"(&lexNumber!RangeType); registerPostProcess!"5"(&lexNumber);
registerPostProcess!"6"(&lexNumber!RangeType); registerPostProcess!"6"(&lexNumber);
registerPostProcess!"7"(&lexNumber!RangeType); registerPostProcess!"7"(&lexNumber);
registerPostProcess!"8"(&lexNumber!RangeType); registerPostProcess!"8"(&lexNumber);
registerPostProcess!"9"(&lexNumber!RangeType); registerPostProcess!"9"(&lexNumber);
registerPostProcess!"#"(&lexNumber!RangeType); registerPostProcess!"#"(&lexNumber);
registerPostProcess!"q\""(&lexDelimitedString!RangeType); registerPostProcess!"q\""(&lexDelimitedString);
registerPostProcess!"q{"(&lexTokenString!RangeType); registerPostProcess!"q{"(&lexTokenString);
registerPostProcess!"r\""(&lexWysiwygString!RangeType); registerPostProcess!"r\""(&lexWysiwygString);
registerPostProcess!"x\""(&lexHexString!RangeType); registerPostProcess!"x\""(&lexHexString);
registerPostProcess!" "(&lexWhitespace!RangeType); registerPostProcess!" "(&lexWhitespace);
registerPostProcess!"\t"(&lexWhitespace!RangeType); registerPostProcess!"\t"(&lexWhitespace);
registerPostProcess!"\r"(&lexWhitespace!RangeType); registerPostProcess!"\r"(&lexWhitespace);
registerPostProcess!"\n"(&lexWhitespace!RangeType); registerPostProcess!"\n"(&lexWhitespace);
registerPostProcess!"\u2028"(&lexLongNewline!RangeType); registerPostProcess!"\u2028"(&lexLongNewline);
registerPostProcess!"\u2029"(&lexLongNewline!RangeType); registerPostProcess!"\u2029"(&lexLongNewline);
this.range = RangeType(range); this.range = RangeType(range);
popFront(); popFront();
} }
static bool isWhitespace(LR)(LR range) bool isWhitespace() pure const nothrow
{ {
switch (range.front) switch (range.front)
{ {
@ -379,7 +379,7 @@ public struct DLexer(R)
} }
} }
static void popFrontWhitespaceAware(LR)(ref LR range) void popFrontWhitespaceAware() pure nothrow
{ {
switch (range.front) switch (range.front)
{ {
@ -418,7 +418,7 @@ public struct DLexer(R)
} }
} }
Token lexWhitespace(LR)(ref LR range) Token lexWhitespace() pure nothrow
{ {
range.mark(); range.mark();
loop: do loop: do
@ -461,7 +461,7 @@ public struct DLexer(R)
range.column, range.index); range.column, range.index);
} }
Token lexNumber(LR)(ref LR range) Token lexNumber() pure nothrow
{ {
range.mark(); range.mark();
if (range.front == '0') if (range.front == '0')
@ -472,21 +472,21 @@ public struct DLexer(R)
case 'X': case 'X':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
return lexHex(range); return lexHex();
case 'b': case 'b':
case 'B': case 'B':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
return lexBinary(range); return lexBinary();
default: default:
return lexDecimal(range); return lexDecimal();
} }
} }
else else
return lexDecimal(range); return lexDecimal();
} }
Token lexHex(LR)(ref LR range) Token lexHex() pure nothrow
{ {
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
bool foundDot; bool foundDot;
@ -502,26 +502,26 @@ public struct DLexer(R)
break; break;
case 'u': case 'u':
case 'U': case 'U':
lexIntSuffix(range, type); lexIntSuffix(type);
break hexLoop; break hexLoop;
case 'i': case 'i':
if (foundDot) if (foundDot)
lexFloatSuffix(range, type); lexFloatSuffix(type);
break hexLoop; break hexLoop;
case 'L': case 'L':
if (foundDot) if (foundDot)
{ {
lexFloatSuffix(range, type); lexFloatSuffix(type);
break hexLoop; break hexLoop;
} }
else else
{ {
lexIntSuffix(range, type); lexIntSuffix(type);
break hexLoop; break hexLoop;
} }
case 'p': case 'p':
case 'P': case 'P':
lexExponent(range, type); lexExponent(type);
break hexLoop; break hexLoop;
case '.': case '.':
if (foundDot) if (foundDot)
@ -540,7 +540,7 @@ public struct DLexer(R)
range.index); range.index);
} }
Token lexBinary(LR)(ref LR range) Token lexBinary() pure nothrow
{ {
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
binaryLoop: while (!range.empty) binaryLoop: while (!range.empty)
@ -555,7 +555,7 @@ public struct DLexer(R)
case 'u': case 'u':
case 'U': case 'U':
case 'L': case 'L':
lexIntSuffix(range, type); lexIntSuffix(type);
break binaryLoop; break binaryLoop;
default: default:
break binaryLoop; break binaryLoop;
@ -565,7 +565,7 @@ public struct DLexer(R)
range.index); range.index);
} }
Token lexDecimal(LR)(ref LR range) Token lexDecimal() pure nothrow
{ {
bool foundDot = range.front == '.'; bool foundDot = range.front == '.';
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
@ -586,24 +586,24 @@ public struct DLexer(R)
case 'u': case 'u':
case 'U': case 'U':
if (!foundDot) if (!foundDot)
lexIntSuffix(range, type); lexIntSuffix(type);
break decimalLoop; break decimalLoop;
case 'i': case 'i':
lexFloatSuffix(range, type); lexFloatSuffix(type);
break decimalLoop; break decimalLoop;
case 'L': case 'L':
if (foundDot) if (foundDot)
lexFloatSuffix(range, type); lexFloatSuffix(type);
else else
lexIntSuffix(range, type); lexIntSuffix(type);
break decimalLoop; break decimalLoop;
case 'f': case 'f':
case 'F': case 'F':
lexFloatSuffix(range, type); lexFloatSuffix(type);
break decimalLoop; break decimalLoop;
case 'e': case 'e':
case 'E': case 'E':
lexExponent(range, type); lexExponent(type);
break decimalLoop; break decimalLoop;
case '.': case '.':
if (foundDot) if (foundDot)
@ -642,7 +642,7 @@ public struct DLexer(R)
range.index); range.index);
} }
static void lexIntSuffix(R)(ref R range, ref IdType type) void lexIntSuffix(ref IdType type) pure nothrow @safe
{ {
bool secondPass; bool secondPass;
if (range.front == 'u' || range.front == 'U') if (range.front == 'u' || range.front == 'U')
@ -676,7 +676,7 @@ public struct DLexer(R)
} }
} }
static void lexFloatSuffix(R)(ref R range, ref IdType type) void lexFloatSuffix(ref IdType type) pure nothrow @safe
{ {
switch (range.front) switch (range.front)
{ {
@ -702,7 +702,7 @@ public struct DLexer(R)
} }
} }
static void lexExponent(R)(ref R range, ref IdType type) void lexExponent(ref IdType type) pure nothrow @safe
{ {
range.popFront(); range.popFront();
bool foundSign = false; bool foundSign = false;
@ -716,7 +716,7 @@ public struct DLexer(R)
if (foundSign) if (foundSign)
{ {
if (!foundDigit) if (!foundDigit)
writeln("Expected an exponent"); error("Expected an exponent");
return; return;
} }
foundSign = true; foundSign = true;
@ -731,23 +731,23 @@ public struct DLexer(R)
case 'f': case 'f':
case 'F': case 'F':
case 'i': case 'i':
lexFloatSuffix(range, type); lexFloatSuffix(type);
return; return;
default: default:
if (!foundDigit) if (!foundDigit)
writeln("Expected an exponent"); error("Expected an exponent");
return; return;
} }
} }
} }
Token lexSpecialTokenSequence(LR)(ref LR range) Token lexSpecialTokenSequence() pure nothrow @safe
{ {
assert (false, "Not implemented"); assert (false, "Not implemented");
} }
Token lexSlashStarComment(LR)(ref LR range) Token lexSlashStarComment() pure
{ {
range.mark(); range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
@ -765,13 +765,13 @@ public struct DLexer(R)
} }
} }
else else
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index); range.index);
} }
Token lexSlashSlashComment(LR)(ref LR range) Token lexSlashSlashComment() pure nothrow
{ {
range.mark(); range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
@ -787,7 +787,7 @@ public struct DLexer(R)
range.index); range.index);
} }
Token lexSlashPlusComment(LR)(ref LR range) Token lexSlashPlusComment() pure nothrow
{ {
range.mark(); range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
@ -815,13 +815,13 @@ public struct DLexer(R)
} }
} }
else else
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index); range.index);
} }
Token lexStringLiteral(LR)(ref LR range) Token lexStringLiteral() pure nothrow
{ {
range.mark(); range.mark();
range.popFront(); range.popFront();
@ -829,7 +829,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: unterminated string literal"); error("Error: unterminated string literal");
return Token(); return Token();
} }
else if (range.front == '"') else if (range.front == '"')
@ -839,18 +839,18 @@ public struct DLexer(R)
} }
else if (range.front == '\\') else if (range.front == '\\')
{ {
lexEscapeSequence(range); lexEscapeSequence();
} }
else else
range.popFront(); range.popFront();
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(range, type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index); range.index);
} }
Token lexWysiwygString(LR)(ref LR range) Token lexWysiwygString() pure nothrow
{ {
range.mark(); range.mark();
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
@ -862,7 +862,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: unterminated string literal"); error("Error: unterminated string literal");
return Token(tok!""); return Token(tok!"");
} }
else if (range.front == '`') else if (range.front == '`')
@ -871,7 +871,7 @@ public struct DLexer(R)
break; break;
} }
else else
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
} }
} }
else else
@ -879,7 +879,7 @@ public struct DLexer(R)
range.popFront(); range.popFront();
if (range.empty) if (range.empty)
{ {
writeln("Error: unterminated string literal"); error("Error: unterminated string literal");
return Token(tok!""); return Token(tok!"");
} }
range.popFront(); range.popFront();
@ -887,7 +887,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: unterminated string literal"); error("Error: unterminated string literal");
return Token(tok!""); return Token(tok!"");
} }
else if (range.front == '"') else if (range.front == '"')
@ -896,15 +896,15 @@ public struct DLexer(R)
break; break;
} }
else else
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
} }
} }
lexStringSuffix(range, type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index); range.index);
} }
static void lexStringSuffix(R)(ref R range, ref IdType type) void lexStringSuffix(ref IdType type) pure
{ {
if (range.empty) if (range.empty)
type = tok!"stringLiteral"; type = tok!"stringLiteral";
@ -920,7 +920,7 @@ public struct DLexer(R)
} }
} }
Token lexDelimitedString(LR)(ref LR range) Token lexDelimitedString() pure nothrow
{ {
range.mark(); range.mark();
range.popFront(); range.popFront();
@ -933,30 +933,29 @@ public struct DLexer(R)
open = '<'; open = '<';
close = '>'; close = '>';
range.popFront(); range.popFront();
return lexNormalDelimitedString!LR(range, open, close); return lexNormalDelimitedString(open, close);
case '{': case '{':
open = '{'; open = '{';
close = '}'; close = '}';
range.popFront(); range.popFront();
return lexNormalDelimitedString!LR(range, open, close); return lexNormalDelimitedString(open, close);
case '[': case '[':
open = '['; open = '[';
close = ']'; close = ']';
range.popFront(); range.popFront();
return lexNormalDelimitedString!LR(range, open, close); return lexNormalDelimitedString(open, close);
case '(': case '(':
open = '('; open = '(';
close = ')'; close = ')';
range.popFront(); range.popFront();
return lexNormalDelimitedString!LR(range, open, close); return lexNormalDelimitedString(open, close);
default: default:
return lexHeredocString(range); return lexHeredocString();
} }
} }
Token lexNormalDelimitedString(LR)(ref LR range, Token lexNormalDelimitedString(ElementEncodingType!RangeType open,
ElementEncodingType!LR open, ElementEncodingType!RangeType close) pure nothrow
ElementEncodingType!LR close)
{ {
int depth = 1; int depth = 1;
while (!range.empty && depth > 0) while (!range.empty && depth > 0)
@ -976,34 +975,37 @@ public struct DLexer(R)
range.popFront(); range.popFront();
else else
{ {
writeln("Error: \" expected to end delimited string literal"); error("Error: \" expected to end delimited string literal");
return Token(tok!""); return Token(tok!"");
} }
} }
} }
else else
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(range, type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, range.index); return Token(type, cast(string) range.getMarked(), range.line, range.column, range.index);
} }
Token lexHeredocString(LR)(ref LR range) Token lexHeredocString() pure nothrow
{ {
assert (false, "unimplemented"); assert (false, "unimplemented");
} }
Token lexTokenString(LR)(ref LR range) Token lexTokenString() pure
{ {
assert(range.front == 'q');
range.popFront(); range.popFront();
assert(range.front == '{');
range.popFront(); range.popFront();
auto app = appender!string(); auto app = appender!string();
app.put("q{"); app.put("q{");
int depth = 1; int depth = 1;
_front = advance();
while (depth > 0 && !empty) while (depth > 0 && !empty)
{ {
popFront();
auto t = front(); auto t = front();
if (t.text is null) if (t.text is null)
app.put(str(t.type)); app.put(str(t.type));
@ -1013,13 +1015,14 @@ public struct DLexer(R)
depth--; depth--;
else if (t.type == tok!"{") else if (t.type == tok!"{")
depth++; depth++;
popFront();
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(range, type); lexStringSuffix(type);
return Token(type, app.data, range.line, range.column, range.index); return Token(type, app.data, range.line, range.column, range.index);
} }
Token lexHexString(LR)(ref LR range) Token lexHexString() pure nothrow
{ {
range.mark(); range.mark();
range.popFront(); range.popFront();
@ -1029,11 +1032,11 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: unterminated hex string literal"); error("Error: unterminated hex string literal");
return Token(); return Token();
} }
else if (isWhitespace(range)) else if (isWhitespace())
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
else switch (range.front) else switch (range.front)
{ {
case '0': .. case '9': case '0': .. case '9':
@ -1045,23 +1048,23 @@ public struct DLexer(R)
range.popFront(); range.popFront();
break loop; break loop;
default: default:
writeln("Error: invalid character in hex string"); error("Error: invalid character in hex string");
return Token(); return Token();
} }
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(range, type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.getMarked(), range.line, range.column,
range.index); range.index);
} }
static bool lexEscapeSequence(LR)(ref LR range) bool lexEscapeSequence() pure nothrow
{ {
range.popFront(); range.popFront();
if (range.empty) if (range.empty)
{ {
writeln("Error: non-terminated character escape sequence."); error("Error: non-terminated character escape sequence.");
return false; return false;
} }
switch (range.front) switch (range.front)
@ -1094,7 +1097,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: at least 4 hex digits expected."); error("Error: at least 4 hex digits expected.");
return false; return false;
} }
switch (range.front) switch (range.front)
@ -1105,7 +1108,7 @@ public struct DLexer(R)
range.popFront(); range.popFront();
break; break;
default: default:
writeln("Error: at least 4 hex digits expected."); error("Error: at least 4 hex digits expected.");
return false; return false;
} }
} }
@ -1116,7 +1119,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: at least 8 hex digits expected."); error("Error: at least 8 hex digits expected.");
return false; return false;
} }
switch (range.front) switch (range.front)
@ -1127,7 +1130,7 @@ public struct DLexer(R)
range.popFront(); range.popFront();
break; break;
default: default:
writeln("Error: at least 8 hex digits expected."); error("Error: at least 8 hex digits expected.");
return false; return false;
} }
} }
@ -1137,7 +1140,7 @@ public struct DLexer(R)
{ {
if (range.empty) if (range.empty)
{ {
writeln("Error: non-terminated character escape sequence."); error("Error: non-terminated character escape sequence.");
return false; return false;
} }
if (range.front == ';') if (range.front == ';')
@ -1149,13 +1152,13 @@ public struct DLexer(R)
return true; return true;
} }
Token lexCharacterLiteral(LR)(ref LR range) Token lexCharacterLiteral() pure nothrow
{ {
range.mark(); range.mark();
range.popFront(); range.popFront();
if (range.front == '\\') if (range.front == '\\')
{ {
lexEscapeSequence(range); lexEscapeSequence();
goto close; goto close;
} }
else if (range.front == '\'') else if (range.front == '\'')
@ -1172,7 +1175,7 @@ public struct DLexer(R)
} }
else else
{ {
popFrontWhitespaceAware(range); popFrontWhitespaceAware();
goto close; goto close;
} }
close: close:
@ -1184,12 +1187,12 @@ public struct DLexer(R)
} }
else else
{ {
writeln("Error: Expected ' to end character literal ", cast(char) range.front); error("Error: Expected ' to end character literal ", cast(char) range.front);
return Token(); return Token();
} }
} }
Token lexIdentifier(LR)(ref LR range) Token lexIdentifier() pure nothrow
{ {
range.mark(); range.mark();
while (!range.empty && !isSeparating(range.front)) while (!range.empty && !isSeparating(range.front))
@ -1200,7 +1203,7 @@ public struct DLexer(R)
range.line, range.column); range.line, range.column);
} }
Token lexDot(LR)(ref LR range) Token lexDot() pure nothrow
{ {
if (!range.canPeek) if (!range.canPeek)
{ {
@ -1210,7 +1213,7 @@ public struct DLexer(R)
switch (range.peek()) switch (range.peek())
{ {
case '0': .. case '9': case '0': .. case '9':
return lexNumber(range); return lexNumber();
case '.': case '.':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -1227,7 +1230,7 @@ public struct DLexer(R)
} }
} }
Token lexLongNewline(LR)(ref LR range) Token lexLongNewline() pure nothrow
{ {
range.mark(); range.mark();
range.popFront(); range.popFront();
@ -1238,7 +1241,7 @@ public struct DLexer(R)
range.column, range.index); range.column, range.index);
} }
static bool isSeparating(C)(C c) nothrow pure bool isSeparating(C)(C c) nothrow pure @safe
{ {
if (c <= 0x2f) return true; if (c <= 0x2f) return true;
if (c >= ':' && c <= '@') return true; if (c >= ':' && c <= '@') return true;
@ -1247,4 +1250,8 @@ public struct DLexer(R)
if (c == '`') return true; if (c == '`') return true;
return false; return false;
} }
void error(...) pure {
}
} }

64
stdx/lexer.d
View File

@ -201,12 +201,12 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
return code; return code;
} }
Token front() @property const(Token) front() pure nothrow const @property
{ {
return _front; return _front;
} }
void popFront() void popFront() pure
{ {
_front = advance(); _front = advance();
} }
@ -216,11 +216,6 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
return _front.type == tok!"\0"; return _front.type == tok!"\0";
} }
void registerPostProcess(alias t)(Token delegate(ref RangeType) fun)
{
post[pseudoTok!t] = fun;
}
template pseudoTok(string symbol) template pseudoTok(string symbol)
{ {
static assert (pseudoTokens.countUntil(symbol) >= 0); static assert (pseudoTokens.countUntil(symbol) >= 0);
@ -247,19 +242,17 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
return rVal; return rVal;
} }
Token advance() Token advance() pure
{ {
if (range.empty) if (range.empty)
return Token(tok!"\0"); return Token(tok!"\0");
auto r = range.save;
lexerLoop: switch (range.front) lexerLoop: switch (range.front)
{ {
mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens)))); mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
// pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens)))); // pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
outer_default: outer_default:
default: default:
range = r; return defaultTokenFunction();
return defaultTokenFunction(range);
} }
} }
@ -274,18 +267,24 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
return rVal; return rVal;
} }
Token postProcess(IDType i) void registerPostProcess(alias t)(Token delegate() pure fun)
{ {
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]); post[pseudoTok!t] = fun;
return post[i](range);
} }
Token delegate(ref RangeType)[pseudoTokens.length] post; Token postProcess(IDType i) pure
{
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]);
return post[i]();
}
Token delegate() pure [pseudoTokens.length] post;
RangeType range; RangeType range;
Token _front; Token _front;
} }
struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R && isForwardRange!R) struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R
&& isForwardRange!R && hasSlicing!R)
{ {
public: public:
@ -294,12 +293,25 @@ public:
this.range = range; this.range = range;
} }
invariant()
{
import std.string;
if (range.length != 6190)
assert (false, format("range.length = %d %s", range.length, cast(char[]) range[0 .. 100]));
}
bool startsWith(string s)
{
return index + s.length < range.length
&& (cast(const(ubyte[])) s) == range[index .. index + s.length];
}
bool empty() pure nothrow const @property bool empty() pure nothrow const @property
{ {
return _index >= range.length; return _index >= range.length;
} }
ElementType!R front() pure nothrow const @property const(ElementType!R) front() pure nothrow const @property
in in
{ {
assert (!empty); assert (!empty);
@ -321,7 +333,7 @@ public:
popFront(); popFront();
} }
ElementType!R peek(int offset = 1) pure nothrow const const(ElementType!R) peek(int offset = 1) pure nothrow const
in in
{ {
assert (canPeek(offset)); assert (canPeek(offset));
@ -336,24 +348,14 @@ public:
return _index + offset < range.length; return _index + offset < range.length;
} }
typeof(this) save() @property
{
typeof(this) copy;
copy.range = range;
copy._index = _index;
copy._column = _column;
copy._line = _line;
return copy;
}
void mark() nothrow pure void mark() nothrow pure
{ {
markBegin = index; markBegin = _index;
} }
R getMarked() nothrow pure const(R) getMarked() pure nothrow const
{ {
return range[markBegin .. index]; return range[markBegin .. _index];
} }
void incrementLine() pure nothrow void incrementLine() pure nothrow