forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wysiwyg strings and hex strings

This commit is contained in:
Hackerpilot 2013-01-19 13:28:05 -08:00
parent f4a8c97e18
commit e077c633da
1 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
tokenizer.d
View File

@ -506,7 +506,7 @@ Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Escaped) const StringStyle style = StringStyle.Escaped)
in in
{ {
assert (input.front == '\'' || input.front == '"' || input.front == '`'); assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r');
} }
body body
{ {
@ -514,10 +514,17 @@ body
t.lineNumber = lineNumber; t.lineNumber = lineNumber;
t.startIndex = index; t.startIndex = index;
t.type = TokenType.StringLiteral; t.type = TokenType.StringLiteral;
auto app = appender!(char[])();
if (input.front == 'r')
{
if (style & StringStyle.IncludeQuotes)
app.put('r');
input.popFront();
}
auto quote = input.front; auto quote = input.front;
input.popFront(); input.popFront();
++index; ++index;
auto app = appender!(char[])();
if (style & StringStyle.IncludeQuotes) if (style & StringStyle.IncludeQuotes)
app.put(quote); app.put(quote);
while (!isEoF(input)) while (!isEoF(input))
@ -1180,6 +1187,7 @@ pure nothrow bool isSeparating(C)(C ch) if (isSomeChar!C)
case ':': .. case '@': case ':': .. case '@':
case '[': .. case '^': case '[': .. case '^':
case '{': .. case '~': case '{': .. case '~':
case '`':
case 0x20: // space case 0x20: // space
case 0x09: // tab case 0x09: // tab
case 0x0a: .. case 0x0d: // newline, vertical tab, form feed, carriage return case 0x0a: .. case 0x0d: // newline, vertical tab, form feed, carriage return
@ -1197,9 +1205,9 @@ enum IterationStyle
/// Only include code, not whitespace or comments /// Only include code, not whitespace or comments
CodeOnly = 0, CodeOnly = 0,
/// Includes comments /// Includes comments
IncludeComments = 1, IncludeComments = 0b01,
/// Includes whitespace /// Includes whitespace
IncludeWhitespace = 2 << 1, IncludeWhitespace = 0b10,
/// Include everything /// Include everything
Everything = IncludeComments | IncludeWhitespace Everything = IncludeComments | IncludeWhitespace
} }
@ -1382,14 +1390,20 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
auto r = range.save(); auto r = range.save();
r.popFront(); r.popFront();
if (!r.isEoF() && r.front == '"') if (!r.isEoF() && r.front == '"')
writeln("parse wysiwyg string"); {
current = lexString(range, index, lineNumber, StringStyle.NotEscaped);
break;
}
else else
goto default; goto default;
case 'x': case 'x':
auto r = range.save(); auto r = range.save();
r.popFront(); r.popFront();
if (!r.isEoF() && r.front == '"') if (!r.isEoF() && r.front == '"')
writeln("parse hex string"); {
current = lexHexString(range, index, lineNumber);
break;
}
else else
goto default; goto default;
default: default:
@ -1418,7 +1432,7 @@ private:
unittest unittest
{ {
auto c = ">><==>)(*)\"TestString\"if import ifire 0,10.4f `\n`@property void//comment\ntest/* comment *//+comment/+moar comment+/+/"; auto c = "rust r\"\\ntest\" r`eh?`";
foreach (t; byToken(c)) foreach (t; byToken(c))
writeln(t); writeln(t);
} }