diff --git a/cache.d b/cache.d
index be68016..c2ea3fa 100644
--- a/cache.d
+++ b/cache.d
@@ -105,7 +105,7 @@ void updateCache(string dirs[], string moduleNames[])
continue;
// re-parse the module
- Module m = parseModule(tokenize(readText(filePath)));
+ Module m = parseModule(byToken(readText(filePath)).array());
updateCache(m);
diff --git a/editors/textadept/modules/dmd/init.lua b/editors/textadept/modules/dmd/init.lua
index a7e54c5..6f883cb 100755
--- a/editors/textadept/modules/dmd/init.lua
+++ b/editors/textadept/modules/dmd/init.lua
@@ -367,34 +367,34 @@ local function showCompletionList(r)
buffer.auto_c_choose_single = setting
end
-events.connect(events.CHAR_ADDED, function(ch)
- if buffer:get_lexer() ~= "dmd" then return end
- if ch > 255 then return end
- local character = string.char(ch)
- if character == "." or character == "(" then
- local fileName = os.tmpname()
- local tmpFile = io.open(fileName, "w")
- tmpFile:write(buffer:get_text())
- local command = M.PATH_TO_DSCANNER
- .. (character == "." and " --dotComplete " or " --parenComplete ")
- .. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
- local p = io.popen(command)
- local r = p:read("*a")
- if r ~= "\n" then
- if character == "." then
- showCompletionList(r)
- elseif character == "(" then
- if r:find("^completions\n") then
- showCompletionList(r)
- elseif r:find("^calltips\n.*") then
- r = r:gsub("^calltips\n", "")
- buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
- end
- end
- end
- os.remove(fileName)
- end
-end)
+--events.connect(events.CHAR_ADDED, function(ch)
+-- if buffer:get_lexer() ~= "dmd" then return end
+-- if ch > 255 then return end
+-- local character = string.char(ch)
+-- if character == "." or character == "(" then
+-- local fileName = os.tmpname()
+-- local tmpFile = io.open(fileName, "w")
+-- tmpFile:write(buffer:get_text())
+-- local command = M.PATH_TO_DSCANNER
+-- .. (character == "." and " --dotComplete " or " --parenComplete ")
+-- .. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
+-- local p = io.popen(command)
+-- local r = p:read("*a")
+-- if r ~= "\n" then
+-- if character == "." then
+-- showCompletionList(r)
+-- elseif character == "(" then
+-- if r:find("^completions\n") then
+-- showCompletionList(r)
+-- elseif r:find("^calltips\n.*") then
+-- r = r:gsub("^calltips\n", "")
+-- buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
+-- end
+-- end
+-- end
+-- os.remove(fileName)
+-- end
+--end)
local function autocomplete()
diff --git a/highlighter.d b/highlighter.d
index ee43747..1478a1c 100644
--- a/highlighter.d
+++ b/highlighter.d
@@ -12,10 +12,10 @@ import std.array;
void writeSpan(string cssClass, string value)
{
- stdout.write(``, value.replace("<", "<"), ``);
+ stdout.write(``, value.replace("&", "&").replace("<", "<"), ``);
}
-void highlight(Token[] tokens)
+void highlight(R)(R tokens)
{
stdout.writeln(q"[
@@ -56,9 +56,6 @@ html { background-color: #111; color: #ccc; }
case TokenType.OPERATORS_BEGIN: .. case TokenType.OPERATORS_END:
writeSpan("operator", t.value);
break;
- case TokenType.PROPERTIES_BEGIN: .. case TokenType.PROPERTIES_END:
- writeSpan("property", t.value);
- break;
default:
stdout.write(t.value.replace("<", "<"));
break;
diff --git a/main.d b/main.d
index 3c75cd1..39e3488 100644
--- a/main.d
+++ b/main.d
@@ -137,11 +137,11 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
- writeln(f.data.tokenize().count!(a => isLineOfCode(a.type))());
+ writeln(f.data.byToken().count!(a => isLineOfCode(a.type))());
}
else
{
- writeln(args[1..$].map!(a => a.readText().tokenize())().joiner()
+ writeln(args[1..$].map!(a => a.readText().byToken())().joiner()
.count!(a => isLineOfCode(a.type))());
}
return 0;
@@ -155,11 +155,13 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
- highlighter.highlight(f.data.tokenize(IterationStyle.EVERYTHING));
+ highlighter.highlight(f.data.byToken(IterationStyle.Everything,
+ StringStyle.Source));
}
else
{
- highlighter.highlight(args[1].readText().tokenize(IterationStyle.EVERYTHING));
+ highlighter.highlight(args[1].readText().byToken(
+ IterationStyle.Everything, StringStyle.Source));
}
return 0;
}
@@ -178,11 +180,11 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
- tokens = f.data.tokenize();
+ tokens = f.data.byToken().array();
}
catch(ConvException e)
{
- tokens = args[1].readText().tokenize();
+ tokens = args[1].readText().byToken().array();
args.popFront();
}
auto mod = parseModule(tokens);
@@ -193,7 +195,7 @@ int main(string[] args)
auto p = findAbsPath(importDirs, im);
if (p is null || !p.exists())
continue;
- context.addModule(p.readText().tokenize().parseModule());
+ context.addModule(p.readText().byToken().array().parseModule());
}
auto complete = AutoComplete(tokens, context);
if (parenComplete)
@@ -213,12 +215,12 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
- tokens = tokenize(f.data);
+ tokens = byToken(f.data).array();
}
else
{
// read given file
- tokens = tokenize(readText(args[1]));
+ tokens = byToken(readText(args[1])).array();
}
auto mod = parseModule(tokens);
mod.writeJSONTo(stdout);
@@ -229,8 +231,8 @@ int main(string[] args)
{
if (!recursiveCtags)
{
- auto tokens = tokenize(readText(args[1]));
- auto mod = parseModule(tokens);
+ auto tokens = byToken(readText(args[1]));
+ auto mod = parseModule(tokens.array());
mod.writeCtagsTo(stdout, args[1]);
}
else
@@ -241,12 +243,12 @@ int main(string[] args)
if (!dirEntry.name.endsWith(".d", ".di"))
continue;
stderr.writeln("Generating tags for ", dirEntry.name);
- auto tokens = tokenize(readText(dirEntry.name));
+ auto tokens = byToken(readText(dirEntry.name));
if (m is null)
- m = parseModule(tokens);
+ m = parseModule(tokens.array());
else
{
- auto mod = parseModule(tokens);
+ auto mod = parseModule(tokens.array());
m.merge(mod);
}
}
diff --git a/parser.d b/parser.d
index f1d4e8f..1db9481 100644
--- a/parser.d
+++ b/parser.d
@@ -302,10 +302,6 @@ Module parseModule(const Token[] tokens, string protection = "public", string[]
case TokenType.Nothrow:
case TokenType.Override:
case TokenType.Synchronized:
- case TokenType.AtDisable:
- case TokenType.AtProperty:
- case TokenType.AtSafe:
- case TokenType.AtSystem:
case TokenType.Abstract:
case TokenType.Final:
case TokenType.Gshared:
@@ -635,8 +631,6 @@ body
case TokenType.Immutable:
case TokenType.Const:
case TokenType.Pure:
- case TokenType.AtTrusted:
- case TokenType.AtProperty:
case TokenType.Nothrow:
case TokenType.Final:
case TokenType.Override:
diff --git a/tokenizer.d b/tokenizer.d
index daf70ef..dae129e 100644
--- a/tokenizer.d
+++ b/tokenizer.d
@@ -396,7 +396,7 @@ unittest
}
Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber,
- const StringStyle style = StringStyle.Escaped)
+ const StringStyle style = StringStyle.Default)
in
{
assert (input.front == 'x');
@@ -426,7 +426,7 @@ body
input.popFront();
++index;
}
- else if (std.uni.isWhite(input.front) && !(style & StringStyle.Escaped))
+ else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped))
{
app.put(input.front);
input.popFront();
@@ -465,15 +465,17 @@ body
break;
}
}
- if (style & StringStyle.Escaped)
+ if (style & StringStyle.NotEscaped)
+ t.value = to!string(app.data);
+ else
{
auto a = appender!(char[])();
foreach (b; std.range.chunks(app.data, 2))
a.put(to!string(cast(dchar) parse!uint(b, 16)));
t.value = to!string(a.data);
}
- else
- t.value = to!string(app.data);
+
+
return t;
}
@@ -503,7 +505,7 @@ unittest
}
Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
- const StringStyle style = StringStyle.Escaped)
+ const StringStyle style = StringStyle.Default)
in
{
assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r');
@@ -534,8 +536,38 @@ body
app.put(popNewline(input, index));
lineNumber++;
}
- else if (input.front == '\\' && style & StringStyle.Escaped)
- app.put(interpretEscapeSequence(input, index));
+ else if (input.front == '\\')
+ {
+ if (style & StringStyle.NotEscaped)
+ {
+ auto r = input.save();
+ r.popFront();
+ if (r.front == quote)
+ {
+ app.put('\\');
+ app.put(quote);
+ input.popFront();
+ input.popFront();
+ index += 2;
+ }
+ else if (r.front == '\\')
+ {
+ app.put('\\');
+ app.put('\\');
+ input.popFront();
+ input.popFront();
+ index += 2;
+ }
+ else
+ {
+ app.put('\\');
+ input.popFront();
+ ++index;
+ }
+ }
+ else
+ app.put(interpretEscapeSequence(input, index));
+ }
else if (input.front == quote)
{
if (style & StringStyle.IncludeQuotes)
@@ -1217,20 +1249,28 @@ enum IterationStyle
*/
enum StringStyle : uint
{
- NotEscaped = 0,
- /// String escape sequences will be processed and enclosing quote characters
- /// will not be preserved.
- Escaped = 1,
+ /// Escape sequences will be replaced with their equivalent characters.
+ /// Quote characters will not be included
+ Default = 0b0000,
+
+ /// Escape sequences will not be processed
+ NotEscaped = 0b0001,
+
+ /// Strings will include their opening and closing quote characters as well
+ /// as any prefixes or suffixes (e.g.: "abcde"w will include the 'w'
+ /// character)
+ IncludeQuotes = 0x0010,
+
/// Strings will be read exactly as they appeared in the source, including
/// their opening and closing quote characters. Useful for syntax highlighting.
- IncludeQuotes = 2,
+ Source = NotEscaped | IncludeQuotes,
}
-TokenRange!(R) byToken(R)(ref R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly,
- const StringStyle tokenStyle = StringStyle.Escaped) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
+TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly,
+ const StringStyle stringStyle = StringStyle.Default) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{
auto r = TokenRange!(R)(range);
- r.tokenStyle = tokenStyle;
+ r.stringStyle = stringStyle;
r.iterStyle = iterationStyle;
r.lineNumber = 1;
r.popFront();
@@ -1273,7 +1313,7 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
if (iterStyle == IterationStyle.Everything)
{
current = lexWhitespace(range, index, lineNumber);
- break;
+ return c;
}
else
lexWhitespace(range, index, lineNumber);
@@ -1348,16 +1388,19 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
break;
case '\'':
case '"':
- current = lexString(range, index, lineNumber);
+ current = lexString(range, index, lineNumber, stringStyle);
break;
case '`':
- current = lexString(range, index, lineNumber, StringStyle.NotEscaped);
+ current = lexString(range, index, lineNumber, stringStyle);
break;
case 'q':
auto r = range.save;
r.popFront();
if (!r.isEoF() && r.front == '{')
+ {
writeln("ParseTokenString");
+ break;
+ }
else
goto default;
case '/':
@@ -1427,7 +1470,7 @@ private:
R range;
bool _empty;
IterationStyle iterStyle;
- StringStyle tokenStyle;
+ StringStyle stringStyle;
}
unittest