forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tokenizer and highlighter functional

This commit is contained in:
Hackerpilot 2013-01-19 14:49:24 +00:00
parent e077c633da
commit 36ff7d043c
6 changed files with 110 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cache.d
View File

@ -105,7 +105,7 @@ void updateCache(string dirs[], string moduleNames[])
continue;
// re-parse the module
Module m = parseModule(tokenize(readText(filePath)));
Module m = parseModule(byToken(readText(filePath)).array());
updateCache(m);

56
editors/textadept/modules/dmd/init.lua
View File

@ -367,34 +367,34 @@ local function showCompletionList(r)
buffer.auto_c_choose_single = setting
end
events.connect(events.CHAR_ADDED, function(ch)
if buffer:get_lexer() ~= "dmd" then return end
if ch > 255 then return end
local character = string.char(ch)
if character == "." or character == "(" then
local fileName = os.tmpname()
local tmpFile = io.open(fileName, "w")
tmpFile:write(buffer:get_text())
local command = M.PATH_TO_DSCANNER
.. (character == "." and " --dotComplete " or " --parenComplete ")
.. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
local p = io.popen(command)
local r = p:read("*a")
if r ~= "\n" then
if character == "." then
showCompletionList(r)
elseif character == "(" then
if r:find("^completions\n") then
showCompletionList(r)
elseif r:find("^calltips\n.*") then
r = r:gsub("^calltips\n", "")
buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
end
end
end
os.remove(fileName)
end
end)
--events.connect(events.CHAR_ADDED, function(ch)
-- if buffer:get_lexer() ~= "dmd" then return end
-- if ch > 255 then return end
-- local character = string.char(ch)
-- if character == "." or character == "(" then
-- local fileName = os.tmpname()
-- local tmpFile = io.open(fileName, "w")
-- tmpFile:write(buffer:get_text())
-- local command = M.PATH_TO_DSCANNER
-- .. (character == "." and " --dotComplete " or " --parenComplete ")
-- .. fileName .. " " .. buffer.current_pos .. " -I" .. buffer.filename:match(".+[\\/]")
-- local p = io.popen(command)
-- local r = p:read("*a")
-- if r ~= "\n" then
-- if character == "." then
-- showCompletionList(r)
-- elseif character == "(" then
-- if r:find("^completions\n") then
-- showCompletionList(r)
-- elseif r:find("^calltips\n.*") then
-- r = r:gsub("^calltips\n", "")
-- buffer:call_tip_show(buffer.current_pos, r:gsub("\\n", "\n"):gsub("\\t", "\t"):match("(.*)%s+$"))
-- end
-- end
-- end
-- os.remove(fileName)
-- end
--end)
local function autocomplete()

7
highlighter.d
View File

@ -12,10 +12,10 @@ import std.array;
void writeSpan(string cssClass, string value)
{
stdout.write(`<span class="`, cssClass, `">`, value.replace("<", "&lt;"), `</span>`);
stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
}
void highlight(Token[] tokens)
void highlight(R)(R tokens)
{
stdout.writeln(q"[<!DOCTYPE html>
<html>
@ -56,9 +56,6 @@ html { background-color: #111; color: #ccc; }
case TokenType.OPERATORS_BEGIN: .. case TokenType.OPERATORS_END:
writeSpan("operator", t.value);
break;
case TokenType.PROPERTIES_BEGIN: .. case TokenType.PROPERTIES_END:
writeSpan("property", t.value);
break;
default:
stdout.write(t.value.replace("<", "&lt;"));
break;

30
main.d
View File

@ -137,11 +137,11 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
writeln(f.data.tokenize().count!(a => isLineOfCode(a.type))());
writeln(f.data.byToken().count!(a => isLineOfCode(a.type))());
}
else
{
writeln(args[1..$].map!(a => a.readText().tokenize())().joiner()
writeln(args[1..$].map!(a => a.readText().byToken())().joiner()
.count!(a => isLineOfCode(a.type))());
}
return 0;
@ -155,11 +155,13 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
highlighter.highlight(f.data.tokenize(IterationStyle.EVERYTHING));
highlighter.highlight(f.data.byToken(IterationStyle.Everything,
StringStyle.Source));
}
else
{
highlighter.highlight(args[1].readText().tokenize(IterationStyle.EVERYTHING));
highlighter.highlight(args[1].readText().byToken(
IterationStyle.Everything, StringStyle.Source));
}
return 0;
}
@ -178,11 +180,11 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
tokens = f.data.tokenize();
tokens = f.data.byToken().array();
}
catch(ConvException e)
{
tokens = args[1].readText().tokenize();
tokens = args[1].readText().byToken().array();
args.popFront();
}
auto mod = parseModule(tokens);
@ -193,7 +195,7 @@ int main(string[] args)
auto p = findAbsPath(importDirs, im);
if (p is null || !p.exists())
continue;
context.addModule(p.readText().tokenize().parseModule());
context.addModule(p.readText().byToken().array().parseModule());
}
auto complete = AutoComplete(tokens, context);
if (parenComplete)
@ -213,12 +215,12 @@ int main(string[] args)
char[] buf;
while (stdin.readln(buf))
f.put(buf);
tokens = tokenize(f.data);
tokens = byToken(f.data).array();
}
else
{
// read given file
tokens = tokenize(readText(args[1]));
tokens = byToken(readText(args[1])).array();
}
auto mod = parseModule(tokens);
mod.writeJSONTo(stdout);
@ -229,8 +231,8 @@ int main(string[] args)
{
if (!recursiveCtags)
{
auto tokens = tokenize(readText(args[1]));
auto mod = parseModule(tokens);
auto tokens = byToken(readText(args[1]));
auto mod = parseModule(tokens.array());
mod.writeCtagsTo(stdout, args[1]);
}
else
@ -241,12 +243,12 @@ int main(string[] args)
if (!dirEntry.name.endsWith(".d", ".di"))
continue;
stderr.writeln("Generating tags for ", dirEntry.name);
auto tokens = tokenize(readText(dirEntry.name));
auto tokens = byToken(readText(dirEntry.name));
if (m is null)
m = parseModule(tokens);
m = parseModule(tokens.array());
else
{
auto mod = parseModule(tokens);
auto mod = parseModule(tokens.array());
m.merge(mod);
}
}

6
parser.d
View File

@ -302,10 +302,6 @@ Module parseModule(const Token[] tokens, string protection = "public", string[]
case TokenType.Nothrow:
case TokenType.Override:
case TokenType.Synchronized:
case TokenType.AtDisable:
case TokenType.AtProperty:
case TokenType.AtSafe:
case TokenType.AtSystem:
case TokenType.Abstract:
case TokenType.Final:
case TokenType.Gshared:
@ -635,8 +631,6 @@ body
case TokenType.Immutable:
case TokenType.Const:
case TokenType.Pure:
case TokenType.AtTrusted:
case TokenType.AtProperty:
case TokenType.Nothrow:
case TokenType.Final:
case TokenType.Override:

83
tokenizer.d
View File

@ -396,7 +396,7 @@ unittest
}
Token lexHexString(R, C = ElementType!R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Escaped)
const StringStyle style = StringStyle.Default)
in
{
assert (input.front == 'x');
@ -426,7 +426,7 @@ body
input.popFront();
++index;
}
else if (std.uni.isWhite(input.front) && !(style & StringStyle.Escaped))
else if (std.uni.isWhite(input.front) && (style & StringStyle.NotEscaped))
{
app.put(input.front);
input.popFront();
@ -465,15 +465,17 @@ body
break;
}
}
if (style & StringStyle.Escaped)
if (style & StringStyle.NotEscaped)
t.value = to!string(app.data);
else
{
auto a = appender!(char[])();
foreach (b; std.range.chunks(app.data, 2))
a.put(to!string(cast(dchar) parse!uint(b, 16)));
t.value = to!string(a.data);
}
else
t.value = to!string(app.data);
return t;
}
@ -503,7 +505,7 @@ unittest
}
Token lexString(R)(ref R input, ref uint index, ref uint lineNumber,
const StringStyle style = StringStyle.Escaped)
const StringStyle style = StringStyle.Default)
in
{
assert (input.front == '\'' || input.front == '"' || input.front == '`' || input.front == 'r');
@ -534,8 +536,38 @@ body
app.put(popNewline(input, index));
lineNumber++;
}
else if (input.front == '\\' && style & StringStyle.Escaped)
app.put(interpretEscapeSequence(input, index));
else if (input.front == '\\')
{
if (style & StringStyle.NotEscaped)
{
auto r = input.save();
r.popFront();
if (r.front == quote)
{
app.put('\\');
app.put(quote);
input.popFront();
input.popFront();
index += 2;
}
else if (r.front == '\\')
{
app.put('\\');
app.put('\\');
input.popFront();
input.popFront();
index += 2;
}
else
{
app.put('\\');
input.popFront();
++index;
}
}
else
app.put(interpretEscapeSequence(input, index));
}
else if (input.front == quote)
{
if (style & StringStyle.IncludeQuotes)
@ -1217,20 +1249,28 @@ enum IterationStyle
*/
enum StringStyle : uint
{
NotEscaped = 0,
/// String escape sequences will be processed and enclosing quote characters
/// will not be preserved.
Escaped = 1,
/// Escape sequences will be replaced with their equivalent characters.
/// Quote characters will not be included
Default = 0b0000,
/// Escape sequences will not be processed
NotEscaped = 0b0001,
/// Strings will include their opening and closing quote characters as well
/// as any prefixes or suffixes (e.g.: "abcde"w will include the 'w'
/// character)
IncludeQuotes = 0x0010,
/// Strings will be read exactly as they appeared in the source, including
/// their opening and closing quote characters. Useful for syntax highlighting.
IncludeQuotes = 2,
Source = NotEscaped | IncludeQuotes,
}
TokenRange!(R) byToken(R)(ref R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly,
const StringStyle tokenStyle = StringStyle.Escaped) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
TokenRange!(R) byToken(R)(R range, const IterationStyle iterationStyle = IterationStyle.CodeOnly,
const StringStyle stringStyle = StringStyle.Default) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
{
auto r = TokenRange!(R)(range);
r.tokenStyle = tokenStyle;
r.stringStyle = stringStyle;
r.iterStyle = iterationStyle;
r.lineNumber = 1;
r.popFront();
@ -1273,7 +1313,7 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
if (iterStyle == IterationStyle.Everything)
{
current = lexWhitespace(range, index, lineNumber);
break;
return c;
}
else
lexWhitespace(range, index, lineNumber);
@ -1348,16 +1388,19 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
break;
case '\'':
case '"':
current = lexString(range, index, lineNumber);
current = lexString(range, index, lineNumber, stringStyle);
break;
case '`':
current = lexString(range, index, lineNumber, StringStyle.NotEscaped);
current = lexString(range, index, lineNumber, stringStyle);
break;
case 'q':
auto r = range.save;
r.popFront();
if (!r.isEoF() && r.front == '{')
{
writeln("ParseTokenString");
break;
}
else
goto default;
case '/':
@ -1427,7 +1470,7 @@ private:
R range;
bool _empty;
IterationStyle iterStyle;
StringStyle tokenStyle;
StringStyle stringStyle;
}
unittest