Migrated over to Dmitry's buffer range. Lots of stuff is disabled for now
This commit is contained in:
Hackerpilot
parent
1febda6fc7
commit
b092840c70
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "datapicked"]
|
||||
path = datapicked
|
||||
url = ./datapicked/
|
||||
|
|
@ -469,9 +469,9 @@ class XMLPrinter : ASTVisitor
|
|||
output.writeln("<forStatement>");
|
||||
if (forStatement.declarationOrStatement !is null)
|
||||
{
|
||||
output.writeln("<initialize>");
|
||||
visit(forStatement.declarationOrStatement);
|
||||
output.writeln("</initialize>");
|
||||
output.writeln("<initialization>");
|
||||
visit(forStatement.initialization);
|
||||
output.writeln("</initialization>");
|
||||
}
|
||||
if (forStatement.test !is null)
|
||||
{
|
||||
|
|
@ -485,7 +485,7 @@ class XMLPrinter : ASTVisitor
|
|||
visit(forStatement.increment);
|
||||
output.writeln("</increment>");
|
||||
}
|
||||
visit(forStatement.statementNoCaseNoDefault);
|
||||
visit(forStatement.declarationOrStatement);
|
||||
output.writeln("</forStatement>");
|
||||
}
|
||||
|
||||
|
|
|
|||
15
build.sh
15
build.sh
|
|
@ -1,5 +1,18 @@
|
|||
#dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
|
||||
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -g -m64 -wi -ofdscanner
|
||||
dmd\
|
||||
main.d\
|
||||
stats.d\
|
||||
imports.d\
|
||||
highlighter.d\
|
||||
ctags.d\
|
||||
astprinter.d\
|
||||
formatter.d\
|
||||
outliner.d\
|
||||
stdx/*.d\
|
||||
stdx/d/*.d\
|
||||
datapicked/dpick/buffer/*.d\
|
||||
-Idatapicked\
|
||||
-g -m64 -wi -ofdscanner
|
||||
#ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq
|
||||
#ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g
|
||||
#/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d
|
||||
|
|
|
|||
3
ctags.d
3
ctags.d
|
|
@ -24,7 +24,8 @@ void printCtags(File output, string[] fileNames)
|
|||
File f = File(fileName);
|
||||
auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
|
||||
f.rawRead(bytes);
|
||||
Module m = parseModule(byToken!(ubyte[])(bytes).array, fileName, &doNothing);
|
||||
auto tokens = DLexer!(typeof(bytes))(bytes);
|
||||
Module m = parseModule(tokens.array, fileName, &doNothing);
|
||||
auto printer = new CTagsPrinter;
|
||||
printer.fileName = fileName;
|
||||
printer.visit(m);
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit eb14a5244153c0e13ceca79f292838dfe2ac9bfb
|
||||
|
|
@ -11,7 +11,7 @@ import std.array;
|
|||
import stdx.d.lexer;
|
||||
|
||||
// http://ethanschoonover.com/solarized
|
||||
void highlight(R)(R tokens, string fileName)
|
||||
void highlight(R)(ref R tokens, string fileName)
|
||||
{
|
||||
stdout.writeln(q"[
|
||||
<!DOCTYPE html>
|
||||
|
|
@ -33,8 +33,10 @@ html { background-color: #fdf6e3; color: #002b36; }
|
|||
</style>
|
||||
<pre>]");
|
||||
|
||||
foreach (Token t; tokens)
|
||||
while (!tokens.empty)
|
||||
{
|
||||
auto t = tokens.front;
|
||||
tokens.popFront();
|
||||
if (isBasicType(t.type))
|
||||
writeSpan("type", str(t.type));
|
||||
else if (isKeyword(t.type))
|
||||
|
|
|
|||
133
main.d
133
main.d
|
|
@ -17,13 +17,14 @@ import std.stdio;
|
|||
import std.range;
|
||||
import stdx.d.lexer;
|
||||
import stdx.d.parser;
|
||||
import dpick.buffer.buffer;
|
||||
|
||||
import highlighter;
|
||||
import stats;
|
||||
import ctags;
|
||||
import astprinter;
|
||||
import imports;
|
||||
import outliner;
|
||||
//import stats;
|
||||
//import ctags;
|
||||
//import astprinter;
|
||||
//import imports;
|
||||
//import outliner;
|
||||
|
||||
int main(string[] args)
|
||||
{
|
||||
|
|
@ -91,69 +92,69 @@ int main(string[] args)
|
|||
{
|
||||
bool usingStdin = args.length == 1;
|
||||
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
|
||||
highlighter.highlight(byToken!(typeof(bytes), false, false)(bytes),
|
||||
args.length == 1 ? "stdin" : args[1]);
|
||||
auto tokens = DLexer!(ubyte[])(bytes);
|
||||
highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
|
||||
return 0;
|
||||
}
|
||||
else if (ctags)
|
||||
{
|
||||
stdout.printCtags(expandArgs(args, recursive));
|
||||
}
|
||||
else
|
||||
{
|
||||
bool usingStdin = args.length == 1;
|
||||
if (sloc || tokenCount)
|
||||
{
|
||||
if (usingStdin)
|
||||
{
|
||||
auto tokens = byToken!(ubyte[], false, false)(readStdin());
|
||||
if (tokenCount)
|
||||
printTokenCount(stdout, "stdin", tokens);
|
||||
else
|
||||
printLineCount(stdout, "stdin", tokens);
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong count;
|
||||
foreach (f; expandArgs(args, recursive))
|
||||
{
|
||||
auto tokens = byToken!(ubyte[])(readFile(f));
|
||||
if (tokenCount)
|
||||
count += printTokenCount(stdout, f, tokens);
|
||||
else
|
||||
count += printLineCount(stdout, f, tokens);
|
||||
}
|
||||
writefln("total:\t%d", count);
|
||||
}
|
||||
}
|
||||
else if (syntaxCheck)
|
||||
{
|
||||
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
}
|
||||
else if (imports)
|
||||
{
|
||||
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
auto visitor = new ImportPrinter;
|
||||
visitor.visit(mod);
|
||||
}
|
||||
else if (ast)
|
||||
{
|
||||
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
auto printer = new XMLPrinter;
|
||||
printer.output = stdout;
|
||||
printer.visit(mod);
|
||||
}
|
||||
else if (outline)
|
||||
{
|
||||
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
auto outliner = new Outliner(stdout);
|
||||
outliner.visit(mod);
|
||||
}
|
||||
}
|
||||
// else if (ctags)
|
||||
// {
|
||||
// stdout.printCtags(expandArgs(args, recursive));
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// bool usingStdin = args.length == 1;
|
||||
// if (sloc || tokenCount)
|
||||
// {
|
||||
// if (usingStdin)
|
||||
// {
|
||||
// auto tokens = byToken!(ubyte[], false, false)(readStdin());
|
||||
// if (tokenCount)
|
||||
// printTokenCount(stdout, "stdin", tokens);
|
||||
// else
|
||||
// printLineCount(stdout, "stdin", tokens);
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// ulong count;
|
||||
// foreach (f; expandArgs(args, recursive))
|
||||
// {
|
||||
// auto tokens = byToken!(ubyte[])(readFile(f));
|
||||
// if (tokenCount)
|
||||
// count += printTokenCount(stdout, f, tokens);
|
||||
// else
|
||||
// count += printLineCount(stdout, f, tokens);
|
||||
// }
|
||||
// writefln("total:\t%d", count);
|
||||
// }
|
||||
// }
|
||||
// else if (syntaxCheck)
|
||||
// {
|
||||
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
// parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
// }
|
||||
// else if (imports)
|
||||
// {
|
||||
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
// auto visitor = new ImportPrinter;
|
||||
// visitor.visit(mod);
|
||||
// }
|
||||
// else if (ast)
|
||||
// {
|
||||
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
// auto printer = new XMLPrinter;
|
||||
// printer.output = stdout;
|
||||
// printer.visit(mod);
|
||||
// }
|
||||
// else if (outline)
|
||||
// {
|
||||
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
|
||||
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
|
||||
// auto outliner = new Outliner(stdout);
|
||||
// outliner.visit(mod);
|
||||
// }
|
||||
// }
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
276
main.html
276
main.html
|
|
@ -1,276 +0,0 @@
|
|||
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
|
||||
<title>main.d</title>
|
||||
</head>
|
||||
<body>
|
||||
<style type="text/css">
|
||||
html { background-color: #fdf6e3; color: #002b36; }
|
||||
.kwrd { color: #b58900; font-weight: bold; }
|
||||
.com { color: #93a1a1; font-style: italic; }
|
||||
.num { color: #dc322f; font-weigth: bold; }
|
||||
.str { color: #2aa198; font-style: italic; }
|
||||
.op { color: #586e75; font-weight: bold; }
|
||||
.type { color: #268bd2; font-weight: bold; }
|
||||
.cons { color: #859900; font-weight: bold; }
|
||||
</style>
|
||||
<pre>
|
||||
<span class="com">// Copyright Brian Schott (Sir Alaran) 2012.</span>
|
||||
<span class="com">// Distributed under the Boost Software License, Version 1.0.</span>
|
||||
<span class="com">// (See accompanying file LICENSE_1_0.txt or copy at</span>
|
||||
<span class="com">// http://www.boost.org/LICENSE_1_0.txt)</span>
|
||||
|
||||
<span class="kwrd">module</span> main<span class="op">;</span>
|
||||
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>algorithm<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>array<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>conv<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>file<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>getopt<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>parallelism<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>path<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>regex<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>stdio<span class="op">;</span>
|
||||
<span class="kwrd">import</span> std<span class="op">.</span>range<span class="op">;</span>
|
||||
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>lexer<span class="op">;</span>
|
||||
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>parser<span class="op">;</span>
|
||||
|
||||
<span class="kwrd">import</span> highlighter<span class="op">;</span>
|
||||
<span class="kwrd">import</span> stats<span class="op">;</span>
|
||||
<span class="kwrd">import</span> ctags<span class="op">;</span>
|
||||
<span class="kwrd">import</span> astprinter<span class="op">;</span>
|
||||
<span class="kwrd">import</span> imports<span class="op">;</span>
|
||||
<span class="kwrd">import</span> outliner<span class="op">;</span>
|
||||
|
||||
<span class="type">int</span> main<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="type">bool</span> sloc<span class="op">;</span>
|
||||
<span class="type">bool</span> highlight<span class="op">;</span>
|
||||
<span class="type">bool</span> ctags<span class="op">;</span>
|
||||
<span class="type">bool</span> recursive<span class="op">;</span>
|
||||
<span class="type">bool</span> format<span class="op">;</span>
|
||||
<span class="type">bool</span> help<span class="op">;</span>
|
||||
<span class="type">bool</span> tokenCount<span class="op">;</span>
|
||||
<span class="type">bool</span> syntaxCheck<span class="op">;</span>
|
||||
<span class="type">bool</span> ast<span class="op">;</span>
|
||||
<span class="type">bool</span> imports<span class="op">;</span>
|
||||
<span class="type">bool</span> muffin<span class="op">;</span>
|
||||
<span class="type">bool</span> outline<span class="op">;</span>
|
||||
|
||||
<span class="kwrd">try</span>
|
||||
<span class="op">{</span>
|
||||
getopt<span class="op">(</span>args<span class="op">,</span> <span class="str">"sloc|l"</span><span class="op">,</span> <span class="op">&</span>sloc<span class="op">,</span> <span class="str">"highlight"</span><span class="op">,</span> <span class="op">&</span>highlight<span class="op">,</span>
|
||||
<span class="str">"ctags|c"</span><span class="op">,</span> <span class="op">&</span>ctags<span class="op">,</span> <span class="str">"recursive|r|R"</span><span class="op">,</span> <span class="op">&</span>recursive<span class="op">,</span> <span class="str">"help|h"</span><span class="op">,</span> <span class="op">&</span>help<span class="op">,</span>
|
||||
<span class="str">"tokenCount|t"</span><span class="op">,</span> <span class="op">&</span>tokenCount<span class="op">,</span> <span class="str">"syntaxCheck|s"</span><span class="op">,</span> <span class="op">&</span>syntaxCheck<span class="op">,</span>
|
||||
<span class="str">"ast|xml"</span><span class="op">,</span> <span class="op">&</span>ast<span class="op">,</span> <span class="str">"imports|i"</span><span class="op">,</span> <span class="op">&</span>imports<span class="op">,</span> <span class="str">"outline|o"</span><span class="op">,</span> <span class="op">&</span>outline<span class="op">,</span>
|
||||
<span class="str">"muffinButton"</span><span class="op">,</span> <span class="op">&</span>muffin<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">catch</span> <span class="op">(</span>Exception e<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stderr<span class="op">.</span>writeln<span class="op">(</span>e<span class="op">.</span>msg<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="kwrd">if</span> <span class="op">(</span>muffin<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stdout<span class="op">.</span>writeln<span class="op">(</span>
|
||||
<span class="str">` ___________
|
||||
__(#*O 0** @%*)__
|
||||
_(%*o#*O%*0 #O#%##@)_
|
||||
(*#@%#o*@ #o%O*%@ #o #)
|
||||
\=====================/
|
||||
|I|I|I|I|I|I|I|I|I|I|
|
||||
|I|I|I|I|I|I|I|I|I|I|
|
||||
|I|I|I|I|I|I|I|I|I|I|
|
||||
|I|I|I|I|I|I|I|I|I|I|`</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="kwrd">if</span> <span class="op">(</span>help<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="kwrd">auto</span> optionCount <span class="op">=</span> count<span class="op">!</span><span class="str">"a"</span><span class="op">(</span><span class="op">[</span>sloc<span class="op">,</span> highlight<span class="op">,</span> ctags<span class="op">,</span> tokenCount<span class="op">,</span>
|
||||
syntaxCheck<span class="op">,</span> ast<span class="op">,</span> imports<span class="op">,</span> outline<span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">></span> <span class="num">1</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stderr<span class="op">.</span>writeln<span class="op">(</span><span class="str">"Too many options specified"</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op"><</span> <span class="num">1</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="kwrd">if</span> <span class="op">(</span>highlight<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
|
||||
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> bytes <span class="op">=</span> usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
highlighter<span class="op">.</span>highlight<span class="op">(</span>byToken<span class="op">!</span><span class="op">(</span><span class="kwrd">typeof</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span>
|
||||
args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span> <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ctags<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stdout<span class="op">.</span>printCtags<span class="op">(</span>expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span>
|
||||
<span class="op">{</span>
|
||||
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>sloc <span class="op">||</span> tokenCount<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>usingStdin<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>readStdin<span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
|
||||
printTokenCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">else</span>
|
||||
printLineCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span>
|
||||
<span class="op">{</span>
|
||||
<span class="type">ulong</span> count<span class="op">;</span>
|
||||
<span class="kwrd">foreach</span> <span class="op">(</span>f<span class="op">;</span> expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>readFile<span class="op">(</span>f<span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
|
||||
count <span class="op">+=</span> printTokenCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">else</span>
|
||||
count <span class="op">+=</span> printLineCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
writefln<span class="op">(</span><span class="str">"total:\t%d"</span><span class="op">,</span> count<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>syntaxCheck<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>imports<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> visitor <span class="op">=</span> <span class="kwrd">new</span> ImportPrinter<span class="op">;</span>
|
||||
visitor<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ast<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> printer <span class="op">=</span> <span class="kwrd">new</span> XMLPrinter<span class="op">;</span>
|
||||
printer<span class="op">.</span>output <span class="op">=</span> stdout<span class="op">;</span>
|
||||
printer<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>outline<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">auto</span> outliner <span class="op">=</span> <span class="kwrd">new</span> Outliner<span class="op">(</span>stdout<span class="op">)</span><span class="op">;</span>
|
||||
outliner<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
string<span class="op">[</span><span class="op">]</span> expandArgs<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">,</span> <span class="type">bool</span> recursive<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>recursive<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
string<span class="op">[</span><span class="op">]</span> rVal<span class="op">;</span>
|
||||
<span class="kwrd">foreach</span> <span class="op">(</span>arg<span class="op">;</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span><span class="op">$</span><span class="op">]</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>arg<span class="op">)</span> <span class="op">&&</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span>
|
||||
rVal <span class="kwrd">abstract</span> arg<span class="op">;</span>
|
||||
<span class="kwrd">else</span> <span class="kwrd">foreach</span> <span class="op">(</span>item<span class="op">;</span> dirEntries<span class="op">(</span>arg<span class="op">,</span> SpanMode<span class="op">.</span>breadth<span class="op">)</span><span class="op">.</span>map<span class="op">!</span><span class="op">(</span>a <span class="op">=></span> a<span class="op">.</span>name<span class="op">)</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>item<span class="op">)</span> <span class="op">&&</span> <span class="op">(</span>item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
|
||||
rVal <span class="kwrd">abstract</span> item<span class="op">;</span>
|
||||
<span class="kwrd">else</span>
|
||||
<span class="kwrd">continue</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">return</span> rVal<span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">else</span>
|
||||
<span class="kwrd">return</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span> <span class="op">$</span><span class="op">]</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readStdin<span class="op">(</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> sourceCode <span class="op">=</span> appender<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span><span class="op">)</span><span class="op">;</span>
|
||||
<span class="type">ubyte</span><span class="op">[</span><span class="num">4096</span><span class="op">]</span> buf<span class="op">;</span>
|
||||
<span class="kwrd">while</span> <span class="op">(</span><span class="kwrd">true</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">auto</span> b <span class="op">=</span> stdin<span class="op">.</span>rawRead<span class="op">(</span>buf<span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span>b<span class="op">.</span>length <span class="op">==</span> <span class="num">0</span><span class="op">)</span>
|
||||
<span class="kwrd">break</span><span class="op">;</span>
|
||||
sourceCode<span class="op">.</span>put<span class="op">(</span>b<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
<span class="kwrd">return</span> sourceCode<span class="op">.</span>data<span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readFile<span class="op">(</span>string fileName<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
<span class="kwrd">if</span> <span class="op">(</span><span class="op">!</span>exists<span class="op">(</span>fileName<span class="op">)</span><span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stderr<span class="op">.</span>writefln<span class="op">(</span><span class="str">"%s does not exist"</span><span class="op">,</span> fileName<span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> <span class="op">[</span><span class="op">]</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
File f <span class="op">=</span> File<span class="op">(</span>fileName<span class="op">)</span><span class="op">;</span>
|
||||
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> sourceCode <span class="op">=</span> uninitializedArray<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>to<span class="op">!</span>size_t<span class="op">(</span>f<span class="op">.</span>size<span class="op">)</span><span class="op">)</span><span class="op">;</span>
|
||||
f<span class="op">.</span>rawRead<span class="op">(</span>sourceCode<span class="op">)</span><span class="op">;</span>
|
||||
<span class="kwrd">return</span> sourceCode<span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
|
||||
<span class="type">void</span> printHelp<span class="op">(</span>string programName<span class="op">)</span>
|
||||
<span class="op">{</span>
|
||||
stderr<span class="op">.</span>writefln<span class="op">(</span>
|
||||
<span class="str">`
|
||||
Usage: %s options
|
||||
|
||||
options:
|
||||
--help | -h
|
||||
Prints this help message
|
||||
|
||||
--sloc | -l [sourceFiles]
|
||||
Prints the number of logical lines of code in the given
|
||||
source files. If no files are specified, input is read from stdin.
|
||||
|
||||
--tokenCount | t [sourceFiles]
|
||||
Prints the number of tokens in the given source files. If no files are
|
||||
specified, input is read from stdin.
|
||||
|
||||
--highlight [sourceFile] - Syntax-highlight the given source file. The
|
||||
resulting HTML will be written to standard output. If no files are
|
||||
specified, input is read from stdin.
|
||||
|
||||
--imports | -i [sourceFile]
|
||||
Prints modules imported by the given source file. If no files are
|
||||
specified, input is read from stdin.
|
||||
|
||||
--syntaxCheck | -s [sourceFile]
|
||||
Lexes and parses sourceFile, printing the line and column number of any
|
||||
syntax errors to stdout. One error or warning is printed per line.
|
||||
If no files are specified, input is read from stdin.
|
||||
|
||||
--ctags | -c sourceFile
|
||||
Generates ctags information from the given source code file. Note that
|
||||
ctags information requires a filename, so stdin cannot be used in place
|
||||
of a filename.
|
||||
|
||||
--ast | --xml sourceFile
|
||||
Generates an XML representation of the source files abstract syntax
|
||||
tree. If no files are specified, input is read from stdin.
|
||||
|
||||
--recursive | -R | -r
|
||||
When used with --ctags, --tokenCount, or --sloc, dscanner will produce
|
||||
ctags output for all .d and .di files contained within the given
|
||||
directories and its sub-directories.`</span><span class="op">,</span>
|
||||
programName<span class="op">)</span><span class="op">;</span>
|
||||
<span class="op">}</span>
|
||||
</pre>
|
||||
</body></html>
|
||||
12
stdx/d/ast.d
12
stdx/d/ast.d
|
|
@ -1203,13 +1203,13 @@ class ForStatement : ASTNode
|
|||
public:
|
||||
override void accept(ASTVisitor visitor)
|
||||
{
|
||||
mixin (visitIfNotNull!(declarationOrStatement, test, increment,
|
||||
statementNoCaseNoDefault));
|
||||
mixin (visitIfNotNull!(initialization, test, increment,
|
||||
declarationOrStatement));
|
||||
}
|
||||
/** */ DeclarationOrStatement declarationOrStatement;
|
||||
/** */ DeclarationOrStatement initialization;
|
||||
/** */ ExpressionStatement test;
|
||||
/** */ Expression increment;
|
||||
/** */ StatementNoCaseNoDefault statementNoCaseNoDefault;
|
||||
/** */ DeclarationOrStatement declarationOrStatement;
|
||||
/** */ size_t startIndex;
|
||||
}
|
||||
|
||||
|
|
@ -2760,11 +2760,11 @@ class WhileStatement : ASTNode
|
|||
public:
|
||||
override void accept(ASTVisitor visitor)
|
||||
{
|
||||
mixin (visitIfNotNull!(expression, statementNoCaseNoDefault));
|
||||
mixin (visitIfNotNull!(expression, declarationOrStatement));
|
||||
}
|
||||
|
||||
/** */ Expression expression;
|
||||
/** */ StatementNoCaseNoDefault statementNoCaseNoDefault;
|
||||
/** */ DeclarationOrStatement declarationOrStatement;
|
||||
/** */ size_t startIndex;
|
||||
}
|
||||
|
||||
|
|
|
|||
247
stdx/d/lexer.d
247
stdx/d/lexer.d
|
|
@ -17,7 +17,7 @@ private enum staticTokens = [
|
|||
|
||||
private enum pseudoTokens = [
|
||||
"\"", "`", "//", "/*", "/+", ".", "'", "0", "1", "2", "3", "4", "5", "6",
|
||||
"7", "8", "9", "#", "q\"", "q{", "r\"", "x\"", " ", "\t", "\r", "\n", "#!",
|
||||
"7", "8", "9", "q\"", "q{", "r\"", "x\"", " ", "\t", "\r", "\n", "#!",
|
||||
"\u2028", "\u2029"
|
||||
];
|
||||
|
||||
|
|
@ -57,24 +57,24 @@ public template tok(string token)
|
|||
}
|
||||
public alias stdx.lexer.TokenStructure!(IdType) Token;
|
||||
|
||||
public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
|
||||
{
|
||||
pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
|
||||
pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
|
||||
pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
|
||||
|
||||
static if (skipComments)
|
||||
{
|
||||
static if (skipWhitespace)
|
||||
return DLexer!(R)(range).filter!isNotEither;
|
||||
else
|
||||
return DLexer!(R)(range).filter!isNotComment;
|
||||
}
|
||||
else static if (skipWhitespace)
|
||||
return DLexer!(R)(range).filter!isNotWhitespace;
|
||||
else
|
||||
return DLexer!(R)(range);
|
||||
}
|
||||
//public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
|
||||
//{
|
||||
// pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
|
||||
// pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
|
||||
// pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
|
||||
// return new DLexer!(R)(range);
|
||||
// static if (skipComments)
|
||||
// {
|
||||
// static if (skipWhitespace)
|
||||
// return filter!isNotEither(tokens);
|
||||
// else
|
||||
// return filter!isNotComment(tokens);
|
||||
// }
|
||||
// else static if (skipWhitespace)
|
||||
// return filter!isNotWhitespace(tokens);
|
||||
// else
|
||||
// return tokens;
|
||||
//}
|
||||
|
||||
public bool isBasicType(IdType type) nothrow pure @safe
|
||||
{
|
||||
|
|
@ -322,45 +322,50 @@ public struct DLexer(R)
|
|||
{
|
||||
import std.conv;
|
||||
import core.vararg;
|
||||
|
||||
mixin Lexer!(R, IdType, Token, isSeparating, lexIdentifier, staticTokens,
|
||||
dynamicTokens, pseudoTokens, possibleDefaultTokens);
|
||||
import dpick.buffer.buffer;
|
||||
|
||||
private enum pseudoTokenHandlers = [
|
||||
"\"", "lexStringLiteral",
|
||||
"`", "lexWysiwygString",
|
||||
"//", "lexSlashSlashComment",
|
||||
"/*", "lexSlashStarComment",
|
||||
"/+", "lexSlashPlusComment",
|
||||
".", "lexDot",
|
||||
"'", "lexCharacterLiteral",
|
||||
"0", "lexNumber",
|
||||
"1", "lexNumber",
|
||||
"2", "lexNumber",
|
||||
"3", "lexNumber",
|
||||
"4", "lexNumber",
|
||||
"5", "lexNumber",
|
||||
"6", "lexNumber",
|
||||
"7", "lexNumber",
|
||||
"8", "lexNumber",
|
||||
"9", "lexNumber",
|
||||
"q\"", "lexDelimitedString",
|
||||
"q{", "lexTokenString",
|
||||
"r\"", "lexWysiwygString",
|
||||
"x\"", "lexHexString",
|
||||
" ", "lexWhitespace",
|
||||
"\t", "lexWhitespace",
|
||||
"\r", "lexWhitespace",
|
||||
"\n", "lexWhitespace",
|
||||
"\u2028", "lexLongNewline",
|
||||
"\u2029", "lexLongNewline",
|
||||
"#!", "lexScriptLine"
|
||||
];
|
||||
|
||||
mixin Lexer!(R, IdType, Token, lexIdentifier, staticTokens,
|
||||
dynamicTokens, pseudoTokens, pseudoTokenHandlers, possibleDefaultTokens);
|
||||
|
||||
private alias typeof(range).Mark Mark;
|
||||
|
||||
this(R range)
|
||||
{
|
||||
registerPostProcess!"\""(&lexStringLiteral);
|
||||
registerPostProcess!"`"(&lexWysiwygString);
|
||||
registerPostProcess!"//"(&lexSlashSlashComment);
|
||||
registerPostProcess!"/*"(&lexSlashStarComment);
|
||||
registerPostProcess!"/+"(&lexSlashPlusComment);
|
||||
registerPostProcess!"."(&lexDot);
|
||||
registerPostProcess!"'"(&lexCharacterLiteral);
|
||||
registerPostProcess!"0"(&lexNumber);
|
||||
registerPostProcess!"1"(&lexNumber);
|
||||
registerPostProcess!"2"(&lexNumber);
|
||||
registerPostProcess!"3"(&lexNumber);
|
||||
registerPostProcess!"4"(&lexNumber);
|
||||
registerPostProcess!"5"(&lexNumber);
|
||||
registerPostProcess!"6"(&lexNumber);
|
||||
registerPostProcess!"7"(&lexNumber);
|
||||
registerPostProcess!"8"(&lexNumber);
|
||||
registerPostProcess!"9"(&lexNumber);
|
||||
registerPostProcess!"#"(&lexNumber);
|
||||
registerPostProcess!"q\""(&lexDelimitedString);
|
||||
registerPostProcess!"q{"(&lexTokenString);
|
||||
registerPostProcess!"r\""(&lexWysiwygString);
|
||||
registerPostProcess!"x\""(&lexHexString);
|
||||
registerPostProcess!" "(&lexWhitespace);
|
||||
registerPostProcess!"\t"(&lexWhitespace);
|
||||
registerPostProcess!"\r"(&lexWhitespace);
|
||||
registerPostProcess!"\n"(&lexWhitespace);
|
||||
registerPostProcess!"\u2028"(&lexLongNewline);
|
||||
registerPostProcess!"\u2029"(&lexLongNewline);
|
||||
this.range = RangeType(range);
|
||||
popFront();
|
||||
this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
|
||||
}
|
||||
|
||||
bool isWhitespace() pure const nothrow
|
||||
bool isWhitespace() pure /*const*/ nothrow
|
||||
{
|
||||
switch (range.front)
|
||||
{
|
||||
|
|
@ -370,10 +375,10 @@ public struct DLexer(R)
|
|||
case '\t':
|
||||
return true;
|
||||
case 0xe2:
|
||||
if (!range.canPeek(2))
|
||||
return false;
|
||||
return range.peek() == 0x80
|
||||
&& (range.peek(2) == 0xa8 || range.peek(2) == 0xa9);
|
||||
auto peek = range.lookahead(2);
|
||||
return peek.length == 2
|
||||
&& peek[0] == 0x80
|
||||
&& (peek[1] == 0xa8 || peek[1] == 0xa9);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
|
@ -398,8 +403,9 @@ public struct DLexer(R)
|
|||
range.incrementLine();
|
||||
return;
|
||||
case 0xe2:
|
||||
if (range.canPeek(2) && range.peek() == 0x80
|
||||
&& (range.peek(2) == 0xa8 || range.peek(2) == 0xa9))
|
||||
auto lookahead = range.lookahead(3);
|
||||
if (lookahead.length == 3 && lookahead[1] == 0x80
|
||||
&& (lookahead[2] == 0xa8 || lookahead[2] == 0xa9))
|
||||
{
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
|
@ -420,7 +426,7 @@ public struct DLexer(R)
|
|||
|
||||
Token lexWhitespace() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
loop: do
|
||||
{
|
||||
switch (range.front)
|
||||
|
|
@ -440,11 +446,12 @@ public struct DLexer(R)
|
|||
range.popFront();
|
||||
break;
|
||||
case 0xe2:
|
||||
if (!range.canPeek(2))
|
||||
auto lookahead = range.lookahead(3);
|
||||
if (lookahead.length != 3)
|
||||
break loop;
|
||||
if (range.peek() != 0x80)
|
||||
if (lookahead[1] != 0x80)
|
||||
break loop;
|
||||
if (range.peek(2) == 0xa8 || range.peek(2) == 0xa9)
|
||||
if (lookahead[2] == 0xa8 || lookahead[2] == 0xa9)
|
||||
{
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
|
@ -457,36 +464,43 @@ public struct DLexer(R)
|
|||
break loop;
|
||||
}
|
||||
} while (!range.empty);
|
||||
return Token(tok!"whitespace", cast(string) range.getMarked(), range.line,
|
||||
return Token(tok!"whitespace", cast(string) range.slice(mark), range.line,
|
||||
range.column, range.index);
|
||||
}
|
||||
|
||||
Token lexNumber() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
if (range.front == '0')
|
||||
auto mark = range.mark();
|
||||
auto lookahead = range.lookahead(1);
|
||||
if (range.front == '0' && lookahead.length == 1)
|
||||
{
|
||||
switch (range.peek())
|
||||
switch (lookahead[0])
|
||||
{
|
||||
case 'x':
|
||||
case 'X':
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
return lexHex();
|
||||
return lexHex(mark);
|
||||
case 'b':
|
||||
case 'B':
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
return lexBinary();
|
||||
return lexBinary(mark);
|
||||
default:
|
||||
return lexDecimal();
|
||||
return lexDecimal(mark);
|
||||
}
|
||||
}
|
||||
else
|
||||
return lexDecimal();
|
||||
return lexDecimal(mark);
|
||||
}
|
||||
|
||||
Token lexHex() pure nothrow
|
||||
{
|
||||
auto mark = range.mark();
|
||||
return lexHex(mark);
|
||||
}
|
||||
|
||||
Token lexHex(Mark mark) pure nothrow
|
||||
{
|
||||
IdType type = tok!"intLiteral";
|
||||
bool foundDot;
|
||||
|
|
@ -526,7 +540,7 @@ public struct DLexer(R)
|
|||
case '.':
|
||||
if (foundDot)
|
||||
break hexLoop;
|
||||
if (range.canPeek() && range.peek() == '.')
|
||||
if (range.lookahead(1).length && range.lookahead(1)[0] == '.')
|
||||
break hexLoop;
|
||||
range.popFront();
|
||||
foundDot = true;
|
||||
|
|
@ -536,11 +550,17 @@ public struct DLexer(R)
|
|||
break hexLoop;
|
||||
}
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexBinary() pure nothrow
|
||||
{
|
||||
auto mark = range.mark();
|
||||
return lexBinary(mark);
|
||||
}
|
||||
|
||||
Token lexBinary(Mark mark) pure nothrow
|
||||
{
|
||||
IdType type = tok!"intLiteral";
|
||||
binaryLoop: while (!range.empty)
|
||||
|
|
@ -561,11 +581,11 @@ public struct DLexer(R)
|
|||
break binaryLoop;
|
||||
}
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexDecimal() pure nothrow
|
||||
Token lexDecimal(Mark mark) pure nothrow
|
||||
{
|
||||
bool foundDot = range.front == '.';
|
||||
IdType type = tok!"intLiteral";
|
||||
|
|
@ -608,16 +628,17 @@ public struct DLexer(R)
|
|||
case '.':
|
||||
if (foundDot)
|
||||
break decimalLoop;
|
||||
if (range.canPeek() && range.peek() == '.')
|
||||
auto lookahead = range.lookahead(1);
|
||||
if (lookahead.length == 1 && lookahead[0] == '.')
|
||||
break decimalLoop;
|
||||
else
|
||||
{
|
||||
// The following bit of silliness tries to tell the
|
||||
// difference between "int dot identifier" and
|
||||
// "double identifier".
|
||||
if (range.canPeek())
|
||||
if (lookahead.length == 1)
|
||||
{
|
||||
switch (range.peek())
|
||||
switch (lookahead[0])
|
||||
{
|
||||
case '0': .. case '9':
|
||||
goto doubleLiteral;
|
||||
|
|
@ -638,7 +659,7 @@ public struct DLexer(R)
|
|||
break decimalLoop;
|
||||
}
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
|
|
@ -749,7 +770,7 @@ public struct DLexer(R)
|
|||
|
||||
Token lexSlashStarComment() pure
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
IdType type = tok!"comment";
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
|
@ -767,13 +788,13 @@ public struct DLexer(R)
|
|||
else
|
||||
popFrontWhitespaceAware();
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexSlashSlashComment() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
IdType type = tok!"comment";
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
|
@ -783,13 +804,13 @@ public struct DLexer(R)
|
|||
break;
|
||||
range.popFront();
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexSlashPlusComment() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
IdType type = tok!"comment";
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
|
@ -817,13 +838,13 @@ public struct DLexer(R)
|
|||
else
|
||||
popFrontWhitespaceAware();
|
||||
}
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexStringLiteral() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
range.popFront();
|
||||
while (true)
|
||||
{
|
||||
|
|
@ -846,13 +867,13 @@ public struct DLexer(R)
|
|||
}
|
||||
IdType type = tok!"stringLiteral";
|
||||
lexStringSuffix(type);
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
Token lexWysiwygString() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
IdType type = tok!"stringLiteral";
|
||||
bool backtick = range.front == '`';
|
||||
if (backtick)
|
||||
|
|
@ -900,7 +921,7 @@ public struct DLexer(R)
|
|||
}
|
||||
}
|
||||
lexStringSuffix(type);
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
|
|
@ -922,7 +943,7 @@ public struct DLexer(R)
|
|||
|
||||
Token lexDelimitedString() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
ElementEncodingType!R open;
|
||||
|
|
@ -933,29 +954,29 @@ public struct DLexer(R)
|
|||
open = '<';
|
||||
close = '>';
|
||||
range.popFront();
|
||||
return lexNormalDelimitedString(open, close);
|
||||
return lexNormalDelimitedString(mark, open, close);
|
||||
case '{':
|
||||
open = '{';
|
||||
close = '}';
|
||||
range.popFront();
|
||||
return lexNormalDelimitedString(open, close);
|
||||
return lexNormalDelimitedString(mark, open, close);
|
||||
case '[':
|
||||
open = '[';
|
||||
close = ']';
|
||||
range.popFront();
|
||||
return lexNormalDelimitedString(open, close);
|
||||
return lexNormalDelimitedString(mark, open, close);
|
||||
case '(':
|
||||
open = '(';
|
||||
close = ')';
|
||||
range.popFront();
|
||||
return lexNormalDelimitedString(open, close);
|
||||
return lexNormalDelimitedString(mark, open, close);
|
||||
default:
|
||||
return lexHeredocString();
|
||||
}
|
||||
}
|
||||
|
||||
Token lexNormalDelimitedString(ElementEncodingType!RangeType open,
|
||||
ElementEncodingType!RangeType close) pure nothrow
|
||||
Token lexNormalDelimitedString(Mark mark, ElementEncodingType!R open,
|
||||
ElementEncodingType!R close) pure nothrow
|
||||
{
|
||||
int depth = 1;
|
||||
while (!range.empty && depth > 0)
|
||||
|
|
@ -985,7 +1006,7 @@ public struct DLexer(R)
|
|||
}
|
||||
IdType type = tok!"stringLiteral";
|
||||
lexStringSuffix(type);
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column, range.index);
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column, range.index);
|
||||
}
|
||||
|
||||
Token lexHeredocString() pure nothrow
|
||||
|
|
@ -1024,7 +1045,7 @@ public struct DLexer(R)
|
|||
|
||||
Token lexHexString() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
|
||||
|
|
@ -1055,7 +1076,7 @@ public struct DLexer(R)
|
|||
|
||||
IdType type = tok!"stringLiteral";
|
||||
lexStringSuffix(type);
|
||||
return Token(type, cast(string) range.getMarked(), range.line, range.column,
|
||||
return Token(type, cast(string) range.slice(mark), range.line, range.column,
|
||||
range.index);
|
||||
}
|
||||
|
||||
|
|
@ -1154,7 +1175,7 @@ public struct DLexer(R)
|
|||
|
||||
Token lexCharacterLiteral() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
range.popFront();
|
||||
if (range.front == '\\')
|
||||
{
|
||||
|
|
@ -1164,7 +1185,7 @@ public struct DLexer(R)
|
|||
else if (range.front == '\'')
|
||||
{
|
||||
range.popFront();
|
||||
return Token(tok!"characterLiteral", cast(string) range.getMarked(),
|
||||
return Token(tok!"characterLiteral", cast(string) range.slice(mark),
|
||||
range.line, range.column, range.index);
|
||||
}
|
||||
else if (range.front & 0x80)
|
||||
|
|
@ -1182,7 +1203,7 @@ public struct DLexer(R)
|
|||
if (range.front == '\'')
|
||||
{
|
||||
range.popFront();
|
||||
return Token(tok!"characterLiteral", cast(string) range.getMarked(),
|
||||
return Token(tok!"characterLiteral", cast(string) range.slice(mark),
|
||||
range.line, range.column, range.index);
|
||||
}
|
||||
else
|
||||
|
|
@ -1194,30 +1215,31 @@ public struct DLexer(R)
|
|||
|
||||
Token lexIdentifier() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
while (!range.empty && !isSeparating(range.front))
|
||||
{
|
||||
range.popFront();
|
||||
}
|
||||
return Token(tok!"identifier", cast(string) range.getMarked(), range.index,
|
||||
return Token(tok!"identifier", cast(string) range.slice(mark), range.index,
|
||||
range.line, range.column);
|
||||
}
|
||||
|
||||
Token lexDot() pure nothrow
|
||||
{
|
||||
if (!range.canPeek)
|
||||
auto lookahead = range.lookahead(1);
|
||||
if (lookahead.length == 0)
|
||||
{
|
||||
range.popFront();
|
||||
return Token(tok!".", null, range.line, range.column, range.index);
|
||||
}
|
||||
switch (range.peek())
|
||||
switch (lookahead[0])
|
||||
{
|
||||
case '0': .. case '9':
|
||||
return lexNumber();
|
||||
case '.':
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
if (range.front == '.')
|
||||
if (!range.empty && range.front == '.')
|
||||
{
|
||||
range.popFront();
|
||||
return Token(tok!"...", null, range.line, range.column, range.index);
|
||||
|
|
@ -1232,16 +1254,21 @@ public struct DLexer(R)
|
|||
|
||||
Token lexLongNewline() pure nothrow
|
||||
{
|
||||
range.mark();
|
||||
auto mark = range.mark();
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
range.popFront();
|
||||
range.incrementLine();
|
||||
return Token(tok!"whitespace", cast(string) range.getMarked(), range.line,
|
||||
return Token(tok!"whitespace", cast(string) range.slice(mark), range.line,
|
||||
range.column, range.index);
|
||||
}
|
||||
|
||||
Token lexScriptLine() pure nothrow
|
||||
{
|
||||
assert(false, "Not implemented");
|
||||
}
|
||||
|
||||
bool isSeparating(C)(C c) nothrow pure @safe
|
||||
bool isSeparating(ElementType!R c) nothrow pure @safe
|
||||
{
|
||||
if (c <= 0x2f) return true;
|
||||
if (c >= ':' && c <= '@') return true;
|
||||
|
|
|
|||
|
|
@ -1,62 +1,5 @@
|
|||
// Written in the D programming language
|
||||
|
||||
/**
|
||||
* This module contains a _parser for D source code.
|
||||
*
|
||||
* Grammar:
|
||||
* The grammar format used in the documentation of this module generally follows
|
||||
* the format used by the ANTLR _parser generator.
|
||||
* $(UL
|
||||
* $(LI Tokens and rules can be grouped by parenthesis.)
|
||||
* $(LI An asterisk (*) indicates that the previous rule, token, or group
|
||||
* can repeat 0 or more times.)
|
||||
* $(LI A question mark (?) indicates that the previous rule, token, or group
|
||||
* will be present either 0 or 1 times.)
|
||||
* $(LI A plus sign (+) indicates that the previous rule, token, or group
|
||||
* repeats one or more times. (i.e. it is optional))
|
||||
* $(LI If there is more than one way to match a rule, the alternatives will be
|
||||
* separated by a pipe character (|).)
|
||||
* $(LI Rule definitions begin with the rule name followed by a colon (:). Rule
|
||||
* definitions end with a semicolon (;).)
|
||||
* )
|
||||
*
|
||||
* The grammar for D starts with the $(LINK2 #module, module) rule.
|
||||
*
|
||||
* Examples:
|
||||
* ---
|
||||
* import std.d.lexer;
|
||||
* import std.d.parser;
|
||||
* import std.d.ast;
|
||||
* import std.array;
|
||||
*
|
||||
* string sourceCode = q{
|
||||
* import std.stdio;
|
||||
*
|
||||
* void main()
|
||||
* {
|
||||
* writeln("Hello, World.");
|
||||
* }
|
||||
* }c;
|
||||
* void main()
|
||||
* {
|
||||
* LexerConfig config;
|
||||
* auto tokens = byToken(cast(ubyte[]) sourceCode, config).array();
|
||||
* Module mod = parseModule(tokens);
|
||||
* // Use module here...
|
||||
* }
|
||||
* ---
|
||||
*
|
||||
* Copyright: Brian Schott 2013
|
||||
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
|
||||
* Authors: Brian Schott
|
||||
* Source: $(PHOBOSSRC std/d/_parser.d)
|
||||
* Macros:
|
||||
* GRAMMAR = <pre>$0</pre>
|
||||
* RULEDEF = <a name="$0"><span style="font-weight: bold;">$0</span></a>
|
||||
* RULE = <a href="#$0"><span style="font-weight: bold;">$0</span></a>
|
||||
* LITERAL = <span style="color: green;">$0</span>
|
||||
*/
|
||||
|
||||
module stdx.d.parser;
|
||||
|
||||
import stdx.d.lexer;
|
||||
|
|
@ -2162,7 +2105,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
|
|||
* Parses a ForStatement
|
||||
*
|
||||
* $(GRAMMAR $(RULEDEF forStatement):
|
||||
* $(LITERAL 'for') $(LITERAL '$(LPAREN)') $(RULE declarationOrStatement) $(RULE expression)? $(LITERAL ';') $(RULE expression)? $(LITERAL '$(RPAREN)') $(RULE statementNoCaseNoDefault)
|
||||
* $(LITERAL 'for') $(LITERAL '$(LPAREN)') $(RULE declarationOrStatement) $(RULE expression)? $(LITERAL ';') $(RULE expression)? $(LITERAL '$(RPAREN)') $(RULE declarationOrStatement)
|
||||
* ;)
|
||||
*/
|
||||
ForStatement parseForStatement()
|
||||
|
|
@ -2176,7 +2119,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
|
|||
if (currentIs(tok!";"))
|
||||
advance();
|
||||
else
|
||||
node.declarationOrStatement = parseDeclarationOrStatement();
|
||||
node.initialization = parseDeclarationOrStatement();
|
||||
|
||||
if (currentIs(tok!";"))
|
||||
advance();
|
||||
|
|
@ -2192,8 +2135,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
|
|||
error("Statement expected", false);
|
||||
return node; // this line makes DCD better
|
||||
}
|
||||
node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
|
||||
if (node.statementNoCaseNoDefault is null) return null;
|
||||
node.declarationOrStatement = parseDeclarationOrStatement();
|
||||
if (node.declarationOrStatement is null) return null;
|
||||
return node;
|
||||
}
|
||||
|
||||
|
|
@ -5917,7 +5860,7 @@ q{doStuff(5)}c;
|
|||
error("Statement expected", false);
|
||||
return node; // this line makes DCD better
|
||||
}
|
||||
node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault();
|
||||
node.declarationOrStatement = parseDeclarationOrStatement();
|
||||
return node;
|
||||
}
|
||||
|
||||
|
|
|
|||
277
stdx/lexer.d
277
stdx/lexer.d
|
|
@ -10,12 +10,15 @@
|
|||
*/
|
||||
|
||||
module stdx.lexer;
|
||||
|
||||
import std.typecons;
|
||||
import std.algorithm;
|
||||
import std.range;
|
||||
import std.traits;
|
||||
import std.conv;
|
||||
import std.math;
|
||||
import dpick.buffer.buffer;
|
||||
import dpick.buffer.traits;
|
||||
|
||||
template TokenIdType(alias staticTokens, alias dynamicTokens,
|
||||
alias possibleDefaultTokens)
|
||||
|
|
@ -34,12 +37,12 @@ string TokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens
|
|||
{
|
||||
if (type == 0)
|
||||
return "!ERROR!";
|
||||
else if (type < staticTokens.length)
|
||||
else if (type < staticTokens.length + 1)
|
||||
return staticTokens[type - 1];
|
||||
else if (type < staticTokens.length + possibleDefaultTokens.length)
|
||||
return possibleDefaultTokens[type - staticTokens.length];
|
||||
else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length)
|
||||
return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length];
|
||||
else if (type < staticTokens.length + possibleDefaultTokens.length + 1)
|
||||
return possibleDefaultTokens[type - staticTokens.length - 1];
|
||||
else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
|
||||
return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length - 1];
|
||||
else
|
||||
return null;
|
||||
}
|
||||
|
|
@ -70,14 +73,16 @@ template TokenId(IdType, alias staticTokens, alias dynamicTokens,
|
|||
enum ii = possibleDefaultTokens.countUntil(symbol);
|
||||
static if (ii >= 0)
|
||||
{
|
||||
enum id = ii + staticTokens.length;
|
||||
enum id = ii + staticTokens.length + 1;
|
||||
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
|
||||
alias id TokenId;
|
||||
}
|
||||
else
|
||||
{
|
||||
enum dynamicId = dynamicTokens.countUntil(symbol);
|
||||
enum id = dynamicId >= 0 ? i + staticTokens.length + possibleDefaultTokens.length + dynamicId : -1;
|
||||
enum id = dynamicId >= 0
|
||||
? i + staticTokens.length + possibleDefaultTokens.length + dynamicId + 1
|
||||
: -1;
|
||||
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
|
||||
alias id TokenId;
|
||||
}
|
||||
|
|
@ -113,13 +118,10 @@ struct TokenStructure(IDType)
|
|||
IDType type;
|
||||
}
|
||||
|
||||
mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFunction,
|
||||
mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
|
||||
alias staticTokens, alias dynamicTokens, alias pseudoTokens,
|
||||
alias possibleDefaultTokens) if (isForwardRange!R)
|
||||
alias pseudoTokenHandlers, alias possibleDefaultTokens)
|
||||
{
|
||||
enum size_t lookAhead = chain(staticTokens, pseudoTokens).map!"a.length".reduce!"max(a, b)"();
|
||||
alias PeekRange!(R, lookAhead) RangeType;
|
||||
|
||||
static string generateCaseStatements(string[] tokens, size_t offset = 0)
|
||||
{
|
||||
string code;
|
||||
|
|
@ -141,9 +143,9 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
code ~= generateLeaf(tokens[i], indent ~ " ");
|
||||
else
|
||||
{
|
||||
code ~= indent ~ " if (!range.canPeek(" ~ text(tokens[i].length - 1) ~ "))\n";
|
||||
code ~= indent ~ " if (range.lookahead(" ~ text(tokens[i].length) ~ ").length == 0)\n";
|
||||
code ~= indent ~ " goto outer_default;\n";
|
||||
code ~= indent ~ " if (range.startsWith(\"" ~ escape(tokens[i]) ~ "\"))\n";
|
||||
code ~= indent ~ " if (range.lookahead(" ~ text(tokens[i].length) ~ ") == \"" ~ escape(tokens[i]) ~ "\")\n";
|
||||
code ~= indent ~ " {\n";
|
||||
code ~= generateLeaf(tokens[i], indent ~ " ");
|
||||
code ~= indent ~ " }\n";
|
||||
|
|
@ -153,11 +155,11 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
}
|
||||
else
|
||||
{
|
||||
code ~= indent ~ " if (!range.canPeek(" ~ text(offset + 1) ~ "))\n";
|
||||
code ~= indent ~ " if (range.lookahead(" ~ text(offset + 2) ~ ").length == 0)\n";
|
||||
code ~= indent ~ " {\n";
|
||||
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
|
||||
code ~= indent ~ " }\n";
|
||||
code ~= indent ~ " switch (range.peek(" ~ text(offset + 1) ~ "))\n";
|
||||
code ~= indent ~ " switch (range.lookahead(" ~ text(offset + 2) ~ ")[" ~ text(offset + 1) ~ "])\n";
|
||||
code ~= indent ~ " {\n";
|
||||
code ~= generateCaseStatements(tokens[i .. j], offset + 1);
|
||||
code ~= indent ~ " default:\n";
|
||||
|
|
@ -172,6 +174,8 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
|
||||
static string generateLeaf(string token, string indent)
|
||||
{
|
||||
static assert (pseudoTokenHandlers.length % 2 == 0,
|
||||
"Each pseudo-token must have a matching function name.");
|
||||
string code;
|
||||
if (staticTokens.countUntil(token) >= 0)
|
||||
{
|
||||
|
|
@ -179,13 +183,13 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
code ~= indent ~ "range.popFront();\n";
|
||||
else
|
||||
code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
|
||||
code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
|
||||
code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~ "\", null, range.line, range.column, range.index);\n";
|
||||
}
|
||||
else if (pseudoTokens.countUntil(token) >= 0)
|
||||
code ~= indent ~ "return postProcess(pseudoTok!\"" ~ escape(token) ~"\");\n";
|
||||
code ~= indent ~ "return " ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] ~ "();\n";
|
||||
else if (possibleDefaultTokens.countUntil(token) >= 0)
|
||||
{
|
||||
code ~= indent ~ "if (!range.canPeek(" ~ text(token.length) ~ ") || isSeparating(range.peek(" ~ text(token.length) ~ ")))\n";
|
||||
code ~= indent ~ "if (range.lookahead(" ~ text(token.length + 1) ~ ").length == 0 || isSeparating(range.lookahead(" ~ text(token.length + 1) ~ ")[" ~ text(token.length) ~ "]))\n";
|
||||
code ~= indent ~ "{\n";
|
||||
if (token.length == 1)
|
||||
code ~= indent ~ " range.popFront();\n";
|
||||
|
|
@ -211,18 +215,11 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
_front = advance();
|
||||
}
|
||||
|
||||
bool empty() const nothrow @property
|
||||
bool empty() pure const nothrow @property
|
||||
{
|
||||
return _front.type == tok!"\0";
|
||||
}
|
||||
|
||||
template pseudoTok(string symbol)
|
||||
{
|
||||
static assert (pseudoTokens.countUntil(symbol) >= 0);
|
||||
enum index = cast(IDType) pseudoTokens.countUntil(symbol);
|
||||
alias index pseudoTok;
|
||||
}
|
||||
|
||||
static string escape(string input)
|
||||
{
|
||||
string rVal;
|
||||
|
|
@ -267,224 +264,36 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
|
|||
return rVal;
|
||||
}
|
||||
|
||||
void registerPostProcess(alias t)(Token delegate() pure fun)
|
||||
{
|
||||
post[pseudoTok!t] = fun;
|
||||
}
|
||||
|
||||
Token postProcess(IDType i) pure
|
||||
{
|
||||
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]);
|
||||
return post[i]();
|
||||
}
|
||||
|
||||
Token delegate() pure [pseudoTokens.length] post;
|
||||
RangeType range;
|
||||
LexerRange!(typeof(buffer(R.init))) range;
|
||||
Token _front;
|
||||
}
|
||||
|
||||
struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R
|
||||
&& isForwardRange!R && hasSlicing!R)
|
||||
struct LexerRange(BufferType) if (isBuffer!BufferType)
|
||||
{
|
||||
public:
|
||||
|
||||
this(R range)
|
||||
this(BufferType r)
|
||||
{
|
||||
this.range = range;
|
||||
this.range = r;
|
||||
index = 0;
|
||||
column = 1;
|
||||
line = 1;
|
||||
}
|
||||
|
||||
invariant()
|
||||
void popFront() pure
|
||||
{
|
||||
import std.string;
|
||||
if (range.length != 6190)
|
||||
assert (false, format("range.length = %d %s", range.length, cast(char[]) range[0 .. 100]));
|
||||
index++;
|
||||
column++;
|
||||
range.popFront();
|
||||
}
|
||||
|
||||
bool startsWith(string s)
|
||||
{
|
||||
return index + s.length < range.length
|
||||
&& (cast(const(ubyte[])) s) == range[index .. index + s.length];
|
||||
}
|
||||
|
||||
bool empty() pure nothrow const @property
|
||||
{
|
||||
return _index >= range.length;
|
||||
}
|
||||
|
||||
const(ElementType!R) front() pure nothrow const @property
|
||||
in
|
||||
{
|
||||
assert (!empty);
|
||||
}
|
||||
body
|
||||
{
|
||||
return range[_index];
|
||||
}
|
||||
|
||||
void popFront() pure nothrow
|
||||
{
|
||||
_index++;
|
||||
_column++;
|
||||
}
|
||||
|
||||
void popFrontN(size_t n) pure nothrow
|
||||
{
|
||||
foreach (i; 0 .. n)
|
||||
popFront();
|
||||
}
|
||||
|
||||
const(ElementType!R) peek(int offset = 1) pure nothrow const
|
||||
in
|
||||
{
|
||||
assert (canPeek(offset));
|
||||
}
|
||||
body
|
||||
{
|
||||
return range[_index + offset];
|
||||
}
|
||||
|
||||
bool canPeek(size_t offset = 1) pure nothrow const
|
||||
{
|
||||
return _index + offset < range.length;
|
||||
}
|
||||
|
||||
void mark() nothrow pure
|
||||
{
|
||||
markBegin = _index;
|
||||
}
|
||||
|
||||
const(R) getMarked() pure nothrow const
|
||||
{
|
||||
return range[markBegin .. _index];
|
||||
}
|
||||
|
||||
void incrementLine() pure nothrow
|
||||
{
|
||||
_column = 1;
|
||||
_line++;
|
||||
column = 1;
|
||||
line++;
|
||||
}
|
||||
|
||||
size_t line() pure nothrow const @property { return _line; }
|
||||
size_t column() pure nothrow const @property { return _column; }
|
||||
size_t index() pure nothrow const @property { return _index; }
|
||||
|
||||
private:
|
||||
size_t markBegin;
|
||||
size_t _column = 1;
|
||||
size_t _line = 1;
|
||||
size_t _index = 0;
|
||||
R range;
|
||||
|
||||
BufferType range;
|
||||
alias range this;
|
||||
size_t index;
|
||||
size_t column;
|
||||
size_t line;
|
||||
}
|
||||
|
||||
//struct PeekRange(R, size_t peekSupported = 1)
|
||||
// if (!isRandomAccessRange!R && isForwardRange!R)
|
||||
//{
|
||||
//public:
|
||||
//
|
||||
// this(R range)
|
||||
// {
|
||||
// this.range = range;
|
||||
// for (size_t i = 0; !this.range.empty && i < peekSupported; i++)
|
||||
// {
|
||||
// rangeSizeCount++;
|
||||
// buffer[i] = this.range.front;
|
||||
// range.popFront();
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// ElementType!R front() const @property
|
||||
// in
|
||||
// {
|
||||
// assert (!empty);
|
||||
// }
|
||||
// body
|
||||
// {
|
||||
// return buffer[bufferIndex];
|
||||
// }
|
||||
//
|
||||
// void popFront()
|
||||
// in
|
||||
// {
|
||||
// assert (!empty);
|
||||
// }
|
||||
// body
|
||||
// {
|
||||
// index++;
|
||||
// column++;
|
||||
// count++;
|
||||
// bufferIndex = bufferIndex + 1 > buffer.length ? 0 : bufferIndex + 1;
|
||||
// if (marking)
|
||||
// markBuffer.put(buffer[bufferIndex]);
|
||||
// if (!range.empty)
|
||||
// {
|
||||
// buffer[bufferIndex + peekSupported % buffer.length] = range.front();
|
||||
// range.popFront();
|
||||
// rangeSizeCount++;
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// bool empty() const nothrow pure @property
|
||||
// {
|
||||
// return rangeSizeCount == count;
|
||||
// }
|
||||
//
|
||||
// ElementType!R peek(int offset = 1) pure nothrow const
|
||||
// in
|
||||
// {
|
||||
// assert (canPeek(offset));
|
||||
// }
|
||||
// body
|
||||
// {
|
||||
// return buffer[(bufferIndex + offset) % buffer.length];
|
||||
// }
|
||||
//
|
||||
// bool canPeek(size_t int offset = 1) pure nothrow const
|
||||
// {
|
||||
// return offset <= peekSupported && count + offset <= rangeSizeCount;
|
||||
// }
|
||||
//
|
||||
// typeof(this) save() @property
|
||||
// {
|
||||
// typeof(this) newRange;
|
||||
// newRange.count = count;
|
||||
// newRange.rangeSizeCount = count;
|
||||
// newRange.buffer = buffer.dup;
|
||||
// newRange.bufferIndex = bufferIndex;
|
||||
// newRange.range = range.save;
|
||||
// return newRange;
|
||||
// }
|
||||
//
|
||||
// void mark()
|
||||
// {
|
||||
// marking = true;
|
||||
// markBuffer.clear();
|
||||
// }
|
||||
//
|
||||
// ElementEncodingType!R[] getMarked()
|
||||
// {
|
||||
// marking = false;
|
||||
// return markBuffer.data;
|
||||
// }
|
||||
//
|
||||
// void incrementLine() pure nothrow
|
||||
// {
|
||||
// _column = 1;
|
||||
// _line++;
|
||||
// }
|
||||
//
|
||||
// size_t line() pure nothrow const @property { return _line; }
|
||||
// size_t column() pure nothrow const @property { return _column; }
|
||||
// size_t index() pure nothrow const @property { return _index; }
|
||||
//
|
||||
//private:
|
||||
// auto markBuffer = appender!(ElementType!R[])();
|
||||
// bool marking;
|
||||
// size_t count;
|
||||
// size_t rangeSizeCount;
|
||||
// ElementType!(R)[peekSupported + 1] buffer;
|
||||
// size_t bufferIndex;
|
||||
// size_t _column = 1;
|
||||
// size_t _line = 1;
|
||||
// size_t _index = 0;
|
||||
// R range;
|
||||
//}
|
||||
|
|
|
|||
Loading…
Reference in New Issue