forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrated over to Dmitry's buffer range. Lots of stuff is disabled for now

This commit is contained in:
Hackerpilot 2014-01-09 01:17:47 +00:00
parent 1febda6fc7
commit b092840c70
12 changed files with 286 additions and 762 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored Executable file → Normal file
View File

@ -0,0 +1,3 @@
[submodule "datapicked"]
path = datapicked
url = ./datapicked/

8
astprinter.d
View File

@ -469,9 +469,9 @@ class XMLPrinter : ASTVisitor
output.writeln("<forStatement>"); output.writeln("<forStatement>");
if (forStatement.declarationOrStatement !is null) if (forStatement.declarationOrStatement !is null)
{ {
output.writeln("<initialize>"); output.writeln("<initialization>");
visit(forStatement.declarationOrStatement); visit(forStatement.initialization);
output.writeln("</initialize>"); output.writeln("</initialization>");
} }
if (forStatement.test !is null) if (forStatement.test !is null)
{ {
@ -485,7 +485,7 @@ class XMLPrinter : ASTVisitor
visit(forStatement.increment); visit(forStatement.increment);
output.writeln("</increment>"); output.writeln("</increment>");
} }
visit(forStatement.statementNoCaseNoDefault); visit(forStatement.declarationOrStatement);
output.writeln("</forStatement>"); output.writeln("</forStatement>");
} }

15
build.sh
View File

@ -1,5 +1,18 @@
#dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd #dmd *.d stdx/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner-dmd
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -g -m64 -wi -ofdscanner dmd\
main.d\
stats.d\
imports.d\
highlighter.d\
ctags.d\
astprinter.d\
formatter.d\
outliner.d\
stdx/*.d\
stdx/d/*.d\
datapicked/dpick/buffer/*.d\
-Idatapicked\
-g -m64 -wi -ofdscanner
#ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq #ldc2 main.d stats.d imports.d highlighter.d ctags.d astprinter.d formatter.d outliner.d stdx/*.d stdx/d/*.d -of=dscanner-ldc -m64 -oq
#ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g #ldc2 *.d stdx/d/*.d -of=dscanner -unittest -m64 -g
#/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d #/opt/gdc/bin/gdc -O3 -odscanner-gdc -fno-bounds-check -frelease -m64 *.d stdx/d/*.d

3
ctags.d
View File

@ -24,7 +24,8 @@ void printCtags(File output, string[] fileNames)
File f = File(fileName); File f = File(fileName);
auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size)); auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
f.rawRead(bytes); f.rawRead(bytes);
Module m = parseModule(byToken!(ubyte[])(bytes).array, fileName, &doNothing); auto tokens = DLexer!(typeof(bytes))(bytes);
Module m = parseModule(tokens.array, fileName, &doNothing);
auto printer = new CTagsPrinter; auto printer = new CTagsPrinter;
printer.fileName = fileName; printer.fileName = fileName;
printer.visit(m); printer.visit(m);

1
datapicked Submodule

@ -0,0 +1 @@
Subproject commit eb14a5244153c0e13ceca79f292838dfe2ac9bfb

6
highlighter.d
View File

@ -11,7 +11,7 @@ import std.array;
import stdx.d.lexer; import stdx.d.lexer;
// http://ethanschoonover.com/solarized // http://ethanschoonover.com/solarized
void highlight(R)(R tokens, string fileName) void highlight(R)(ref R tokens, string fileName)
{ {
stdout.writeln(q"[ stdout.writeln(q"[
<!DOCTYPE html> <!DOCTYPE html>
@ -33,8 +33,10 @@ html { background-color: #fdf6e3; color: #002b36; }
</style> </style>
<pre>]"); <pre>]");
foreach (Token t; tokens) while (!tokens.empty)
{ {
auto t = tokens.front;
tokens.popFront();
if (isBasicType(t.type)) if (isBasicType(t.type))
writeSpan("type", str(t.type)); writeSpan("type", str(t.type));
else if (isKeyword(t.type)) else if (isKeyword(t.type))

133
main.d
View File

@ -17,13 +17,14 @@ import std.stdio;
import std.range; import std.range;
import stdx.d.lexer; import stdx.d.lexer;
import stdx.d.parser; import stdx.d.parser;
import dpick.buffer.buffer;
import highlighter; import highlighter;
import stats; //import stats;
import ctags; //import ctags;
import astprinter; //import astprinter;
import imports; //import imports;
import outliner; //import outliner;
int main(string[] args) int main(string[] args)
{ {
@ -91,69 +92,69 @@ int main(string[] args)
{ {
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
highlighter.highlight(byToken!(typeof(bytes), false, false)(bytes), auto tokens = DLexer!(ubyte[])(bytes);
args.length == 1 ? "stdin" : args[1]); highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
return 0; return 0;
} }
else if (ctags) // else if (ctags)
{ // {
stdout.printCtags(expandArgs(args, recursive)); // stdout.printCtags(expandArgs(args, recursive));
} // }
else // else
{ // {
bool usingStdin = args.length == 1; // bool usingStdin = args.length == 1;
if (sloc || tokenCount) // if (sloc || tokenCount)
{ // {
if (usingStdin) // if (usingStdin)
{ // {
auto tokens = byToken!(ubyte[], false, false)(readStdin()); // auto tokens = byToken!(ubyte[], false, false)(readStdin());
if (tokenCount) // if (tokenCount)
printTokenCount(stdout, "stdin", tokens); // printTokenCount(stdout, "stdin", tokens);
else // else
printLineCount(stdout, "stdin", tokens); // printLineCount(stdout, "stdin", tokens);
} // }
else // else
{ // {
ulong count; // ulong count;
foreach (f; expandArgs(args, recursive)) // foreach (f; expandArgs(args, recursive))
{ // {
auto tokens = byToken!(ubyte[])(readFile(f)); // auto tokens = byToken!(ubyte[])(readFile(f));
if (tokenCount) // if (tokenCount)
count += printTokenCount(stdout, f, tokens); // count += printTokenCount(stdout, f, tokens);
else // else
count += printLineCount(stdout, f, tokens); // count += printLineCount(stdout, f, tokens);
} // }
writefln("total:\t%d", count); // writefln("total:\t%d", count);
} // }
} // }
else if (syntaxCheck) // else if (syntaxCheck)
{ // {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); // auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); // parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
} // }
else if (imports) // else if (imports)
{ // {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); // auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); // auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto visitor = new ImportPrinter; // auto visitor = new ImportPrinter;
visitor.visit(mod); // visitor.visit(mod);
} // }
else if (ast) // else if (ast)
{ // {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); // auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); // auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto printer = new XMLPrinter; // auto printer = new XMLPrinter;
printer.output = stdout; // printer.output = stdout;
printer.visit(mod); // printer.visit(mod);
} // }
else if (outline) // else if (outline)
{ // {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); // auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); // auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto outliner = new Outliner(stdout); // auto outliner = new Outliner(stdout);
outliner.visit(mod); // outliner.visit(mod);
} // }
} // }
return 0; return 0;
} }

276
main.html
View File

@ -1,276 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<title>main.d</title>
</head>
<body>
<style type="text/css">
html { background-color: #fdf6e3; color: #002b36; }
.kwrd { color: #b58900; font-weight: bold; }
.com { color: #93a1a1; font-style: italic; }
.num { color: #dc322f; font-weigth: bold; }
.str { color: #2aa198; font-style: italic; }
.op { color: #586e75; font-weight: bold; }
.type { color: #268bd2; font-weight: bold; }
.cons { color: #859900; font-weight: bold; }
</style>
<pre>
<span class="com">// Copyright Brian Schott (Sir Alaran) 2012.</span>
<span class="com">// Distributed under the Boost Software License, Version 1.0.</span>
<span class="com">// (See accompanying file LICENSE_1_0.txt or copy at</span>
<span class="com">// http://www.boost.org/LICENSE_1_0.txt)</span>
<span class="kwrd">module</span> main<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>algorithm<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>array<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>conv<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>file<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>getopt<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>parallelism<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>path<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>regex<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>stdio<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>range<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>lexer<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>parser<span class="op">;</span>
<span class="kwrd">import</span> highlighter<span class="op">;</span>
<span class="kwrd">import</span> stats<span class="op">;</span>
<span class="kwrd">import</span> ctags<span class="op">;</span>
<span class="kwrd">import</span> astprinter<span class="op">;</span>
<span class="kwrd">import</span> imports<span class="op">;</span>
<span class="kwrd">import</span> outliner<span class="op">;</span>
<span class="type">int</span> main<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> sloc<span class="op">;</span>
<span class="type">bool</span> highlight<span class="op">;</span>
<span class="type">bool</span> ctags<span class="op">;</span>
<span class="type">bool</span> recursive<span class="op">;</span>
<span class="type">bool</span> format<span class="op">;</span>
<span class="type">bool</span> help<span class="op">;</span>
<span class="type">bool</span> tokenCount<span class="op">;</span>
<span class="type">bool</span> syntaxCheck<span class="op">;</span>
<span class="type">bool</span> ast<span class="op">;</span>
<span class="type">bool</span> imports<span class="op">;</span>
<span class="type">bool</span> muffin<span class="op">;</span>
<span class="type">bool</span> outline<span class="op">;</span>
<span class="kwrd">try</span>
<span class="op">{</span>
getopt<span class="op">(</span>args<span class="op">,</span> <span class="str">"sloc|l"</span><span class="op">,</span> <span class="op">&amp;</span>sloc<span class="op">,</span> <span class="str">"highlight"</span><span class="op">,</span> <span class="op">&amp;</span>highlight<span class="op">,</span>
<span class="str">"ctags|c"</span><span class="op">,</span> <span class="op">&amp;</span>ctags<span class="op">,</span> <span class="str">"recursive|r|R"</span><span class="op">,</span> <span class="op">&amp;</span>recursive<span class="op">,</span> <span class="str">"help|h"</span><span class="op">,</span> <span class="op">&amp;</span>help<span class="op">,</span>
<span class="str">"tokenCount|t"</span><span class="op">,</span> <span class="op">&amp;</span>tokenCount<span class="op">,</span> <span class="str">"syntaxCheck|s"</span><span class="op">,</span> <span class="op">&amp;</span>syntaxCheck<span class="op">,</span>
<span class="str">"ast|xml"</span><span class="op">,</span> <span class="op">&amp;</span>ast<span class="op">,</span> <span class="str">"imports|i"</span><span class="op">,</span> <span class="op">&amp;</span>imports<span class="op">,</span> <span class="str">"outline|o"</span><span class="op">,</span> <span class="op">&amp;</span>outline<span class="op">,</span>
<span class="str">"muffinButton"</span><span class="op">,</span> <span class="op">&amp;</span>muffin<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">catch</span> <span class="op">(</span>Exception e<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span>e<span class="op">.</span>msg<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>muffin<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>writeln<span class="op">(</span>
<span class="str">` ___________
__(#*O 0** @%*)__
_(%*o#*O%*0 #O#%##@)_
(*#@%#o*@ #o%O*%@ #o #)
\=====================/
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|`</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>help<span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">auto</span> optionCount <span class="op">=</span> count<span class="op">!</span><span class="str">"a"</span><span class="op">(</span><span class="op">[</span>sloc<span class="op">,</span> highlight<span class="op">,</span> ctags<span class="op">,</span> tokenCount<span class="op">,</span>
syntaxCheck<span class="op">,</span> ast<span class="op">,</span> imports<span class="op">,</span> outline<span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">></span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span><span class="str">"Too many options specified"</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">&lt;</span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>highlight<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> bytes <span class="op">=</span> usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
highlighter<span class="op">.</span>highlight<span class="op">(</span>byToken<span class="op">!</span><span class="op">(</span><span class="kwrd">typeof</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span>
args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span> <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ctags<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>printCtags<span class="op">(</span>expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>sloc <span class="op">||</span> tokenCount<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>usingStdin<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>readStdin<span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
printTokenCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
printLineCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">ulong</span> count<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>f<span class="op">;</span> expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>readFile<span class="op">(</span>f<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
count <span class="op">+=</span> printTokenCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
count <span class="op">+=</span> printLineCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
writefln<span class="op">(</span><span class="str">"total:\t%d"</span><span class="op">,</span> count<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>syntaxCheck<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>imports<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> visitor <span class="op">=</span> <span class="kwrd">new</span> ImportPrinter<span class="op">;</span>
visitor<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ast<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> printer <span class="op">=</span> <span class="kwrd">new</span> XMLPrinter<span class="op">;</span>
printer<span class="op">.</span>output <span class="op">=</span> stdout<span class="op">;</span>
printer<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>outline<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> outliner <span class="op">=</span> <span class="kwrd">new</span> Outliner<span class="op">(</span>stdout<span class="op">)</span><span class="op">;</span>
outliner<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
string<span class="op">[</span><span class="op">]</span> expandArgs<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">,</span> <span class="type">bool</span> recursive<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>recursive<span class="op">)</span>
<span class="op">{</span>
string<span class="op">[</span><span class="op">]</span> rVal<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>arg<span class="op">;</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span><span class="op">$</span><span class="op">]</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>arg<span class="op">)</span> <span class="op">&amp;&amp;</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> arg<span class="op">;</span>
<span class="kwrd">else</span> <span class="kwrd">foreach</span> <span class="op">(</span>item<span class="op">;</span> dirEntries<span class="op">(</span>arg<span class="op">,</span> SpanMode<span class="op">.</span>breadth<span class="op">)</span><span class="op">.</span>map<span class="op">!</span><span class="op">(</span>a <span class="op">=></span> a<span class="op">.</span>name<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>item<span class="op">)</span> <span class="op">&amp;&amp;</span> <span class="op">(</span>item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> item<span class="op">;</span>
<span class="kwrd">else</span>
<span class="kwrd">continue</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> rVal<span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="kwrd">return</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span> <span class="op">$</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readStdin<span class="op">(</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> sourceCode <span class="op">=</span> appender<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span><span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="num">4096</span><span class="op">]</span> buf<span class="op">;</span>
<span class="kwrd">while</span> <span class="op">(</span><span class="kwrd">true</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> b <span class="op">=</span> stdin<span class="op">.</span>rawRead<span class="op">(</span>buf<span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>b<span class="op">.</span>length <span class="op">==</span> <span class="num">0</span><span class="op">)</span>
<span class="kwrd">break</span><span class="op">;</span>
sourceCode<span class="op">.</span>put<span class="op">(</span>b<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">return</span> sourceCode<span class="op">.</span>data<span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readFile<span class="op">(</span>string fileName<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span><span class="op">!</span>exists<span class="op">(</span>fileName<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span><span class="str">"%s does not exist"</span><span class="op">,</span> fileName<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="op">[</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
File f <span class="op">=</span> File<span class="op">(</span>fileName<span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> sourceCode <span class="op">=</span> uninitializedArray<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>to<span class="op">!</span>size_t<span class="op">(</span>f<span class="op">.</span>size<span class="op">)</span><span class="op">)</span><span class="op">;</span>
f<span class="op">.</span>rawRead<span class="op">(</span>sourceCode<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> sourceCode<span class="op">;</span>
<span class="op">}</span>
<span class="type">void</span> printHelp<span class="op">(</span>string programName<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span>
<span class="str">`
Usage: %s options
options:
--help | -h
Prints this help message
--sloc | -l [sourceFiles]
Prints the number of logical lines of code in the given
source files. If no files are specified, input is read from stdin.
--tokenCount | t [sourceFiles]
Prints the number of tokens in the given source files. If no files are
specified, input is read from stdin.
--highlight [sourceFile] - Syntax-highlight the given source file. The
resulting HTML will be written to standard output. If no files are
specified, input is read from stdin.
--imports | -i [sourceFile]
Prints modules imported by the given source file. If no files are
specified, input is read from stdin.
--syntaxCheck | -s [sourceFile]
Lexes and parses sourceFile, printing the line and column number of any
syntax errors to stdout. One error or warning is printed per line.
If no files are specified, input is read from stdin.
--ctags | -c sourceFile
Generates ctags information from the given source code file. Note that
ctags information requires a filename, so stdin cannot be used in place
of a filename.
--ast | --xml sourceFile
Generates an XML representation of the source files abstract syntax
tree. If no files are specified, input is read from stdin.
--recursive | -R | -r
When used with --ctags, --tokenCount, or --sloc, dscanner will produce
ctags output for all .d and .di files contained within the given
directories and its sub-directories.`</span><span class="op">,</span>
programName<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
</pre>
</body></html>

12
stdx/d/ast.d
View File

@ -1203,13 +1203,13 @@ class ForStatement : ASTNode
public: public:
override void accept(ASTVisitor visitor) override void accept(ASTVisitor visitor)
{ {
mixin (visitIfNotNull!(declarationOrStatement, test, increment, mixin (visitIfNotNull!(initialization, test, increment,
statementNoCaseNoDefault)); declarationOrStatement));
} }
/** */ DeclarationOrStatement declarationOrStatement; /** */ DeclarationOrStatement initialization;
/** */ ExpressionStatement test; /** */ ExpressionStatement test;
/** */ Expression increment; /** */ Expression increment;
/** */ StatementNoCaseNoDefault statementNoCaseNoDefault; /** */ DeclarationOrStatement declarationOrStatement;
/** */ size_t startIndex; /** */ size_t startIndex;
} }
@ -2760,11 +2760,11 @@ class WhileStatement : ASTNode
public: public:
override void accept(ASTVisitor visitor) override void accept(ASTVisitor visitor)
{ {
mixin (visitIfNotNull!(expression, statementNoCaseNoDefault)); mixin (visitIfNotNull!(expression, declarationOrStatement));
} }
/** */ Expression expression; /** */ Expression expression;
/** */ StatementNoCaseNoDefault statementNoCaseNoDefault; /** */ DeclarationOrStatement declarationOrStatement;
/** */ size_t startIndex; /** */ size_t startIndex;
} }

247
stdx/d/lexer.d
View File

@ -17,7 +17,7 @@ private enum staticTokens = [
private enum pseudoTokens = [ private enum pseudoTokens = [
"\"", "`", "//", "/*", "/+", ".", "'", "0", "1", "2", "3", "4", "5", "6", "\"", "`", "//", "/*", "/+", ".", "'", "0", "1", "2", "3", "4", "5", "6",
"7", "8", "9", "#", "q\"", "q{", "r\"", "x\"", " ", "\t", "\r", "\n", "#!", "7", "8", "9", "q\"", "q{", "r\"", "x\"", " ", "\t", "\r", "\n", "#!",
"\u2028", "\u2029" "\u2028", "\u2029"
]; ];
@ -57,24 +57,24 @@ public template tok(string token)
} }
public alias stdx.lexer.TokenStructure!(IdType) Token; public alias stdx.lexer.TokenStructure!(IdType) Token;
public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range) //public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
{ //{
pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; } // pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; } // pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; } // pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
// return new DLexer!(R)(range);
static if (skipComments) // static if (skipComments)
{ // {
static if (skipWhitespace) // static if (skipWhitespace)
return DLexer!(R)(range).filter!isNotEither; // return filter!isNotEither(tokens);
else // else
return DLexer!(R)(range).filter!isNotComment; // return filter!isNotComment(tokens);
} // }
else static if (skipWhitespace) // else static if (skipWhitespace)
return DLexer!(R)(range).filter!isNotWhitespace; // return filter!isNotWhitespace(tokens);
else // else
return DLexer!(R)(range); // return tokens;
} //}
public bool isBasicType(IdType type) nothrow pure @safe public bool isBasicType(IdType type) nothrow pure @safe
{ {
@ -322,45 +322,50 @@ public struct DLexer(R)
{ {
import std.conv; import std.conv;
import core.vararg; import core.vararg;
import dpick.buffer.buffer;
mixin Lexer!(R, IdType, Token, isSeparating, lexIdentifier, staticTokens,
dynamicTokens, pseudoTokens, possibleDefaultTokens); private enum pseudoTokenHandlers = [
"\"", "lexStringLiteral",
"`", "lexWysiwygString",
"//", "lexSlashSlashComment",
"/*", "lexSlashStarComment",
"/+", "lexSlashPlusComment",
".", "lexDot",
"'", "lexCharacterLiteral",
"0", "lexNumber",
"1", "lexNumber",
"2", "lexNumber",
"3", "lexNumber",
"4", "lexNumber",
"5", "lexNumber",
"6", "lexNumber",
"7", "lexNumber",
"8", "lexNumber",
"9", "lexNumber",
"q\"", "lexDelimitedString",
"q{", "lexTokenString",
"r\"", "lexWysiwygString",
"x\"", "lexHexString",
" ", "lexWhitespace",
"\t", "lexWhitespace",
"\r", "lexWhitespace",
"\n", "lexWhitespace",
"\u2028", "lexLongNewline",
"\u2029", "lexLongNewline",
"#!", "lexScriptLine"
];
mixin Lexer!(R, IdType, Token, lexIdentifier, staticTokens,
dynamicTokens, pseudoTokens, pseudoTokenHandlers, possibleDefaultTokens);
private alias typeof(range).Mark Mark;
this(R range) this(R range)
{ {
registerPostProcess!"\""(&lexStringLiteral); this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
registerPostProcess!"`"(&lexWysiwygString);
registerPostProcess!"//"(&lexSlashSlashComment);
registerPostProcess!"/*"(&lexSlashStarComment);
registerPostProcess!"/+"(&lexSlashPlusComment);
registerPostProcess!"."(&lexDot);
registerPostProcess!"'"(&lexCharacterLiteral);
registerPostProcess!"0"(&lexNumber);
registerPostProcess!"1"(&lexNumber);
registerPostProcess!"2"(&lexNumber);
registerPostProcess!"3"(&lexNumber);
registerPostProcess!"4"(&lexNumber);
registerPostProcess!"5"(&lexNumber);
registerPostProcess!"6"(&lexNumber);
registerPostProcess!"7"(&lexNumber);
registerPostProcess!"8"(&lexNumber);
registerPostProcess!"9"(&lexNumber);
registerPostProcess!"#"(&lexNumber);
registerPostProcess!"q\""(&lexDelimitedString);
registerPostProcess!"q{"(&lexTokenString);
registerPostProcess!"r\""(&lexWysiwygString);
registerPostProcess!"x\""(&lexHexString);
registerPostProcess!" "(&lexWhitespace);
registerPostProcess!"\t"(&lexWhitespace);
registerPostProcess!"\r"(&lexWhitespace);
registerPostProcess!"\n"(&lexWhitespace);
registerPostProcess!"\u2028"(&lexLongNewline);
registerPostProcess!"\u2029"(&lexLongNewline);
this.range = RangeType(range);
popFront();
} }
bool isWhitespace() pure const nothrow bool isWhitespace() pure /*const*/ nothrow
{ {
switch (range.front) switch (range.front)
{ {
@ -370,10 +375,10 @@ public struct DLexer(R)
case '\t': case '\t':
return true; return true;
case 0xe2: case 0xe2:
if (!range.canPeek(2)) auto peek = range.lookahead(2);
return false; return peek.length == 2
return range.peek() == 0x80 && peek[0] == 0x80
&& (range.peek(2) == 0xa8 || range.peek(2) == 0xa9); && (peek[1] == 0xa8 || peek[1] == 0xa9);
default: default:
return false; return false;
} }
@ -398,8 +403,9 @@ public struct DLexer(R)
range.incrementLine(); range.incrementLine();
return; return;
case 0xe2: case 0xe2:
if (range.canPeek(2) && range.peek() == 0x80 auto lookahead = range.lookahead(3);
&& (range.peek(2) == 0xa8 || range.peek(2) == 0xa9)) if (lookahead.length == 3 && lookahead[1] == 0x80
&& (lookahead[2] == 0xa8 || lookahead[2] == 0xa9))
{ {
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -420,7 +426,7 @@ public struct DLexer(R)
Token lexWhitespace() pure nothrow Token lexWhitespace() pure nothrow
{ {
range.mark(); auto mark = range.mark();
loop: do loop: do
{ {
switch (range.front) switch (range.front)
@ -440,11 +446,12 @@ public struct DLexer(R)
range.popFront(); range.popFront();
break; break;
case 0xe2: case 0xe2:
if (!range.canPeek(2)) auto lookahead = range.lookahead(3);
if (lookahead.length != 3)
break loop; break loop;
if (range.peek() != 0x80) if (lookahead[1] != 0x80)
break loop; break loop;
if (range.peek(2) == 0xa8 || range.peek(2) == 0xa9) if (lookahead[2] == 0xa8 || lookahead[2] == 0xa9)
{ {
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -457,36 +464,43 @@ public struct DLexer(R)
break loop; break loop;
} }
} while (!range.empty); } while (!range.empty);
return Token(tok!"whitespace", cast(string) range.getMarked(), range.line, return Token(tok!"whitespace", cast(string) range.slice(mark), range.line,
range.column, range.index); range.column, range.index);
} }
Token lexNumber() pure nothrow Token lexNumber() pure nothrow
{ {
range.mark(); auto mark = range.mark();
if (range.front == '0') auto lookahead = range.lookahead(1);
if (range.front == '0' && lookahead.length == 1)
{ {
switch (range.peek()) switch (lookahead[0])
{ {
case 'x': case 'x':
case 'X': case 'X':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
return lexHex(); return lexHex(mark);
case 'b': case 'b':
case 'B': case 'B':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
return lexBinary(); return lexBinary(mark);
default: default:
return lexDecimal(); return lexDecimal(mark);
} }
} }
else else
return lexDecimal(); return lexDecimal(mark);
} }
Token lexHex() pure nothrow Token lexHex() pure nothrow
{
auto mark = range.mark();
return lexHex(mark);
}
Token lexHex(Mark mark) pure nothrow
{ {
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
bool foundDot; bool foundDot;
@ -526,7 +540,7 @@ public struct DLexer(R)
case '.': case '.':
if (foundDot) if (foundDot)
break hexLoop; break hexLoop;
if (range.canPeek() && range.peek() == '.') if (range.lookahead(1).length && range.lookahead(1)[0] == '.')
break hexLoop; break hexLoop;
range.popFront(); range.popFront();
foundDot = true; foundDot = true;
@ -536,11 +550,17 @@ public struct DLexer(R)
break hexLoop; break hexLoop;
} }
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexBinary() pure nothrow Token lexBinary() pure nothrow
{
auto mark = range.mark();
return lexBinary(mark);
}
Token lexBinary(Mark mark) pure nothrow
{ {
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
binaryLoop: while (!range.empty) binaryLoop: while (!range.empty)
@ -561,11 +581,11 @@ public struct DLexer(R)
break binaryLoop; break binaryLoop;
} }
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexDecimal() pure nothrow Token lexDecimal(Mark mark) pure nothrow
{ {
bool foundDot = range.front == '.'; bool foundDot = range.front == '.';
IdType type = tok!"intLiteral"; IdType type = tok!"intLiteral";
@ -608,16 +628,17 @@ public struct DLexer(R)
case '.': case '.':
if (foundDot) if (foundDot)
break decimalLoop; break decimalLoop;
if (range.canPeek() && range.peek() == '.') auto lookahead = range.lookahead(1);
if (lookahead.length == 1 && lookahead[0] == '.')
break decimalLoop; break decimalLoop;
else else
{ {
// The following bit of silliness tries to tell the // The following bit of silliness tries to tell the
// difference between "int dot identifier" and // difference between "int dot identifier" and
// "double identifier". // "double identifier".
if (range.canPeek()) if (lookahead.length == 1)
{ {
switch (range.peek()) switch (lookahead[0])
{ {
case '0': .. case '9': case '0': .. case '9':
goto doubleLiteral; goto doubleLiteral;
@ -638,7 +659,7 @@ public struct DLexer(R)
break decimalLoop; break decimalLoop;
} }
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
@ -749,7 +770,7 @@ public struct DLexer(R)
Token lexSlashStarComment() pure Token lexSlashStarComment() pure
{ {
range.mark(); auto mark = range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -767,13 +788,13 @@ public struct DLexer(R)
else else
popFrontWhitespaceAware(); popFrontWhitespaceAware();
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexSlashSlashComment() pure nothrow Token lexSlashSlashComment() pure nothrow
{ {
range.mark(); auto mark = range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -783,13 +804,13 @@ public struct DLexer(R)
break; break;
range.popFront(); range.popFront();
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexSlashPlusComment() pure nothrow Token lexSlashPlusComment() pure nothrow
{ {
range.mark(); auto mark = range.mark();
IdType type = tok!"comment"; IdType type = tok!"comment";
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -817,13 +838,13 @@ public struct DLexer(R)
else else
popFrontWhitespaceAware(); popFrontWhitespaceAware();
} }
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexStringLiteral() pure nothrow Token lexStringLiteral() pure nothrow
{ {
range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
while (true) while (true)
{ {
@ -846,13 +867,13 @@ public struct DLexer(R)
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
Token lexWysiwygString() pure nothrow Token lexWysiwygString() pure nothrow
{ {
range.mark(); auto mark = range.mark();
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
bool backtick = range.front == '`'; bool backtick = range.front == '`';
if (backtick) if (backtick)
@ -900,7 +921,7 @@ public struct DLexer(R)
} }
} }
lexStringSuffix(type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
@ -922,7 +943,7 @@ public struct DLexer(R)
Token lexDelimitedString() pure nothrow Token lexDelimitedString() pure nothrow
{ {
range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
range.popFront(); range.popFront();
ElementEncodingType!R open; ElementEncodingType!R open;
@ -933,29 +954,29 @@ public struct DLexer(R)
open = '<'; open = '<';
close = '>'; close = '>';
range.popFront(); range.popFront();
return lexNormalDelimitedString(open, close); return lexNormalDelimitedString(mark, open, close);
case '{': case '{':
open = '{'; open = '{';
close = '}'; close = '}';
range.popFront(); range.popFront();
return lexNormalDelimitedString(open, close); return lexNormalDelimitedString(mark, open, close);
case '[': case '[':
open = '['; open = '[';
close = ']'; close = ']';
range.popFront(); range.popFront();
return lexNormalDelimitedString(open, close); return lexNormalDelimitedString(mark, open, close);
case '(': case '(':
open = '('; open = '(';
close = ')'; close = ')';
range.popFront(); range.popFront();
return lexNormalDelimitedString(open, close); return lexNormalDelimitedString(mark, open, close);
default: default:
return lexHeredocString(); return lexHeredocString();
} }
} }
Token lexNormalDelimitedString(ElementEncodingType!RangeType open, Token lexNormalDelimitedString(Mark mark, ElementEncodingType!R open,
ElementEncodingType!RangeType close) pure nothrow ElementEncodingType!R close) pure nothrow
{ {
int depth = 1; int depth = 1;
while (!range.empty && depth > 0) while (!range.empty && depth > 0)
@ -985,7 +1006,7 @@ public struct DLexer(R)
} }
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, range.index); return Token(type, cast(string) range.slice(mark), range.line, range.column, range.index);
} }
Token lexHeredocString() pure nothrow Token lexHeredocString() pure nothrow
@ -1024,7 +1045,7 @@ public struct DLexer(R)
Token lexHexString() pure nothrow Token lexHexString() pure nothrow
{ {
range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
range.popFront(); range.popFront();
@ -1055,7 +1076,7 @@ public struct DLexer(R)
IdType type = tok!"stringLiteral"; IdType type = tok!"stringLiteral";
lexStringSuffix(type); lexStringSuffix(type);
return Token(type, cast(string) range.getMarked(), range.line, range.column, return Token(type, cast(string) range.slice(mark), range.line, range.column,
range.index); range.index);
} }
@ -1154,7 +1175,7 @@ public struct DLexer(R)
Token lexCharacterLiteral() pure nothrow Token lexCharacterLiteral() pure nothrow
{ {
range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
if (range.front == '\\') if (range.front == '\\')
{ {
@ -1164,7 +1185,7 @@ public struct DLexer(R)
else if (range.front == '\'') else if (range.front == '\'')
{ {
range.popFront(); range.popFront();
return Token(tok!"characterLiteral", cast(string) range.getMarked(), return Token(tok!"characterLiteral", cast(string) range.slice(mark),
range.line, range.column, range.index); range.line, range.column, range.index);
} }
else if (range.front & 0x80) else if (range.front & 0x80)
@ -1182,7 +1203,7 @@ public struct DLexer(R)
if (range.front == '\'') if (range.front == '\'')
{ {
range.popFront(); range.popFront();
return Token(tok!"characterLiteral", cast(string) range.getMarked(), return Token(tok!"characterLiteral", cast(string) range.slice(mark),
range.line, range.column, range.index); range.line, range.column, range.index);
} }
else else
@ -1194,30 +1215,31 @@ public struct DLexer(R)
Token lexIdentifier() pure nothrow Token lexIdentifier() pure nothrow
{ {
range.mark(); auto mark = range.mark();
while (!range.empty && !isSeparating(range.front)) while (!range.empty && !isSeparating(range.front))
{ {
range.popFront(); range.popFront();
} }
return Token(tok!"identifier", cast(string) range.getMarked(), range.index, return Token(tok!"identifier", cast(string) range.slice(mark), range.index,
range.line, range.column); range.line, range.column);
} }
Token lexDot() pure nothrow Token lexDot() pure nothrow
{ {
if (!range.canPeek) auto lookahead = range.lookahead(1);
if (lookahead.length == 0)
{ {
range.popFront(); range.popFront();
return Token(tok!".", null, range.line, range.column, range.index); return Token(tok!".", null, range.line, range.column, range.index);
} }
switch (range.peek()) switch (lookahead[0])
{ {
case '0': .. case '9': case '0': .. case '9':
return lexNumber(); return lexNumber();
case '.': case '.':
range.popFront(); range.popFront();
range.popFront(); range.popFront();
if (range.front == '.') if (!range.empty && range.front == '.')
{ {
range.popFront(); range.popFront();
return Token(tok!"...", null, range.line, range.column, range.index); return Token(tok!"...", null, range.line, range.column, range.index);
@ -1232,16 +1254,21 @@ public struct DLexer(R)
Token lexLongNewline() pure nothrow Token lexLongNewline() pure nothrow
{ {
range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
range.popFront(); range.popFront();
range.popFront(); range.popFront();
range.incrementLine(); range.incrementLine();
return Token(tok!"whitespace", cast(string) range.getMarked(), range.line, return Token(tok!"whitespace", cast(string) range.slice(mark), range.line,
range.column, range.index); range.column, range.index);
} }
Token lexScriptLine() pure nothrow
{
assert(false, "Not implemented");
}
bool isSeparating(C)(C c) nothrow pure @safe bool isSeparating(ElementType!R c) nothrow pure @safe
{ {
if (c <= 0x2f) return true; if (c <= 0x2f) return true;
if (c >= ':' && c <= '@') return true; if (c >= ':' && c <= '@') return true;

67
stdx/d/parser.d
View File

@ -1,62 +1,5 @@
// Written in the D programming language // Written in the D programming language
/**
* This module contains a _parser for D source code.
*
* Grammar:
* The grammar format used in the documentation of this module generally follows
* the format used by the ANTLR _parser generator.
* $(UL
* $(LI Tokens and rules can be grouped by parenthesis.)
* $(LI An asterisk (*) indicates that the previous rule, token, or group
* can repeat 0 or more times.)
* $(LI A question mark (?) indicates that the previous rule, token, or group
* will be present either 0 or 1 times.)
* $(LI A plus sign (+) indicates that the previous rule, token, or group
* repeats one or more times. (i.e. it is optional))
* $(LI If there is more than one way to match a rule, the alternatives will be
* separated by a pipe character (|).)
* $(LI Rule definitions begin with the rule name followed by a colon (:). Rule
* definitions end with a semicolon (;).)
* )
*
* The grammar for D starts with the $(LINK2 #module, module) rule.
*
* Examples:
* ---
* import std.d.lexer;
* import std.d.parser;
* import std.d.ast;
* import std.array;
*
* string sourceCode = q{
* import std.stdio;
*
* void main()
* {
* writeln("Hello, World.");
* }
* }c;
* void main()
* {
* LexerConfig config;
* auto tokens = byToken(cast(ubyte[]) sourceCode, config).array();
* Module mod = parseModule(tokens);
* // Use module here...
* }
* ---
*
* Copyright: Brian Schott 2013
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
* Authors: Brian Schott
* Source: $(PHOBOSSRC std/d/_parser.d)
* Macros:
* GRAMMAR = <pre>$0</pre>
* RULEDEF = <a name="$0"><span style="font-weight: bold;">$0</span></a>
* RULE = <a href="#$0"><span style="font-weight: bold;">$0</span></a>
* LITERAL = <span style="color: green;">$0</span>
*/
module stdx.d.parser; module stdx.d.parser;
import stdx.d.lexer; import stdx.d.lexer;
@ -2162,7 +2105,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
* Parses a ForStatement * Parses a ForStatement
* *
* $(GRAMMAR $(RULEDEF forStatement): * $(GRAMMAR $(RULEDEF forStatement):
* $(LITERAL 'for') $(LITERAL '$(LPAREN)') $(RULE declarationOrStatement) $(RULE expression)? $(LITERAL ';') $(RULE expression)? $(LITERAL '$(RPAREN)') $(RULE statementNoCaseNoDefault) * $(LITERAL 'for') $(LITERAL '$(LPAREN)') $(RULE declarationOrStatement) $(RULE expression)? $(LITERAL ';') $(RULE expression)? $(LITERAL '$(RPAREN)') $(RULE declarationOrStatement)
* ;) * ;)
*/ */
ForStatement parseForStatement() ForStatement parseForStatement()
@ -2176,7 +2119,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
if (currentIs(tok!";")) if (currentIs(tok!";"))
advance(); advance();
else else
node.declarationOrStatement = parseDeclarationOrStatement(); node.initialization = parseDeclarationOrStatement();
if (currentIs(tok!";")) if (currentIs(tok!";"))
advance(); advance();
@ -2192,8 +2135,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
error("Statement expected", false); error("Statement expected", false);
return node; // this line makes DCD better return node; // this line makes DCD better
} }
node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault(); node.declarationOrStatement = parseDeclarationOrStatement();
if (node.statementNoCaseNoDefault is null) return null; if (node.declarationOrStatement is null) return null;
return node; return node;
} }
@ -5917,7 +5860,7 @@ q{doStuff(5)}c;
error("Statement expected", false); error("Statement expected", false);
return node; // this line makes DCD better return node; // this line makes DCD better
} }
node.statementNoCaseNoDefault = parseStatementNoCaseNoDefault(); node.declarationOrStatement = parseDeclarationOrStatement();
return node; return node;
} }

277
stdx/lexer.d
View File

@ -10,12 +10,15 @@
*/ */
module stdx.lexer; module stdx.lexer;
import std.typecons; import std.typecons;
import std.algorithm; import std.algorithm;
import std.range; import std.range;
import std.traits; import std.traits;
import std.conv; import std.conv;
import std.math; import std.math;
import dpick.buffer.buffer;
import dpick.buffer.traits;
template TokenIdType(alias staticTokens, alias dynamicTokens, template TokenIdType(alias staticTokens, alias dynamicTokens,
alias possibleDefaultTokens) alias possibleDefaultTokens)
@ -34,12 +37,12 @@ string TokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens
{ {
if (type == 0) if (type == 0)
return "!ERROR!"; return "!ERROR!";
else if (type < staticTokens.length) else if (type < staticTokens.length + 1)
return staticTokens[type - 1]; return staticTokens[type - 1];
else if (type < staticTokens.length + possibleDefaultTokens.length) else if (type < staticTokens.length + possibleDefaultTokens.length + 1)
return possibleDefaultTokens[type - staticTokens.length]; return possibleDefaultTokens[type - staticTokens.length - 1];
else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length) else if (type < staticTokens.length + possibleDefaultTokens.length + dynamicTokens.length + 1)
return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length]; return dynamicTokens[type - staticTokens.length - possibleDefaultTokens.length - 1];
else else
return null; return null;
} }
@ -70,14 +73,16 @@ template TokenId(IdType, alias staticTokens, alias dynamicTokens,
enum ii = possibleDefaultTokens.countUntil(symbol); enum ii = possibleDefaultTokens.countUntil(symbol);
static if (ii >= 0) static if (ii >= 0)
{ {
enum id = ii + staticTokens.length; enum id = ii + staticTokens.length + 1;
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol); static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
alias id TokenId; alias id TokenId;
} }
else else
{ {
enum dynamicId = dynamicTokens.countUntil(symbol); enum dynamicId = dynamicTokens.countUntil(symbol);
enum id = dynamicId >= 0 ? i + staticTokens.length + possibleDefaultTokens.length + dynamicId : -1; enum id = dynamicId >= 0
? i + staticTokens.length + possibleDefaultTokens.length + dynamicId + 1
: -1;
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol); static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
alias id TokenId; alias id TokenId;
} }
@ -113,13 +118,10 @@ struct TokenStructure(IDType)
IDType type; IDType type;
} }
mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFunction, mixin template Lexer(R, IDType, Token, alias defaultTokenFunction,
alias staticTokens, alias dynamicTokens, alias pseudoTokens, alias staticTokens, alias dynamicTokens, alias pseudoTokens,
alias possibleDefaultTokens) if (isForwardRange!R) alias pseudoTokenHandlers, alias possibleDefaultTokens)
{ {
enum size_t lookAhead = chain(staticTokens, pseudoTokens).map!"a.length".reduce!"max(a, b)"();
alias PeekRange!(R, lookAhead) RangeType;
static string generateCaseStatements(string[] tokens, size_t offset = 0) static string generateCaseStatements(string[] tokens, size_t offset = 0)
{ {
string code; string code;
@ -141,9 +143,9 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
code ~= generateLeaf(tokens[i], indent ~ " "); code ~= generateLeaf(tokens[i], indent ~ " ");
else else
{ {
code ~= indent ~ " if (!range.canPeek(" ~ text(tokens[i].length - 1) ~ "))\n"; code ~= indent ~ " if (range.lookahead(" ~ text(tokens[i].length) ~ ").length == 0)\n";
code ~= indent ~ " goto outer_default;\n"; code ~= indent ~ " goto outer_default;\n";
code ~= indent ~ " if (range.startsWith(\"" ~ escape(tokens[i]) ~ "\"))\n"; code ~= indent ~ " if (range.lookahead(" ~ text(tokens[i].length) ~ ") == \"" ~ escape(tokens[i]) ~ "\")\n";
code ~= indent ~ " {\n"; code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i], indent ~ " "); code ~= generateLeaf(tokens[i], indent ~ " ");
code ~= indent ~ " }\n"; code ~= indent ~ " }\n";
@ -153,11 +155,11 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
} }
else else
{ {
code ~= indent ~ " if (!range.canPeek(" ~ text(offset + 1) ~ "))\n"; code ~= indent ~ " if (range.lookahead(" ~ text(offset + 2) ~ ").length == 0)\n";
code ~= indent ~ " {\n"; code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " "); code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
code ~= indent ~ " }\n"; code ~= indent ~ " }\n";
code ~= indent ~ " switch (range.peek(" ~ text(offset + 1) ~ "))\n"; code ~= indent ~ " switch (range.lookahead(" ~ text(offset + 2) ~ ")[" ~ text(offset + 1) ~ "])\n";
code ~= indent ~ " {\n"; code ~= indent ~ " {\n";
code ~= generateCaseStatements(tokens[i .. j], offset + 1); code ~= generateCaseStatements(tokens[i .. j], offset + 1);
code ~= indent ~ " default:\n"; code ~= indent ~ " default:\n";
@ -172,6 +174,8 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
static string generateLeaf(string token, string indent) static string generateLeaf(string token, string indent)
{ {
static assert (pseudoTokenHandlers.length % 2 == 0,
"Each pseudo-token must have a matching function name.");
string code; string code;
if (staticTokens.countUntil(token) >= 0) if (staticTokens.countUntil(token) >= 0)
{ {
@ -179,13 +183,13 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
code ~= indent ~ "range.popFront();\n"; code ~= indent ~ "range.popFront();\n";
else else
code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n"; code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n"; code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~ "\", null, range.line, range.column, range.index);\n";
} }
else if (pseudoTokens.countUntil(token) >= 0) else if (pseudoTokens.countUntil(token) >= 0)
code ~= indent ~ "return postProcess(pseudoTok!\"" ~ escape(token) ~"\");\n"; code ~= indent ~ "return " ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] ~ "();\n";
else if (possibleDefaultTokens.countUntil(token) >= 0) else if (possibleDefaultTokens.countUntil(token) >= 0)
{ {
code ~= indent ~ "if (!range.canPeek(" ~ text(token.length) ~ ") || isSeparating(range.peek(" ~ text(token.length) ~ ")))\n"; code ~= indent ~ "if (range.lookahead(" ~ text(token.length + 1) ~ ").length == 0 || isSeparating(range.lookahead(" ~ text(token.length + 1) ~ ")[" ~ text(token.length) ~ "]))\n";
code ~= indent ~ "{\n"; code ~= indent ~ "{\n";
if (token.length == 1) if (token.length == 1)
code ~= indent ~ " range.popFront();\n"; code ~= indent ~ " range.popFront();\n";
@ -211,18 +215,11 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
_front = advance(); _front = advance();
} }
bool empty() const nothrow @property bool empty() pure const nothrow @property
{ {
return _front.type == tok!"\0"; return _front.type == tok!"\0";
} }
template pseudoTok(string symbol)
{
static assert (pseudoTokens.countUntil(symbol) >= 0);
enum index = cast(IDType) pseudoTokens.countUntil(symbol);
alias index pseudoTok;
}
static string escape(string input) static string escape(string input)
{ {
string rVal; string rVal;
@ -267,224 +264,36 @@ mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFun
return rVal; return rVal;
} }
void registerPostProcess(alias t)(Token delegate() pure fun) LexerRange!(typeof(buffer(R.init))) range;
{
post[pseudoTok!t] = fun;
}
Token postProcess(IDType i) pure
{
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]);
return post[i]();
}
Token delegate() pure [pseudoTokens.length] post;
RangeType range;
Token _front; Token _front;
} }
struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R struct LexerRange(BufferType) if (isBuffer!BufferType)
&& isForwardRange!R && hasSlicing!R)
{ {
public: this(BufferType r)
this(R range)
{ {
this.range = range; this.range = r;
index = 0;
column = 1;
line = 1;
} }
invariant() void popFront() pure
{ {
import std.string; index++;
if (range.length != 6190) column++;
assert (false, format("range.length = %d %s", range.length, cast(char[]) range[0 .. 100])); range.popFront();
} }
bool startsWith(string s)
{
return index + s.length < range.length
&& (cast(const(ubyte[])) s) == range[index .. index + s.length];
}
bool empty() pure nothrow const @property
{
return _index >= range.length;
}
const(ElementType!R) front() pure nothrow const @property
in
{
assert (!empty);
}
body
{
return range[_index];
}
void popFront() pure nothrow
{
_index++;
_column++;
}
void popFrontN(size_t n) pure nothrow
{
foreach (i; 0 .. n)
popFront();
}
const(ElementType!R) peek(int offset = 1) pure nothrow const
in
{
assert (canPeek(offset));
}
body
{
return range[_index + offset];
}
bool canPeek(size_t offset = 1) pure nothrow const
{
return _index + offset < range.length;
}
void mark() nothrow pure
{
markBegin = _index;
}
const(R) getMarked() pure nothrow const
{
return range[markBegin .. _index];
}
void incrementLine() pure nothrow void incrementLine() pure nothrow
{ {
_column = 1; column = 1;
_line++; line++;
} }
size_t line() pure nothrow const @property { return _line; } BufferType range;
size_t column() pure nothrow const @property { return _column; } alias range this;
size_t index() pure nothrow const @property { return _index; } size_t index;
size_t column;
private: size_t line;
size_t markBegin;
size_t _column = 1;
size_t _line = 1;
size_t _index = 0;
R range;
} }
//struct PeekRange(R, size_t peekSupported = 1)
// if (!isRandomAccessRange!R && isForwardRange!R)
//{
//public:
//
// this(R range)
// {
// this.range = range;
// for (size_t i = 0; !this.range.empty && i < peekSupported; i++)
// {
// rangeSizeCount++;
// buffer[i] = this.range.front;
// range.popFront();
// }
// }
//
// ElementType!R front() const @property
// in
// {
// assert (!empty);
// }
// body
// {
// return buffer[bufferIndex];
// }
//
// void popFront()
// in
// {
// assert (!empty);
// }
// body
// {
// index++;
// column++;
// count++;
// bufferIndex = bufferIndex + 1 > buffer.length ? 0 : bufferIndex + 1;
// if (marking)
// markBuffer.put(buffer[bufferIndex]);
// if (!range.empty)
// {
// buffer[bufferIndex + peekSupported % buffer.length] = range.front();
// range.popFront();
// rangeSizeCount++;
// }
// }
//
// bool empty() const nothrow pure @property
// {
// return rangeSizeCount == count;
// }
//
// ElementType!R peek(int offset = 1) pure nothrow const
// in
// {
// assert (canPeek(offset));
// }
// body
// {
// return buffer[(bufferIndex + offset) % buffer.length];
// }
//
// bool canPeek(size_t int offset = 1) pure nothrow const
// {
// return offset <= peekSupported && count + offset <= rangeSizeCount;
// }
//
// typeof(this) save() @property
// {
// typeof(this) newRange;
// newRange.count = count;
// newRange.rangeSizeCount = count;
// newRange.buffer = buffer.dup;
// newRange.bufferIndex = bufferIndex;
// newRange.range = range.save;
// return newRange;
// }
//
// void mark()
// {
// marking = true;
// markBuffer.clear();
// }
//
// ElementEncodingType!R[] getMarked()
// {
// marking = false;
// return markBuffer.data;
// }
//
// void incrementLine() pure nothrow
// {
// _column = 1;
// _line++;
// }
//
// size_t line() pure nothrow const @property { return _line; }
// size_t column() pure nothrow const @property { return _column; }
// size_t index() pure nothrow const @property { return _index; }
//
//private:
// auto markBuffer = appender!(ElementType!R[])();
// bool marking;
// size_t count;
// size_t rangeSizeCount;
// ElementType!(R)[peekSupported + 1] buffer;
// size_t bufferIndex;
// size_t _column = 1;
// size_t _line = 1;
// size_t _index = 0;
// R range;
//}