forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix unit tests

This commit is contained in:
Hackerpilot 2014-01-11 15:36:35 -08:00
parent b092840c70
commit 9fbd106af5
4 changed files with 151 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
main.d
View File

@ -20,11 +20,11 @@ import stdx.d.parser;
import dpick.buffer.buffer; import dpick.buffer.buffer;
import highlighter; import highlighter;
//import stats; import stats;
//import ctags; import ctags;
//import astprinter; import astprinter;
//import imports; import imports;
//import outliner; import outliner;
int main(string[] args) int main(string[] args)
{ {
@ -92,69 +92,69 @@ int main(string[] args)
{ {
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
auto tokens = DLexer!(ubyte[])(bytes); auto tokens = byToken!(ubyte[], false, false)(bytes);
highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]); highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]);
return 0; return 0;
} }
// else if (ctags) else if (ctags)
// { {
// stdout.printCtags(expandArgs(args, recursive)); stdout.printCtags(expandArgs(args, recursive));
// } }
// else else
// { {
// bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
// if (sloc || tokenCount) if (sloc || tokenCount)
// { {
// if (usingStdin) if (usingStdin)
// { {
// auto tokens = byToken!(ubyte[], false, false)(readStdin()); auto tokens = byToken!(ubyte[], false, false)(readStdin());
// if (tokenCount) if (tokenCount)
// printTokenCount(stdout, "stdin", tokens); printTokenCount(stdout, "stdin", tokens);
// else else
// printLineCount(stdout, "stdin", tokens); printLineCount(stdout, "stdin", tokens);
// } }
// else else
// { {
// ulong count; ulong count;
// foreach (f; expandArgs(args, recursive)) foreach (f; expandArgs(args, recursive))
// { {
// auto tokens = byToken!(ubyte[])(readFile(f)); auto tokens = byToken!(ubyte[])(readFile(f));
// if (tokenCount) if (tokenCount)
// count += printTokenCount(stdout, f, tokens); count += printTokenCount(stdout, f, tokens);
// else else
// count += printLineCount(stdout, f, tokens); count += printLineCount(stdout, f, tokens);
// } }
// writefln("total:\t%d", count); writefln("total:\t%d", count);
// } }
// } }
// else if (syntaxCheck) else if (syntaxCheck)
// { {
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
// parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
// } }
// else if (imports) else if (imports)
// { {
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
// auto visitor = new ImportPrinter; auto visitor = new ImportPrinter;
// visitor.visit(mod); visitor.visit(mod);
// } }
// else if (ast) else if (ast)
// { {
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
// auto printer = new XMLPrinter; auto printer = new XMLPrinter;
// printer.output = stdout; printer.output = stdout;
// printer.visit(mod); printer.visit(mod);
// } }
// else if (outline) else if (outline)
// { {
// auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1])); auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
// auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
// auto outliner = new Outliner(stdout); auto outliner = new Outliner(stdout);
// outliner.visit(mod); outliner.visit(mod);
// } }
// } }
return 0; return 0;
} }

2
stdx/d/entities.d
View File

@ -6,7 +6,7 @@
* Copyright: Brian Schott 2013 * Copyright: Brian Schott 2013
* License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>. * License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
* Authors: Brian Schott * Authors: Brian Schott
* Source: $(PHOBOSSRC std/d/_lexer.d) * Source: $(PHOBOSSRC std/d/_entities.d)
*/ */
module stdx.d.entities; module stdx.d.entities;

63
stdx/d/lexer.d
View File

@ -57,24 +57,34 @@ public template tok(string token)
} }
public alias stdx.lexer.TokenStructure!(IdType) Token; public alias stdx.lexer.TokenStructure!(IdType) Token;
//public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range) pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; }
//{ pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
// pure nothrow bool isNotComment(const Token t) { return t.type != tok!"comment"; } pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; }
// pure nothrow bool isNotWhitespace(const Token t) { return t.type != tok!"whitespace"; }
// pure nothrow bool isNotEither(const Token t) { return t.type != tok!"whitespace" && t.type != tok!"comment"; } public auto byToken(R, bool skipComments = true, bool skipWhitespace = true)(R range)
// return new DLexer!(R)(range); {
// static if (skipComments) auto tokens = DLexer!(R)(range);
// { static if (skipComments)
// static if (skipWhitespace) {
// return filter!isNotEither(tokens); static if (skipWhitespace)
// else return filter!isNotEither(tokens);
// return filter!isNotComment(tokens); else
// } return filter!isNotComment(tokens);
// else static if (skipWhitespace) }
// return filter!isNotWhitespace(tokens); else static if (skipWhitespace)
// else return filter!isNotWhitespace(tokens);
// return tokens; else
//} return tokens;
}
unittest
{
import std.stdio;
auto source = cast(ubyte[]) q{ import std.stdio;}c;
auto tokens = byToken(source);
assert (tokens.map!"a.type"().equal([tok!"import", tok!"identifier", tok!".",
tok!"identifier", tok!";"]));
}
public bool isBasicType(IdType type) nothrow pure @safe public bool isBasicType(IdType type) nothrow pure @safe
{ {
@ -363,6 +373,7 @@ public struct DLexer(R)
this(R range) this(R range)
{ {
this.range = LexerRange!(typeof(buffer(range)))(buffer(range)); this.range = LexerRange!(typeof(buffer(range)))(buffer(range));
popFront();
} }
bool isWhitespace() pure /*const*/ nothrow bool isWhitespace() pure /*const*/ nothrow
@ -524,15 +535,10 @@ public struct DLexer(R)
break hexLoop; break hexLoop;
case 'L': case 'L':
if (foundDot) if (foundDot)
{
lexFloatSuffix(type); lexFloatSuffix(type);
break hexLoop;
}
else else
{
lexIntSuffix(type); lexIntSuffix(type);
break hexLoop; break hexLoop;
}
case 'p': case 'p':
case 'P': case 'P':
lexExponent(type); lexExponent(type);
@ -943,11 +949,12 @@ public struct DLexer(R)
Token lexDelimitedString() pure nothrow Token lexDelimitedString() pure nothrow
{ {
import std.traits;
auto mark = range.mark(); auto mark = range.mark();
range.popFront(); range.popFront();
range.popFront(); range.popFront();
ElementEncodingType!R open; Unqual!(ElementEncodingType!R) open;
ElementEncodingType!R close; Unqual!(ElementEncodingType!R) close;
switch (range.front) switch (range.front)
{ {
case '<': case '<':
@ -1220,8 +1227,8 @@ public struct DLexer(R)
{ {
range.popFront(); range.popFront();
} }
return Token(tok!"identifier", cast(string) range.slice(mark), range.index, return Token(tok!"identifier", cast(string) range.slice(mark), range.line,
range.line, range.column); range.column, range.index);
} }
Token lexDot() pure nothrow Token lexDot() pure nothrow

80
stdx/d/parser.d
View File

@ -12,7 +12,7 @@ import std.string : format;
// Uncomment this if you want ALL THE OUTPUT // Uncomment this if you want ALL THE OUTPUT
// Caution: generates 180 megabytes of logging for std.datetime // Caution: generates 180 megabytes of logging for std.datetime
//version = std_parser_verbose; version = std_parser_verbose;
/** /**
* Params: * Params:
@ -96,6 +96,7 @@ class Parser
unittest unittest
{ {
stderr.writeln("Running unittest for parseAliasDeclaration.");
auto sourceCode = auto sourceCode =
q{ q{
alias core.sys.posix.stdio.fileno fileno; alias core.sys.posix.stdio.fileno fileno;
@ -288,7 +289,7 @@ alias core.sys.posix.stdio.fileno fileno;
switch (current.type) switch (current.type)
{ {
case tok!"{": case tok!"{":
case tok!"}": case tok!"[":
node.nonVoidInitializer = parseNonVoidInitializer(); node.nonVoidInitializer = parseNonVoidInitializer();
if (node.nonVoidInitializer is null) return null; if (node.nonVoidInitializer is null) return null;
break; break;
@ -1028,45 +1029,41 @@ incorrect;
CastQualifier one = p.parseCastQualifier(); CastQualifier one = p.parseCastQualifier();
assert (one.first == tok!"const"); assert (one.first == tok!"const");
assert (!one.hasSecond); assert (one.second == tok!"");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier two = p.parseCastQualifier(); CastQualifier two = p.parseCastQualifier();
assert (two.first == tok!"const"); assert (two.first == tok!"const");
assert (two.hasSecond);
assert (two.second == tok!"shared"); assert (two.second == tok!"shared");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier three = p.parseCastQualifier(); CastQualifier three = p.parseCastQualifier();
assert (three.first == tok!"immutable"); assert (three.first == tok!"immutable");
assert (!three.hasSecond); assert (three.second == tok!"");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier four = p.parseCastQualifier(); CastQualifier four = p.parseCastQualifier();
assert (four.first == tok!"inout"); assert (four.first == tok!"inout");
assert (!four.hasSecond); assert (four.second == tok!"");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier five = p.parseCastQualifier(); CastQualifier five = p.parseCastQualifier();
assert (five.first == tok!"inout"); assert (five.first == tok!"inout");
assert (five.hasSecond);
assert (five.second == tok!"shared"); assert (five.second == tok!"shared");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier six = p.parseCastQualifier(); CastQualifier six = p.parseCastQualifier();
assert (six.first == tok!"shared"); assert (six.first == tok!"shared");
assert (!six.hasSecond); assert (six.second == tok!"");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier seven = p.parseCastQualifier(); CastQualifier seven = p.parseCastQualifier();
assert (seven.first == tok!"shared"); assert (seven.first == tok!"shared");
assert (seven.hasSecond);
assert (seven.second == tok!"const"); assert (seven.second == tok!"const");
p.expect(tok!";"); p.expect(tok!";");
CastQualifier eight = p.parseCastQualifier(); CastQualifier eight = p.parseCastQualifier();
assert (eight.first == tok!"shared"); assert (eight.first == tok!"shared");
assert (eight.hasSecond);
assert (eight.second == tok!"inout"); assert (eight.second == tok!"inout");
p.expect(tok!";"); p.expect(tok!";");
@ -1170,14 +1167,14 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
Parser p = getParserForUnittest(sourceCode, "parseClassDeclaration"); Parser p = getParserForUnittest(sourceCode, "parseClassDeclaration");
auto classOne = p.parseClassDeclaration(); auto classOne = p.parseClassDeclaration();
assert (classOne.name == "ClassOne"); assert (classOne.name.text == "ClassOne");
assert (classOne.structBody.declarations.length == 0); assert (classOne.structBody.declarations.length == 0);
assert (classOne.baseClassList is null); assert (classOne.baseClassList is null);
assert (classOne.constraint is null); assert (classOne.constraint is null);
assert (classOne.templateParameters is null); assert (classOne.templateParameters is null);
auto classTwo = p.parseClassDeclaration(); auto classTwo = p.parseClassDeclaration();
assert (classTwo.name == "ClassTwo", classTwo.name.value); assert (classTwo.name.text == "ClassTwo", classTwo.name.text);
assert (classTwo.baseClassList !is null); assert (classTwo.baseClassList !is null);
assert (classTwo.baseClassList.items.length == 1, assert (classTwo.baseClassList.items.length == 1,
to!string(classTwo.baseClassList.items.length)); to!string(classTwo.baseClassList.items.length));
@ -1185,7 +1182,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
to!string(classTwo.structBody.declarations.length)); to!string(classTwo.structBody.declarations.length));
auto classThree = p.parseClassDeclaration(); auto classThree = p.parseClassDeclaration();
assert (classThree.name == "ClassThree", classThree.name.value); assert (classThree.name.text == "ClassThree", classThree.name.text);
assert (classThree.templateParameters !is null); assert (classThree.templateParameters !is null);
assert (classThree.templateParameters.templateParameterList.items.length == 2); assert (classThree.templateParameters.templateParameterList.items.length == 2);
assert (classThree.baseClassList !is null); assert (classThree.baseClassList !is null);
@ -1194,7 +1191,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
to!string(classThree.structBody.declarations.length)); to!string(classThree.structBody.declarations.length));
//auto classFour = p.parseClassDeclaration(); //auto classFour = p.parseClassDeclaration();
//assert (classFour.name == "ClassFour", classFour.name.value); //assert (classFour.name == "ClassFour", classFour.name.text);
//assert (classFour.templateParameters !is null); //assert (classFour.templateParameters !is null);
//assert (classFour.baseClassList !is null); //assert (classFour.baseClassList !is null);
//assert (classFour.constraint !is null); //assert (classFour.constraint !is null);
@ -1551,7 +1548,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c;
switch (current.type) switch (current.type)
{ {
case tok!":": case tok!";":
// http://d.puremagic.com/issues/show_bug.cgi?id=4559 // http://d.puremagic.com/issues/show_bug.cgi?id=4559
warn("Empty declaration"); warn("Empty declaration");
advance(); advance();
@ -2595,8 +2592,8 @@ body {} // six
{ {
mixin(traceEnterAndExit!(__FUNCTION__)); mixin(traceEnterAndExit!(__FUNCTION__));
auto node = new IdentifierOrTemplateInstance; auto node = new IdentifierOrTemplateInstance;
if (peekIs(tok!".") && !startsWith(tok!"identifier", if (peekIs(tok!"!") && !startsWith(tok!"identifier",
tok!".", tok!"is") tok!"!", tok!"is")
&& !startsWith(tok!"identifier", tok!"!", tok!"in")) && !startsWith(tok!"identifier", tok!"!", tok!"in"))
{ {
node.templateInstance = parseTemplateInstance(); node.templateInstance = parseTemplateInstance();
@ -2665,7 +2662,7 @@ body {} // six
auto b = setBookmark(); auto b = setBookmark();
auto t = parseType(); auto t = parseType();
if (t is null || !currentIs(tok!"identifier") if (t is null || !currentIs(tok!"identifier")
|| !peekIs(tok!"=")) || !peekIs(tok!"="))
{ {
goToBookmark(b); goToBookmark(b);
node.expression = parseExpression(); node.expression = parseExpression();
@ -2684,10 +2681,10 @@ body {} // six
if (expect(tok!")") is null) return null; if (expect(tok!")") is null) return null;
if (currentIs(tok!"}")) if (currentIs(tok!"}"))
{ {
error("Statement expected", false); error("Statement expected", false);
return node; // this line makes DCD better return node; // this line makes DCD better
} }
node.thenStatement = parseDeclarationOrStatement(); node.thenStatement = parseDeclarationOrStatement();
if (currentIs(tok!"else")) if (currentIs(tok!"else"))
{ {
@ -2994,7 +2991,7 @@ interface "Four"
InterfaceDeclaration one = p.parseInterfaceDeclaration(); InterfaceDeclaration one = p.parseInterfaceDeclaration();
assert (one !is null); assert (one !is null);
assert (one.identifier == "One"); assert (one.name.text == "One");
assert (one.constraint is null); assert (one.constraint is null);
assert (one.templateParameters is null); assert (one.templateParameters is null);
assert (one.structBody !is null); assert (one.structBody !is null);
@ -3003,7 +3000,7 @@ interface "Four"
InterfaceDeclaration two = p.parseInterfaceDeclaration(); InterfaceDeclaration two = p.parseInterfaceDeclaration();
assert (two !is null); assert (two !is null);
assert (two.identifier == "Two"); assert (two.name.text == "Two");
assert (two.constraint is null); assert (two.constraint is null);
assert (two.templateParameters is null); assert (two.templateParameters is null);
assert (two.structBody !is null); assert (two.structBody !is null);
@ -3012,7 +3009,7 @@ interface "Four"
InterfaceDeclaration three = p.parseInterfaceDeclaration(); InterfaceDeclaration three = p.parseInterfaceDeclaration();
assert (three !is null); assert (three !is null);
assert (three.identifier == "Three"); assert (three.name.text == "Three");
assert (three.constraint !is null); assert (three.constraint !is null);
assert (three.templateParameters !is null); assert (three.templateParameters !is null);
assert (three.structBody !is null); assert (three.structBody !is null);
@ -3382,6 +3379,7 @@ invariant() foo();
*/ */
Module parseModule() Module parseModule()
{ {
mixin(traceEnterAndExit!(__FUNCTION__));
Module m = new Module; Module m = new Module;
if (currentIs(tok!"scriptLine")) if (currentIs(tok!"scriptLine"))
advance(); advance();
@ -3857,7 +3855,7 @@ q{(int a, ...)
Parameters params1 = p.parseParameters(); Parameters params1 = p.parseParameters();
assert (params1.hasVarargs); assert (params1.hasVarargs);
assert (params1.parameters.length == 1); assert (params1.parameters.length == 1);
assert (params1.parameters[0].name == "a"); assert (params1.parameters[0].name.text == "a");
Parameters params2 = p.parseParameters(); Parameters params2 = p.parseParameters();
assert (params2.parameters.length == 1); assert (params2.parameters.length == 1);
@ -5150,7 +5148,7 @@ q{(int a, ...)
{ {
advance(); advance();
node.expression = parseExpression(); node.expression = parseExpression();
expect(tok!":"); if (expect(tok!":") is null) return null;
node.ternaryExpression = parseTernaryExpression(); node.ternaryExpression = parseTernaryExpression();
} }
return node; return node;
@ -5454,7 +5452,7 @@ q{(int a, ...)
advance(); advance();
if (currentIs(tok!"]")) if (currentIs(tok!"]"))
{ {
if (expect(tok!"]") is null) return null; advance();
return node; return node;
} }
auto bookmark = setBookmark(); auto bookmark = setBookmark();
@ -6356,8 +6354,10 @@ protected:
&& (tokens[index].type == tok!")" && (tokens[index].type == tok!")"
|| tokens[index].type == tok!";" || tokens[index].type == tok!";"
|| tokens[index].type == tok!"}"); || tokens[index].type == tok!"}");
error("Expected " ~ tokenString ~ " instead of " auto token = (index < tokens.length
~ (index < tokens.length ? tokens[index].text: "EOF"), ? (tokens[index].text is null ? str(tokens[index].type) : tokens[index].text)
: "EOF");
error("Expected " ~ tokenString ~ " instead of " ~ token,
!shouldNotAdvance); !shouldNotAdvance);
return null; return null;
} }
@ -6437,9 +6437,8 @@ protected:
version (unittest) static Parser getParserForUnittest(string sourceCode, version (unittest) static Parser getParserForUnittest(string sourceCode,
string testName) string testName)
{ {
LexerConfig config; auto r = byToken(cast(ubyte[]) sourceCode);
auto r = byToken(cast(const(ubyte)[]) sourceCode, config); Parser p = new Parser;
Parser p;
//p.messageFunction = &doNothingErrorFunction; //p.messageFunction = &doNothingErrorFunction;
p.fileName = testName ~ ".d"; p.fileName = testName ~ ".d";
p.tokens = r.array(); p.tokens = r.array();
@ -6484,10 +6483,10 @@ protected:
template traceEnterAndExit(string fun) template traceEnterAndExit(string fun)
{ {
enum traceEnterAndExit = `version (std_parser_verbose) trace("` enum traceEnterAndExit = `version (std_parser_verbose) { _traceDepth++; trace("`
~ "\033[01;32m" ~ fun ~ "\033[0m" ~ ` ");` ~ "\033[01;32m" ~ fun ~ "\033[0m" ~ ` "); }`
~ `version (std_parser_verbose) scope(exit) trace("` ~ `version (std_parser_verbose) scope(exit) { trace("`
~ "\033[01;31m" ~ fun ~ "\033[0m" ~ ` ");`; ~ "\033[01;31m" ~ fun ~ "\033[0m" ~ ` "); _traceDepth--; }`;
} }
version (std_parser_verbose) version (std_parser_verbose)
@ -6497,9 +6496,9 @@ protected:
if (suppressMessages > 0) if (suppressMessages > 0)
return; return;
if (index < tokens.length) if (index < tokens.length)
writeln(message, "(", current.line, ":", current.column, ")"); writeln(_traceDepth, " ", message, "(", current.line, ":", current.column, ")");
else else
writeln(message, "(EOF:0)"); writeln(_traceDepth, " ", message, "(EOF:0)");
} }
} }
else else
@ -6560,8 +6559,9 @@ protected:
case tok!"__TIMESTAMP__": case tok!"__TIMESTAMP__":
case tok!"__VENDOR__": case tok!"__VENDOR__":
case tok!"__VERSION__": case tok!"__VERSION__":
}; };
const(Token)[] tokens; const(Token)[] tokens;
int suppressMessages; int suppressMessages;
size_t index; size_t index;
int _traceDepth;
} }