forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Converted everything to the new lexer

This commit is contained in:
Hackerpilot 2013-12-15 03:02:52 -08:00
parent 4bac2671b4
commit 8c4a87f563
16 changed files with 3219 additions and 4863 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
astprinter.d
View File

@ -21,7 +21,7 @@ class XMLPrinter : ASTVisitor
{
override void visit(AddExpression addExpression)
{
output.writeln("<addExpression operator=\"", getTokenValue(addExpression.operator) ,"\">");
output.writeln("<addExpression operator=\"", str(addExpression.operator) ,"\">");
output.writeln("<left>");
addExpression.left.accept(this);
output.writeln("</left>");
@ -51,7 +51,7 @@ class XMLPrinter : ASTVisitor
override void visit(AlignAttribute alignAttribute)
{
output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.value, "\">");
output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.text, "\">");
}
override void visit(AndAndExpression andAndExpression)
@ -130,7 +130,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<assignExpression>");
else
output.writeln("<assignExpression operator=\"",
getTokenValue(assignExpression.operator), "\">");
str(assignExpression.operator), "\">");
assignExpression.accept(this);
output.writeln("</assignExpression>");
}
@ -143,20 +143,20 @@ class XMLPrinter : ASTVisitor
override void visit(AtAttribute atAttribute)
{
output.writeln("<atAttribute>");
if (atAttribute.identifier.type == TokenType.invalid)
if (atAttribute.identifier.type == tok!"")
atAttribute.accept(this);
else
output.writeln("<identifier>", atAttribute.identifier.value, "</identifier>");
output.writeln("<identifier>", atAttribute.identifier.text, "</identifier>");
output.writeln("</atAttribute>");
}
override void visit(Attribute attribute)
{
output.writeln("<attribute>");
if (attribute.attribute == TokenType.invalid)
if (attribute.attribute == tok!"")
attribute.accept(this);
else
output.writeln(getTokenValue(attribute.attribute));
output.writeln(str(attribute.attribute));
output.writeln("</attribute>");
}
@ -173,7 +173,7 @@ class XMLPrinter : ASTVisitor
{
output.writeln("<item>");
output.writeln("<name line=\"", autoDec.identifiers[i].line, "\">",
autoDec.identifiers[i].value, "</name>");
autoDec.identifiers[i].text, "</name>");
visit(autoDec.initializers[i]);
output.writeln("</item>");
}
@ -196,10 +196,10 @@ class XMLPrinter : ASTVisitor
override void visit(BreakStatement breakStatement)
{
if (breakStatement.label.type == TokenType.invalid)
if (breakStatement.label.type == tok!"")
output.writeln("<breakStatement/>");
else
output.writeln("<breakStatement label=\"", breakStatement.label.value, "\"/>");
output.writeln("<breakStatement label=\"", breakStatement.label.text, "\"/>");
}
override void visit(BaseClass baseClass)
@ -256,7 +256,7 @@ class XMLPrinter : ASTVisitor
override void visit(ClassDeclaration classDec)
{
output.writeln("<classDeclaration line=\"", classDec.name.line, "\">");
output.writeln("<name>", classDec.name.value, "</name>");
output.writeln("<name>", classDec.name.text, "</name>");
classDec.accept(this);
output.writeln("</classDeclaration>");
}
@ -318,29 +318,29 @@ class XMLPrinter : ASTVisitor
override void visit(ContinueStatement continueStatement)
{
if (continueStatement.label.type == TokenType.invalid)
if (continueStatement.label.type == tok!"")
output.writeln("<continueStatement/>");
else
output.writeln("<continueStatement label=\"",
continueStatement.label.value, "\"/>");
continueStatement.label.text, "\"/>");
}
override void visit(DebugCondition debugCondition)
{
if (debugCondition.identifierOrInteger.type == TokenType.invalid)
if (debugCondition.identifierOrInteger.type == tok!"")
output.writeln("<debugCondition/>");
else
output.writeln("<debugCondition condition=\"",
debugCondition.identifierOrInteger.value, "\"/>");
debugCondition.identifierOrInteger.text, "\"/>");
}
override void visit(DebugSpecification debugSpecification)
{
if (debugSpecification.identifierOrInteger.type == TokenType.invalid)
if (debugSpecification.identifierOrInteger.type == tok!"")
output.writeln("<debugSpecification/>");
else
output.writeln("<debugSpecification condition=\"",
debugSpecification.identifierOrInteger.value, "\"/>");
debugSpecification.identifierOrInteger.text, "\"/>");
}
override void visit(Declaration declaration)
@ -361,7 +361,7 @@ class XMLPrinter : ASTVisitor
override void visit(Declarator declarator)
{
output.writeln("<declarator line=\"", declarator.name.line, "\">");
output.writeln("<name>", declarator.name.value, "</name>");
output.writeln("<name>", declarator.name.text, "</name>");
declarator.accept(this);
output.writeln("</declarator>");
}
@ -411,8 +411,8 @@ class XMLPrinter : ASTVisitor
override void visit(EnumDeclaration enumDec)
{
output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">");
if (enumDec.name.type == TokenType.identifier)
output.writeln("<name>", enumDec.name.value, "</name>");
if (enumDec.name.type == tok!"identifier")
output.writeln("<name>", enumDec.name.text, "</name>");
enumDec.accept(this);
output.writeln("</enumDeclaration>");
}
@ -426,7 +426,7 @@ class XMLPrinter : ASTVisitor
override void visit(EqualExpression equalExpression)
{
output.writeln("<enumMember operator=\"", getTokenValue(equalExpression.operator), "\">");
output.writeln("<enumMember operator=\"", str(equalExpression.operator), "\">");
output.writeln("<left>");
visit(equalExpression.left);
output.writeln("</left>");
@ -491,7 +491,7 @@ class XMLPrinter : ASTVisitor
override void visit(ForeachStatement foreachStatement)
{
output.writeln("<foreachStatement type=\"", getTokenValue(
output.writeln("<foreachStatement type=\"", str(
foreachStatement.type), "\">");
if (foreachStatement.foreachType !is null)
visit(foreachStatement.foreachType);
@ -515,7 +515,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<foreachType>");
foreach (constructor; foreachType.typeConstructors)
{
output.writeln("<typeConstructor>", getTokenValue(constructor), "</typeConstructor>");
output.writeln("<typeConstructor>", str(constructor), "</typeConstructor>");
}
if (foreachType.type !is null)
visit(foreachType.type);
@ -552,7 +552,7 @@ class XMLPrinter : ASTVisitor
override void visit(FunctionDeclaration functionDec)
{
output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">");
output.writeln("<name>", functionDec.name.value, "</name>");
output.writeln("<name>", functionDec.name.text, "</name>");
if (functionDec.hasAuto)
output.writeln("<auto/>");
if (functionDec.hasRef)
@ -564,17 +564,17 @@ class XMLPrinter : ASTVisitor
override void visit(FunctionLiteralExpression functionLiteralExpression)
{
output.writeln("<functionLiteralExpression type=\"",
getTokenValue(functionLiteralExpression.functionOrDelegate), "\">");
str(functionLiteralExpression.functionOrDelegate), "\">");
functionLiteralExpression.accept(this);
output.writeln("</functionLiteralExpression>");
}
override void visit(GotoStatement gotoStatement)
{
if (gotoStatement.label.type == TokenType.default_)
if (gotoStatement.label.type == tok!"default")
output.writeln("<gotoStatement default=\"true\"/>");
else if (gotoStatement.label.type == TokenType.identifier)
output.writeln("<gotoStatement label=\"", gotoStatement.label.value, "\"/>");
else if (gotoStatement.label.type == tok!"identifier")
output.writeln("<gotoStatement label=\"", gotoStatement.label.text, "\"/>");
else
{
output.writeln("<gotoStatement>");
@ -625,7 +625,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<ifStatement>");
output.writeln("<condition>");
if (ifStatement.identifier.type != TokenType.invalid)
if (ifStatement.identifier.type != tok!"")
{
if (ifStatement.type is null)
output.writeln("<auto/>");
@ -651,11 +651,11 @@ class XMLPrinter : ASTVisitor
override void visit(ImportBind importBind)
{
if (importBind.right.type == TokenType.invalid)
output.writeln("<importBind symbol=\"", importBind.left.value, "\">");
if (importBind.right.type == tok!"")
output.writeln("<importBind symbol=\"", importBind.left.text, "\">");
else
output.writeln("<importBind symbol=\"", importBind.right.value,
"\" rename=\"", importBind.left.value, "\">");
output.writeln("<importBind symbol=\"", importBind.right.text,
"\" rename=\"", importBind.left.text, "\">");
}
override void visit(ImportBindings importBindings)
@ -725,7 +725,7 @@ class XMLPrinter : ASTVisitor
override void visit(InterfaceDeclaration interfaceDec)
{
output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">");
output.writeln("<name>", interfaceDec.name.value, "</name>");
output.writeln("<name>", interfaceDec.name.text, "</name>");
interfaceDec.accept(this);
output.writeln("</interfaceDeclaration>");
}
@ -741,11 +741,11 @@ class XMLPrinter : ASTVisitor
{
output.writeln("<isExpression>");
visit(isExpression.type);
if (isExpression.identifier.type != TokenType.invalid)
if (isExpression.identifier.type != tok!"")
visit(isExpression.identifier);
if (isExpression.typeSpecialization !is null)
{
if (isExpression.equalsOrColon == TokenType.colon)
if (isExpression.equalsOrColon == tok!":")
output.writeln("<colon/>");
else
output.writeln("<equals/>");
@ -776,7 +776,7 @@ class XMLPrinter : ASTVisitor
override void visit (LabeledStatement labeledStatement)
{
output.writeln("<labeledStatement label=\"",
labeledStatement.identifier.value ,"\">");
labeledStatement.identifier.text ,"\">");
visit(labeledStatement.declarationOrStatement);
output.writeln("</labeledStatement>");
}
@ -784,9 +784,9 @@ class XMLPrinter : ASTVisitor
override void visit(LambdaExpression lambdaExpression)
{
output.writeln("<lambdaExpression>");
if (lambdaExpression.functionType == TokenType.function_)
if (lambdaExpression.functionType == tok!"function")
output.writeln("<function/>");
if (lambdaExpression.functionType == TokenType.delegate_)
if (lambdaExpression.functionType == tok!"delegate")
output.writeln("<delegate/>");
lambdaExpression.accept(this);
output.writeln("</lambdaExpression>");
@ -803,14 +803,14 @@ class XMLPrinter : ASTVisitor
output.writeln("<linkageAttribute linkage=\"c++\"/>");
else
output.writeln("<linkageAttribute linkage=\"",
linkageAttribute.identifier.value, "\"/>");
linkageAttribute.identifier.text, "\"/>");
}
override void visit(MemberFunctionAttribute memberFunctionAttribute)
{
output.writeln("<memberFunctionAttribute>");
if (memberFunctionAttribute.atAttribute is null)
output.writeln(getTokenValue(memberFunctionAttribute.tokenType));
output.writeln(str(memberFunctionAttribute.tokenType));
else
memberFunctionAttribute.accept(this);
output.writeln("</memberFunctionAttribute>");
@ -850,7 +850,7 @@ class XMLPrinter : ASTVisitor
override void visit(MulExpression mulExpression)
{
output.writeln("<mulExpression operator=\"", getTokenValue(mulExpression.operator) ,"\">");
output.writeln("<mulExpression operator=\"", str(mulExpression.operator) ,"\">");
output.writeln("<left>");
mulExpression.left.accept(this);
output.writeln("</left>");
@ -901,11 +901,11 @@ class XMLPrinter : ASTVisitor
override void visit(Parameter param)
{
output.writeln("<parameter>");
if (param.name.type == TokenType.identifier)
output.writeln("<name>", param.name.value, "</name>");
if (param.name.type == tok!"identifier")
output.writeln("<name>", param.name.text, "</name>");
foreach (attribute; param.parameterAttributes)
{
output.writeln("<parameterAttribute>", getTokenValue(attribute), "</parameterAttribute>");
output.writeln("<parameterAttribute>", str(attribute), "</parameterAttribute>");
}
param.accept(this);
if (param.vararg)
@ -926,7 +926,7 @@ class XMLPrinter : ASTVisitor
override void visit(PostIncDecExpression postIncDecExpression)
{
output.writeln("<postIncDecExpression operator=\"",
getTokenValue(postIncDecExpression.operator), "\">");
str(postIncDecExpression.operator), "\">");
postIncDecExpression.accept(this);
output.writeln("</postIncDecExpression>");
}
@ -959,7 +959,7 @@ class XMLPrinter : ASTVisitor
override void visit(PreIncDecExpression preIncDecExpression)
{
output.writeln("<preIncDecExpression operator=\"",
getTokenValue(preIncDecExpression.operator), "\">");
str(preIncDecExpression.operator), "\">");
preIncDecExpression.accept(this);
output.writeln("</preIncDecExpression>");
}
@ -974,7 +974,7 @@ class XMLPrinter : ASTVisitor
override void visit(RelExpression relExpression)
{
output.writeln("<relExpression operator=\"",
xmlEscape(getTokenValue(relExpression.operator)), "\">");
xmlEscape(str(relExpression.operator)), "\">");
output.writeln("<left>");
visit(relExpression.left);
output.writeln("</left>");
@ -1014,7 +1014,7 @@ class XMLPrinter : ASTVisitor
override void visit(ShiftExpression shiftExpression)
{
output.writeln("<shiftExpression operator=\"",
xmlEscape(getTokenValue(shiftExpression.operator)), "\">");
xmlEscape(str(shiftExpression.operator)), "\">");
output.writeln("<left>");
visit(shiftExpression.left);
output.writeln("</left>");
@ -1026,10 +1026,10 @@ class XMLPrinter : ASTVisitor
override void visit(SingleImport singleImport)
{
if (singleImport.rename.type == TokenType.invalid)
if (singleImport.rename.type == tok!"")
output.writeln("<singleImport>");
else
output.writeln("<singleImport rename=\"", singleImport.rename.value, "\">");
output.writeln("<singleImport rename=\"", singleImport.rename.text, "\">");
visit(singleImport.identifierChain);
output.writeln("</singleImport>");
}
@ -1094,7 +1094,7 @@ class XMLPrinter : ASTVisitor
override void visit(StructDeclaration structDec)
{
output.writeln("<structDeclaration line=\"", structDec.name.line, "\">");
output.writeln("<name>", structDec.name.value, "</name>");
output.writeln("<name>", structDec.name.text, "</name>");
structDec.accept(this);
output.writeln("</structDeclaration>");
}
@ -1196,7 +1196,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<templateDeclaration line=\"",
templateDeclaration.name.line, "\">");
output.writeln("<name>", templateDeclaration.name.value, "</name>");
output.writeln("<name>", templateDeclaration.name.text, "</name>");
visit(templateDeclaration.templateParameters);
if (templateDeclaration.constraint !is null)
visit(templateDeclaration.constraint);
@ -1275,28 +1275,28 @@ class XMLPrinter : ASTVisitor
override void visit(Token token)
{
string tagName;
with (TokenType) switch (token.type)
switch (token.type)
{
case invalid: return;
case identifier: tagName = "identifier"; break;
case doubleLiteral: tagName = "doubleLiteral"; break;
case idoubleLiteral: tagName = "idoubleLiteral"; break;
case floatLiteral: tagName = "floatLiteral"; break;
case ifloatLiteral: tagName = "ifloatLiteral"; break;
case intLiteral: tagName = "intLiteral"; break;
case uintLiteral: tagName = "uintLiteral"; break;
case longLiteral: tagName = "longLiteral"; break;
case ulongLiteral: tagName = "ulongLiteral"; break;
case realLiteral: tagName = "realLiteral"; break;
case irealLiteral: tagName = "irealLiteral"; break;
case characterLiteral: tagName = "characterLiteral"; break;
case stringLiteral: tagName = "stringLiteral"; break;
case dstringLiteral: tagName = "dstringLiteral"; break;
case wstringLiteral: tagName = "wstringLiteral"; break;
case dollar: output.writeln("<dollar/>"); return;
default: output.writeln("<", getTokenValue(token.type), "/>"); return;
case tok!"": return;
case tok!"identifier": tagName = "identifier"; break;
case tok!"doubleLiteral": tagName = "doubleLiteral"; break;
case tok!"idoubleLiteral": tagName = "idoubleLiteral"; break;
case tok!"floatLiteral": tagName = "floatLiteral"; break;
case tok!"ifloatLiteral": tagName = "ifloatLiteral"; break;
case tok!"intLiteral": tagName = "intLiteral"; break;
case tok!"uintLiteral": tagName = "uintLiteral"; break;
case tok!"longLiteral": tagName = "longLiteral"; break;
case tok!"ulongLiteral": tagName = "ulongLiteral"; break;
case tok!"realLiteral": tagName = "realLiteral"; break;
case tok!"irealLiteral": tagName = "irealLiteral"; break;
case tok!"characterLiteral": tagName = "characterLiteral"; break;
case tok!"stringLiteral": tagName = "stringLiteral"; break;
case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
case tok!"$": output.writeln("<dollar/>"); return;
default: output.writeln("<", str(token.type), "/>"); return;
}
output.writeln("<", tagName, ">", xmlEscape(token.value), "</", tagName, ">");
output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
}
override void visit(TraitsExpression traitsExpression)
@ -1322,8 +1322,8 @@ class XMLPrinter : ASTVisitor
override void visit(Type2 type2)
{
output.writeln("<type2>");
if (type2.builtinType != TokenType.invalid)
output.writeln(getTokenValue(type2.builtinType));
if (type2.builtinType != tok!"")
output.writeln(str(type2.builtinType));
else
type2.accept(this);
output.writeln("</type2>");
@ -1392,16 +1392,16 @@ class XMLPrinter : ASTVisitor
override void visit(UnaryExpression unaryExpression)
{
output.writeln("<unaryExpression>");
if (unaryExpression.prefix != TokenType.invalid)
if (unaryExpression.prefix != tok!"")
{
output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.value),
output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.text),
"</prefix>");
visit(unaryExpression.unaryExpression);
}
if (unaryExpression.suffix != TokenType.invalid)
if (unaryExpression.suffix != tok!"")
{
visit(unaryExpression.unaryExpression);
output.writeln("<suffix>", unaryExpression.suffix.value,
output.writeln("<suffix>", unaryExpression.suffix.text,
"</suffix>");
}
else
@ -1412,8 +1412,8 @@ class XMLPrinter : ASTVisitor
override void visit(UnionDeclaration unionDeclaration)
{
output.writeln("<unionDeclaration line=\"", unionDeclaration.name.line, "\">");
if (unionDeclaration.name != TokenType.invalid)
output.writeln("<name>", unionDeclaration.name.value, "</name>");
if (unionDeclaration.name != tok!"")
output.writeln("<name>", unionDeclaration.name.text, "</name>");
if (unionDeclaration.templateParameters !is null)
visit(unionDeclaration.templateParameters);
if (unionDeclaration.constraint !is null)

2
build.bat
View File

@ -1 +1 @@
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d -wi -ofdscanner
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d stdx/lexer.d -wi -ofdscanner

40
ctags.d
View File

@ -24,9 +24,7 @@ void printCtags(File output, string[] fileNames)
File f = File(fileName);
auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
f.rawRead(bytes);
LexerConfig config;
auto tokens = byToken(bytes, config);
Module m = parseModule(tokens.array(), fileName, &doNothing);
Module m = parseModule(byToken!(ubyte[])(bytes).array, fileName, &doNothing);
auto printer = new CTagsPrinter;
printer.fileName = fileName;
printer.visit(m);
@ -46,83 +44,83 @@ class CTagsPrinter : ASTVisitor
override void visit(ClassDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context);
tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context;
context = "\tclass:" ~ dec.name.value;
context = "\tclass:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(StructDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.value, fileName, dec.name.line, context);
tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context;
context = "\tstruct:" ~ dec.name.value;
context = "\tstruct:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(InterfaceDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context);
tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context;
context = "\tclass:" ~ dec.name.value;
context = "\tclass:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(TemplateDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context);
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context;
context = "\ttemplate:" ~ dec.name.value;
context = "\ttemplate:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(FunctionDeclaration dec)
{
tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName,
tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.text, fileName,
dec.name.line, dec.parameters.parameters.length, context);
auto c = context;
context = "\tfunction:" ~ dec.name.value;
context = "\tfunction:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(EnumDeclaration dec)
{
if (dec.name == TokenType.invalid)
if (dec.name == tok!"")
{
dec.accept(this);
return;
}
tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName,
tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.text, fileName,
dec.name.line, context);
auto c = context;
context = "\tenum:" ~ dec.name.value;
context = "\tenum:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(UnionDeclaration dec)
{
if (dec.name == TokenType.invalid)
if (dec.name == tok!"")
{
dec.accept(this);
return;
}
tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName,
tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.text, fileName,
dec.name.line, context);
auto c = context;
context = "\tunion:" ~ dec.name.value;
context = "\tunion:" ~ dec.name.text;
dec.accept(this);
context = c;
}
override void visit(EnumMember mem)
{
tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName,
tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.text, fileName,
mem.name.line, context);
}
@ -130,7 +128,7 @@ class CTagsPrinter : ASTVisitor
{
foreach (d; dec.declarators)
{
tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.value, fileName,
tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.text, fileName,
d.name.line, context);
}
dec.accept(this);

BIN
dscanner.exe Normal file
View File

Binary file not shown.

BIN
dscanner.obj Normal file
View File

Binary file not shown.

28
formatter.d
View File

@ -70,7 +70,7 @@ class Formatter(Sink)
if (addExpression.right is null)
return;
sink.put(" ");
sink.put(getTokenValue(addExpression.operator));
sink.put(str(addExpression.operator));
sink.put(" ");
format(addExpression.right);
}
@ -333,8 +333,8 @@ class Formatter(Sink)
void format(const GotoStatement gotoStatement)
{
sink.put("goto ");
if (gotoStatement.label != TokenType.invalid)
sink.put(gotoStatement.label.value);
if (gotoStatement.label != tok!"")
sink.put(gotoStatement.label.text);
else
format(gotoStatement.expression);
sink.put(";");
@ -348,7 +348,7 @@ class Formatter(Sink)
if (!first)
sink.put(".");
first = false;
sink.put(ident.value);
sink.put(ident.text);
}
}
@ -372,7 +372,7 @@ class Formatter(Sink)
if (identifierOrTemplateInstance.templateInstance !is null)
format(identifierOrTemplateInstance.templateInstance);
else
sink.put(identifierOrTemplateInstance.identifier.value);
sink.put(identifierOrTemplateInstance.identifier.text);
}
@ -488,14 +488,14 @@ class Formatter(Sink)
{
foreach (attribute; parameter.parameterAttributes)
{
sink.put(getTokenValue(attribute));
sink.put(str(attribute));
}
if (parameter.type !is null)
format(parameter.type);
if (parameter.name.type != TokenType.invalid)
if (parameter.name.type != tok!"")
{
sink.put(" ");
sink.put(parameter.name.value);
sink.put(parameter.name.text);
}
if (parameter.vararg)
sink.put(" ...");
@ -607,7 +607,7 @@ class Formatter(Sink)
void format(const Symbol symbol)
{
if (symbol.dot != TokenType.invalid)
if (symbol.dot != tok!"")
sink.put(".");
format(symbol.identifierOrTemplateChain);
}
@ -671,7 +671,7 @@ class Formatter(Sink)
void format(const Token token)
{
sink.put(token.value);
sink.put(token.text);
}
void format(const TraitsExpression traitsExpression)
@ -688,7 +688,7 @@ class Formatter(Sink)
if (first)
sink.put(" ");
first = false;
sink.put(getTokenValue(constructor));
sink.put(str(constructor));
}
if (type.typeConstructors.length > 0)
sink.put(" ");
@ -711,16 +711,16 @@ class Formatter(Sink)
format(type2.typeofExpression);
return;
}
else if (type2.typeConstructor != TokenType.invalid)
else if (type2.typeConstructor != tok!"")
{
sink.put(getTokenValue(type2.typeConstructor));
sink.put(str(type2.typeConstructor));
sink.put("(");
format(type2.type);
sink.put(")");
return;
}
else
sink.put(getTokenValue(type2.builtinType));
sink.put(str(type2.builtinType));
}
void format(const TypeSpecialization typeSpecialization)

32
highlighter.d
View File

@ -11,7 +11,7 @@ import std.array;
import stdx.d.lexer;
// http://ethanschoonover.com/solarized
void highlight(R)(TokenRange!R tokens, string fileName)
void highlight(R)(R tokens, string fileName)
{
stdout.writeln(q"[
<!DOCTYPE html>
@ -36,27 +36,27 @@ html { background-color: #fdf6e3; color: #002b36; }
foreach (Token t; tokens)
{
if (isBasicType(t.type))
writeSpan("type", t.value);
writeSpan("type", str(t.type));
else if (isKeyword(t.type))
writeSpan("kwrd", t.value);
else if (t.type == TokenType.comment)
writeSpan("com", t.value);
else if (isStringLiteral(t.type) || t.type == TokenType.characterLiteral)
writeSpan("str", t.value);
writeSpan("kwrd", str(t.type));
else if (t.type == tok!"comment")
writeSpan("com", t.text);
else if (isStringLiteral(t.type) || t.type == tok!"characterLiteral")
writeSpan("str", t.text);
else if (isNumberLiteral(t.type))
writeSpan("num", t.value);
writeSpan("num", t.text);
else if (isOperator(t.type))
writeSpan("op", t.value);
writeSpan("op", str(t.type));
else
{
version(Windows)
{
// Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong.
// Strip out the CR characters here to avoid this.
stdout.write(t.value.replace("<", "&lt;").replace("\r", ""));
stdout.write(t.text.replace("<", "&lt;").replace("\r", ""));
}
else
stdout.write(t.value.replace("<", "&lt;"));
stdout.write(t.text.replace("<", "&lt;"));
}
}
@ -70,13 +70,3 @@ void writeSpan(string cssClass, string value)
else
stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
}
/+void main(string[] args)
{
LexerConfig config;
config.tokenStyle = TokenStyle.source;
config.iterStyle = IterationStyle.everything;
config.fileName = args[1];
auto f = File(args[1]);
(cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config).highlight();
}+/

2
imports.d
View File

@ -26,7 +26,7 @@ class ImportPrinter : ASTVisitor
{
if (!first)
write(".");
write(ident.value);
write(ident.text);
first = false;
}
}

46
main.d
View File

@ -89,12 +89,9 @@ int main(string[] args)
if (highlight)
{
LexerConfig config;
config.iterStyle = IterationStyle.everything;
config.tokenStyle = TokenStyle.source;
bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
highlighter.highlight(byToken(bytes, config),
highlighter.highlight(byToken!(typeof(bytes), false, false)(bytes),
args.length == 1 ? "stdin" : args[1]);
return 0;
}
@ -104,13 +101,12 @@ int main(string[] args)
}
else
{
LexerConfig config;
bool usingStdin = args.length == 1;
if (sloc || tokenCount)
{
if (usingStdin)
{
auto tokens = byToken(readStdin(), config);
auto tokens = byToken!(ubyte[], false, false)(readStdin());
if (tokenCount)
printTokenCount(stdout, "stdin", tokens);
else
@ -121,7 +117,7 @@ int main(string[] args)
ulong count;
foreach (f; expandArgs(args, recursive))
{
auto tokens = byToken(readFile(f), config);
auto tokens = byToken!(ubyte[])(readFile(f));
if (tokenCount)
count += printTokenCount(stdout, f, tokens);
else
@ -132,48 +128,28 @@ int main(string[] args)
}
else if (syntaxCheck)
{
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
config);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
parseModule(tokens.array(), config.fileName);
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
}
else if (imports)
{
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
config);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto visitor = new ImportPrinter;
visitor.visit(mod);
}
else if (ast)
{
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
config);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto printer = new XMLPrinter;
printer.output = stdout;
printer.visit(mod);
}
else if (outline)
{
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]),
config);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
auto outliner = new Outliner(stdout);
outliner.visit(mod);
}

276
main.html Normal file
View File

@ -0,0 +1,276 @@
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<title>main.d</title>
</head>
<body>
<style type="text/css">
html { background-color: #fdf6e3; color: #002b36; }
.kwrd { color: #b58900; font-weight: bold; }
.com { color: #93a1a1; font-style: italic; }
.num { color: #dc322f; font-weigth: bold; }
.str { color: #2aa198; font-style: italic; }
.op { color: #586e75; font-weight: bold; }
.type { color: #268bd2; font-weight: bold; }
.cons { color: #859900; font-weight: bold; }
</style>
<pre>
<span class="com">// Copyright Brian Schott (Sir Alaran) 2012.</span>
<span class="com">// Distributed under the Boost Software License, Version 1.0.</span>
<span class="com">// (See accompanying file LICENSE_1_0.txt or copy at</span>
<span class="com">// http://www.boost.org/LICENSE_1_0.txt)</span>
<span class="kwrd">module</span> main<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>algorithm<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>array<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>conv<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>file<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>getopt<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>parallelism<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>path<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>regex<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>stdio<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>range<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>lexer<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>parser<span class="op">;</span>
<span class="kwrd">import</span> highlighter<span class="op">;</span>
<span class="kwrd">import</span> stats<span class="op">;</span>
<span class="kwrd">import</span> ctags<span class="op">;</span>
<span class="kwrd">import</span> astprinter<span class="op">;</span>
<span class="kwrd">import</span> imports<span class="op">;</span>
<span class="kwrd">import</span> outliner<span class="op">;</span>
<span class="type">int</span> main<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> sloc<span class="op">;</span>
<span class="type">bool</span> highlight<span class="op">;</span>
<span class="type">bool</span> ctags<span class="op">;</span>
<span class="type">bool</span> recursive<span class="op">;</span>
<span class="type">bool</span> format<span class="op">;</span>
<span class="type">bool</span> help<span class="op">;</span>
<span class="type">bool</span> tokenCount<span class="op">;</span>
<span class="type">bool</span> syntaxCheck<span class="op">;</span>
<span class="type">bool</span> ast<span class="op">;</span>
<span class="type">bool</span> imports<span class="op">;</span>
<span class="type">bool</span> muffin<span class="op">;</span>
<span class="type">bool</span> outline<span class="op">;</span>
<span class="kwrd">try</span>
<span class="op">{</span>
getopt<span class="op">(</span>args<span class="op">,</span> <span class="str">"sloc|l"</span><span class="op">,</span> <span class="op">&amp;</span>sloc<span class="op">,</span> <span class="str">"highlight"</span><span class="op">,</span> <span class="op">&amp;</span>highlight<span class="op">,</span>
<span class="str">"ctags|c"</span><span class="op">,</span> <span class="op">&amp;</span>ctags<span class="op">,</span> <span class="str">"recursive|r|R"</span><span class="op">,</span> <span class="op">&amp;</span>recursive<span class="op">,</span> <span class="str">"help|h"</span><span class="op">,</span> <span class="op">&amp;</span>help<span class="op">,</span>
<span class="str">"tokenCount|t"</span><span class="op">,</span> <span class="op">&amp;</span>tokenCount<span class="op">,</span> <span class="str">"syntaxCheck|s"</span><span class="op">,</span> <span class="op">&amp;</span>syntaxCheck<span class="op">,</span>
<span class="str">"ast|xml"</span><span class="op">,</span> <span class="op">&amp;</span>ast<span class="op">,</span> <span class="str">"imports|i"</span><span class="op">,</span> <span class="op">&amp;</span>imports<span class="op">,</span> <span class="str">"outline|o"</span><span class="op">,</span> <span class="op">&amp;</span>outline<span class="op">,</span>
<span class="str">"muffinButton"</span><span class="op">,</span> <span class="op">&amp;</span>muffin<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">catch</span> <span class="op">(</span>Exception e<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span>e<span class="op">.</span>msg<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>muffin<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>writeln<span class="op">(</span>
<span class="str">` ___________
__(#*O 0** @%*)__
_(%*o#*O%*0 #O#%##@)_
(*#@%#o*@ #o%O*%@ #o #)
\=====================/
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|`</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>help<span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">auto</span> optionCount <span class="op">=</span> count<span class="op">!</span><span class="str">"a"</span><span class="op">(</span><span class="op">[</span>sloc<span class="op">,</span> highlight<span class="op">,</span> ctags<span class="op">,</span> tokenCount<span class="op">,</span>
syntaxCheck<span class="op">,</span> ast<span class="op">,</span> imports<span class="op">,</span> outline<span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">></span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span><span class="str">"Too many options specified"</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">&lt;</span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>highlight<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> bytes <span class="op">=</span> usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
highlighter<span class="op">.</span>highlight<span class="op">(</span>byToken<span class="op">!</span><span class="op">(</span><span class="kwrd">typeof</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span>
args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span> <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ctags<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>printCtags<span class="op">(</span>expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>sloc <span class="op">||</span> tokenCount<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>usingStdin<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>readStdin<span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
printTokenCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
printLineCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">ulong</span> count<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>f<span class="op">;</span> expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>readFile<span class="op">(</span>f<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
count <span class="op">+=</span> printTokenCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
count <span class="op">+=</span> printLineCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
writefln<span class="op">(</span><span class="str">"total:\t%d"</span><span class="op">,</span> count<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>syntaxCheck<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>imports<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> visitor <span class="op">=</span> <span class="kwrd">new</span> ImportPrinter<span class="op">;</span>
visitor<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ast<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> printer <span class="op">=</span> <span class="kwrd">new</span> XMLPrinter<span class="op">;</span>
printer<span class="op">.</span>output <span class="op">=</span> stdout<span class="op">;</span>
printer<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>outline<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> outliner <span class="op">=</span> <span class="kwrd">new</span> Outliner<span class="op">(</span>stdout<span class="op">)</span><span class="op">;</span>
outliner<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
string<span class="op">[</span><span class="op">]</span> expandArgs<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">,</span> <span class="type">bool</span> recursive<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>recursive<span class="op">)</span>
<span class="op">{</span>
string<span class="op">[</span><span class="op">]</span> rVal<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>arg<span class="op">;</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span><span class="op">$</span><span class="op">]</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>arg<span class="op">)</span> <span class="op">&amp;&amp;</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> arg<span class="op">;</span>
<span class="kwrd">else</span> <span class="kwrd">foreach</span> <span class="op">(</span>item<span class="op">;</span> dirEntries<span class="op">(</span>arg<span class="op">,</span> SpanMode<span class="op">.</span>breadth<span class="op">)</span><span class="op">.</span>map<span class="op">!</span><span class="op">(</span>a <span class="op">=></span> a<span class="op">.</span>name<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>item<span class="op">)</span> <span class="op">&amp;&amp;</span> <span class="op">(</span>item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> item<span class="op">;</span>
<span class="kwrd">else</span>
<span class="kwrd">continue</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> rVal<span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="kwrd">return</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span> <span class="op">$</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readStdin<span class="op">(</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> sourceCode <span class="op">=</span> appender<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span><span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="num">4096</span><span class="op">]</span> buf<span class="op">;</span>
<span class="kwrd">while</span> <span class="op">(</span><span class="kwrd">true</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> b <span class="op">=</span> stdin<span class="op">.</span>rawRead<span class="op">(</span>buf<span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>b<span class="op">.</span>length <span class="op">==</span> <span class="num">0</span><span class="op">)</span>
<span class="kwrd">break</span><span class="op">;</span>
sourceCode<span class="op">.</span>put<span class="op">(</span>b<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">return</span> sourceCode<span class="op">.</span>data<span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readFile<span class="op">(</span>string fileName<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span><span class="op">!</span>exists<span class="op">(</span>fileName<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span><span class="str">"%s does not exist"</span><span class="op">,</span> fileName<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="op">[</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
File f <span class="op">=</span> File<span class="op">(</span>fileName<span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> sourceCode <span class="op">=</span> uninitializedArray<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>to<span class="op">!</span>size_t<span class="op">(</span>f<span class="op">.</span>size<span class="op">)</span><span class="op">)</span><span class="op">;</span>
f<span class="op">.</span>rawRead<span class="op">(</span>sourceCode<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> sourceCode<span class="op">;</span>
<span class="op">}</span>
<span class="type">void</span> printHelp<span class="op">(</span>string programName<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span>
<span class="str">`
Usage: %s options
options:
--help | -h
Prints this help message
--sloc | -l [sourceFiles]
Prints the number of logical lines of code in the given
source files. If no files are specified, input is read from stdin.
--tokenCount | t [sourceFiles]
Prints the number of tokens in the given source files. If no files are
specified, input is read from stdin.
--highlight [sourceFile] - Syntax-highlight the given source file. The
resulting HTML will be written to standard output. If no files are
specified, input is read from stdin.
--imports | -i [sourceFile]
Prints modules imported by the given source file. If no files are
specified, input is read from stdin.
--syntaxCheck | -s [sourceFile]
Lexes and parses sourceFile, printing the line and column number of any
syntax errors to stdout. One error or warning is printed per line.
If no files are specified, input is read from stdin.
--ctags | -c sourceFile
Generates ctags information from the given source code file. Note that
ctags information requires a filename, so stdin cannot be used in place
of a filename.
--ast | --xml sourceFile
Generates an XML representation of the source files abstract syntax
tree. If no files are specified, input is read from stdin.
--recursive | -R | -r
When used with --ctags, --tokenCount, or --sloc, dscanner will produce
ctags output for all .d and .di files contained within the given
directories and its sub-directories.`</span><span class="op">,</span>
programName<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
</pre>
</body></html>

18
outliner.d
View File

@ -21,7 +21,7 @@ class Outliner : ASTVisitor
override void visit(ClassDeclaration classDec)
{
printIndentation();
output.writeln("class ", classDec.name.value, " : ", classDec.name.line);
output.writeln("class ", classDec.name.text, " : ", classDec.name.line);
indent();
classDec.accept(this);
outdent();
@ -31,7 +31,7 @@ class Outliner : ASTVisitor
override void visit(EnumDeclaration enumDec)
{
printIndentation();
output.writeln("enum ", enumDec.name.value, " : ", enumDec.name.line);
output.writeln("enum ", enumDec.name.text, " : ", enumDec.name.line);
indent();
enumDec.accept(this);
outdent();
@ -41,7 +41,7 @@ class Outliner : ASTVisitor
override void visit(EnumMember enumMem)
{
printIndentation();
output.writeln(enumMem.name.value, " : ", enumMem.name.line);
output.writeln(enumMem.name.text, " : ", enumMem.name.line);
finish();
}
@ -57,7 +57,7 @@ class Outliner : ASTVisitor
if (functionDec.returnType !is null)
f.format(functionDec.returnType);
app.put(" ");
app.put(functionDec.name.value);
app.put(functionDec.name.text);
f.format(functionDec.parameters);
app.put(" : ");
app.put(to!string(functionDec.name.line));
@ -68,7 +68,7 @@ class Outliner : ASTVisitor
override void visit(InterfaceDeclaration interfaceDec)
{
printIndentation();
output.writeln("interface ", interfaceDec.name.value, " : ",
output.writeln("interface ", interfaceDec.name.text, " : ",
interfaceDec.name.line);
indent();
interfaceDec.accept(this);
@ -79,7 +79,7 @@ class Outliner : ASTVisitor
override void visit(StructDeclaration structDec)
{
printIndentation();
output.writeln("struct ", structDec.name.value, " : ",
output.writeln("struct ", structDec.name.text, " : ",
structDec.name.line);
indent();
structDec.accept(this);
@ -90,7 +90,7 @@ class Outliner : ASTVisitor
override void visit(TemplateDeclaration templateDeclaration)
{
printIndentation();
output.writeln("template", templateDeclaration.name.value, " : ",
output.writeln("template", templateDeclaration.name.text, " : ",
templateDeclaration.name.line);
finish();
}
@ -105,7 +105,7 @@ class Outliner : ASTVisitor
override void visit(UnionDeclaration unionDeclaration)
{
printIndentation();
output.writeln("union ", unionDeclaration.name.value, " : ",
output.writeln("union ", unionDeclaration.name.text, " : ",
unionDeclaration.name.line);
indent();
unionDeclaration.accept(this);
@ -125,7 +125,7 @@ class Outliner : ASTVisitor
f.format(variableDeclaration.type);
}
app.put(" ");
app.put(d.name.value);
app.put(d.name.text);
app.put(" : ");
app.put(to!string(d.name.line));
output.writeln(app.data);

26
stats.d
View File

@ -9,21 +9,21 @@ import std.stdio;
import std.algorithm;
import stdx.d.lexer;
pure nothrow bool isLineOfCode(TokenType t)
pure nothrow bool isLineOfCode(IdType t)
{
with (TokenType) switch(t)
switch(t)
{
case semicolon:
case while_:
case if_:
case do_:
case else_:
case switch_:
case for_:
case foreach_:
case foreach_reverse_:
case default_:
case case_:
case tok!";":
case tok!"while":
case tok!"if":
case tok!"do":
case tok!"else":
case tok!"switch":
case tok!"for":
case tok!"foreach":
case tok!"foreach_reverse":
case tok!"default":
case tok!"case":
return true;
default:
return false;

42
stdx/d/ast.d
View File

@ -275,7 +275,7 @@ public:
{
mixin (visitIfNotNull!(left, right));
}
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -410,7 +410,7 @@ class AsmAddExp : ExpressionNode
{
public:
mixin (DEFAULT_ACCEPT);
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -483,7 +483,7 @@ class AsmMulExp : ExpressionNode
{
public:
mixin (DEFAULT_ACCEPT);
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -583,7 +583,7 @@ public:
}
/** */ ExpressionNode ternaryExpression;
/** */ ExpressionNode assignExpression;
/** */ TokenType operator;
/** */ IdType operator;
}
///
@ -623,7 +623,7 @@ public:
/** */ AlignAttribute alignAttribute;
/** */ PragmaExpression pragmaExpression;
/** */ StorageClass storageClass;
/** */ TokenType attribute;
/** */ IdType attribute;
}
///
@ -1149,7 +1149,7 @@ public:
{
mixin (visitIfNotNull!(left, right));
}
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -1222,7 +1222,7 @@ public:
mixin (visitIfNotNull!(foreachType, foreachTypeList, low, high,
declarationOrStatement));
}
/** */ TokenType type;
/** */ IdType type;
/** */ ForeachTypeList foreachTypeList;
/** */ ForeachType foreachType;
/** */ Expression low;
@ -1239,7 +1239,7 @@ public:
{
mixin (visitIfNotNull!(type, identifier));
}
/** */ TokenType[] typeConstructors;
/** */ IdType[] typeConstructors;
/** */ Type type;
/** */ Token identifier;
}
@ -1336,7 +1336,7 @@ public:
mixin (visitIfNotNull!(type, parameters, functionAttributes,
functionBody));
}
/** */ TokenType functionOrDelegate;
/** */ IdType functionOrDelegate;
/** */ Type type;
/** */ Parameters parameters;
/** */ FunctionAttribute[] functionAttributes;
@ -1575,7 +1575,7 @@ public:
/** */ Token identifier;
/** */ TypeSpecialization typeSpecialization;
/** */ TemplateParameterList templateParameterList;
/** */ TokenType equalsOrColon;
/** */ IdType equalsOrColon;
}
///
@ -1622,7 +1622,7 @@ public:
mixin (visitIfNotNull!(identifier, parameters, functionAttributes,
assignExpression));
}
/** */ TokenType functionType;
/** */ IdType functionType;
/** */ Token identifier;
/** */ Parameters parameters;
/** */ FunctionAttribute[] functionAttributes;
@ -1660,7 +1660,7 @@ public:
{
mixin (visitIfNotNull!(atAttribute));
}
/** */ TokenType tokenType;
/** */ IdType tokenType;
/** */ AtAttribute atAttribute;
}
@ -1743,7 +1743,7 @@ public:
{
mixin (visitIfNotNull!(left, right));
}
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -1893,7 +1893,7 @@ public:
mixin (visitIfNotNull!(type, name, default_));
}
/** */ TokenType[] parameterAttributes;
/** */ IdType[] parameterAttributes;
/** */ Type type;
/** */ Token name;
/** */ bool vararg;
@ -1932,7 +1932,7 @@ public:
{
mixin (visitIfNotNull!(unaryExpression));
}
/** */ TokenType operator;
/** */ IdType operator;
/** */ UnaryExpression unaryExpression;
}
@ -1978,7 +1978,7 @@ public:
{
mixin (visitIfNotNull!(unaryExpression));
}
/** */ TokenType operator;
/** */ IdType operator;
/** */ UnaryExpression unaryExpression;
}
@ -2030,7 +2030,7 @@ public:
{
mixin (visitIfNotNull!(left, right));
}
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -2089,7 +2089,7 @@ public:
{
mixin (visitIfNotNull!(left, right));
}
/** */ TokenType operator;
/** */ IdType operator;
mixin BinaryExpressionBody;
}
@ -2573,7 +2573,7 @@ public:
mixin (visitIfNotNull!(type2, typeSuffixes));
}
/** */ TokenType[] typeConstructors;
/** */ IdType[] typeConstructors;
/** */ TypeSuffix[] typeSuffixes;
/** */ Type2 type2;
}
@ -2588,11 +2588,11 @@ public:
identifierOrTemplateChain, type));
}
/** */ TokenType builtinType;
/** */ IdType builtinType;
/** */ Symbol symbol;
/** */ TypeofExpression typeofExpression;
/** */ IdentifierOrTemplateChain identifierOrTemplateChain;
/** */ TokenType typeConstructor;
/** */ IdType typeConstructor;
/** */ Type type;
}

4802
stdx/d/lexer.d
View File

File diff suppressed because it is too large Load Diff

2149
stdx/d/parser.d
View File

File diff suppressed because it is too large Load Diff

457
stdx/lexer.d Normal file
View File

@ -0,0 +1,457 @@
// Written in the D programming language
/**
* This module contains a range-based _lexer generator.
*
* Copyright: Brian Schott 2013
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
* Authors: Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu
* Source: $(PHOBOSSRC std/_lexer.d)
*/
module stdx.lexer;
import std.typecons;
import std.algorithm;
import std.range;
import std.traits;
import std.conv;
import std.math;
template TokenIdType(alias staticTokens, alias dynamicTokens,
alias possibleDefaultTokens)
{
static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ubyte.max)
alias TokenIdType = ubyte;
else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ushort.max)
alias TokenIdType = ushort;
else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= uint.max)
alias TokenIdType = uint;
else
static assert (false);
}
string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefaultTokens)(IdType type) @property
{
if (type == 0)
return "!ERROR!";
else if (type < staticTokens.length)
return staticTokens[type - 1];
else if (type < staticTokens.length + possibleDefaultTokens.length)
return possibleDefaultTokens[type - staticTokens.length];
else
return null;
}
IdType TokenId(IdType, alias staticTokens, alias dynamicTokens,
alias possibleDefaultTokens, string symbol)() @property
{
static if (symbol == "")
return 0;
else static if (symbol == "\0")
return 1 + staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length;
else
{
enum i = staticTokens.countUntil(symbol);
static if (i >= 0)
enum id = i + 1;
else
{
enum ii = possibleDefaultTokens.countUntil(symbol);
static if (ii >= 0)
enum id = ii + staticTokens.length;
else
{
enum dynamicId = dynamicTokens.countUntil(symbol);
enum id = dynamicId >= 0 ? i + staticTokens.length + possibleDefaultTokens.length + dynamicId : -1;
}
}
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
return id;
}
}
struct TokenStructure(IDType)
{
bool opEquals(IDType type) const pure nothrow @safe
{
return this.type == type;
}
IDType type;
string text;
size_t line;
size_t column;
size_t index;
}
mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFunction,
alias staticTokens, alias dynamicTokens, alias pseudoTokens,
alias possibleDefaultTokens) if (isForwardRange!R)
{
enum size_t lookAhead = chain(staticTokens, pseudoTokens).map!"a.length".reduce!"max(a, b)"();
alias PeekRange!(R, lookAhead) RangeType;
static string generateCaseStatements(string[] tokens, size_t offset = 0)
{
string code;
for (size_t i = 0; i < tokens.length; i++)
{
auto indent = "";
foreach (k; 0 .. offset)
indent ~= " ";
size_t j = i + 1;
if (offset < tokens[i].length)
{
while (j < tokens.length && offset < tokens[j].length
&& tokens[i][offset] == tokens[j][offset]) j++;
code ~= indent ~ "case " ~ text(cast(ubyte) tokens[i][offset]) ~ ":\n";
if (i + 1 >= j)
{
if (offset + 1 == tokens[i].length)
code ~= generateLeaf(tokens[i], indent ~ " ");
else
{
code ~= indent ~ " if (!range.canPeek(" ~ text(tokens[i].length - 1) ~ "))\n";
code ~= indent ~ " goto outer_default;\n";
code ~= indent ~ " if (range.startsWith(\"" ~ escape(tokens[i]) ~ "\"))\n";
code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i], indent ~ " ");
code ~= indent ~ " }\n";
code ~= indent ~ " else\n";
code ~= indent ~ " goto outer_default;\n";
}
}
else
{
code ~= indent ~ " if (!range.canPeek(" ~ text(offset + 1) ~ "))\n";
code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
code ~= indent ~ " }\n";
code ~= indent ~ " switch (range.peek(" ~ text(offset + 1) ~ "))\n";
code ~= indent ~ " {\n";
code ~= generateCaseStatements(tokens[i .. j], offset + 1);
code ~= indent ~ " default:\n";
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
code ~= indent ~ " }\n";
}
}
i = j - 1;
}
return code;
}
static string generateLeaf(string token, string indent)
{
string code;
if (staticTokens.countUntil(token) >= 0)
{
code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
}
else if (pseudoTokens.countUntil(token) >= 0)
code ~= indent ~ "return postProcess(pseudoTok!\"" ~ escape(token) ~"\");\n";
else if (possibleDefaultTokens.countUntil(token) >= 0)
{
code ~= indent ~ "if (!range.canPeek(" ~ text(token.length) ~ ") || isSeparating(range.peek(" ~ text(token.length) ~ ")))\n";
code ~= indent ~ "{\n";
code ~= indent ~ " range.popFrontN(" ~ text(token.length) ~ ");\n";
code ~= indent ~ " return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
code ~= indent ~ "}\n";
code ~= indent ~ "else\n";
code ~= indent ~ " goto outer_default;\n";
}
else
code ~= indent ~ "goto outer_default;\n";
return code;
}
Token front() @property
{
return _front;
}
void popFront()
{
_front = advance();
}
bool empty() const nothrow @property
{
return _front.type == tok!"\0";
}
void registerPostProcess(alias t)(Token delegate(ref RangeType) fun)
{
post[pseudoTok!t] = fun;
}
static IDType pseudoTok(string symbol)() nothrow pure @property
{
static assert (pseudoTokens.countUntil(symbol) >= 0);
enum index = cast(IDType) pseudoTokens.countUntil(symbol);
return index;
}
static string escape(string input)
{
string rVal;
foreach (ubyte c; cast(ubyte[]) input)
{
switch (c)
{
case '\\': rVal ~= `\\`; break;
case '"': rVal ~= `\"`; break;
case '\'': rVal ~= `\'`; break;
case '\t': rVal ~= `\t`; break;
case '\n': rVal ~= `\n`; break;
case '\r': rVal ~= `\r`; break;
default: rVal ~= c; break;
}
}
return rVal;
}
Token advance()
{
if (range.empty)
return Token(tok!"\0");
auto r = range.save;
lexerLoop: switch (range.front)
{
mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
//pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
outer_default:
default:
range = r;
return defaultTokenFunction(range);
}
}
/**
* This only exists because the real array() can't be called at compile-time
*/
static T[] stupidToArray(R, T = ElementType!R)(R range)
{
T[] rVal;
foreach (v; range)
rVal ~= v;
return rVal;
}
Token postProcess(IDType i)
{
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]);
return post[i](range);
}
Token delegate(ref RangeType)[pseudoTokens.length] post;
RangeType range;
Token _front;
}
struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R && isForwardRange!R)
{
public:
this(R range)
{
this.range = range;
}
bool empty() pure nothrow const @property
{
return _index >= range.length;
}
ElementType!R front() pure nothrow const @property
in
{
assert (!empty);
}
body
{
return range[_index];
}
void popFront() pure nothrow
{
_index++;
_column++;
}
void popFrontN(size_t n) pure nothrow
{
foreach (i; 0 .. n)
popFront();
}
ElementType!R peek(int offset = 1) pure nothrow const
in
{
assert (canPeek(offset));
}
body
{
return range[_index + offset];
}
bool canPeek(int offset = 1) pure nothrow const
{
return _index + offset >= 0 && _index + offset < range.length;
}
typeof(this) save() @property
{
typeof(this) copy;
copy.range = range;
copy._index = _index;
copy._column = _column;
copy._line = _line;
return copy;
}
void mark() nothrow pure
{
markBegin = index;
}
R getMarked() nothrow pure
{
return range[markBegin .. index];
}
void incrementLine() pure nothrow
{
_column = 1;
_line++;
}
size_t line() pure nothrow const @property { return _line; }
size_t column() pure nothrow const @property { return _column; }
size_t index() pure nothrow const @property { return _index; }
private:
size_t markBegin;
size_t _column = 1;
size_t _line = 1;
size_t _index = 0;
R range;
}
struct PeekRange(R, size_t peekSupported = 1)
if (!isRandomAccessRange!R && isForwardRange!R)
{
public:
this(R range)
{
this.range = range;
for (size_t i = 0; !this.range.empty && i < peekSupported; i++)
{
rangeSizeCount++;
buffer[i] = this.range.front;
range.popFront();
}
}
ElementType!R front() const @property
in
{
assert (!empty);
}
body
{
return buffer[bufferIndex];
}
void popFront()
in
{
assert (!empty);
}
body
{
index++;
column++;
count++;
bufferIndex = bufferIndex + 1 > buffer.length ? 0 : bufferIndex + 1;
if (marking)
markBuffer.put(buffer[bufferIndex]);
if (!range.empty)
{
buffer[bufferIndex + peekSupported % buffer.length] = range.front();
range.popFront();
rangeSizeCount++;
}
}
bool empty() const nothrow pure @property
{
return rangeSizeCount == count;
}
ElementType!R peek(int offset = 1) pure nothrow const
in
{
assert (canPeek(offset));
}
body
{
return buffer[(bufferIndex + offset) % buffer.length];
}
bool canPeek(int offset = 1) pure nothrow const
{
return offset >= 0
? offset <= peekSupported && count + offset <= rangeSizeCount
: abs(offset) <= peekSupported && (count - abs(offset)) >= 0;
}
typeof(this) save() @property
{
typeof(this) newRange;
newRange.count = count;
newRange.rangeSizeCount = count;
newRange.buffer = buffer.dup;
newRange.bufferIndex = bufferIndex;
newRange.range = range.save;
return newRange;
}
void mark()
{
marking = true;
markBuffer.clear();
}
ElementEncodingType!R[] getMarked()
{
marking = false;
return markBuffer.data;
}
void incrementLine() pure nothrow
{
_column = 1;
_line++;
}
size_t line() pure nothrow const @property { return _line; }
size_t column() pure nothrow const @property { return _column; }
size_t index() pure nothrow const @property { return _index; }
private:
auto markBuffer = appender!(ElementType!R[])();
bool marking;
size_t count;
size_t rangeSizeCount;
ElementType!(R)[(peekSupported * 2) + 1] buffer;
size_t bufferIndex;
size_t _column = 1;
size_t _line = 1;
size_t _index = 0;
R range;
}