forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Converted everything to the new lexer

This commit is contained in:
Hackerpilot 2013-12-15 03:02:52 -08:00
parent 4bac2671b4
commit 8c4a87f563
16 changed files with 3219 additions and 4863 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
astprinter.d
View File

@ -21,7 +21,7 @@ class XMLPrinter : ASTVisitor
{ {
override void visit(AddExpression addExpression) override void visit(AddExpression addExpression)
{ {
output.writeln("<addExpression operator=\"", getTokenValue(addExpression.operator) ,"\">"); output.writeln("<addExpression operator=\"", str(addExpression.operator) ,"\">");
output.writeln("<left>"); output.writeln("<left>");
addExpression.left.accept(this); addExpression.left.accept(this);
output.writeln("</left>"); output.writeln("</left>");
@ -51,7 +51,7 @@ class XMLPrinter : ASTVisitor
override void visit(AlignAttribute alignAttribute) override void visit(AlignAttribute alignAttribute)
{ {
output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.value, "\">"); output.writeln("<alignAttribute align=\"", alignAttribute.intLiteral.text, "\">");
} }
override void visit(AndAndExpression andAndExpression) override void visit(AndAndExpression andAndExpression)
@ -130,7 +130,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<assignExpression>"); output.writeln("<assignExpression>");
else else
output.writeln("<assignExpression operator=\"", output.writeln("<assignExpression operator=\"",
getTokenValue(assignExpression.operator), "\">"); str(assignExpression.operator), "\">");
assignExpression.accept(this); assignExpression.accept(this);
output.writeln("</assignExpression>"); output.writeln("</assignExpression>");
} }
@ -143,20 +143,20 @@ class XMLPrinter : ASTVisitor
override void visit(AtAttribute atAttribute) override void visit(AtAttribute atAttribute)
{ {
output.writeln("<atAttribute>"); output.writeln("<atAttribute>");
if (atAttribute.identifier.type == TokenType.invalid) if (atAttribute.identifier.type == tok!"")
atAttribute.accept(this); atAttribute.accept(this);
else else
output.writeln("<identifier>", atAttribute.identifier.value, "</identifier>"); output.writeln("<identifier>", atAttribute.identifier.text, "</identifier>");
output.writeln("</atAttribute>"); output.writeln("</atAttribute>");
} }
override void visit(Attribute attribute) override void visit(Attribute attribute)
{ {
output.writeln("<attribute>"); output.writeln("<attribute>");
if (attribute.attribute == TokenType.invalid) if (attribute.attribute == tok!"")
attribute.accept(this); attribute.accept(this);
else else
output.writeln(getTokenValue(attribute.attribute)); output.writeln(str(attribute.attribute));
output.writeln("</attribute>"); output.writeln("</attribute>");
} }
@ -173,7 +173,7 @@ class XMLPrinter : ASTVisitor
{ {
output.writeln("<item>"); output.writeln("<item>");
output.writeln("<name line=\"", autoDec.identifiers[i].line, "\">", output.writeln("<name line=\"", autoDec.identifiers[i].line, "\">",
autoDec.identifiers[i].value, "</name>"); autoDec.identifiers[i].text, "</name>");
visit(autoDec.initializers[i]); visit(autoDec.initializers[i]);
output.writeln("</item>"); output.writeln("</item>");
} }
@ -196,10 +196,10 @@ class XMLPrinter : ASTVisitor
override void visit(BreakStatement breakStatement) override void visit(BreakStatement breakStatement)
{ {
if (breakStatement.label.type == TokenType.invalid) if (breakStatement.label.type == tok!"")
output.writeln("<breakStatement/>"); output.writeln("<breakStatement/>");
else else
output.writeln("<breakStatement label=\"", breakStatement.label.value, "\"/>"); output.writeln("<breakStatement label=\"", breakStatement.label.text, "\"/>");
} }
override void visit(BaseClass baseClass) override void visit(BaseClass baseClass)
@ -256,7 +256,7 @@ class XMLPrinter : ASTVisitor
override void visit(ClassDeclaration classDec) override void visit(ClassDeclaration classDec)
{ {
output.writeln("<classDeclaration line=\"", classDec.name.line, "\">"); output.writeln("<classDeclaration line=\"", classDec.name.line, "\">");
output.writeln("<name>", classDec.name.value, "</name>"); output.writeln("<name>", classDec.name.text, "</name>");
classDec.accept(this); classDec.accept(this);
output.writeln("</classDeclaration>"); output.writeln("</classDeclaration>");
} }
@ -318,29 +318,29 @@ class XMLPrinter : ASTVisitor
override void visit(ContinueStatement continueStatement) override void visit(ContinueStatement continueStatement)
{ {
if (continueStatement.label.type == TokenType.invalid) if (continueStatement.label.type == tok!"")
output.writeln("<continueStatement/>"); output.writeln("<continueStatement/>");
else else
output.writeln("<continueStatement label=\"", output.writeln("<continueStatement label=\"",
continueStatement.label.value, "\"/>"); continueStatement.label.text, "\"/>");
} }
override void visit(DebugCondition debugCondition) override void visit(DebugCondition debugCondition)
{ {
if (debugCondition.identifierOrInteger.type == TokenType.invalid) if (debugCondition.identifierOrInteger.type == tok!"")
output.writeln("<debugCondition/>"); output.writeln("<debugCondition/>");
else else
output.writeln("<debugCondition condition=\"", output.writeln("<debugCondition condition=\"",
debugCondition.identifierOrInteger.value, "\"/>"); debugCondition.identifierOrInteger.text, "\"/>");
} }
override void visit(DebugSpecification debugSpecification) override void visit(DebugSpecification debugSpecification)
{ {
if (debugSpecification.identifierOrInteger.type == TokenType.invalid) if (debugSpecification.identifierOrInteger.type == tok!"")
output.writeln("<debugSpecification/>"); output.writeln("<debugSpecification/>");
else else
output.writeln("<debugSpecification condition=\"", output.writeln("<debugSpecification condition=\"",
debugSpecification.identifierOrInteger.value, "\"/>"); debugSpecification.identifierOrInteger.text, "\"/>");
} }
override void visit(Declaration declaration) override void visit(Declaration declaration)
@ -361,7 +361,7 @@ class XMLPrinter : ASTVisitor
override void visit(Declarator declarator) override void visit(Declarator declarator)
{ {
output.writeln("<declarator line=\"", declarator.name.line, "\">"); output.writeln("<declarator line=\"", declarator.name.line, "\">");
output.writeln("<name>", declarator.name.value, "</name>"); output.writeln("<name>", declarator.name.text, "</name>");
declarator.accept(this); declarator.accept(this);
output.writeln("</declarator>"); output.writeln("</declarator>");
} }
@ -411,8 +411,8 @@ class XMLPrinter : ASTVisitor
override void visit(EnumDeclaration enumDec) override void visit(EnumDeclaration enumDec)
{ {
output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">"); output.writeln("<enumDeclaration line=\"", enumDec.name.line, "\">");
if (enumDec.name.type == TokenType.identifier) if (enumDec.name.type == tok!"identifier")
output.writeln("<name>", enumDec.name.value, "</name>"); output.writeln("<name>", enumDec.name.text, "</name>");
enumDec.accept(this); enumDec.accept(this);
output.writeln("</enumDeclaration>"); output.writeln("</enumDeclaration>");
} }
@ -426,7 +426,7 @@ class XMLPrinter : ASTVisitor
override void visit(EqualExpression equalExpression) override void visit(EqualExpression equalExpression)
{ {
output.writeln("<enumMember operator=\"", getTokenValue(equalExpression.operator), "\">"); output.writeln("<enumMember operator=\"", str(equalExpression.operator), "\">");
output.writeln("<left>"); output.writeln("<left>");
visit(equalExpression.left); visit(equalExpression.left);
output.writeln("</left>"); output.writeln("</left>");
@ -491,7 +491,7 @@ class XMLPrinter : ASTVisitor
override void visit(ForeachStatement foreachStatement) override void visit(ForeachStatement foreachStatement)
{ {
output.writeln("<foreachStatement type=\"", getTokenValue( output.writeln("<foreachStatement type=\"", str(
foreachStatement.type), "\">"); foreachStatement.type), "\">");
if (foreachStatement.foreachType !is null) if (foreachStatement.foreachType !is null)
visit(foreachStatement.foreachType); visit(foreachStatement.foreachType);
@ -515,7 +515,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<foreachType>"); output.writeln("<foreachType>");
foreach (constructor; foreachType.typeConstructors) foreach (constructor; foreachType.typeConstructors)
{ {
output.writeln("<typeConstructor>", getTokenValue(constructor), "</typeConstructor>"); output.writeln("<typeConstructor>", str(constructor), "</typeConstructor>");
} }
if (foreachType.type !is null) if (foreachType.type !is null)
visit(foreachType.type); visit(foreachType.type);
@ -552,7 +552,7 @@ class XMLPrinter : ASTVisitor
override void visit(FunctionDeclaration functionDec) override void visit(FunctionDeclaration functionDec)
{ {
output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">"); output.writeln("<functionDeclaration line=\"", functionDec.name.line, "\">");
output.writeln("<name>", functionDec.name.value, "</name>"); output.writeln("<name>", functionDec.name.text, "</name>");
if (functionDec.hasAuto) if (functionDec.hasAuto)
output.writeln("<auto/>"); output.writeln("<auto/>");
if (functionDec.hasRef) if (functionDec.hasRef)
@ -564,17 +564,17 @@ class XMLPrinter : ASTVisitor
override void visit(FunctionLiteralExpression functionLiteralExpression) override void visit(FunctionLiteralExpression functionLiteralExpression)
{ {
output.writeln("<functionLiteralExpression type=\"", output.writeln("<functionLiteralExpression type=\"",
getTokenValue(functionLiteralExpression.functionOrDelegate), "\">"); str(functionLiteralExpression.functionOrDelegate), "\">");
functionLiteralExpression.accept(this); functionLiteralExpression.accept(this);
output.writeln("</functionLiteralExpression>"); output.writeln("</functionLiteralExpression>");
} }
override void visit(GotoStatement gotoStatement) override void visit(GotoStatement gotoStatement)
{ {
if (gotoStatement.label.type == TokenType.default_) if (gotoStatement.label.type == tok!"default")
output.writeln("<gotoStatement default=\"true\"/>"); output.writeln("<gotoStatement default=\"true\"/>");
else if (gotoStatement.label.type == TokenType.identifier) else if (gotoStatement.label.type == tok!"identifier")
output.writeln("<gotoStatement label=\"", gotoStatement.label.value, "\"/>"); output.writeln("<gotoStatement label=\"", gotoStatement.label.text, "\"/>");
else else
{ {
output.writeln("<gotoStatement>"); output.writeln("<gotoStatement>");
@ -625,7 +625,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<ifStatement>"); output.writeln("<ifStatement>");
output.writeln("<condition>"); output.writeln("<condition>");
if (ifStatement.identifier.type != TokenType.invalid) if (ifStatement.identifier.type != tok!"")
{ {
if (ifStatement.type is null) if (ifStatement.type is null)
output.writeln("<auto/>"); output.writeln("<auto/>");
@ -651,11 +651,11 @@ class XMLPrinter : ASTVisitor
override void visit(ImportBind importBind) override void visit(ImportBind importBind)
{ {
if (importBind.right.type == TokenType.invalid) if (importBind.right.type == tok!"")
output.writeln("<importBind symbol=\"", importBind.left.value, "\">"); output.writeln("<importBind symbol=\"", importBind.left.text, "\">");
else else
output.writeln("<importBind symbol=\"", importBind.right.value, output.writeln("<importBind symbol=\"", importBind.right.text,
"\" rename=\"", importBind.left.value, "\">"); "\" rename=\"", importBind.left.text, "\">");
} }
override void visit(ImportBindings importBindings) override void visit(ImportBindings importBindings)
@ -725,7 +725,7 @@ class XMLPrinter : ASTVisitor
override void visit(InterfaceDeclaration interfaceDec) override void visit(InterfaceDeclaration interfaceDec)
{ {
output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">"); output.writeln("<interfaceDeclaration line=\"", interfaceDec.name.line, "\">");
output.writeln("<name>", interfaceDec.name.value, "</name>"); output.writeln("<name>", interfaceDec.name.text, "</name>");
interfaceDec.accept(this); interfaceDec.accept(this);
output.writeln("</interfaceDeclaration>"); output.writeln("</interfaceDeclaration>");
} }
@ -741,11 +741,11 @@ class XMLPrinter : ASTVisitor
{ {
output.writeln("<isExpression>"); output.writeln("<isExpression>");
visit(isExpression.type); visit(isExpression.type);
if (isExpression.identifier.type != TokenType.invalid) if (isExpression.identifier.type != tok!"")
visit(isExpression.identifier); visit(isExpression.identifier);
if (isExpression.typeSpecialization !is null) if (isExpression.typeSpecialization !is null)
{ {
if (isExpression.equalsOrColon == TokenType.colon) if (isExpression.equalsOrColon == tok!":")
output.writeln("<colon/>"); output.writeln("<colon/>");
else else
output.writeln("<equals/>"); output.writeln("<equals/>");
@ -776,7 +776,7 @@ class XMLPrinter : ASTVisitor
override void visit (LabeledStatement labeledStatement) override void visit (LabeledStatement labeledStatement)
{ {
output.writeln("<labeledStatement label=\"", output.writeln("<labeledStatement label=\"",
labeledStatement.identifier.value ,"\">"); labeledStatement.identifier.text ,"\">");
visit(labeledStatement.declarationOrStatement); visit(labeledStatement.declarationOrStatement);
output.writeln("</labeledStatement>"); output.writeln("</labeledStatement>");
} }
@ -784,9 +784,9 @@ class XMLPrinter : ASTVisitor
override void visit(LambdaExpression lambdaExpression) override void visit(LambdaExpression lambdaExpression)
{ {
output.writeln("<lambdaExpression>"); output.writeln("<lambdaExpression>");
if (lambdaExpression.functionType == TokenType.function_) if (lambdaExpression.functionType == tok!"function")
output.writeln("<function/>"); output.writeln("<function/>");
if (lambdaExpression.functionType == TokenType.delegate_) if (lambdaExpression.functionType == tok!"delegate")
output.writeln("<delegate/>"); output.writeln("<delegate/>");
lambdaExpression.accept(this); lambdaExpression.accept(this);
output.writeln("</lambdaExpression>"); output.writeln("</lambdaExpression>");
@ -803,14 +803,14 @@ class XMLPrinter : ASTVisitor
output.writeln("<linkageAttribute linkage=\"c++\"/>"); output.writeln("<linkageAttribute linkage=\"c++\"/>");
else else
output.writeln("<linkageAttribute linkage=\"", output.writeln("<linkageAttribute linkage=\"",
linkageAttribute.identifier.value, "\"/>"); linkageAttribute.identifier.text, "\"/>");
} }
override void visit(MemberFunctionAttribute memberFunctionAttribute) override void visit(MemberFunctionAttribute memberFunctionAttribute)
{ {
output.writeln("<memberFunctionAttribute>"); output.writeln("<memberFunctionAttribute>");
if (memberFunctionAttribute.atAttribute is null) if (memberFunctionAttribute.atAttribute is null)
output.writeln(getTokenValue(memberFunctionAttribute.tokenType)); output.writeln(str(memberFunctionAttribute.tokenType));
else else
memberFunctionAttribute.accept(this); memberFunctionAttribute.accept(this);
output.writeln("</memberFunctionAttribute>"); output.writeln("</memberFunctionAttribute>");
@ -850,7 +850,7 @@ class XMLPrinter : ASTVisitor
override void visit(MulExpression mulExpression) override void visit(MulExpression mulExpression)
{ {
output.writeln("<mulExpression operator=\"", getTokenValue(mulExpression.operator) ,"\">"); output.writeln("<mulExpression operator=\"", str(mulExpression.operator) ,"\">");
output.writeln("<left>"); output.writeln("<left>");
mulExpression.left.accept(this); mulExpression.left.accept(this);
output.writeln("</left>"); output.writeln("</left>");
@ -901,11 +901,11 @@ class XMLPrinter : ASTVisitor
override void visit(Parameter param) override void visit(Parameter param)
{ {
output.writeln("<parameter>"); output.writeln("<parameter>");
if (param.name.type == TokenType.identifier) if (param.name.type == tok!"identifier")
output.writeln("<name>", param.name.value, "</name>"); output.writeln("<name>", param.name.text, "</name>");
foreach (attribute; param.parameterAttributes) foreach (attribute; param.parameterAttributes)
{ {
output.writeln("<parameterAttribute>", getTokenValue(attribute), "</parameterAttribute>"); output.writeln("<parameterAttribute>", str(attribute), "</parameterAttribute>");
} }
param.accept(this); param.accept(this);
if (param.vararg) if (param.vararg)
@ -926,7 +926,7 @@ class XMLPrinter : ASTVisitor
override void visit(PostIncDecExpression postIncDecExpression) override void visit(PostIncDecExpression postIncDecExpression)
{ {
output.writeln("<postIncDecExpression operator=\"", output.writeln("<postIncDecExpression operator=\"",
getTokenValue(postIncDecExpression.operator), "\">"); str(postIncDecExpression.operator), "\">");
postIncDecExpression.accept(this); postIncDecExpression.accept(this);
output.writeln("</postIncDecExpression>"); output.writeln("</postIncDecExpression>");
} }
@ -959,7 +959,7 @@ class XMLPrinter : ASTVisitor
override void visit(PreIncDecExpression preIncDecExpression) override void visit(PreIncDecExpression preIncDecExpression)
{ {
output.writeln("<preIncDecExpression operator=\"", output.writeln("<preIncDecExpression operator=\"",
getTokenValue(preIncDecExpression.operator), "\">"); str(preIncDecExpression.operator), "\">");
preIncDecExpression.accept(this); preIncDecExpression.accept(this);
output.writeln("</preIncDecExpression>"); output.writeln("</preIncDecExpression>");
} }
@ -974,7 +974,7 @@ class XMLPrinter : ASTVisitor
override void visit(RelExpression relExpression) override void visit(RelExpression relExpression)
{ {
output.writeln("<relExpression operator=\"", output.writeln("<relExpression operator=\"",
xmlEscape(getTokenValue(relExpression.operator)), "\">"); xmlEscape(str(relExpression.operator)), "\">");
output.writeln("<left>"); output.writeln("<left>");
visit(relExpression.left); visit(relExpression.left);
output.writeln("</left>"); output.writeln("</left>");
@ -1014,7 +1014,7 @@ class XMLPrinter : ASTVisitor
override void visit(ShiftExpression shiftExpression) override void visit(ShiftExpression shiftExpression)
{ {
output.writeln("<shiftExpression operator=\"", output.writeln("<shiftExpression operator=\"",
xmlEscape(getTokenValue(shiftExpression.operator)), "\">"); xmlEscape(str(shiftExpression.operator)), "\">");
output.writeln("<left>"); output.writeln("<left>");
visit(shiftExpression.left); visit(shiftExpression.left);
output.writeln("</left>"); output.writeln("</left>");
@ -1026,10 +1026,10 @@ class XMLPrinter : ASTVisitor
override void visit(SingleImport singleImport) override void visit(SingleImport singleImport)
{ {
if (singleImport.rename.type == TokenType.invalid) if (singleImport.rename.type == tok!"")
output.writeln("<singleImport>"); output.writeln("<singleImport>");
else else
output.writeln("<singleImport rename=\"", singleImport.rename.value, "\">"); output.writeln("<singleImport rename=\"", singleImport.rename.text, "\">");
visit(singleImport.identifierChain); visit(singleImport.identifierChain);
output.writeln("</singleImport>"); output.writeln("</singleImport>");
} }
@ -1094,7 +1094,7 @@ class XMLPrinter : ASTVisitor
override void visit(StructDeclaration structDec) override void visit(StructDeclaration structDec)
{ {
output.writeln("<structDeclaration line=\"", structDec.name.line, "\">"); output.writeln("<structDeclaration line=\"", structDec.name.line, "\">");
output.writeln("<name>", structDec.name.value, "</name>"); output.writeln("<name>", structDec.name.text, "</name>");
structDec.accept(this); structDec.accept(this);
output.writeln("</structDeclaration>"); output.writeln("</structDeclaration>");
} }
@ -1196,7 +1196,7 @@ class XMLPrinter : ASTVisitor
output.writeln("<templateDeclaration line=\"", output.writeln("<templateDeclaration line=\"",
templateDeclaration.name.line, "\">"); templateDeclaration.name.line, "\">");
output.writeln("<name>", templateDeclaration.name.value, "</name>"); output.writeln("<name>", templateDeclaration.name.text, "</name>");
visit(templateDeclaration.templateParameters); visit(templateDeclaration.templateParameters);
if (templateDeclaration.constraint !is null) if (templateDeclaration.constraint !is null)
visit(templateDeclaration.constraint); visit(templateDeclaration.constraint);
@ -1275,28 +1275,28 @@ class XMLPrinter : ASTVisitor
override void visit(Token token) override void visit(Token token)
{ {
string tagName; string tagName;
with (TokenType) switch (token.type) switch (token.type)
{ {
case invalid: return; case tok!"": return;
case identifier: tagName = "identifier"; break; case tok!"identifier": tagName = "identifier"; break;
case doubleLiteral: tagName = "doubleLiteral"; break; case tok!"doubleLiteral": tagName = "doubleLiteral"; break;
case idoubleLiteral: tagName = "idoubleLiteral"; break; case tok!"idoubleLiteral": tagName = "idoubleLiteral"; break;
case floatLiteral: tagName = "floatLiteral"; break; case tok!"floatLiteral": tagName = "floatLiteral"; break;
case ifloatLiteral: tagName = "ifloatLiteral"; break; case tok!"ifloatLiteral": tagName = "ifloatLiteral"; break;
case intLiteral: tagName = "intLiteral"; break; case tok!"intLiteral": tagName = "intLiteral"; break;
case uintLiteral: tagName = "uintLiteral"; break; case tok!"uintLiteral": tagName = "uintLiteral"; break;
case longLiteral: tagName = "longLiteral"; break; case tok!"longLiteral": tagName = "longLiteral"; break;
case ulongLiteral: tagName = "ulongLiteral"; break; case tok!"ulongLiteral": tagName = "ulongLiteral"; break;
case realLiteral: tagName = "realLiteral"; break; case tok!"realLiteral": tagName = "realLiteral"; break;
case irealLiteral: tagName = "irealLiteral"; break; case tok!"irealLiteral": tagName = "irealLiteral"; break;
case characterLiteral: tagName = "characterLiteral"; break; case tok!"characterLiteral": tagName = "characterLiteral"; break;
case stringLiteral: tagName = "stringLiteral"; break; case tok!"stringLiteral": tagName = "stringLiteral"; break;
case dstringLiteral: tagName = "dstringLiteral"; break; case tok!"dstringLiteral": tagName = "dstringLiteral"; break;
case wstringLiteral: tagName = "wstringLiteral"; break; case tok!"wstringLiteral": tagName = "wstringLiteral"; break;
case dollar: output.writeln("<dollar/>"); return; case tok!"$": output.writeln("<dollar/>"); return;
default: output.writeln("<", getTokenValue(token.type), "/>"); return; default: output.writeln("<", str(token.type), "/>"); return;
} }
output.writeln("<", tagName, ">", xmlEscape(token.value), "</", tagName, ">"); output.writeln("<", tagName, ">", xmlEscape(token.text), "</", tagName, ">");
} }
override void visit(TraitsExpression traitsExpression) override void visit(TraitsExpression traitsExpression)
@ -1322,8 +1322,8 @@ class XMLPrinter : ASTVisitor
override void visit(Type2 type2) override void visit(Type2 type2)
{ {
output.writeln("<type2>"); output.writeln("<type2>");
if (type2.builtinType != TokenType.invalid) if (type2.builtinType != tok!"")
output.writeln(getTokenValue(type2.builtinType)); output.writeln(str(type2.builtinType));
else else
type2.accept(this); type2.accept(this);
output.writeln("</type2>"); output.writeln("</type2>");
@ -1392,16 +1392,16 @@ class XMLPrinter : ASTVisitor
override void visit(UnaryExpression unaryExpression) override void visit(UnaryExpression unaryExpression)
{ {
output.writeln("<unaryExpression>"); output.writeln("<unaryExpression>");
if (unaryExpression.prefix != TokenType.invalid) if (unaryExpression.prefix != tok!"")
{ {
output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.value), output.writeln("<prefix>", xmlEscape(unaryExpression.prefix.text),
"</prefix>"); "</prefix>");
visit(unaryExpression.unaryExpression); visit(unaryExpression.unaryExpression);
} }
if (unaryExpression.suffix != TokenType.invalid) if (unaryExpression.suffix != tok!"")
{ {
visit(unaryExpression.unaryExpression); visit(unaryExpression.unaryExpression);
output.writeln("<suffix>", unaryExpression.suffix.value, output.writeln("<suffix>", unaryExpression.suffix.text,
"</suffix>"); "</suffix>");
} }
else else
@ -1412,8 +1412,8 @@ class XMLPrinter : ASTVisitor
override void visit(UnionDeclaration unionDeclaration) override void visit(UnionDeclaration unionDeclaration)
{ {
output.writeln("<unionDeclaration line=\"", unionDeclaration.name.line, "\">"); output.writeln("<unionDeclaration line=\"", unionDeclaration.name.line, "\">");
if (unionDeclaration.name != TokenType.invalid) if (unionDeclaration.name != tok!"")
output.writeln("<name>", unionDeclaration.name.value, "</name>"); output.writeln("<name>", unionDeclaration.name.text, "</name>");
if (unionDeclaration.templateParameters !is null) if (unionDeclaration.templateParameters !is null)
visit(unionDeclaration.templateParameters); visit(unionDeclaration.templateParameters);
if (unionDeclaration.constraint !is null) if (unionDeclaration.constraint !is null)

2
build.bat
View File

@ -1 +1 @@
dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d -wi -ofdscanner dmd main.d stats.d imports.d highlighter.d ctags.d astprinter.d outliner.d formatter.d stdx/d/lexer.d stdx/d/parser.d stdx/d/entities.d stdx/d/ast.d stdx/lexer.d -wi -ofdscanner

40
ctags.d
View File

@ -24,9 +24,7 @@ void printCtags(File output, string[] fileNames)
File f = File(fileName); File f = File(fileName);
auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size)); auto bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
f.rawRead(bytes); f.rawRead(bytes);
LexerConfig config; Module m = parseModule(byToken!(ubyte[])(bytes).array, fileName, &doNothing);
auto tokens = byToken(bytes, config);
Module m = parseModule(tokens.array(), fileName, &doNothing);
auto printer = new CTagsPrinter; auto printer = new CTagsPrinter;
printer.fileName = fileName; printer.fileName = fileName;
printer.visit(m); printer.visit(m);
@ -46,83 +44,83 @@ class CTagsPrinter : ASTVisitor
override void visit(ClassDeclaration dec) override void visit(ClassDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.value, fileName, dec.name.line, context); tagLines ~= "%s\t%s\t%d;\"\tc%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context; auto c = context;
context = "\tclass:" ~ dec.name.value; context = "\tclass:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(StructDeclaration dec) override void visit(StructDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.value, fileName, dec.name.line, context); tagLines ~= "%s\t%s\t%d;\"\ts%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context; auto c = context;
context = "\tstruct:" ~ dec.name.value; context = "\tstruct:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(InterfaceDeclaration dec) override void visit(InterfaceDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.value, fileName, dec.name.line, context); tagLines ~= "%s\t%s\t%d;\"\ti%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context; auto c = context;
context = "\tclass:" ~ dec.name.value; context = "\tclass:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(TemplateDeclaration dec) override void visit(TemplateDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.value, fileName, dec.name.line, context); tagLines ~= "%s\t%s\t%d;\"\tT%s\n".format(dec.name.text, fileName, dec.name.line, context);
auto c = context; auto c = context;
context = "\ttemplate:" ~ dec.name.value; context = "\ttemplate:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(FunctionDeclaration dec) override void visit(FunctionDeclaration dec)
{ {
tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.value, fileName, tagLines ~= "%s\t%s\t%d;\"\tf\tarity:%d%s\n".format(dec.name.text, fileName,
dec.name.line, dec.parameters.parameters.length, context); dec.name.line, dec.parameters.parameters.length, context);
auto c = context; auto c = context;
context = "\tfunction:" ~ dec.name.value; context = "\tfunction:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(EnumDeclaration dec) override void visit(EnumDeclaration dec)
{ {
if (dec.name == TokenType.invalid) if (dec.name == tok!"")
{ {
dec.accept(this); dec.accept(this);
return; return;
} }
tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.value, fileName, tagLines ~= "%s\t%s\t%d;\"\tg%s\n".format(dec.name.text, fileName,
dec.name.line, context); dec.name.line, context);
auto c = context; auto c = context;
context = "\tenum:" ~ dec.name.value; context = "\tenum:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(UnionDeclaration dec) override void visit(UnionDeclaration dec)
{ {
if (dec.name == TokenType.invalid) if (dec.name == tok!"")
{ {
dec.accept(this); dec.accept(this);
return; return;
} }
tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.value, fileName, tagLines ~= "%s\t%s\t%d;\"\tu%s\n".format(dec.name.text, fileName,
dec.name.line, context); dec.name.line, context);
auto c = context; auto c = context;
context = "\tunion:" ~ dec.name.value; context = "\tunion:" ~ dec.name.text;
dec.accept(this); dec.accept(this);
context = c; context = c;
} }
override void visit(EnumMember mem) override void visit(EnumMember mem)
{ {
tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.value, fileName, tagLines ~= "%s\t%s\t%d;\"\te%s\n".format(mem.name.text, fileName,
mem.name.line, context); mem.name.line, context);
} }
@ -130,7 +128,7 @@ class CTagsPrinter : ASTVisitor
{ {
foreach (d; dec.declarators) foreach (d; dec.declarators)
{ {
tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.value, fileName, tagLines ~= "%s\t%s\t%d;\"\tv%s\n".format(d.name.text, fileName,
d.name.line, context); d.name.line, context);
} }
dec.accept(this); dec.accept(this);

BIN
dscanner.exe Normal file
View File

Binary file not shown.

BIN
dscanner.obj Normal file
View File

Binary file not shown.

28
formatter.d
View File

@ -70,7 +70,7 @@ class Formatter(Sink)
if (addExpression.right is null) if (addExpression.right is null)
return; return;
sink.put(" "); sink.put(" ");
sink.put(getTokenValue(addExpression.operator)); sink.put(str(addExpression.operator));
sink.put(" "); sink.put(" ");
format(addExpression.right); format(addExpression.right);
} }
@ -333,8 +333,8 @@ class Formatter(Sink)
void format(const GotoStatement gotoStatement) void format(const GotoStatement gotoStatement)
{ {
sink.put("goto "); sink.put("goto ");
if (gotoStatement.label != TokenType.invalid) if (gotoStatement.label != tok!"")
sink.put(gotoStatement.label.value); sink.put(gotoStatement.label.text);
else else
format(gotoStatement.expression); format(gotoStatement.expression);
sink.put(";"); sink.put(";");
@ -348,7 +348,7 @@ class Formatter(Sink)
if (!first) if (!first)
sink.put("."); sink.put(".");
first = false; first = false;
sink.put(ident.value); sink.put(ident.text);
} }
} }
@ -372,7 +372,7 @@ class Formatter(Sink)
if (identifierOrTemplateInstance.templateInstance !is null) if (identifierOrTemplateInstance.templateInstance !is null)
format(identifierOrTemplateInstance.templateInstance); format(identifierOrTemplateInstance.templateInstance);
else else
sink.put(identifierOrTemplateInstance.identifier.value); sink.put(identifierOrTemplateInstance.identifier.text);
} }
@ -488,14 +488,14 @@ class Formatter(Sink)
{ {
foreach (attribute; parameter.parameterAttributes) foreach (attribute; parameter.parameterAttributes)
{ {
sink.put(getTokenValue(attribute)); sink.put(str(attribute));
} }
if (parameter.type !is null) if (parameter.type !is null)
format(parameter.type); format(parameter.type);
if (parameter.name.type != TokenType.invalid) if (parameter.name.type != tok!"")
{ {
sink.put(" "); sink.put(" ");
sink.put(parameter.name.value); sink.put(parameter.name.text);
} }
if (parameter.vararg) if (parameter.vararg)
sink.put(" ..."); sink.put(" ...");
@ -607,7 +607,7 @@ class Formatter(Sink)
void format(const Symbol symbol) void format(const Symbol symbol)
{ {
if (symbol.dot != TokenType.invalid) if (symbol.dot != tok!"")
sink.put("."); sink.put(".");
format(symbol.identifierOrTemplateChain); format(symbol.identifierOrTemplateChain);
} }
@ -671,7 +671,7 @@ class Formatter(Sink)
void format(const Token token) void format(const Token token)
{ {
sink.put(token.value); sink.put(token.text);
} }
void format(const TraitsExpression traitsExpression) void format(const TraitsExpression traitsExpression)
@ -688,7 +688,7 @@ class Formatter(Sink)
if (first) if (first)
sink.put(" "); sink.put(" ");
first = false; first = false;
sink.put(getTokenValue(constructor)); sink.put(str(constructor));
} }
if (type.typeConstructors.length > 0) if (type.typeConstructors.length > 0)
sink.put(" "); sink.put(" ");
@ -711,16 +711,16 @@ class Formatter(Sink)
format(type2.typeofExpression); format(type2.typeofExpression);
return; return;
} }
else if (type2.typeConstructor != TokenType.invalid) else if (type2.typeConstructor != tok!"")
{ {
sink.put(getTokenValue(type2.typeConstructor)); sink.put(str(type2.typeConstructor));
sink.put("("); sink.put("(");
format(type2.type); format(type2.type);
sink.put(")"); sink.put(")");
return; return;
} }
else else
sink.put(getTokenValue(type2.builtinType)); sink.put(str(type2.builtinType));
} }
void format(const TypeSpecialization typeSpecialization) void format(const TypeSpecialization typeSpecialization)

32
highlighter.d
View File

@ -11,7 +11,7 @@ import std.array;
import stdx.d.lexer; import stdx.d.lexer;
// http://ethanschoonover.com/solarized // http://ethanschoonover.com/solarized
void highlight(R)(TokenRange!R tokens, string fileName) void highlight(R)(R tokens, string fileName)
{ {
stdout.writeln(q"[ stdout.writeln(q"[
<!DOCTYPE html> <!DOCTYPE html>
@ -36,27 +36,27 @@ html { background-color: #fdf6e3; color: #002b36; }
foreach (Token t; tokens) foreach (Token t; tokens)
{ {
if (isBasicType(t.type)) if (isBasicType(t.type))
writeSpan("type", t.value); writeSpan("type", str(t.type));
else if (isKeyword(t.type)) else if (isKeyword(t.type))
writeSpan("kwrd", t.value); writeSpan("kwrd", str(t.type));
else if (t.type == TokenType.comment) else if (t.type == tok!"comment")
writeSpan("com", t.value); writeSpan("com", t.text);
else if (isStringLiteral(t.type) || t.type == TokenType.characterLiteral) else if (isStringLiteral(t.type) || t.type == tok!"characterLiteral")
writeSpan("str", t.value); writeSpan("str", t.text);
else if (isNumberLiteral(t.type)) else if (isNumberLiteral(t.type))
writeSpan("num", t.value); writeSpan("num", t.text);
else if (isOperator(t.type)) else if (isOperator(t.type))
writeSpan("op", t.value); writeSpan("op", str(t.type));
else else
{ {
version(Windows) version(Windows)
{ {
// Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong. // Stupid Windows automatically does a LF → CRLF, so CRLF → CRCRLF, which is obviously wrong.
// Strip out the CR characters here to avoid this. // Strip out the CR characters here to avoid this.
stdout.write(t.value.replace("<", "&lt;").replace("\r", "")); stdout.write(t.text.replace("<", "&lt;").replace("\r", ""));
} }
else else
stdout.write(t.value.replace("<", "&lt;")); stdout.write(t.text.replace("<", "&lt;"));
} }
} }
@ -70,13 +70,3 @@ void writeSpan(string cssClass, string value)
else else
stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`); stdout.write(`<span class="`, cssClass, `">`, value.replace("&", "&amp;").replace("<", "&lt;"), `</span>`);
} }
/+void main(string[] args)
{
LexerConfig config;
config.tokenStyle = TokenStyle.source;
config.iterStyle = IterationStyle.everything;
config.fileName = args[1];
auto f = File(args[1]);
(cast(ubyte[]) f.byLine(KeepTerminator.yes).join()).byToken(config).highlight();
}+/

2
imports.d
View File

@ -26,7 +26,7 @@ class ImportPrinter : ASTVisitor
{ {
if (!first) if (!first)
write("."); write(".");
write(ident.value); write(ident.text);
first = false; first = false;
} }
} }

46
main.d
View File

@ -89,12 +89,9 @@ int main(string[] args)
if (highlight) if (highlight)
{ {
LexerConfig config;
config.iterStyle = IterationStyle.everything;
config.tokenStyle = TokenStyle.source;
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]);
highlighter.highlight(byToken(bytes, config), highlighter.highlight(byToken!(typeof(bytes), false, false)(bytes),
args.length == 1 ? "stdin" : args[1]); args.length == 1 ? "stdin" : args[1]);
return 0; return 0;
} }
@ -104,13 +101,12 @@ int main(string[] args)
} }
else else
{ {
LexerConfig config;
bool usingStdin = args.length == 1; bool usingStdin = args.length == 1;
if (sloc || tokenCount) if (sloc || tokenCount)
{ {
if (usingStdin) if (usingStdin)
{ {
auto tokens = byToken(readStdin(), config); auto tokens = byToken!(ubyte[], false, false)(readStdin());
if (tokenCount) if (tokenCount)
printTokenCount(stdout, "stdin", tokens); printTokenCount(stdout, "stdin", tokens);
else else
@ -121,7 +117,7 @@ int main(string[] args)
ulong count; ulong count;
foreach (f; expandArgs(args, recursive)) foreach (f; expandArgs(args, recursive))
{ {
auto tokens = byToken(readFile(f), config); auto tokens = byToken!(ubyte[])(readFile(f));
if (tokenCount) if (tokenCount)
count += printTokenCount(stdout, f, tokens); count += printTokenCount(stdout, f, tokens);
else else
@ -132,48 +128,28 @@ int main(string[] args)
} }
else if (syntaxCheck) else if (syntaxCheck)
{ {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]), auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
config); parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
parseModule(tokens.array(), config.fileName);
} }
else if (imports) else if (imports)
{ {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]), auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
config); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto visitor = new ImportPrinter; auto visitor = new ImportPrinter;
visitor.visit(mod); visitor.visit(mod);
} }
else if (ast) else if (ast)
{ {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]), auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
config); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto printer = new XMLPrinter; auto printer = new XMLPrinter;
printer.output = stdout; printer.output = stdout;
printer.visit(mod); printer.visit(mod);
} }
else if (outline) else if (outline)
{ {
auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]), auto tokens = byToken(usingStdin ? readStdin() : readFile(args[1]));
config); auto mod = parseModule(tokens.array(), usingStdin ? "stdin" : args[1]);
if (usingStdin)
config.fileName = "stdin";
else
config.fileName = args[1];
auto mod = parseModule(tokens.array(), config.fileName);
auto outliner = new Outliner(stdout); auto outliner = new Outliner(stdout);
outliner.visit(mod); outliner.visit(mod);
} }

276
main.html Normal file
View File

@ -0,0 +1,276 @@
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
<title>main.d</title>
</head>
<body>
<style type="text/css">
html { background-color: #fdf6e3; color: #002b36; }
.kwrd { color: #b58900; font-weight: bold; }
.com { color: #93a1a1; font-style: italic; }
.num { color: #dc322f; font-weigth: bold; }
.str { color: #2aa198; font-style: italic; }
.op { color: #586e75; font-weight: bold; }
.type { color: #268bd2; font-weight: bold; }
.cons { color: #859900; font-weight: bold; }
</style>
<pre>
<span class="com">// Copyright Brian Schott (Sir Alaran) 2012.</span>
<span class="com">// Distributed under the Boost Software License, Version 1.0.</span>
<span class="com">// (See accompanying file LICENSE_1_0.txt or copy at</span>
<span class="com">// http://www.boost.org/LICENSE_1_0.txt)</span>
<span class="kwrd">module</span> main<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>algorithm<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>array<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>conv<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>file<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>getopt<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>parallelism<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>path<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>regex<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>stdio<span class="op">;</span>
<span class="kwrd">import</span> std<span class="op">.</span>range<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>lexer<span class="op">;</span>
<span class="kwrd">import</span> stdx<span class="op">.</span>d<span class="op">.</span>parser<span class="op">;</span>
<span class="kwrd">import</span> highlighter<span class="op">;</span>
<span class="kwrd">import</span> stats<span class="op">;</span>
<span class="kwrd">import</span> ctags<span class="op">;</span>
<span class="kwrd">import</span> astprinter<span class="op">;</span>
<span class="kwrd">import</span> imports<span class="op">;</span>
<span class="kwrd">import</span> outliner<span class="op">;</span>
<span class="type">int</span> main<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> sloc<span class="op">;</span>
<span class="type">bool</span> highlight<span class="op">;</span>
<span class="type">bool</span> ctags<span class="op">;</span>
<span class="type">bool</span> recursive<span class="op">;</span>
<span class="type">bool</span> format<span class="op">;</span>
<span class="type">bool</span> help<span class="op">;</span>
<span class="type">bool</span> tokenCount<span class="op">;</span>
<span class="type">bool</span> syntaxCheck<span class="op">;</span>
<span class="type">bool</span> ast<span class="op">;</span>
<span class="type">bool</span> imports<span class="op">;</span>
<span class="type">bool</span> muffin<span class="op">;</span>
<span class="type">bool</span> outline<span class="op">;</span>
<span class="kwrd">try</span>
<span class="op">{</span>
getopt<span class="op">(</span>args<span class="op">,</span> <span class="str">"sloc|l"</span><span class="op">,</span> <span class="op">&amp;</span>sloc<span class="op">,</span> <span class="str">"highlight"</span><span class="op">,</span> <span class="op">&amp;</span>highlight<span class="op">,</span>
<span class="str">"ctags|c"</span><span class="op">,</span> <span class="op">&amp;</span>ctags<span class="op">,</span> <span class="str">"recursive|r|R"</span><span class="op">,</span> <span class="op">&amp;</span>recursive<span class="op">,</span> <span class="str">"help|h"</span><span class="op">,</span> <span class="op">&amp;</span>help<span class="op">,</span>
<span class="str">"tokenCount|t"</span><span class="op">,</span> <span class="op">&amp;</span>tokenCount<span class="op">,</span> <span class="str">"syntaxCheck|s"</span><span class="op">,</span> <span class="op">&amp;</span>syntaxCheck<span class="op">,</span>
<span class="str">"ast|xml"</span><span class="op">,</span> <span class="op">&amp;</span>ast<span class="op">,</span> <span class="str">"imports|i"</span><span class="op">,</span> <span class="op">&amp;</span>imports<span class="op">,</span> <span class="str">"outline|o"</span><span class="op">,</span> <span class="op">&amp;</span>outline<span class="op">,</span>
<span class="str">"muffinButton"</span><span class="op">,</span> <span class="op">&amp;</span>muffin<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">catch</span> <span class="op">(</span>Exception e<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span>e<span class="op">.</span>msg<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>muffin<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>writeln<span class="op">(</span>
<span class="str">` ___________
__(#*O 0** @%*)__
_(%*o#*O%*0 #O#%##@)_
(*#@%#o*@ #o%O*%@ #o #)
\=====================/
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|
|I|I|I|I|I|I|I|I|I|I|`</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>help<span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">auto</span> optionCount <span class="op">=</span> count<span class="op">!</span><span class="str">"a"</span><span class="op">(</span><span class="op">[</span>sloc<span class="op">,</span> highlight<span class="op">,</span> ctags<span class="op">,</span> tokenCount<span class="op">,</span>
syntaxCheck<span class="op">,</span> ast<span class="op">,</span> imports<span class="op">,</span> outline<span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">></span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writeln<span class="op">(</span><span class="str">"Too many options specified"</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>optionCount <span class="op">&lt;</span> <span class="num">1</span><span class="op">)</span>
<span class="op">{</span>
printHelp<span class="op">(</span>args<span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">1</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">if</span> <span class="op">(</span>highlight<span class="op">)</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> bytes <span class="op">=</span> usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
highlighter<span class="op">.</span>highlight<span class="op">(</span>byToken<span class="op">!</span><span class="op">(</span><span class="kwrd">typeof</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>bytes<span class="op">)</span><span class="op">,</span>
args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span> <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ctags<span class="op">)</span>
<span class="op">{</span>
stdout<span class="op">.</span>printCtags<span class="op">(</span>expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">bool</span> usingStdin <span class="op">=</span> args<span class="op">.</span>length <span class="op">==</span> <span class="num">1</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>sloc <span class="op">||</span> tokenCount<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>usingStdin<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">,</span> <span class="kwrd">false</span><span class="op">)</span><span class="op">(</span>readStdin<span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
printTokenCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
printLineCount<span class="op">(</span>stdout<span class="op">,</span> <span class="str">"stdin"</span><span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="op">{</span>
<span class="type">ulong</span> count<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>f<span class="op">;</span> expandArgs<span class="op">(</span>args<span class="op">,</span> recursive<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>readFile<span class="op">(</span>f<span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>tokenCount<span class="op">)</span>
count <span class="op">+=</span> printTokenCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="kwrd">else</span>
count <span class="op">+=</span> printLineCount<span class="op">(</span>stdout<span class="op">,</span> f<span class="op">,</span> tokens<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
writefln<span class="op">(</span><span class="str">"total:\t%d"</span><span class="op">,</span> count<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>syntaxCheck<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>imports<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> visitor <span class="op">=</span> <span class="kwrd">new</span> ImportPrinter<span class="op">;</span>
visitor<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>ast<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> printer <span class="op">=</span> <span class="kwrd">new</span> XMLPrinter<span class="op">;</span>
printer<span class="op">.</span>output <span class="op">=</span> stdout<span class="op">;</span>
printer<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span> <span class="kwrd">if</span> <span class="op">(</span>outline<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> tokens <span class="op">=</span> byToken<span class="op">(</span>usingStdin <span class="op">?</span> readStdin<span class="op">(</span><span class="op">)</span> <span class="op">:</span> readFile<span class="op">(</span>args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> mod <span class="op">=</span> parseModule<span class="op">(</span>tokens<span class="op">.</span>array<span class="op">(</span><span class="op">)</span><span class="op">,</span> usingStdin <span class="op">?</span> <span class="str">"stdin"</span> <span class="op">:</span> args<span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">)</span><span class="op">;</span>
<span class="kwrd">auto</span> outliner <span class="op">=</span> <span class="kwrd">new</span> Outliner<span class="op">(</span>stdout<span class="op">)</span><span class="op">;</span>
outliner<span class="op">.</span>visit<span class="op">(</span>mod<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> <span class="num">0</span><span class="op">;</span>
<span class="op">}</span>
string<span class="op">[</span><span class="op">]</span> expandArgs<span class="op">(</span>string<span class="op">[</span><span class="op">]</span> args<span class="op">,</span> <span class="type">bool</span> recursive<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>recursive<span class="op">)</span>
<span class="op">{</span>
string<span class="op">[</span><span class="op">]</span> rVal<span class="op">;</span>
<span class="kwrd">foreach</span> <span class="op">(</span>arg<span class="op">;</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span><span class="op">$</span><span class="op">]</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>arg<span class="op">)</span> <span class="op">&amp;&amp;</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> arg<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> arg<span class="op">;</span>
<span class="kwrd">else</span> <span class="kwrd">foreach</span> <span class="op">(</span>item<span class="op">;</span> dirEntries<span class="op">(</span>arg<span class="op">,</span> SpanMode<span class="op">.</span>breadth<span class="op">)</span><span class="op">.</span>map<span class="op">!</span><span class="op">(</span>a <span class="op">=></span> a<span class="op">.</span>name<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span>isFile<span class="op">(</span>item<span class="op">)</span> <span class="op">&amp;&amp;</span> <span class="op">(</span>item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.d`</span><span class="op">)</span> <span class="op">||</span> item<span class="op">.</span>endsWith<span class="op">(</span><span class="str">`.di`</span><span class="op">)</span><span class="op">)</span><span class="op">)</span>
rVal <span class="kwrd">abstract</span> item<span class="op">;</span>
<span class="kwrd">else</span>
<span class="kwrd">continue</span><span class="op">;</span>
<span class="op">}</span>
<span class="op">}</span>
<span class="kwrd">return</span> rVal<span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">else</span>
<span class="kwrd">return</span> args<span class="op">[</span><span class="num">1</span> <span class="op">..</span> <span class="op">$</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readStdin<span class="op">(</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> sourceCode <span class="op">=</span> appender<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span><span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="num">4096</span><span class="op">]</span> buf<span class="op">;</span>
<span class="kwrd">while</span> <span class="op">(</span><span class="kwrd">true</span><span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">auto</span> b <span class="op">=</span> stdin<span class="op">.</span>rawRead<span class="op">(</span>buf<span class="op">)</span><span class="op">;</span>
<span class="kwrd">if</span> <span class="op">(</span>b<span class="op">.</span>length <span class="op">==</span> <span class="num">0</span><span class="op">)</span>
<span class="kwrd">break</span><span class="op">;</span>
sourceCode<span class="op">.</span>put<span class="op">(</span>b<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
<span class="kwrd">return</span> sourceCode<span class="op">.</span>data<span class="op">;</span>
<span class="op">}</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> readFile<span class="op">(</span>string fileName<span class="op">)</span>
<span class="op">{</span>
<span class="kwrd">if</span> <span class="op">(</span><span class="op">!</span>exists<span class="op">(</span>fileName<span class="op">)</span><span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span><span class="str">"%s does not exist"</span><span class="op">,</span> fileName<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> <span class="op">[</span><span class="op">]</span><span class="op">;</span>
<span class="op">}</span>
File f <span class="op">=</span> File<span class="op">(</span>fileName<span class="op">)</span><span class="op">;</span>
<span class="type">ubyte</span><span class="op">[</span><span class="op">]</span> sourceCode <span class="op">=</span> uninitializedArray<span class="op">!</span><span class="op">(</span><span class="type">ubyte</span><span class="op">[</span><span class="op">]</span><span class="op">)</span><span class="op">(</span>to<span class="op">!</span>size_t<span class="op">(</span>f<span class="op">.</span>size<span class="op">)</span><span class="op">)</span><span class="op">;</span>
f<span class="op">.</span>rawRead<span class="op">(</span>sourceCode<span class="op">)</span><span class="op">;</span>
<span class="kwrd">return</span> sourceCode<span class="op">;</span>
<span class="op">}</span>
<span class="type">void</span> printHelp<span class="op">(</span>string programName<span class="op">)</span>
<span class="op">{</span>
stderr<span class="op">.</span>writefln<span class="op">(</span>
<span class="str">`
Usage: %s options
options:
--help | -h
Prints this help message
--sloc | -l [sourceFiles]
Prints the number of logical lines of code in the given
source files. If no files are specified, input is read from stdin.
--tokenCount | t [sourceFiles]
Prints the number of tokens in the given source files. If no files are
specified, input is read from stdin.
--highlight [sourceFile] - Syntax-highlight the given source file. The
resulting HTML will be written to standard output. If no files are
specified, input is read from stdin.
--imports | -i [sourceFile]
Prints modules imported by the given source file. If no files are
specified, input is read from stdin.
--syntaxCheck | -s [sourceFile]
Lexes and parses sourceFile, printing the line and column number of any
syntax errors to stdout. One error or warning is printed per line.
If no files are specified, input is read from stdin.
--ctags | -c sourceFile
Generates ctags information from the given source code file. Note that
ctags information requires a filename, so stdin cannot be used in place
of a filename.
--ast | --xml sourceFile
Generates an XML representation of the source files abstract syntax
tree. If no files are specified, input is read from stdin.
--recursive | -R | -r
When used with --ctags, --tokenCount, or --sloc, dscanner will produce
ctags output for all .d and .di files contained within the given
directories and its sub-directories.`</span><span class="op">,</span>
programName<span class="op">)</span><span class="op">;</span>
<span class="op">}</span>
</pre>
</body></html>

18
outliner.d
View File

@ -21,7 +21,7 @@ class Outliner : ASTVisitor
override void visit(ClassDeclaration classDec) override void visit(ClassDeclaration classDec)
{ {
printIndentation(); printIndentation();
output.writeln("class ", classDec.name.value, " : ", classDec.name.line); output.writeln("class ", classDec.name.text, " : ", classDec.name.line);
indent(); indent();
classDec.accept(this); classDec.accept(this);
outdent(); outdent();
@ -31,7 +31,7 @@ class Outliner : ASTVisitor
override void visit(EnumDeclaration enumDec) override void visit(EnumDeclaration enumDec)
{ {
printIndentation(); printIndentation();
output.writeln("enum ", enumDec.name.value, " : ", enumDec.name.line); output.writeln("enum ", enumDec.name.text, " : ", enumDec.name.line);
indent(); indent();
enumDec.accept(this); enumDec.accept(this);
outdent(); outdent();
@ -41,7 +41,7 @@ class Outliner : ASTVisitor
override void visit(EnumMember enumMem) override void visit(EnumMember enumMem)
{ {
printIndentation(); printIndentation();
output.writeln(enumMem.name.value, " : ", enumMem.name.line); output.writeln(enumMem.name.text, " : ", enumMem.name.line);
finish(); finish();
} }
@ -57,7 +57,7 @@ class Outliner : ASTVisitor
if (functionDec.returnType !is null) if (functionDec.returnType !is null)
f.format(functionDec.returnType); f.format(functionDec.returnType);
app.put(" "); app.put(" ");
app.put(functionDec.name.value); app.put(functionDec.name.text);
f.format(functionDec.parameters); f.format(functionDec.parameters);
app.put(" : "); app.put(" : ");
app.put(to!string(functionDec.name.line)); app.put(to!string(functionDec.name.line));
@ -68,7 +68,7 @@ class Outliner : ASTVisitor
override void visit(InterfaceDeclaration interfaceDec) override void visit(InterfaceDeclaration interfaceDec)
{ {
printIndentation(); printIndentation();
output.writeln("interface ", interfaceDec.name.value, " : ", output.writeln("interface ", interfaceDec.name.text, " : ",
interfaceDec.name.line); interfaceDec.name.line);
indent(); indent();
interfaceDec.accept(this); interfaceDec.accept(this);
@ -79,7 +79,7 @@ class Outliner : ASTVisitor
override void visit(StructDeclaration structDec) override void visit(StructDeclaration structDec)
{ {
printIndentation(); printIndentation();
output.writeln("struct ", structDec.name.value, " : ", output.writeln("struct ", structDec.name.text, " : ",
structDec.name.line); structDec.name.line);
indent(); indent();
structDec.accept(this); structDec.accept(this);
@ -90,7 +90,7 @@ class Outliner : ASTVisitor
override void visit(TemplateDeclaration templateDeclaration) override void visit(TemplateDeclaration templateDeclaration)
{ {
printIndentation(); printIndentation();
output.writeln("template", templateDeclaration.name.value, " : ", output.writeln("template", templateDeclaration.name.text, " : ",
templateDeclaration.name.line); templateDeclaration.name.line);
finish(); finish();
} }
@ -105,7 +105,7 @@ class Outliner : ASTVisitor
override void visit(UnionDeclaration unionDeclaration) override void visit(UnionDeclaration unionDeclaration)
{ {
printIndentation(); printIndentation();
output.writeln("union ", unionDeclaration.name.value, " : ", output.writeln("union ", unionDeclaration.name.text, " : ",
unionDeclaration.name.line); unionDeclaration.name.line);
indent(); indent();
unionDeclaration.accept(this); unionDeclaration.accept(this);
@ -125,7 +125,7 @@ class Outliner : ASTVisitor
f.format(variableDeclaration.type); f.format(variableDeclaration.type);
} }
app.put(" "); app.put(" ");
app.put(d.name.value); app.put(d.name.text);
app.put(" : "); app.put(" : ");
app.put(to!string(d.name.line)); app.put(to!string(d.name.line));
output.writeln(app.data); output.writeln(app.data);

26
stats.d
View File

@ -9,21 +9,21 @@ import std.stdio;
import std.algorithm; import std.algorithm;
import stdx.d.lexer; import stdx.d.lexer;
pure nothrow bool isLineOfCode(TokenType t) pure nothrow bool isLineOfCode(IdType t)
{ {
with (TokenType) switch(t) switch(t)
{ {
case semicolon: case tok!";":
case while_: case tok!"while":
case if_: case tok!"if":
case do_: case tok!"do":
case else_: case tok!"else":
case switch_: case tok!"switch":
case for_: case tok!"for":
case foreach_: case tok!"foreach":
case foreach_reverse_: case tok!"foreach_reverse":
case default_: case tok!"default":
case case_: case tok!"case":
return true; return true;
default: default:
return false; return false;

42
stdx/d/ast.d
View File

@ -275,7 +275,7 @@ public:
{ {
mixin (visitIfNotNull!(left, right)); mixin (visitIfNotNull!(left, right));
} }
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -410,7 +410,7 @@ class AsmAddExp : ExpressionNode
{ {
public: public:
mixin (DEFAULT_ACCEPT); mixin (DEFAULT_ACCEPT);
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -483,7 +483,7 @@ class AsmMulExp : ExpressionNode
{ {
public: public:
mixin (DEFAULT_ACCEPT); mixin (DEFAULT_ACCEPT);
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -583,7 +583,7 @@ public:
} }
/** */ ExpressionNode ternaryExpression; /** */ ExpressionNode ternaryExpression;
/** */ ExpressionNode assignExpression; /** */ ExpressionNode assignExpression;
/** */ TokenType operator; /** */ IdType operator;
} }
/// ///
@ -623,7 +623,7 @@ public:
/** */ AlignAttribute alignAttribute; /** */ AlignAttribute alignAttribute;
/** */ PragmaExpression pragmaExpression; /** */ PragmaExpression pragmaExpression;
/** */ StorageClass storageClass; /** */ StorageClass storageClass;
/** */ TokenType attribute; /** */ IdType attribute;
} }
/// ///
@ -1149,7 +1149,7 @@ public:
{ {
mixin (visitIfNotNull!(left, right)); mixin (visitIfNotNull!(left, right));
} }
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -1222,7 +1222,7 @@ public:
mixin (visitIfNotNull!(foreachType, foreachTypeList, low, high, mixin (visitIfNotNull!(foreachType, foreachTypeList, low, high,
declarationOrStatement)); declarationOrStatement));
} }
/** */ TokenType type; /** */ IdType type;
/** */ ForeachTypeList foreachTypeList; /** */ ForeachTypeList foreachTypeList;
/** */ ForeachType foreachType; /** */ ForeachType foreachType;
/** */ Expression low; /** */ Expression low;
@ -1239,7 +1239,7 @@ public:
{ {
mixin (visitIfNotNull!(type, identifier)); mixin (visitIfNotNull!(type, identifier));
} }
/** */ TokenType[] typeConstructors; /** */ IdType[] typeConstructors;
/** */ Type type; /** */ Type type;
/** */ Token identifier; /** */ Token identifier;
} }
@ -1336,7 +1336,7 @@ public:
mixin (visitIfNotNull!(type, parameters, functionAttributes, mixin (visitIfNotNull!(type, parameters, functionAttributes,
functionBody)); functionBody));
} }
/** */ TokenType functionOrDelegate; /** */ IdType functionOrDelegate;
/** */ Type type; /** */ Type type;
/** */ Parameters parameters; /** */ Parameters parameters;
/** */ FunctionAttribute[] functionAttributes; /** */ FunctionAttribute[] functionAttributes;
@ -1575,7 +1575,7 @@ public:
/** */ Token identifier; /** */ Token identifier;
/** */ TypeSpecialization typeSpecialization; /** */ TypeSpecialization typeSpecialization;
/** */ TemplateParameterList templateParameterList; /** */ TemplateParameterList templateParameterList;
/** */ TokenType equalsOrColon; /** */ IdType equalsOrColon;
} }
/// ///
@ -1622,7 +1622,7 @@ public:
mixin (visitIfNotNull!(identifier, parameters, functionAttributes, mixin (visitIfNotNull!(identifier, parameters, functionAttributes,
assignExpression)); assignExpression));
} }
/** */ TokenType functionType; /** */ IdType functionType;
/** */ Token identifier; /** */ Token identifier;
/** */ Parameters parameters; /** */ Parameters parameters;
/** */ FunctionAttribute[] functionAttributes; /** */ FunctionAttribute[] functionAttributes;
@ -1660,7 +1660,7 @@ public:
{ {
mixin (visitIfNotNull!(atAttribute)); mixin (visitIfNotNull!(atAttribute));
} }
/** */ TokenType tokenType; /** */ IdType tokenType;
/** */ AtAttribute atAttribute; /** */ AtAttribute atAttribute;
} }
@ -1743,7 +1743,7 @@ public:
{ {
mixin (visitIfNotNull!(left, right)); mixin (visitIfNotNull!(left, right));
} }
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -1893,7 +1893,7 @@ public:
mixin (visitIfNotNull!(type, name, default_)); mixin (visitIfNotNull!(type, name, default_));
} }
/** */ TokenType[] parameterAttributes; /** */ IdType[] parameterAttributes;
/** */ Type type; /** */ Type type;
/** */ Token name; /** */ Token name;
/** */ bool vararg; /** */ bool vararg;
@ -1932,7 +1932,7 @@ public:
{ {
mixin (visitIfNotNull!(unaryExpression)); mixin (visitIfNotNull!(unaryExpression));
} }
/** */ TokenType operator; /** */ IdType operator;
/** */ UnaryExpression unaryExpression; /** */ UnaryExpression unaryExpression;
} }
@ -1978,7 +1978,7 @@ public:
{ {
mixin (visitIfNotNull!(unaryExpression)); mixin (visitIfNotNull!(unaryExpression));
} }
/** */ TokenType operator; /** */ IdType operator;
/** */ UnaryExpression unaryExpression; /** */ UnaryExpression unaryExpression;
} }
@ -2030,7 +2030,7 @@ public:
{ {
mixin (visitIfNotNull!(left, right)); mixin (visitIfNotNull!(left, right));
} }
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -2089,7 +2089,7 @@ public:
{ {
mixin (visitIfNotNull!(left, right)); mixin (visitIfNotNull!(left, right));
} }
/** */ TokenType operator; /** */ IdType operator;
mixin BinaryExpressionBody; mixin BinaryExpressionBody;
} }
@ -2573,7 +2573,7 @@ public:
mixin (visitIfNotNull!(type2, typeSuffixes)); mixin (visitIfNotNull!(type2, typeSuffixes));
} }
/** */ TokenType[] typeConstructors; /** */ IdType[] typeConstructors;
/** */ TypeSuffix[] typeSuffixes; /** */ TypeSuffix[] typeSuffixes;
/** */ Type2 type2; /** */ Type2 type2;
} }
@ -2588,11 +2588,11 @@ public:
identifierOrTemplateChain, type)); identifierOrTemplateChain, type));
} }
/** */ TokenType builtinType; /** */ IdType builtinType;
/** */ Symbol symbol; /** */ Symbol symbol;
/** */ TypeofExpression typeofExpression; /** */ TypeofExpression typeofExpression;
/** */ IdentifierOrTemplateChain identifierOrTemplateChain; /** */ IdentifierOrTemplateChain identifierOrTemplateChain;
/** */ TokenType typeConstructor; /** */ IdType typeConstructor;
/** */ Type type; /** */ Type type;
} }

4802
stdx/d/lexer.d
View File

File diff suppressed because it is too large Load Diff

2149
stdx/d/parser.d
View File

File diff suppressed because it is too large Load Diff

457
stdx/lexer.d Normal file
View File

@ -0,0 +1,457 @@
// Written in the D programming language
/**
* This module contains a range-based _lexer generator.
*
* Copyright: Brian Schott 2013
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
* Authors: Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu
* Source: $(PHOBOSSRC std/_lexer.d)
*/
module stdx.lexer;
import std.typecons;
import std.algorithm;
import std.range;
import std.traits;
import std.conv;
import std.math;
template TokenIdType(alias staticTokens, alias dynamicTokens,
alias possibleDefaultTokens)
{
static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ubyte.max)
alias TokenIdType = ubyte;
else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= ushort.max)
alias TokenIdType = ushort;
else static if ((staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length) <= uint.max)
alias TokenIdType = uint;
else
static assert (false);
}
string TokenStringRepresentation(IdType, alias staticTokens, alias possibleDefaultTokens)(IdType type) @property
{
if (type == 0)
return "!ERROR!";
else if (type < staticTokens.length)
return staticTokens[type - 1];
else if (type < staticTokens.length + possibleDefaultTokens.length)
return possibleDefaultTokens[type - staticTokens.length];
else
return null;
}
IdType TokenId(IdType, alias staticTokens, alias dynamicTokens,
alias possibleDefaultTokens, string symbol)() @property
{
static if (symbol == "")
return 0;
else static if (symbol == "\0")
return 1 + staticTokens.length + dynamicTokens.length + possibleDefaultTokens.length;
else
{
enum i = staticTokens.countUntil(symbol);
static if (i >= 0)
enum id = i + 1;
else
{
enum ii = possibleDefaultTokens.countUntil(symbol);
static if (ii >= 0)
enum id = ii + staticTokens.length;
else
{
enum dynamicId = dynamicTokens.countUntil(symbol);
enum id = dynamicId >= 0 ? i + staticTokens.length + possibleDefaultTokens.length + dynamicId : -1;
}
}
static assert (id >= 0 && id < IdType.max, "Invalid token: " ~ symbol);
return id;
}
}
struct TokenStructure(IDType)
{
bool opEquals(IDType type) const pure nothrow @safe
{
return this.type == type;
}
IDType type;
string text;
size_t line;
size_t column;
size_t index;
}
mixin template Lexer(R, IDType, Token, alias isSeparating, alias defaultTokenFunction,
alias staticTokens, alias dynamicTokens, alias pseudoTokens,
alias possibleDefaultTokens) if (isForwardRange!R)
{
enum size_t lookAhead = chain(staticTokens, pseudoTokens).map!"a.length".reduce!"max(a, b)"();
alias PeekRange!(R, lookAhead) RangeType;
static string generateCaseStatements(string[] tokens, size_t offset = 0)
{
string code;
for (size_t i = 0; i < tokens.length; i++)
{
auto indent = "";
foreach (k; 0 .. offset)
indent ~= " ";
size_t j = i + 1;
if (offset < tokens[i].length)
{
while (j < tokens.length && offset < tokens[j].length
&& tokens[i][offset] == tokens[j][offset]) j++;
code ~= indent ~ "case " ~ text(cast(ubyte) tokens[i][offset]) ~ ":\n";
if (i + 1 >= j)
{
if (offset + 1 == tokens[i].length)
code ~= generateLeaf(tokens[i], indent ~ " ");
else
{
code ~= indent ~ " if (!range.canPeek(" ~ text(tokens[i].length - 1) ~ "))\n";
code ~= indent ~ " goto outer_default;\n";
code ~= indent ~ " if (range.startsWith(\"" ~ escape(tokens[i]) ~ "\"))\n";
code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i], indent ~ " ");
code ~= indent ~ " }\n";
code ~= indent ~ " else\n";
code ~= indent ~ " goto outer_default;\n";
}
}
else
{
code ~= indent ~ " if (!range.canPeek(" ~ text(offset + 1) ~ "))\n";
code ~= indent ~ " {\n";
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
code ~= indent ~ " }\n";
code ~= indent ~ " switch (range.peek(" ~ text(offset + 1) ~ "))\n";
code ~= indent ~ " {\n";
code ~= generateCaseStatements(tokens[i .. j], offset + 1);
code ~= indent ~ " default:\n";
code ~= generateLeaf(tokens[i][0 .. offset + 1], indent ~ " ");
code ~= indent ~ " }\n";
}
}
i = j - 1;
}
return code;
}
static string generateLeaf(string token, string indent)
{
string code;
if (staticTokens.countUntil(token) >= 0)
{
code ~= indent ~ "range.popFrontN(" ~ text(token.length) ~ ");\n";
code ~= indent ~ "return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
}
else if (pseudoTokens.countUntil(token) >= 0)
code ~= indent ~ "return postProcess(pseudoTok!\"" ~ escape(token) ~"\");\n";
else if (possibleDefaultTokens.countUntil(token) >= 0)
{
code ~= indent ~ "if (!range.canPeek(" ~ text(token.length) ~ ") || isSeparating(range.peek(" ~ text(token.length) ~ ")))\n";
code ~= indent ~ "{\n";
code ~= indent ~ " range.popFrontN(" ~ text(token.length) ~ ");\n";
code ~= indent ~ " return Token(tok!\"" ~ escape(token) ~"\", null, range.line, range.column, range.index);\n";
code ~= indent ~ "}\n";
code ~= indent ~ "else\n";
code ~= indent ~ " goto outer_default;\n";
}
else
code ~= indent ~ "goto outer_default;\n";
return code;
}
Token front() @property
{
return _front;
}
void popFront()
{
_front = advance();
}
bool empty() const nothrow @property
{
return _front.type == tok!"\0";
}
void registerPostProcess(alias t)(Token delegate(ref RangeType) fun)
{
post[pseudoTok!t] = fun;
}
static IDType pseudoTok(string symbol)() nothrow pure @property
{
static assert (pseudoTokens.countUntil(symbol) >= 0);
enum index = cast(IDType) pseudoTokens.countUntil(symbol);
return index;
}
static string escape(string input)
{
string rVal;
foreach (ubyte c; cast(ubyte[]) input)
{
switch (c)
{
case '\\': rVal ~= `\\`; break;
case '"': rVal ~= `\"`; break;
case '\'': rVal ~= `\'`; break;
case '\t': rVal ~= `\t`; break;
case '\n': rVal ~= `\n`; break;
case '\r': rVal ~= `\r`; break;
default: rVal ~= c; break;
}
}
return rVal;
}
Token advance()
{
if (range.empty)
return Token(tok!"\0");
auto r = range.save;
lexerLoop: switch (range.front)
{
mixin(generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
//pragma(msg, generateCaseStatements(stupidToArray(sort(staticTokens ~ pseudoTokens ~ possibleDefaultTokens))));
outer_default:
default:
range = r;
return defaultTokenFunction(range);
}
}
/**
* This only exists because the real array() can't be called at compile-time
*/
static T[] stupidToArray(R, T = ElementType!R)(R range)
{
T[] rVal;
foreach (v; range)
rVal ~= v;
return rVal;
}
Token postProcess(IDType i)
{
assert (post[i] !is null, "No post-processing function registered for " ~ pseudoTokens[i]);
return post[i](range);
}
Token delegate(ref RangeType)[pseudoTokens.length] post;
RangeType range;
Token _front;
}
struct PeekRange(R, size_t peekSupported = 1) if (isRandomAccessRange!R && isForwardRange!R)
{
public:
this(R range)
{
this.range = range;
}
bool empty() pure nothrow const @property
{
return _index >= range.length;
}
ElementType!R front() pure nothrow const @property
in
{
assert (!empty);
}
body
{
return range[_index];
}
void popFront() pure nothrow
{
_index++;
_column++;
}
void popFrontN(size_t n) pure nothrow
{
foreach (i; 0 .. n)
popFront();
}
ElementType!R peek(int offset = 1) pure nothrow const
in
{
assert (canPeek(offset));
}
body
{
return range[_index + offset];
}
bool canPeek(int offset = 1) pure nothrow const
{
return _index + offset >= 0 && _index + offset < range.length;
}
typeof(this) save() @property
{
typeof(this) copy;
copy.range = range;
copy._index = _index;
copy._column = _column;
copy._line = _line;
return copy;
}
void mark() nothrow pure
{
markBegin = index;
}
R getMarked() nothrow pure
{
return range[markBegin .. index];
}
void incrementLine() pure nothrow
{
_column = 1;
_line++;
}
size_t line() pure nothrow const @property { return _line; }
size_t column() pure nothrow const @property { return _column; }
size_t index() pure nothrow const @property { return _index; }
private:
size_t markBegin;
size_t _column = 1;
size_t _line = 1;
size_t _index = 0;
R range;
}
struct PeekRange(R, size_t peekSupported = 1)
if (!isRandomAccessRange!R && isForwardRange!R)
{
public:
this(R range)
{
this.range = range;
for (size_t i = 0; !this.range.empty && i < peekSupported; i++)
{
rangeSizeCount++;
buffer[i] = this.range.front;
range.popFront();
}
}
ElementType!R front() const @property
in
{
assert (!empty);
}
body
{
return buffer[bufferIndex];
}
void popFront()
in
{
assert (!empty);
}
body
{
index++;
column++;
count++;
bufferIndex = bufferIndex + 1 > buffer.length ? 0 : bufferIndex + 1;
if (marking)
markBuffer.put(buffer[bufferIndex]);
if (!range.empty)
{
buffer[bufferIndex + peekSupported % buffer.length] = range.front();
range.popFront();
rangeSizeCount++;
}
}
bool empty() const nothrow pure @property
{
return rangeSizeCount == count;
}
ElementType!R peek(int offset = 1) pure nothrow const
in
{
assert (canPeek(offset));
}
body
{
return buffer[(bufferIndex + offset) % buffer.length];
}
bool canPeek(int offset = 1) pure nothrow const
{
return offset >= 0
? offset <= peekSupported && count + offset <= rangeSizeCount
: abs(offset) <= peekSupported && (count - abs(offset)) >= 0;
}
typeof(this) save() @property
{
typeof(this) newRange;
newRange.count = count;
newRange.rangeSizeCount = count;
newRange.buffer = buffer.dup;
newRange.bufferIndex = bufferIndex;
newRange.range = range.save;
return newRange;
}
void mark()
{
marking = true;
markBuffer.clear();
}
ElementEncodingType!R[] getMarked()
{
marking = false;
return markBuffer.data;
}
void incrementLine() pure nothrow
{
_column = 1;
_line++;
}
size_t line() pure nothrow const @property { return _line; }
size_t column() pure nothrow const @property { return _column; }
size_t index() pure nothrow const @property { return _index; }
private:
auto markBuffer = appender!(ElementType!R[])();
bool marking;
size_t count;
size_t rangeSizeCount;
ElementType!(R)[(peekSupported * 2) + 1] buffer;
size_t bufferIndex;
size_t _column = 1;
size_t _line = 1;
size_t _index = 0;
R range;
}