From 635ddf79a064cf6227b381e84a0158bf88b9aaae Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Tue, 25 Jun 2013 01:18:09 -0700 Subject: [PATCH] More work on the various expression parsing functions --- std/d/ast.d | 16 ++- std/d/lexer.d | 4 +- std/d/parser.d | 289 ++++++++++++++++++++++++++++++++++++++----------- std/d/tester.d | 7 +- 4 files changed, 247 insertions(+), 69 deletions(-) diff --git a/std/d/ast.d b/std/d/ast.d index c594478..d13026c 100755 --- a/std/d/ast.d +++ b/std/d/ast.d @@ -224,6 +224,7 @@ abstract class ASTVisitor /** */ void visit(UnionDeclaration unionDeclaration) { unionDeclaration.accept(this); } /** */ void visit(Unittest unittest_) { unittest_.accept(this); } /** */ void visit(VariableDeclaration variableDeclaration) { variableDeclaration.accept(this); } + /** */ void visit(Vector vector) { vector.accept(this); } /** */ void visit(VersionCondition versionCondition) { versionCondition.accept(this); } /** */ void visit(VersionSpecification versionSpecification) { versionSpecification.accept(this); } /** */ void visit(WhileStatement whileStatement) { whileStatement.accept(this); } @@ -1641,8 +1642,9 @@ class PrimaryExpression : ASTNode public: mixin(DEFAULT_ACCEPT); /** */ Token primary; + /** */ Token identifier; /** */ IdentifierOrTemplateInstance identifierOrTemplateInstance; - /** */ Type type; + /** */ TokenType basicType; /** */ TypeofExpression typeofExpression; /** */ TypeidExpression typeidExpression; /** */ ArrayLiteral arrayLiteral; @@ -1654,6 +1656,7 @@ public: /** */ TraitsExpression traitsExpression; /** */ MixinExpression mixinExpression; /** */ ImportExpression importExpression; + /** */ Vector vector; } /// @@ -1819,7 +1822,7 @@ class StructDeclaration : ASTNode { public: mixin(DEFAULT_ACCEPT); - /** */ Token identifier; + /** */ Token name; /** */ TemplateParameters templateParameters; /** */ Constraint constraint; /** */ StructBody structBody; @@ -2132,6 +2135,7 @@ public: mixin(DEFAULT_ACCEPT); /** */ Token delegateOrFunction; /** */ bool star; + /** */ bool array; /** */ Type type; /** */ AssignExpression assignExpression; /** */ Parameters parameters; @@ -2204,6 +2208,14 @@ public: /** */ AutoDeclaration autoDeclaration; } +/// +class Vector : ASTNode +{ +public: + mixin(DEFAULT_ACCEPT); + /** */ Type type; +} + /// class VersionCondition : ASTNode { diff --git a/std/d/lexer.d b/std/d/lexer.d index 7205639..075200a 100755 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -2084,7 +2084,6 @@ enum TokenType: ushort dchar_, /// $(D_KEYWORD dchar) double_, /// $(D_KEYWORD double) float_, /// $(D_KEYWORD float) - function_, /// $(D_KEYWORD function) idouble_, /// $(D_KEYWORD idouble) ifloat_, /// $(D_KEYWORD ifloat) int_, /// $(D_KEYWORD int) @@ -2134,6 +2133,7 @@ enum TokenType: ushort debug_, /// $(D_KEYWORD debug) default_, /// $(D_KEYWORD default) delegate_, /// $(D_KEYWORD delegate) + function_, /// $(D_KEYWORD function) delete_, /// $(D_KEYWORD delete) do_, /// $(D_KEYWORD do) else_, /// $(D_KEYWORD else) @@ -2581,7 +2581,6 @@ package immutable(string[TokenType.max + 1]) tokenValues = [ "dchar", "double", "float", - "function", "idouble", "ifloat", "int", @@ -2629,6 +2628,7 @@ package immutable(string[TokenType.max + 1]) tokenValues = [ "debug", "default", "delegate", + "function", "delete", "do", "else", diff --git a/std/d/parser.d b/std/d/parser.d index 20afa6e..23627d5 100755 --- a/std/d/parser.d +++ b/std/d/parser.d @@ -70,7 +70,7 @@ import std.array; version (unittest) import std.stdio; version = development; -//version = verbose; +version = verbose; version(development) import std.stdio; version(verbose) import std.stdio; @@ -81,9 +81,10 @@ version(verbose) import std.stdio; * tokens = the tokens parsed by std.d.lexer * Returns: the parsed module */ -Module parseModule(const(Token)[] tokens) +Module parseModule(const(Token)[] tokens, string fileName) { auto parser = new Parser(); + parser.fileName = fileName; parser.tokens = tokens; return parser.parseModule(); } @@ -3285,6 +3286,7 @@ invariant() foo(); Parameter parseParameter() { auto node = new Parameter; + version (verbose) writeln("parseParameter"); while (moreTokens()) { TokenType type = parseParameterAttribute(false); @@ -3331,6 +3333,10 @@ invariant() foo(); case shared_: case const_: case inout_: + if (peekIs(TokenType.lParen)) + return invalid; + else + goto case auto_; case final_: case in_: case lazy_: @@ -3356,6 +3362,7 @@ invariant() foo(); */ Parameters parseParameters() { + version (verbose) writeln("parseParameters"); auto node = new Parameters; expect(TokenType.lParen); if (currentIs(TokenType.rParen)) @@ -3518,10 +3525,11 @@ q{(int a, ...) * Parses a PrimaryExpression * * $(GRAMMAR $(RULEDEF primaryExpression): - * $(RULE symbol) - * | $(RULE type) $(LITERAL '.') $(LITERAL Identifier) + * $(LITERAL Identifier) + * | $(RULE basicType) $(LITERAL '.') $(LITERAL Identifier) * | $(RULE typeofExpression) * | $(RULE typeidExpression) + * | $(RULE vector) * | $(RULE arrayLiteral) * | $(RULE assocArrayLiteral) * | $(LITERAL '$(LPAREN)') $(RULE expression) $(LITERAL '$(RPAREN)') @@ -3560,8 +3568,25 @@ q{(int a, ...) with (TokenType) switch (current().type) { case identifier: - // TODO - // symbol or type.identifier + if (peekIs(TokenType.goesTo)) + node.lambdaExpression = parseLambdaExpression(); + else + node.identifier = advance(); + break; + mixin (BASIC_TYPE_CASE_RANGE); + node.basicType = advance().type; + expect(dot); + auto t = expect(identifier); + if (t !is null) + node.identifier = *t; + break; + case function_: + case delegate_: + case lBrace: + case in_: + case out_: + case body_: + node.functionLiteralExpression = parseFunctionLiteralExpression(); break; case typeof_: node.typeofExpression = parseTypeofExpression(); @@ -3569,19 +3594,28 @@ q{(int a, ...) case typeid_: node.typeidExpression = parseTypeidExpression(); break; + case vector: + node.vector = parseVector(); + break; case lBracket: - // TODO: array literal or associative array literal + if (isAssociativeArrayLiteral()) + node.assocArrayLiteral = parseAssocArrayLiteral(); + else + node.arrayLiteral = parseArrayLiteral(); break; case lParen: - advance(); - node.expression = parseExpression(); - expect(TokenType.rParen); + if (peekPastParens().type == TokenType.goesTo) + node.lambdaExpression = parseLambdaExpression(); + else + { + advance(); + node.expression = parseExpression(); + expect(TokenType.rParen); + } break; case is_: node.isExpression = parseIsExpression(); break; - // TODO: lambda - // TODO: function literal case traits: node.traitsExpression = parseTraitsExpression(); break; @@ -3596,8 +3630,8 @@ q{(int a, ...) case null_: case true_: case false_: - case specialDate: .. case specialPrettyFunction: - case doubleLiteral: .. case wstringLiteral: + mixin (SPECIAL_CASE_RANGE); + mixin (LITERAL_CASE_RANGE); version(verbose) writeln("special or literal ", index, " ", current()); node.primary = advance(); break; @@ -3954,7 +3988,7 @@ q{(int a, ...) { auto node = new StructDeclaration; expect(TokenType.struct_); - node.identifier = *expect(TokenType.identifier); + node.name = *expect(TokenType.identifier); if (currentIs(TokenType.lParen)) { node.templateParameters = parseTemplateParameters(); @@ -4308,9 +4342,9 @@ q{(int a, ...) case null_: case this_: case identifier: - case specialDate: .. case specialTokenSequence: - case doubleLiteral: .. case wstringLiteral: - case bool_: .. case wchar_: + mixin (SPECIAL_CASE_RANGE); + mixin (LITERAL_CASE_RANGE); + mixin (BASIC_TYPE_CASE_RANGE); node.token = advance(); break; default: @@ -4339,13 +4373,17 @@ q{(int a, ...) * Parses an TemplateTupleParameter * * $(GRAMMAR $(RULEDEF templateTupleParameter): - * $(LITERAL Identifier) '...' + * $(LITERAL Identifier) $(LITERAL '...') * ;) */ TemplateTupleParameter parseTemplateTupleParameter() { auto node = new TemplateTupleParameter; - // TODO + auto i = expect(TokenType.identifier); + if (i is null) + return null; + node.identifier = *i; + expect(TokenType.vararg); return node; } @@ -4522,19 +4560,21 @@ q{(int a, ...) break; } node.type2 = parseType2(); - loop: while (true) + if (node.type2 is null) + return null; + loop: while (true) with (TokenType) switch (current.type) { - switch (current.type) - { - case TokenType.star: - case TokenType.lBracket: - case TokenType.delegate_: - case TokenType.function_: - node.typeSuffixes ~= parseTypeSuffix(); - break; - default: - break loop; - } + case TokenType.star: + case TokenType.lBracket: + case TokenType.delegate_: + case TokenType.function_: + auto suffix = parseTypeSuffix(); + if (suffix !is null) + node.typeSuffixes ~= suffix; + else + return null; + default: + break loop; } return node; } @@ -4552,35 +4592,40 @@ q{(int a, ...) Type2 parseType2() { auto node = new Type2; - switch (current.type) + with (TokenType) switch (current.type) { - case TokenType.identifier: - case TokenType.dot: - node.symbol = parseSymbol(); + case identifier: + case dot: + if ((node.symbol = parseSymbol()) is null) + return null; break; - case TokenType.bool_: .. case TokenType.wchar_: - node.basicType = parseBasicType(); + case bool_: .. case wchar_: + if ((node.basicType = parseBasicType()) == TokenType.invalid) + return null; break; - case TokenType.typeof_: - node.typeofExpression = parseTypeofExpression(); + case typeof_: + if ((node.typeofExpression = parseTypeofExpression()) is null) + return null; if (currentIs(TokenType.dot)) { advance(); node.identifierOrTemplateChain = parseIdentifierOrTemplateChain(); + if (node.identifierOrTemplateChain is null) + return null; } break; - case TokenType.const_: - case TokenType.immutable_: - case TokenType.inout_: - case TokenType.shared_: + case const_: + case immutable_: + case inout_: + case shared_: node.typeConstructor = parseTypeConstructor(); - expect(TokenType.lParen); - node.type = parseType(); - expect(TokenType.rParen); + if (expect(TokenType.lParen) is null) return null; + if ((node.type = parseType()) is null) return null; + if (expect(TokenType.rParen) is null) return null; break; default: error("Basic type, type constructor, symbol, or typeof expected"); - break; + return null; } return node; } @@ -4595,7 +4640,7 @@ q{(int a, ...) * | $(LITERAL 'shared') * ;) */ - TokenType parseTypeConstructor(bool validate = false) + TokenType parseTypeConstructor(bool validate = true) { with (TokenType) switch (current().type) { @@ -4692,19 +4737,32 @@ q{(int a, ...) TypeSuffix parseTypeSuffix() { auto node = new TypeSuffix; - switch(current().type) + version (verbose) writeln("parseTypeSuffix"); + with (TokenType) switch(current().type) { - case TokenType.star: + case star: node.star = true; advance(); return node; - case TokenType.lBracket: + case lBracket: + node.array = true; advance(); - // TODO: magic + if (currentIs(rBracket)) + goto end; + auto bookmark = setBookmark(); + auto type = parseType(); + if (type is null) + { + goToBookmark(bookmark); + node.assignExpression = parseAssignExpression(); + } + else + node.type = type; + end: expect(TokenType.rBracket); return node; - case TokenType.delegate_: - case TokenType.function_: + case delegate_: + case function_: advance(); node.parameters = parseParameters(); while (currentIsMemberFunctionAttribute()) @@ -4928,6 +4986,23 @@ q{(int a, ...) return node; } + /** + * Parses a Vector + * + * $(GRAMMAR $(RULEDEF vector): + * $(LITERAL '__vector') $(LITERAL '$(LPAREN)') $(RULE type) $(LITERAL '$(RPAREN)') + * ;) + */ + Vector parseVector() + { + auto node = new Vector; + expect(TokenType.vector); + expect(TokenType.lParen); + node.type = parseType(); + expect(TokenType.rParen); + return node; + } + /** * Parses a VersionCondition * @@ -5027,7 +5102,74 @@ q{(int a, ...) TokenType.xor)(); } - private bool currentIsMemberFunctionAttribute() const +private: + + bool isAssociativeArrayLiteral() + { + auto i = index; + scope(exit) index = i; + if (!currentIs(TokenType.lBracket)) + return false; + advance(); + while (moreTokens()) with (TokenType) switch (current().type) + { + case lBrace: skipBraceContent(); break; + case lParen: skipParenContent(); break; + case lBracket: skipBracketContent(); break; + case rBracket: return false; + case colon: return true; + default: break; + } + return false; + } + + bool isStatement() const + { + // TODO: Not complete + with (TokenType) switch (current().type) + { + case lBrace: + case if_: + case while_: + case do_: + case for_: + case foreach_: + case foreach_reverse_: + case switch_: + case continue_: + case break_: + case return_: + case goto_: + case with_: + case try_: + case throw_: + case asm_: + case default_: + case case_: + case new_: + mixin (SPECIAL_CASE_RANGE); + mixin (LITERAL_CASE_RANGE); + case true_: + case false_: + return true; + case extern_: + case union_: + case class_: + case interface_: + case function_: + case delegate_: + case typedef_: + case typeof_: + case invariant_: + case alias_: + mixin (BASIC_TYPE_CASE_RANGE); + return false; + default: + return false; + } + } + + bool currentIsMemberFunctionAttribute() const { switch (current.type) { @@ -5044,7 +5186,7 @@ q{(int a, ...) } } - private Left parseLeftAssocBinaryExpression(alias Left, alias Right, Operators ...) + Left parseLeftAssocBinaryExpression(alias Left, alias Right, Operators ...) (Right r = null) { auto node = new Left; @@ -5063,7 +5205,7 @@ q{(int a, ...) return node; } - private ListType parseCommaSeparatedRule(alias ListType, alias ItemType)() + ListType parseCommaSeparatedRule(alias ListType, alias ItemType)() { auto node = new ListType; while (true) @@ -5081,12 +5223,15 @@ q{(int a, ...) { import std.stdio; ++errorCount; - if (errorFunction is null) - stderr.writefln("%s(%d:%d): %s", fileName, tokens[index].line, - tokens[index].column, message); - else - errorFunction(fileName, tokens[index].line, tokens[index].column, - message); + if (!suppressMessages) + { + if (errorFunction is null) + stderr.writefln("%s(%d:%d): %s", fileName, tokens[index].line, + tokens[index].column, message); + else + errorFunction(fileName, tokens[index].line, tokens[index].column, + message); + } while (moreTokens()) { if (currentIsOneOf(TokenType.semicolon, TokenType.rBrace)) @@ -5257,6 +5402,18 @@ q{(int a, ...) return index + 1 < tokens.length; } + size_t setBookmark() + { + suppressMessages = true; + return index; + } + + void goToBookmark(size_t i) + { + suppressMessages = false; + index = i; + } + version (unittest) static void doNothingErrorFunction(string fileName, int line, int column, string message) {} @@ -5277,4 +5434,8 @@ q{(int a, ...) size_t index; string fileName; void function(string, int, int, string) errorFunction; + immutable string BASIC_TYPE_CASE_RANGE = q{case bool_: .. case wchar_:}; + immutable string LITERAL_CASE_RANGE = q{case doubleLiteral: .. case wstringLiteral:}; + immutable string SPECIAL_CASE_RANGE = q{case specialDate: .. case specialPrettyFunction:}; + bool suppressMessages; } diff --git a/std/d/tester.d b/std/d/tester.d index ece38df..e4b2def 100644 --- a/std/d/tester.d +++ b/std/d/tester.d @@ -12,6 +12,11 @@ class TestVisitor : ASTVisitor writeln("class ", classDeclaration.name.value, " on line ", classDeclaration.name.line); } + override void visit(StructDeclaration structDeclaration) + { + writeln("struct ", structDeclaration.name.value, " on line ", structDeclaration.name.line); + } + override void visit(ModuleDeclaration md) { writeln("module declaration found"); @@ -58,7 +63,7 @@ void main(string[] args) ubyte[] rawSource = f.rawRead(sourceBuffer); LexerConfig config; auto tokens = byToken(rawSource, config).array(); - Module m = parseModule(tokens); + Module m = parseModule(tokens, args[1]); ASTVisitor visitor = new TestVisitor; visitor.visit(m); }