diff --git a/main.d b/main.d index 512f5d5..6ec2759 100644 --- a/main.d +++ b/main.d @@ -40,6 +40,7 @@ int main(string[] args) bool imports; bool muffin; bool outline; + bool tokenDump; try { @@ -47,7 +48,7 @@ int main(string[] args) "ctags|c", &ctags, "recursive|r|R", &recursive, "help|h", &help, "tokenCount|t", &tokenCount, "syntaxCheck|s", &syntaxCheck, "ast|xml", &ast, "imports|i", &imports, "outline|o", &outline, - "muffinButton", &muffin); + "tokenDump", &tokenDump, "muffinButton", &muffin); } catch (Exception e) { @@ -76,7 +77,7 @@ int main(string[] args) } auto optionCount = count!"a"([sloc, highlight, ctags, tokenCount, - syntaxCheck, ast, imports, outline]); + syntaxCheck, ast, imports, outline, tokenDump]); if (optionCount > 1) { stderr.writeln("Too many options specified"); @@ -96,6 +97,17 @@ int main(string[] args) highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]); return 0; } + else if (tokenDump) + { + bool usingStdin = args.length == 1; + ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); + auto tokens = byToken!(ubyte[], false, false)(bytes); + foreach (ref token; tokens) + { + writeln("«", token.text is null ? str(token.type) : token.text, + " ", token.index, " ", token.line, " ", token.column, "»"); + } + } else if (ctags) { stdout.printCtags(expandArgs(args, recursive)); diff --git a/stdx/d/lexer.d b/stdx/d/lexer.d index 0652247..735054a 100644 --- a/stdx/d/lexer.d +++ b/stdx/d/lexer.d @@ -482,10 +482,10 @@ public struct DLexer(R) Token lexNumber() pure nothrow { auto mark = range.mark(); - auto lookahead = range.lookahead(1); - if (range.front == '0' && lookahead.length == 1) + auto lookahead = range.lookahead(2); + if (range.front == '0' && lookahead.length == 2) { - switch (lookahead[0]) + switch (lookahead[1]) { case 'x': case 'X': @@ -1040,10 +1040,18 @@ public struct DLexer(R) else app.put(t.text); if (t.type == tok!"}") + { depth--; + if (depth > 0) + popFront(); + } else if (t.type == tok!"{") + { depth++; - popFront(); + popFront(); + } + else + popFront(); } IdType type = tok!"stringLiteral"; lexStringSuffix(type); diff --git a/stdx/d/parser.d b/stdx/d/parser.d index 8fdf874..38ded73 100644 --- a/stdx/d/parser.d +++ b/stdx/d/parser.d @@ -12,7 +12,7 @@ import std.string : format; // Uncomment this if you want ALL THE OUTPUT // Caution: generates 180 megabytes of logging for std.datetime -version = std_parser_verbose; +//version = std_parser_verbose; /** * Params: @@ -1248,7 +1248,7 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; case tok!"!<=": node.relExpression = parseRelExpression(shift); break; - case tok!"=": + case tok!"==": case tok!"!=": node.equalExpression = parseEqualExpression(shift); break; @@ -6547,6 +6547,7 @@ protected: case tok!"stringLiteral": case tok!"wstringLiteral": case tok!"dstringLiteral": + case tok!"characterLiteral": }; enum string SPECIAL_CASES = q{ case tok!"__DATE__":