From 26500bf8768a98684c39653729eb4c1bdfdd70d3 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Mon, 11 Jan 2016 17:41:39 -0800 Subject: [PATCH] Token dump now uses the parser behavior while lexing --- src/main.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.d b/src/main.d index ef7bed9..94fba44 100644 --- a/src/main.d +++ b/src/main.d @@ -155,15 +155,17 @@ int main(string[] args) ubyte[] bytes = usingStdin ? readStdin() : readFile(args[1]); LexerConfig config; config.stringBehavior = StringBehavior.source; - auto tokens = byToken(bytes, config, &cache); + if (highlight) { + auto tokens = byToken(bytes, config, &cache); highlighter.highlight(tokens, args.length == 1 ? "stdin" : args[1]); return 0; } else if (tokenDump) { - writeln("text blank\tindex\tline\tcolumn\ttype\tcomment"); + auto tokens = getTokensForParser(bytes, config, &cache); + writeln("text blank\tindex\tline\tcolumn\ttype\tcomment\ttrailingComment"); foreach (token; tokens) { writefln("<<%20s>>%b\t%d\t%d\t%d\t%d\t%s", token.text is null ? str(token.type) : token.text,