forks
/
D-Scanner
mirror of https://github.com/dlang-community/D-Scanner.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle Unicode byte order marks

This commit is contained in:
Hackerpilot 2014-05-19 13:40:35 -07:00
parent 25f0d93b90
commit db84119e33
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
main.d
View File

@ -121,11 +121,11 @@ int run(string[] args)
} }
else if (tokenDump) else if (tokenDump)
{ {
writeln("text blank\tindex\tline\tcolumn"); writeln("text blank\tindex\tline\tcolumn\ttype");
foreach (token; tokens) foreach (token; tokens)
{ {
writefln("<<%20s>>%b\t%d\t%d\t%d", token.text is null ? str(token.type) : token.text, writefln("<<%20s>>%b\t%d\t%d\t%d\t%d", token.text is null ? str(token.type) : token.text,
token.text !is null, token.index, token.line, token.column); token.text !is null, token.index, token.line, token.column, token.type);
} }
return 0; return 0;
} }

4
std/d/lexer.d
View File

@ -417,7 +417,9 @@ public struct DLexer
this(ubyte[] range, const LexerConfig config, StringCache* cache) this(ubyte[] range, const LexerConfig config, StringCache* cache)
{ {
this.range = LexerRange(range); auto r = (range.length >= 3 && range[0] == 0xef && range[1] == 0xbb && range[2] == 0xbf)
? range[3 .. $] : range;
this.range = LexerRange(r);
this.config = config; this.config = config;
this.cache = cache; this.cache = cache;
popFront(); popFront();