From db84119e33f6ba99f29eff555082901ec0560b16 Mon Sep 17 00:00:00 2001
From: Hackerpilot <briancschott@gmail.com>
Date: Mon, 19 May 2014 13:40:35 -0700
Subject: [PATCH] Handle Unicode byte order marks

---
 main.d        | 6 +++---
 std/d/lexer.d | 4 +++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/main.d b/main.d
index 4072058..50c1304 100644
--- a/main.d
+++ b/main.d
@@ -121,11 +121,11 @@ int run(string[] args)
 		}
 		else if (tokenDump)
 		{
-			writeln("text                    blank\tindex\tline\tcolumn");
+			writeln("text                    blank\tindex\tline\tcolumn\ttype");
 			foreach (token; tokens)
 			{
-				writefln("<<%20s>>%b\t%d\t%d\t%d", token.text is null ? str(token.type) : token.text,
-					token.text !is null, token.index, token.line, token.column);
+				writefln("<<%20s>>%b\t%d\t%d\t%d\t%d", token.text is null ? str(token.type) : token.text,
+					token.text !is null, token.index, token.line, token.column, token.type);
 			}
 			return 0;
 		}
diff --git a/std/d/lexer.d b/std/d/lexer.d
index a40bbe0..3019f2b 100644
--- a/std/d/lexer.d
+++ b/std/d/lexer.d
@@ -417,7 +417,9 @@ public struct DLexer
 
 	this(ubyte[] range, const LexerConfig config, StringCache* cache)
 	{
-		this.range = LexerRange(range);
+		auto r = (range.length >= 3 && range[0] == 0xef && range[1] == 0xbb && range[2] == 0xbf)
+			? range[3 .. $] : range;
+		this.range = LexerRange(r);
 		this.config = config;
 		this.cache = cache;
 		popFront();