From fb66baa36abb12bc280f4c2cb01e46bfa2ff0f64 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sun, 20 Jan 2013 03:51:52 +0000 Subject: [PATCH] Fixed a few infinite loop bugs --- highlighter.d | 21 ++++++++++----------- tokenizer.d | 28 ++++++++++++++++++---------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/highlighter.d b/highlighter.d index 1478a1c..032c3f4 100644 --- a/highlighter.d +++ b/highlighter.d @@ -24,12 +24,11 @@ void highlight(R)(R tokens) 
]");
@@ -39,22 +38,22 @@ html { background-color: #111; color: #ccc; }
 		switch (t.type)
 		{
 		case TokenType.KEYWORDS_BEGIN: .. case TokenType.KEYWORDS_END:
-			writeSpan("keyword", t.value);
+			writeSpan("kwrd", t.value);
 			break;
 		case TokenType.TYPES_BEGIN: .. case TokenType.TYPES_END:
 			writeSpan("type", t.value);
 			break;
 		case TokenType.Comment:
-			writeSpan("comment", t.value);
+			writeSpan("com", t.value);
 			break;
 		case TokenType.STRINGS_BEGIN: .. case TokenType.STRINGS_END:
-			writeSpan("string", t.value);
+			writeSpan("str", t.value);
 			break;
 		case TokenType.NUMBERS_BEGIN: .. case TokenType.NUMBERS_END:
-			writeSpan("number", t.value);
+			writeSpan("num", t.value);
 			break;
 		case TokenType.OPERATORS_BEGIN: .. case TokenType.OPERATORS_END:
-			writeSpan("operator", t.value);
+			writeSpan("op", t.value);
 			break;
 		default:
 			stdout.write(t.value.replace("<", "<"));
diff --git a/tokenizer.d b/tokenizer.d
index dae129e..3dea8ad 100644
--- a/tokenizer.d
+++ b/tokenizer.d
@@ -517,6 +517,7 @@ body
 	t.startIndex = index;
 	t.type = TokenType.StringLiteral;
 	auto app = appender!(char[])();
+	bool isWysiwyg = input.front == 'r' || input.front == '`';
 	if (input.front == 'r')
 	{
 		if (style & StringStyle.IncludeQuotes)
@@ -542,7 +543,7 @@ body
 			{
 				auto r = input.save();
 				r.popFront();
-				if (r.front == quote)
+				if (r.front == quote && !isWysiwyg)
 				{
 					app.put('\\');
 					app.put(quote);
@@ -550,7 +551,7 @@ body
 					input.popFront();
 					index += 2;
 				}
-				else if (r.front == '\\')
+				else if (r.front == '\\' && !isWysiwyg)
 				{
 					app.put('\\');
 					app.put('\\');
@@ -1394,14 +1395,14 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
 			current = lexString(range, index, lineNumber, stringStyle);
 			break;
 		case 'q':
-			auto r = range.save;
+			/+auto r = range.save;
 			r.popFront();
 			if (!r.isEoF() && r.front == '{')
 			{
 				writeln("ParseTokenString");
 				break;
 			}
-			else
+			else+/
 				goto default;
 		case '/':
 			auto r = range.save();
@@ -1410,6 +1411,8 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
 			{
 				current.type = TokenType.Div;
 				current.value = "/";
+				range.popFront();
+				++index;
 				break;
 			}
 			switch (r.front)
@@ -1422,19 +1425,23 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
 			case '=':
 				current.type = TokenType.DivEquals;
 				current.value = "/=";
+				range.popFront();
+				range.popFront();
+				index += 2;
 				break outer;
 			default:
 				current.type = TokenType.Div;
 				current.value = "/";
-				break;
+				++index;
+				range.popFront();
+				break outer;
 			}
-			break;
 		case 'r':
 			auto r = range.save();
 			r.popFront();
 			if (!r.isEoF() && r.front == '"')
 			{
-				current = lexString(range, index, lineNumber, StringStyle.NotEscaped);
+				current = lexString(range, index, lineNumber, stringStyle);
 				break;
 			}
 			else
@@ -1455,6 +1462,7 @@ struct TokenRange(R) if (isForwardRange!(R) && isSomeChar!(ElementType!(R)))
 			{
 				app.put(range.front);
 				range.popFront();
+				++index;
 			}
 			current.value = to!string(app.data);
 			current.type = lookupTokenTypeOptimized(current.value);
@@ -1475,7 +1483,7 @@ private:
 
 unittest
 {
-	auto c = "rust r\"\\ntest\" r`eh?`";
-	foreach (t; byToken(c))
-		writeln(t);
+	auto c = `r"d:\path\foo.bat"`;
+	foreach (t; byToken(c, IterationStyle.CodeOnly, StringStyle.Source))
+		writeln(t.type, ": {", t.value, "}");
 }