diff --git a/build.sh b/build.sh index eb2ae8a..13bf47d 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ #dmd *.d std/d/*.d -release -inline -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline -#dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest +#dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner #-unittest ldc2 -O5 *.d std/d/*.d -of=dscanner -release -vectorize -m64 diff --git a/circularbuffer.d b/circularbuffer.d deleted file mode 100644 index 11e1053..0000000 --- a/circularbuffer.d +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright Brian Schott (Sir Alaran) 2012. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module circularbuffer; - -import std.math; -import std.array; -import std.range; - -class CircularBuffer(T) : InputRange!(T) - -{ -public: - - this (size_t size, InputRange!(T) range) - { - this.range = range; - this.margin = size; - data = new T[(margin * 2) + 1]; - if (range.empty()) - { - _empty = true; - return; - } - for (size_t i = 0; i <= margin && !this.range.empty(); ++i) - { - data[i] = this.range.front(); - this.range.popFront(); - end++; - } - } - - override T front() @property - { - return data[index]; - } - - T peek(int offset = 1) - in - { - assert(canPeek(offset)); - } - body - { - return data[(index + offset) % data.length]; - } - - bool canPeek(int offset = 1) - { - return abs(offset) <= margin && sourceIndex + offset >= 0; - } - - override void popFront() - in - { - assert (!_empty); - } - body - { - index = (index + 1) % data.length; - ++sourceIndex; - if (range.empty()) - { - if (index == end) - _empty = true; - } - else - { - data[end] = range.front(); - end = (end + 1) % data.length; - range.popFront(); - } - } - - bool empty() const @property - { - return _empty; - } - - override T moveFront() - { - auto r = front(); - popFront(); - return r; - } - - override int opApply(int delegate(T) dg) - { - int result = 0; - while (!empty) - { - result = dg(front); - if (result) - break; - popFront(); - } - return result; - } - - override int opApply(int delegate(size_t, T) dg) - { - int result = 0; - int i = 0; - while (!empty) - { - result = dg(i, front); - if (result) - break; - popFront(); - } - return result; - } - -private: - InputRange!(T) range; - immutable size_t margin; - T[] data; - size_t sourceIndex; - size_t end; - size_t index; - bool _empty; -} - -unittest -{ - int[] items = [1, 2]; - auto buf = CircularBuffer!(int, int[])(5, items); - auto result = array(buf); - assert(result == items); -} - -unittest -{ - int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - auto buf = CircularBuffer!(int, int[])(2, arr); - assert (buf.data.length == 5); - auto iterated = array(buf); - assert (iterated == arr); -} - -unittest -{ - int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - auto buf = CircularBuffer!(int, int[])(2, arr); - buf.popFront(); - buf.popFront(); - buf.popFront(); - buf.popFront(); - assert (buf.front == 4); -} - -unittest -{ - int[] arr = [0, 1, 2, 3]; - auto buf = CircularBuffer!(int, int[])(2, arr); - assert (buf.peek(0) == 0); - assert (buf.peek(1) == 1); - assert (buf.peek(2) == 2); - buf.popFront(); - buf.popFront(); - assert (buf.peek(-2) == 0); - assert (buf.peek(-1) == 1); - assert (buf.peek(0) == 2); - assert (buf.peek(1) == 3); -} - diff --git a/langutils.d b/langutils.d deleted file mode 100644 index cfdf689..0000000 --- a/langutils.d +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Brian Schott (Sir Alaran) 2012. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module langutils; - -import std.array; -import std.algorithm; -import std.d.lexer; - -string combineTokens(ref const Token[] tokens) -{ - auto app = appender!string(); - foreach (t; tokens) - app.put(t.value); - return app.data; -} - -pure nothrow string getTypeFromToken(const Token t) -{ - switch (t.type) - { - - case TokenType.doubleLiteral: - return "double"; - case TokenType.floatLiteral: - return "float"; - case TokenType.intLiteral: - return "int"; - case TokenType.realLiteral: - return "real"; - case TokenType.uintLiteral: - return "uint"; - case TokenType.ulongLiteral: - return "ulong"; - case TokenType.longLiteral: - return "long"; - case TokenType.dstringLiteral: - return "dstring"; - case TokenType.stringLiteral: - return "string"; - case TokenType.wstringLiteral: - return "wstring"; - default: - return null; - } -} - -pure bool isDocComment(ref const Token t) -{ - return t.value.startsWith("///") || t.value.startsWith("/**") - || t.value.startsWith("/++"); -} - -pure nothrow bool isIdentifierOrType(const TokenType t) -{ - return isType(t) || t == TokenType.identifier; -} diff --git a/location.d b/location.d deleted file mode 100644 index 502b2d9..0000000 --- a/location.d +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Brian Schott (Sir Alaran) 2012. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -module location; - -import std.string; -import std.path; -import std.file; -import std.stdio; -import std.array; - -/** - * Returns: the absolute path of the given module, or null if it could not be - * found. - */ -string findAbsPath(string[] dirs, string moduleName) -{ - // For file names - if (endsWith(moduleName, ".d") || endsWith(moduleName, ".di")) - { - if (isAbsolute(moduleName)) - return moduleName; - else - return buildPath(getcwd(), moduleName); - } - - // Try to find the file name from a module name like "std.stdio" - foreach(dir; dirs) - { - string fileLocation = buildPath(dir, replace(moduleName, ".", dirSeparator)); - string dfile = fileLocation ~ ".d"; - if (exists(dfile) && isFile(dfile)) - { - return dfile; - } - if (exists(fileLocation ~ ".di") && isFile(fileLocation ~ ".di")) - { - return fileLocation ~ ".di"; - } - } - stderr.writeln("Could not locate import ", moduleName, " in ", dirs); - return null; -} diff --git a/main.d b/main.d index 1dd6c16..7df99cf 100644 --- a/main.d +++ b/main.d @@ -20,8 +20,6 @@ import std.d.lexer; import highlighter; -immutable size_t CIRC_BUFF_SIZE = 4; - pure nothrow bool isLineOfCode(TokenType t) { switch(t) @@ -141,6 +139,7 @@ int main(string[] args) else {+/ LexerConfig config; + config.tokenStyle = TokenStyle.doNotReplaceSpecial; foreach (arg; args[1..$]) { config.fileName = arg; diff --git a/std/d/lexer.d b/std/d/lexer.d index b8741dc..638eb0d 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -438,7 +438,7 @@ private: return; } - outer: switch (currentElement()) + switch (currentElement()) { // pragma(msg, generateCaseTrie( mixin(generateCaseTrie( @@ -501,19 +501,14 @@ private: "^=", "TokenType.xorEquals", )); case '/': - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - if (r.isRangeEoF()) + keepNonNewlineChar(); + if (isEoF()) { current.type = TokenType.div; current.value = "/"; - advanceRange(); return; } - switch (r.front) + switch (currentElement()) { case '/': case '*': @@ -524,59 +519,42 @@ private: current.type = TokenType.divEquals; current.value = "/="; advanceRange(); - advanceRange(); return; default: current.type = TokenType.div; current.value = "/"; - advanceRange(); return; } case '.': - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - if (r.isRangeEoF()) + keepNonNewlineChar(); + if (isEoF()) { current.type = TokenType.dot; current.value = getTokenValue(TokenType.dot); - advanceRange(); return; } - else if (r.front >= '0' && r.front <= '9') + switch (currentElement()) { + case '0': .. case '9': lexNumber(); return; - } - else if (r.front == '.') - { + case '.': current.type = TokenType.slice; - r.popFront(); - if (r.front == '.') + keepNonNewlineChar(); + if (currentElement() == '.') { current.type = TokenType.vararg; - advanceRange(); - advanceRange(); - advanceRange(); - } - else - { - advanceRange(); - advanceRange(); + keepNonNewlineChar(); } current.value = getTokenValue(current.type); return; - } - else - { - advanceRange(); + default: current.type = TokenType.dot; current.value = getTokenValue(TokenType.dot); return; } case '0': .. case '9': + keepNonNewlineChar(); lexNumber(); return; case '\'': @@ -585,30 +563,26 @@ private: lexString(); return; case 'q': - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - if (!r.isRangeEoF() && r.front == '{') + keepNonNewlineChar(); + if (isEoF()) + goto default; + switch (currentElement()) { + case '{': lexTokenString(); return; - } - else if (!r.isRangeEoF() && r.front == '"') - { + case '"': lexDelimitedString(); return; + default: + break; } - else - goto default; + goto default; case 'r': - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - if (!r.isRangeEoF() && r.front == '"') + keepNonNewlineChar(); + if (isEoF()) + goto default; + else if (currentElement() == '"') { lexString(); return; @@ -616,12 +590,10 @@ private: else goto default; case 'x': - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - if (!r.isRangeEoF() && r.front == '"') + keepNonNewlineChar(); + if (isEoF()) + goto default; + else if (currentElement() == '"') { lexHexString(); return; @@ -707,18 +679,17 @@ private: void lexComment() in { - assert (currentElement() == '/'); + assert (currentElement() == '/' || currentElement() == '*' || currentElement() == '+'); } body { current.type = TokenType.comment; - keepChar(); switch(currentElement()) { case '/': while (!isEoF() && !isNewline(currentElement())) { - keepChar(); + keepNonNewlineChar(); } break; case '*': @@ -726,10 +697,10 @@ private: { if (currentElement() == '*') { - keepChar(); + keepNonNewlineChar(); if (currentElement() == '/') { - keepChar(); + keepNonNewlineChar(); break; } } @@ -743,19 +714,19 @@ private: { if (currentElement() == '+') { - keepChar(); + keepNonNewlineChar(); if (currentElement() == '/') { - keepChar(); + keepNonNewlineChar(); --depth; } } else if (currentElement() == '/') { - keepChar(); + keepNonNewlineChar(); if (currentElement() == '+') { - keepChar(); + keepNonNewlineChar(); ++depth; } } @@ -773,13 +744,12 @@ private: void lexHexString() in { - assert (currentElement() == 'x'); + assert (currentElement() == '"' && buffer[0] == 'x'); } body { current.type = TokenType.stringLiteral; keepChar(); - keepChar(); while (true) { if (isEoF()) @@ -789,7 +759,7 @@ private: } else if (isHexDigit(currentElement())) { - keepChar(); + keepNonNewlineChar(); } else if (isWhite(currentElement()) && (config.tokenStyle & TokenStyle.notEscaped)) { @@ -797,7 +767,7 @@ private: } else if (currentElement() == '"') { - keepChar(); + keepNonNewlineChar(); break; } else @@ -813,7 +783,7 @@ private: if (config.tokenStyle & TokenStyle.includeQuotes) setTokenValue(); else - setTokenValue(bufferIndex - 1, 2); + setTokenValue(2, bufferIndex - 1); } else { @@ -830,7 +800,7 @@ private: void lexNumber() in { - assert(isDigit(cast(char) currentElement()) || currentElement() == '.'); + assert(buffer[0] || buffer[0] == '.'); } body { @@ -839,23 +809,16 @@ private: lexDecimal(); else { - static if (isArray!R) - auto r = range[index .. $]; - else - auto r = range.save(); - r.popFront(); - switch (r.front) + switch (currentElement()) { case 'x': case 'X': - keepChar(); - keepChar(); + keepNonNewlineChar(); lexHex(); break; case 'b': case 'B': - keepChar(); - keepChar(); + keepNonNewlineChar(); lexBinary(); break; default: @@ -870,12 +833,12 @@ private: switch (currentElement()) { case 'L': - keepChar(); + keepNonNewlineChar(); current.type = TokenType.doubleLiteral; break; case 'f': case 'F': - keepChar(); + keepNonNewlineChar(); current.type = TokenType.floatLiteral; break; default: @@ -883,7 +846,7 @@ private: } if (!isEoF() && currentElement() == 'i') { - keepChar(); + keepNonNewlineChar(); if (current.type == TokenType.floatLiteral) current.type = TokenType.ifloatLiteral; else @@ -907,11 +870,11 @@ private: { case TokenType.intLiteral: current.type = TokenType.uintLiteral; - keepChar(); + keepNonNewlineChar(); break; case TokenType.longLiteral: current.type = TokenType.ulongLiteral; - keepChar(); + keepNonNewlineChar(); break; default: return; @@ -925,11 +888,11 @@ private: { case TokenType.intLiteral: current.type = TokenType.longLiteral; - keepChar(); + keepNonNewlineChar(); break; case TokenType.uintLiteral: current.type = TokenType.ulongLiteral; - keepChar(); + keepNonNewlineChar(); break; default: return; @@ -950,7 +913,7 @@ private: } body { - keepChar(); + keepNonNewlineChar(); bool foundSign = false; while (!isEoF()) { @@ -961,10 +924,10 @@ private: if (foundSign) return; foundSign = true; - keepChar(); + keepNonNewlineChar(); case '0': .. case '9': case '_': - keepChar(); + keepNonNewlineChar(); break; case 'L': case 'f': @@ -981,7 +944,7 @@ private: void lexDecimal() in { - assert ((currentElement() >= '0' && currentElement() <= '9') || currentElement() == '.'); + assert ((buffer[0] >= '0' && buffer[0] <= '9') || buffer[0] == '.'); } body { @@ -994,7 +957,7 @@ private: { case '0': .. case '9': case '_': - keepChar(); + keepNonNewlineChar(); break; case 'i': case 'L': @@ -1026,7 +989,7 @@ private: break decimalLoop; // possibly slice expression if (foundDot) break decimalLoop; // two dots with other characters between them - keepChar(); + keepNonNewlineChar(); foundDot = true; current.type = TokenType.doubleLiteral; break; @@ -1048,7 +1011,7 @@ private: case '0': case '1': case '_': - keepChar(); + keepNonNewlineChar(); break; case 'u': case 'U': @@ -1074,7 +1037,7 @@ private: case 'A': .. case 'F': case '0': .. case '9': case '_': - keepChar(); + keepNonNewlineChar(); break; case 'i': case 'L': @@ -1102,7 +1065,7 @@ private: break hexLoop; // slice expression if (foundDot) break hexLoop; // two dots with other characters between them - keepChar(); + keepNonNewlineChar(); foundDot = true; current.type = TokenType.doubleLiteral; break; @@ -1126,7 +1089,7 @@ private: current.type = TokenType.dstringLiteral; goto case 'c'; case 'c': - keepChar(); + keepNonNewlineChar(); break; default: break; @@ -1137,14 +1100,12 @@ private: void lexString() in { - assert (currentElement() == '\'' || currentElement() == '"' || currentElement() == '`' || currentElement() == 'r'); + assert (currentElement() == '\'' || currentElement() == '"' || currentElement() == '`'); } body { current.type = TokenType.stringLiteral; - bool isWysiwyg = currentElement() == 'r' || currentElement() == '`'; - if (currentElement() == 'r') - keepChar(); + bool isWysiwyg = buffer[0] == 'r' || currentElement() == '`'; scope (exit) { @@ -1153,9 +1114,9 @@ private: else { if (buffer[0] == 'r') - setTokenValue(bufferIndex - 1, 2); + setTokenValue(2, bufferIndex - 1); else - setTokenValue(bufferIndex - 1, 1); + setTokenValue(1, bufferIndex - 1); } } @@ -1177,20 +1138,20 @@ private: r.popFront(); if (r.front == quote && !isWysiwyg) { - keepChar(); - keepChar(); + keepNonNewlineChar(); + keepNonNewlineChar(); } else if (r.front == '\\' && !isWysiwyg) { - keepChar(); - keepChar(); + keepNonNewlineChar(); + keepNonNewlineChar(); } else keepChar(); } else if (currentElement() == quote) { - keepChar(); + keepNonNewlineChar(); break; } else @@ -1208,7 +1169,6 @@ private: { current.type = TokenType.stringLiteral; - keepChar(); keepChar(); bool heredoc; @@ -1232,7 +1192,7 @@ private: void lexNormalDelimitedString(ubyte open, ubyte close) in { - assert(buffer[0 .. bufferIndex] == "q\""); + assert(currentElement() == '"'); } body { @@ -1244,7 +1204,7 @@ private: if (config.tokenStyle & TokenStyle.includeQuotes) setTokenValue(); else - setTokenValue(bufferIndex - 2, 3); + setTokenValue(3, bufferIndex - 2); } while (true) { @@ -1327,7 +1287,7 @@ private: size_t e = bufferIndex; if (buffer[e - 1] == 'c' || buffer[e - 1] == 'd' || buffer[e - 1] == 'w') --e; - setTokenValue(e, b); + setTokenValue(b, e); } } @@ -1360,13 +1320,12 @@ private: void lexTokenString() in { - assert (currentElement() == 'q'); + assert (currentElement() == '{'); } body { current.type = TokenType.stringLiteral; keepChar(); - keepChar(); LexerConfig c = config; config.iterStyle = IterationStyle.everything; config.tokenStyle = TokenStyle.source; @@ -1392,13 +1351,13 @@ private: config = c; buffer[0] = 'q'; buffer[1] = '{'; - buffer[2 .. bi] = b[0 .. bi]; + buffer[2 .. bi + 2] = b[0 .. bi]; buffer[bi++] = '}'; bufferIndex = bi; if (config.tokenStyle & TokenStyle.includeQuotes) setTokenValue(); else - setTokenValue(bufferIndex - 1, 2); + setTokenValue(2, bufferIndex - 1); lexStringSuffix(); } @@ -1481,7 +1440,7 @@ private: void keepChar() { - if (bufferIndex + 2 >= buffer.length) + while (bufferIndex + 2 >= buffer.length) buffer.length += 1024; bool foundNewline; if (currentElement() == '\r') @@ -1546,7 +1505,7 @@ private: ++index; } - void setTokenValue(size_t endIndex = 0, size_t startIndex = 0) + void setTokenValue(size_t startIndex = 0, size_t endIndex = 0) { if (endIndex == 0) endIndex = bufferIndex; @@ -2347,7 +2306,7 @@ string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString) caseStatement ~= indentString; caseStatement ~= "\t}\n"; caseStatement ~= indentString; - caseStatement ~= "\tswitch (range.front)\n"; + caseStatement ~= "\tswitch (currentElement())\n"; caseStatement ~= indentString; caseStatement ~= "\t{\n"; caseStatement ~= printCaseStatements(v, indentString ~ "\t"); @@ -2363,8 +2322,6 @@ string printCaseStatements(K, V)(TrieNode!(K,V) node, string indentString) caseStatement ~= "\t\treturn;\n"; caseStatement ~= indentString; caseStatement ~= "\t}\n"; -// caseStatement ~= indentString; -// caseStatement ~= "\treturn;\n"; } else {