diff --git a/build.sh b/build.sh index f0f544f..bbf0ff4 100755 --- a/build.sh +++ b/build.sh @@ -1,2 +1,2 @@ -dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline -#dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest +#dmd *.d std/d/*.d -release -noboundscheck -O -w -wi -m64 -property -ofdscanner -L-lsqlite3 #-inline +dmd *.d std/d/*.d -g -m64 -w -wi -property -ofdscanner -L-lsqlite3 #-unittest diff --git a/circularbuffer.d b/circularbuffer.d index 9c45128..11e1053 100644 --- a/circularbuffer.d +++ b/circularbuffer.d @@ -14,28 +14,28 @@ class CircularBuffer(T) : InputRange!(T) { public: - this (size_t size, InputRange!(T) range) - { - this.range = range; - this.margin = size; - data = new T[(margin * 2) + 1]; - if (range.empty()) - { - _empty = true; - return; - } - for (size_t i = 0; i <= margin && !this.range.empty(); ++i) - { - data[i] = this.range.front(); - this.range.popFront(); + this (size_t size, InputRange!(T) range) + { + this.range = range; + this.margin = size; + data = new T[(margin * 2) + 1]; + if (range.empty()) + { + _empty = true; + return; + } + for (size_t i = 0; i <= margin && !this.range.empty(); ++i) + { + data[i] = this.range.front(); + this.range.popFront(); end++; - } - } + } + } - override T front() const @property - { - return data[index]; - } + override T front() @property + { + return data[index]; + } T peek(int offset = 1) in @@ -52,32 +52,32 @@ public: return abs(offset) <= margin && sourceIndex + offset >= 0; } - override void popFront() + override void popFront() in { assert (!_empty); } body - { + { index = (index + 1) % data.length; ++sourceIndex; - if (range.empty()) + if (range.empty()) { if (index == end) _empty = true; } - else - { + else + { data[end] = range.front(); end = (end + 1) % data.length; range.popFront(); - } - } + } + } - bool empty() const @property - { - return _empty; - } + bool empty() const @property + { + return _empty; + } override T moveFront() { @@ -94,6 +94,7 @@ public: result = dg(front); if (result) break; + popFront(); } return result; } @@ -107,18 +108,19 @@ public: result = dg(i, front); if (result) break; + popFront(); } return result; } private: - InputRange!(T) range; - immutable size_t margin; - T[] data; + InputRange!(T) range; + immutable size_t margin; + T[] data; size_t sourceIndex; size_t end; - size_t index; - bool _empty; + size_t index; + bool _empty; } unittest @@ -131,10 +133,10 @@ unittest unittest { - int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - auto buf = CircularBuffer!(int, int[])(2, arr); - assert (buf.data.length == 5); - auto iterated = array(buf); + int[] arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + auto buf = CircularBuffer!(int, int[])(2, arr); + assert (buf.data.length == 5); + auto iterated = array(buf); assert (iterated == arr); } diff --git a/highlighter.d b/highlighter.d index 93ce5f2..ccd1673 100644 --- a/highlighter.d +++ b/highlighter.d @@ -10,8 +10,6 @@ import std.stdio; import std.array; import std.d.lexer; -import langutils; - void writeSpan(string cssClass, string value) { stdout.write(``, value.replace("&", "&").replace("<", "<"), ``); diff --git a/langutils.d b/langutils.d index cf0c995..00bd391 100644 --- a/langutils.d +++ b/langutils.d @@ -6,6 +6,7 @@ module langutils; import std.array; +import std.algorithm; import std.d.lexer; @@ -43,7 +44,7 @@ string combineTokens(ref const Token[] tokens) return app.data; } -pure string getTypeFromToken(const Token t) +pure nothrow string getTypeFromToken(const Token t) { switch (t.type) { @@ -73,8 +74,14 @@ pure string getTypeFromToken(const Token t) } } -pure bool isIdentifierOrType(inout Token t) +pure bool isIdentifierOrType(const Token t) { return t.type == TokenType.Identifier || (t.type > TokenType.TYPES_BEGIN && TokenType.TYPES_END); } + +pure bool isDocComment(ref const Token t) +{ + return t.value.startsWith("///") || t.value.startsWith("/**") + || t.value.startsWith("/++"); +} diff --git a/main.d b/main.d index 55058f1..b1d00ec 100644 --- a/main.d +++ b/main.d @@ -15,6 +15,7 @@ import std.parallelism; import std.path; import std.regex; import std.stdio; +import std.range; import std.d.lexer; import autocomplete; @@ -28,7 +29,7 @@ import circularbuffer; immutable size_t CIRC_BUFF_SIZE = 4; -pure bool isLineOfCode(TokenType t) +pure nothrow bool isLineOfCode(TokenType t) { switch(t) { @@ -138,36 +139,21 @@ int main(string[] args) { if (args.length == 1) { - auto f = appender!string(); - char[] buf; - while (stdin.readln(buf)) - f.put(buf); - writeln(f.data.byToken().count!(a => isLineOfCode(a.type))()); + writeln(stdin.byLine(KeepTerminator.yes).join().byToken().count!(a => isLineOfCode(a.type))()); } else { - writeln(args[1..$].map!(a => a.readText().byToken())().joiner() - .count!(a => isLineOfCode(a.type))()); + writeln(args[1..$].map!(a => File(a).byLine(KeepTerminator.yes).join().byToken(a))() + .joiner().count!(a => isLineOfCode(a.type))()); } return 0; } if (highlight) { - if (args.length == 1) - { - auto f = appender!string(); - char[] buf; - while (stdin.readln(buf)) - f.put(buf); - highlighter.highlight(f.data.byToken("stdin", IterationStyle.Everything, - TokenStyle.Source)); - } - else - { - highlighter.highlight(args[1].readText().byToken(args[1], - IterationStyle.Everything, TokenStyle.Source)); - } + File f = args.length == 1 ? stdin : File(args[1]); + highlighter.highlight(f.byLine(KeepTerminator.yes).join().byToken( + "", IterationStyle.Everything, TokenStyle.Source)); return 0; } @@ -213,20 +199,9 @@ int main(string[] args) if (json) { CircularBuffer!(Token) tokens; - if (args.length == 1) - { - // Read from stdin - auto f = appender!string(); - char[] buf; - while (stdin.readln(buf)) - f.put(buf); - tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(f.data)); - } - else - { - // read given file - tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, byToken!string(readText(args[1]))); - } + File f = args.length == 1 ? stdin : File(args[1]); + tokens = new CircularBuffer!(Token)(CIRC_BUFF_SIZE, + f.byLine(KeepTerminator.yes).join().byToken!(char[])()); auto mod = parseModule(tokens); mod.writeJSONTo(stdout); return 0; diff --git a/parser.d b/parser.d index 70498a8..55f94ba 100644 --- a/parser.d +++ b/parser.d @@ -26,6 +26,7 @@ public: this(InputRange!Token tokens, TokenType open, TokenType close) { super(0, tokens); + this.range = tokens; this.open = open; this.close = close; } @@ -35,7 +36,7 @@ public: return _empty; } - override Token front() const @property + override Token front() @property { return range.front; } @@ -47,14 +48,20 @@ public: ++depth; else if (range.front == close) --depth; - _empty = depth == 0; + _empty = depth == 0 || range.empty; } + invariant() + { + assert (range); + assert (depth >= 0); + } + private: int depth; TokenType open; TokenType close; - TokenBuffer range; + InputRange!(Token) range; bool _empty; } diff --git a/std/d/lexer.d b/std/d/lexer.d index f7073b5..682f04e 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -105,8 +105,8 @@ import std.d.entities; public: /** - * Represents a D token - */ +* Represents a D token +*/ struct Token { /// The token type. @@ -122,28 +122,28 @@ struct Token uint startIndex; /** - * Check to see if the token is of the same type and has the same string - * representation as the given token. - */ + * Check to see if the token is of the same type and has the same string + * representation as the given token. + */ bool opEquals(ref const(Token) other) const { return other.type == type && other.value == value; } /** - * Checks to see if the token's string representation is equal to the given - * string. - */ + * Checks to see if the token's string representation is equal to the given + * string. + */ bool opEquals(string value) const { return this.value == value; } /** - * Checks to see if the token is of the given type. - */ + * Checks to see if the token is of the given type. + */ bool opEquals(TokenType type) const { return type == type; } /** - * Comparison operator orders tokens by start index. - */ + * Comparison operator orders tokens by start index. + */ int opCmp(size_t i) const { if (startIndex < i) return -1; @@ -164,16 +164,16 @@ enum IterationStyle IncludeComments = 0b0001, /// Includes whitespace IncludeWhitespace = 0b0010, - /// Include $(LINK2 http://dlang.org/lex.html#Special%20Tokens%20Sequence, special token sequences) + /// Include $(LINK2 http://dlang.org/lex.html#specialtokens, special tokens) IncludeSpecialTokens = 0b0100, - /// Do not terminate iteration upon reaching the ___EOF__ token + /// Do not stop iteration on reaching the ___EOF__ token IgnoreEOF = 0b1000, - /// Include everything, including the __EOF__ token. + /// Include everything Everything = IncludeComments | IncludeWhitespace | IgnoreEOF } /** - * Configuration of the string lexing style. These flags may be combined with a + * Configuration of the token lexing style. These flags may be combined with a * bitwise or. */ enum TokenStyle : uint @@ -187,10 +187,10 @@ enum TokenStyle : uint Default = 0b0000, /** - * Escape sequences will not be processed. An escaped quote character will - * not terminate string lexing, but it will not be replaced with the quote - * character in the token. - */ + * Escape sequences will not be processed. An escaped quote character will + * not terminate string lexing, but it will not be replaced with the quote + * character in the token. + */ NotEscaped = 0b0001, /** @@ -259,8 +259,8 @@ struct TokenRange(R) if (isForwardRange!(R) && is(ElementType!(R) == char)) } /** - * Returns: the current token - */ + * Returns: the current token + */ override Token front() const @property { enforce(!_empty, "Cannot call front() on empty token range"); @@ -268,8 +268,8 @@ struct TokenRange(R) if (isForwardRange!(R) && is(ElementType!(R) == char)) } /** - * Returns the current token and then removes it from the range - */ + * Returns the current token and then removes it from the range + */ override Token moveFront() { auto r = front(); @@ -2486,7 +2486,6 @@ pure nothrow TokenType lookupTokenType(const string input) default: break; } break; - case 6: switch (input) { @@ -2595,8 +2594,8 @@ pure nothrow TokenType lookupTokenType(const string input) class Trie(K, V) if (isInputRange!K): TrieNode!(K, V) { /** - * Adds the given value to the trie with the given key - */ + * Adds the given value to the trie with the given key + */ void add(K key, V value) pure { TrieNode!(K,V) current = this;