Ddoc / indentation fixes. Also moved the lexer source code below the "private:" line
This commit is contained in:
Hackerpilot
parent
55ad8941b0
commit
bd54c84e3d
579
std/d/lexer.d
579
std/d/lexer.d
|
|
@ -102,7 +102,7 @@
|
|||
*
|
||||
* Copyright: Brian Schott 2013
|
||||
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0)
|
||||
* Authors: Brian Schott
|
||||
* Authors: Brian Schott, Dmitry Olshansky
|
||||
* Source: $(PHOBOSSRC std/d/_lexer.d)
|
||||
*/
|
||||
|
||||
|
|
@ -203,7 +203,7 @@ enum IterationStyle
|
|||
includeSpecialTokens = 0b0100,
|
||||
/// Do not stop iteration on reaching the ___EOF__ token
|
||||
ignoreEOF = 0b1000,
|
||||
/// Include everything
|
||||
/// Include _everything
|
||||
everything = includeComments | includeWhitespace | ignoreEOF
|
||||
}
|
||||
|
||||
|
|
@ -266,7 +266,7 @@ struct LexerConfig
|
|||
TokenStyle tokenStyle = tokenStyle.default_;
|
||||
|
||||
/**
|
||||
* Replacement for the ___VERSION__ token. Defaults to 1.
|
||||
* Replacement for the ___VERSION__ token. Defaults to 100.
|
||||
*/
|
||||
uint versionNumber = 100;
|
||||
|
||||
|
|
@ -289,12 +289,6 @@ struct LexerConfig
|
|||
* and error messsage.
|
||||
*/
|
||||
void delegate(string, size_t, uint, uint, string) errorFunc;
|
||||
|
||||
/**
|
||||
* Initial size of the lexer's internal token buffer in bytes. The lexer
|
||||
* will grow this buffer if necessary.
|
||||
*/
|
||||
size_t bufferSize = 1024 * 4;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -331,287 +325,6 @@ auto byToken(R)(R range, LexerConfig config)
|
|||
return r;
|
||||
}
|
||||
|
||||
// For now a private helper that is tailored to the way lexer works
|
||||
// hides away forwardness of range by buffering
|
||||
// RA-version is strightforward thin wrapping
|
||||
// ATM it is byte-oriented
|
||||
private struct LexSource(R)
|
||||
if(isForwardRange!R && !isRandomAccessRange!R)
|
||||
{
|
||||
bool empty() const { return _empty; }
|
||||
|
||||
auto ref front() const
|
||||
{
|
||||
return accum[accumIdx];
|
||||
}
|
||||
|
||||
auto ref peek() const
|
||||
in
|
||||
{
|
||||
assert (accumIdx + 1 < accum.length);
|
||||
}
|
||||
body
|
||||
{
|
||||
return accum[accumIdx + 1];
|
||||
}
|
||||
|
||||
void popFront()
|
||||
{
|
||||
++_index;
|
||||
range.popFront();
|
||||
// if that was last byte
|
||||
// just advance so that open-righted slice just works
|
||||
accumIdx = (accumIdx+1) & mask;
|
||||
if(range.empty)
|
||||
{
|
||||
_empty = true;
|
||||
return;
|
||||
}
|
||||
if(accumIdx == savedAccumIdx)
|
||||
{
|
||||
// and move stuff around
|
||||
auto oldLen = accum.length;
|
||||
auto toCopy = oldLen - accumIdx;
|
||||
accum.length *= 2; // keep pow of 2
|
||||
// copy starting with last item
|
||||
copy(retro(accum[accumIdx..oldLen]),
|
||||
retro(accum[$-toCopy..$]));
|
||||
savedAccumIdx = accum.length - toCopy;
|
||||
}
|
||||
accum[accumIdx] = range.front;
|
||||
}
|
||||
|
||||
auto save()
|
||||
{
|
||||
typeof(this) copy = this;
|
||||
copy.range = range.save;
|
||||
// sadly need to dup circular buffer, as it overwrites items
|
||||
copy.accum = copy.accum.dup;
|
||||
return copy;
|
||||
}
|
||||
|
||||
// mark a position to slice from later on
|
||||
size_t mark()
|
||||
{
|
||||
savedAccumIdx = accumIdx;
|
||||
return accumIdx;
|
||||
}
|
||||
|
||||
// slice to current position from previously marked position
|
||||
auto slice() @property
|
||||
{
|
||||
// it's an open right range as usual
|
||||
return CircularRange(accum, savedAccumIdx, accumIdx);
|
||||
}
|
||||
|
||||
size_t index() const @property
|
||||
{
|
||||
return _index;
|
||||
}
|
||||
|
||||
private:
|
||||
this(R src, size_t bufferSize)
|
||||
{
|
||||
range = src;
|
||||
assert(bufferSize > 0);
|
||||
assert((bufferSize & (bufferSize-1)) == 0); //is power of 2
|
||||
accum = new ubyte[bufferSize];
|
||||
if(range.empty)
|
||||
_empty = true;
|
||||
else
|
||||
accum[accumIdx] = range.front; // load front
|
||||
}
|
||||
|
||||
// a true RA-range of ubyte
|
||||
struct CircularRange
|
||||
{
|
||||
this(ubyte[] buf, size_t s, size_t e)
|
||||
{
|
||||
assert((buffer.length & (buffer.length-1)) == 0);
|
||||
buffer = buf;
|
||||
start = s;
|
||||
end = e;
|
||||
}
|
||||
//Forward range primitives
|
||||
@property bool empty() const { return start == end; }
|
||||
@property auto ref front() const { return buffer[start]; }
|
||||
void popFront() { start = (start + 1) & mask; }
|
||||
@property auto save() { return this; }
|
||||
|
||||
//Backwards is a bit slower, but should be rarely used (if at all)
|
||||
@property ref back(){ return buffer[(end-1) & mask]; }
|
||||
void popBack() { end = (end - 1) & mask; }
|
||||
|
||||
// RA range primitives
|
||||
ref opIndex(size_t idx){ return buffer[(start+idx) & mask]; }
|
||||
@property size_t length()
|
||||
{
|
||||
return end < start ? end + buffer.length -start : end - start;
|
||||
}
|
||||
alias length opDollar;
|
||||
|
||||
auto opSlice(size_t newStart, size_t newEnd)
|
||||
{
|
||||
size_t maskedStart = (start+newStart) & mask;
|
||||
size_t maskedEnd = (start+newEnd) & mask;
|
||||
return typeof(this)(buffer, maskedStart, maskedEnd);
|
||||
}
|
||||
// @@@bug fwd-ref in ldc0.10 (if placed above previous one)
|
||||
auto opSlice(){ return opSlice(0, length); }
|
||||
private:
|
||||
@property auto mask(){ return buffer.length-1; }
|
||||
size_t start, end;
|
||||
ubyte[] buffer;
|
||||
}
|
||||
|
||||
@property auto mask(){ return accum.length-1; }
|
||||
|
||||
R range;
|
||||
bool _empty;
|
||||
ubyte[] accum; // accumulator buffer for non-RA ranges
|
||||
size_t savedAccumIdx;
|
||||
size_t accumIdx; // current index in accumulator
|
||||
size_t _index; // index of current element in original range
|
||||
}
|
||||
|
||||
// TODO: make sure it's RandomAccess later
|
||||
/*static assert(isRandomAccessRange!(
|
||||
LexSource!(typeof(filter!"true"(cast(ubyte[])null)))
|
||||
.CircularRange)
|
||||
);*/
|
||||
|
||||
//trivial pass-through for RA ranges
|
||||
private struct LexSource(R)
|
||||
if(isRandomAccessRange!R)
|
||||
{
|
||||
bool empty() const @property { return cur >= range.length; }
|
||||
bool canPeek() const { return cur + 1 < range.length; }
|
||||
auto ref front() const @property { return range[cur]; }
|
||||
void popFront(){ cur++; }
|
||||
|
||||
auto ref peek() const
|
||||
in
|
||||
{
|
||||
assert (canPeek());
|
||||
}
|
||||
body
|
||||
{
|
||||
return range[cur + 1];
|
||||
}
|
||||
|
||||
auto save()
|
||||
{
|
||||
typeof(this) copy = this;
|
||||
copy.range = range.save;
|
||||
return copy;
|
||||
}
|
||||
|
||||
auto mark()
|
||||
{
|
||||
saved = cur;
|
||||
}
|
||||
|
||||
// use the underliying range slicing capability
|
||||
auto slice() @property
|
||||
{
|
||||
return range[saved..cur];
|
||||
}
|
||||
|
||||
size_t index() const @property
|
||||
{
|
||||
return cur;
|
||||
}
|
||||
|
||||
private:
|
||||
this(R src)
|
||||
{
|
||||
range = src;
|
||||
}
|
||||
size_t cur, saved;
|
||||
R range;
|
||||
}
|
||||
|
||||
auto lexerSource(Range)(Range range, size_t bufSize=8)
|
||||
if(isForwardRange!Range && !isRandomAccessRange!Range
|
||||
&& is(ElementType!Range : const(ubyte)))
|
||||
{
|
||||
return LexSource!(Range)(range, bufSize);
|
||||
}
|
||||
|
||||
auto lexerSource(Range)(Range range)
|
||||
if(isRandomAccessRange!Range
|
||||
&& is(ElementType!Range : const(ubyte)))
|
||||
{
|
||||
return LexSource!(Range)(range);
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
// test the basic functionality of a "mark-slice" range
|
||||
import std.string, std.stdio;
|
||||
|
||||
static void test_hello(T)(T lexs)
|
||||
{
|
||||
assert(lexs.front == 'H');
|
||||
lexs.popFront();
|
||||
assert(lexs.front == 'e');
|
||||
foreach(i; 0..2)
|
||||
{
|
||||
auto saved = lexs.save;
|
||||
lexs.mark();
|
||||
assert(lexs.slice.equal(""));
|
||||
lexs.popFront();
|
||||
assert(lexs.slice.equal("e"), text(cast(char)lexs.front));
|
||||
lexs.popFrontN(4);
|
||||
auto bytes = lexs.slice.map!"cast(char)a".array();
|
||||
assert(bytes.equal("ello,"), bytes.to!string);
|
||||
lexs.mark();
|
||||
assert(lexs.slice.equal(""));
|
||||
assert(lexs.front == 'w');
|
||||
lexs.popFrontN(6);
|
||||
assert(lexs.empty);
|
||||
auto s = lexs.slice();
|
||||
auto msg = s.save.map!"cast(char)a".array;
|
||||
assert(s[].equal("world!"), msg);
|
||||
assert(s[2..$-1].equal("rld"), msg);
|
||||
assert(s[0] == 'w' && s[$-1] == '!');
|
||||
s.popFront();
|
||||
assert(s.front == 'o' && s.back == '!');
|
||||
s.popBack();
|
||||
assert(s.front == 'o' && s.back == 'd');
|
||||
//restore and repeat again
|
||||
lexs = saved;
|
||||
}
|
||||
}
|
||||
|
||||
static void test_empty(T)(T lexs)
|
||||
{
|
||||
assert(lexs.empty);
|
||||
lexs.mark();
|
||||
assert(lexs.slice().equal(""));
|
||||
}
|
||||
|
||||
auto fwdLex = lexerSource(
|
||||
"Hello, world!"
|
||||
.representation
|
||||
.filter!"a != ' '", 16 // and the one that is more then enough
|
||||
);
|
||||
test_hello(fwdLex);
|
||||
fwdLex = lexerSource(
|
||||
"Hello, world!"
|
||||
.representation
|
||||
.filter!"a != ' '", 1 // try the smallest initial buffer
|
||||
);
|
||||
test_hello(fwdLex);
|
||||
fwdLex = lexerSource("".representation.filter!"a != ' '");
|
||||
auto raLex = lexerSource("".representation);
|
||||
test_empty(raLex);
|
||||
test_empty(fwdLex);
|
||||
raLex = lexerSource("Hello,world!".representation);
|
||||
test_hello(raLex);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Range of tokens. Use byToken$(LPAREN)$(RPAREN) to instantiate.
|
||||
*/
|
||||
|
|
@ -2534,16 +2247,293 @@ enum TokenType: ushort
|
|||
// Implementation details follow
|
||||
private:
|
||||
|
||||
// For now a private helper that is tailored to the way lexer works
|
||||
// hides away forwardness of range by buffering
|
||||
// RA-version is strightforward thin wrapping
|
||||
// ATM it is byte-oriented
|
||||
private struct LexSource(R)
|
||||
if(isForwardRange!R && !isRandomAccessRange!R)
|
||||
{
|
||||
bool empty() const { return _empty; }
|
||||
|
||||
auto ref front() const
|
||||
{
|
||||
return accum[accumIdx];
|
||||
}
|
||||
|
||||
auto ref peek() const
|
||||
in
|
||||
{
|
||||
assert (accumIdx + 1 < accum.length);
|
||||
}
|
||||
body
|
||||
{
|
||||
return accum[accumIdx + 1];
|
||||
}
|
||||
|
||||
void popFront()
|
||||
{
|
||||
++_index;
|
||||
range.popFront();
|
||||
// if that was last byte
|
||||
// just advance so that open-righted slice just works
|
||||
accumIdx = (accumIdx+1) & mask;
|
||||
if(range.empty)
|
||||
{
|
||||
_empty = true;
|
||||
return;
|
||||
}
|
||||
if(accumIdx == savedAccumIdx)
|
||||
{
|
||||
// and move stuff around
|
||||
auto oldLen = accum.length;
|
||||
auto toCopy = oldLen - accumIdx;
|
||||
accum.length *= 2; // keep pow of 2
|
||||
// copy starting with last item
|
||||
copy(retro(accum[accumIdx..oldLen]),
|
||||
retro(accum[$-toCopy..$]));
|
||||
savedAccumIdx = accum.length - toCopy;
|
||||
}
|
||||
accum[accumIdx] = range.front;
|
||||
}
|
||||
|
||||
auto save()
|
||||
{
|
||||
typeof(this) copy = this;
|
||||
copy.range = range.save;
|
||||
// sadly need to dup circular buffer, as it overwrites items
|
||||
copy.accum = copy.accum.dup;
|
||||
return copy;
|
||||
}
|
||||
|
||||
// mark a position to slice from later on
|
||||
size_t mark()
|
||||
{
|
||||
savedAccumIdx = accumIdx;
|
||||
return accumIdx;
|
||||
}
|
||||
|
||||
// slice to current position from previously marked position
|
||||
auto slice() @property
|
||||
{
|
||||
// it's an open right range as usual
|
||||
return CircularRange(accum, savedAccumIdx, accumIdx);
|
||||
}
|
||||
|
||||
size_t index() const @property
|
||||
{
|
||||
return _index;
|
||||
}
|
||||
|
||||
private:
|
||||
this(R src, size_t bufferSize)
|
||||
{
|
||||
range = src;
|
||||
assert(bufferSize > 0);
|
||||
assert((bufferSize & (bufferSize-1)) == 0); //is power of 2
|
||||
accum = new ubyte[bufferSize];
|
||||
if(range.empty)
|
||||
_empty = true;
|
||||
else
|
||||
accum[accumIdx] = range.front; // load front
|
||||
}
|
||||
|
||||
// a true RA-range of ubyte
|
||||
struct CircularRange
|
||||
{
|
||||
this(ubyte[] buf, size_t s, size_t e)
|
||||
{
|
||||
assert((buffer.length & (buffer.length-1)) == 0);
|
||||
buffer = buf;
|
||||
start = s;
|
||||
end = e;
|
||||
}
|
||||
//Forward range primitives
|
||||
@property bool empty() const { return start == end; }
|
||||
@property auto ref front() const { return buffer[start]; }
|
||||
void popFront() { start = (start + 1) & mask; }
|
||||
@property auto save() { return this; }
|
||||
|
||||
//Backwards is a bit slower, but should be rarely used (if at all)
|
||||
@property ref back(){ return buffer[(end-1) & mask]; }
|
||||
void popBack() { end = (end - 1) & mask; }
|
||||
|
||||
// RA range primitives
|
||||
ref opIndex(size_t idx){ return buffer[(start+idx) & mask]; }
|
||||
@property size_t length()
|
||||
{
|
||||
return end < start ? end + buffer.length -start : end - start;
|
||||
}
|
||||
alias length opDollar;
|
||||
|
||||
auto opSlice(size_t newStart, size_t newEnd)
|
||||
{
|
||||
size_t maskedStart = (start+newStart) & mask;
|
||||
size_t maskedEnd = (start+newEnd) & mask;
|
||||
return typeof(this)(buffer, maskedStart, maskedEnd);
|
||||
}
|
||||
// @@@bug fwd-ref in ldc0.10 (if placed above previous one)
|
||||
auto opSlice(){ return opSlice(0, length); }
|
||||
private:
|
||||
@property auto mask(){ return buffer.length-1; }
|
||||
size_t start, end;
|
||||
ubyte[] buffer;
|
||||
}
|
||||
|
||||
@property auto mask(){ return accum.length-1; }
|
||||
|
||||
R range;
|
||||
bool _empty;
|
||||
ubyte[] accum; // accumulator buffer for non-RA ranges
|
||||
size_t savedAccumIdx;
|
||||
size_t accumIdx; // current index in accumulator
|
||||
size_t _index; // index of current element in original range
|
||||
}
|
||||
|
||||
// TODO: make sure it's RandomAccess later
|
||||
/*static assert(isRandomAccessRange!(
|
||||
LexSource!(typeof(filter!"true"(cast(ubyte[])null)))
|
||||
.CircularRange)
|
||||
);*/
|
||||
|
||||
//trivial pass-through for RA ranges
|
||||
private struct LexSource(R)
|
||||
if(isRandomAccessRange!R)
|
||||
{
|
||||
bool empty() const @property { return cur >= range.length; }
|
||||
bool canPeek() const { return cur + 1 < range.length; }
|
||||
auto ref front() const @property { return range[cur]; }
|
||||
void popFront(){ cur++; }
|
||||
|
||||
auto ref peek() const
|
||||
in
|
||||
{
|
||||
assert (canPeek());
|
||||
}
|
||||
body
|
||||
{
|
||||
return range[cur + 1];
|
||||
}
|
||||
|
||||
auto save()
|
||||
{
|
||||
typeof(this) copy = this;
|
||||
copy.range = range.save;
|
||||
return copy;
|
||||
}
|
||||
|
||||
auto mark()
|
||||
{
|
||||
saved = cur;
|
||||
}
|
||||
|
||||
// use the underliying range slicing capability
|
||||
auto slice() @property
|
||||
{
|
||||
return range[saved..cur];
|
||||
}
|
||||
|
||||
size_t index() const @property
|
||||
{
|
||||
return cur;
|
||||
}
|
||||
|
||||
private:
|
||||
this(R src)
|
||||
{
|
||||
range = src;
|
||||
}
|
||||
size_t cur, saved;
|
||||
R range;
|
||||
}
|
||||
|
||||
auto lexerSource(Range)(Range range, size_t bufSize=8)
|
||||
if(isForwardRange!Range && !isRandomAccessRange!Range
|
||||
&& is(ElementType!Range : const(ubyte)))
|
||||
{
|
||||
return LexSource!(Range)(range, bufSize);
|
||||
}
|
||||
|
||||
auto lexerSource(Range)(Range range)
|
||||
if(isRandomAccessRange!Range
|
||||
&& is(ElementType!Range : const(ubyte)))
|
||||
{
|
||||
return LexSource!(Range)(range);
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
// test the basic functionality of a "mark-slice" range
|
||||
import std.string, std.stdio;
|
||||
|
||||
static void test_hello(T)(T lexs)
|
||||
{
|
||||
assert(lexs.front == 'H');
|
||||
lexs.popFront();
|
||||
assert(lexs.front == 'e');
|
||||
foreach(i; 0..2)
|
||||
{
|
||||
auto saved = lexs.save;
|
||||
lexs.mark();
|
||||
assert(lexs.slice.equal(""));
|
||||
lexs.popFront();
|
||||
assert(lexs.slice.equal("e"), text(cast(char)lexs.front));
|
||||
lexs.popFrontN(4);
|
||||
auto bytes = lexs.slice.map!"cast(char)a".array();
|
||||
assert(bytes.equal("ello,"), bytes.to!string);
|
||||
lexs.mark();
|
||||
assert(lexs.slice.equal(""));
|
||||
assert(lexs.front == 'w');
|
||||
lexs.popFrontN(6);
|
||||
assert(lexs.empty);
|
||||
auto s = lexs.slice();
|
||||
auto msg = s.save.map!"cast(char)a".array;
|
||||
assert(s[].equal("world!"), msg);
|
||||
assert(s[2..$-1].equal("rld"), msg);
|
||||
assert(s[0] == 'w' && s[$-1] == '!');
|
||||
s.popFront();
|
||||
assert(s.front == 'o' && s.back == '!');
|
||||
s.popBack();
|
||||
assert(s.front == 'o' && s.back == 'd');
|
||||
//restore and repeat again
|
||||
lexs = saved;
|
||||
}
|
||||
}
|
||||
|
||||
static void test_empty(T)(T lexs)
|
||||
{
|
||||
assert(lexs.empty);
|
||||
lexs.mark();
|
||||
assert(lexs.slice().equal(""));
|
||||
}
|
||||
|
||||
auto fwdLex = lexerSource(
|
||||
"Hello, world!"
|
||||
.representation
|
||||
.filter!"a != ' '", 16 // and the one that is more then enough
|
||||
);
|
||||
test_hello(fwdLex);
|
||||
fwdLex = lexerSource(
|
||||
"Hello, world!"
|
||||
.representation
|
||||
.filter!"a != ' '", 1 // try the smallest initial buffer
|
||||
);
|
||||
test_hello(fwdLex);
|
||||
fwdLex = lexerSource("".representation.filter!"a != ' '");
|
||||
auto raLex = lexerSource("".representation);
|
||||
test_empty(raLex);
|
||||
test_empty(fwdLex);
|
||||
raLex = lexerSource("Hello,world!".representation);
|
||||
test_hello(raLex);
|
||||
}
|
||||
|
||||
// uses auto-detection for pure, safe nothrow
|
||||
bool isRangeEoF(R)(ref R range)
|
||||
{
|
||||
return range.empty || range.front == 0 || range.front == 0x1a;
|
||||
}
|
||||
|
||||
/*
|
||||
* Slices of the above string to save memory. This array is automatically
|
||||
* generated.
|
||||
*/
|
||||
// Lookup table for token values
|
||||
immutable(string[TokenType.max + 1]) tokenValues = [
|
||||
"=",
|
||||
"@",
|
||||
|
|
@ -3394,5 +3384,4 @@ unittest
|
|||
assert (tokenCount == 16);
|
||||
}
|
||||
|
||||
|
||||
//void main(string[] args){}
|
||||
|
|
|
|||
Loading…
Reference in New Issue