forks
/
dlangide
mirror of https://github.com/buggins/dlangide.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tokenized and source edit fixes

This commit is contained in:
Vadim Lopatin 2015-01-20 18:10:07 +03:00
parent 1c514145b6
commit 03f06f5c64
3 changed files with 149 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/app.d
View File

@ -38,7 +38,7 @@ extern (C) int UIAppMain(string[] args) {
// you can override default hinting mode here // you can override default hinting mode here
FontManager.instance.hintingMode = HintingMode.Normal; FontManager.instance.hintingMode = HintingMode.Normal;
// you can override antialiasing setting here // you can override antialiasing setting here
FontManager.instance.minAnitialiasedFontSize = 0; FontManager.instance.minAnitialiasedFontSize = 25;
// create window // create window
Window window = Platform.instance.createWindow("Dlang IDE", null); Window window = Platform.instance.createWindow("Dlang IDE", null);

174
src/ddc/lexer/Tokenizer.d
View File

@ -897,22 +897,24 @@ class Token {
protected int _line; protected int _line;
protected int _pos; protected int _pos;
protected TokenType _type; protected TokenType _type;
public @property TokenType type() { return _type; } @property TokenType type() { return _type; }
public @property string filename() { return _file.filename; } @property string filename() { return _file.filename; }
public @property int line() { return _line; } @property int line() { return _line; }
public @property int pos() { return _pos; } @property int pos() { return _pos; }
public @property dchar[] text() { return null; } @property dchar[] text() { return null; }
public @property dchar literalType() { return 0; } @property dchar literalType() { return 0; }
public @property ulong intValue() { return 0; } @property ulong intValue() { return 0; }
public @property bool isUnsigned() { return false; } @property bool isUnsigned() { return false; }
public @property ulong isLong() { return false; } @property ulong isLong() { return false; }
public @property real realValue() { return 0; } @property real realValue() { return 0; }
public @property double doubleValue() { return 0; } @property double doubleValue() { return 0; }
public @property float floatValue() { return 0; } @property float floatValue() { return 0; }
public @property byte precision() { return 0; } @property byte precision() { return 0; }
public @property bool isImaginary() { return false; } @property bool isImaginary() { return false; }
public @property OpCode opCode() { return OpCode.NONE; } @property OpCode opCode() { return OpCode.NONE; }
public @property Keyword keyword() { return Keyword.NONE; } @property Keyword keyword() { return Keyword.NONE; }
@property bool isDocumentationComment() { return false; }
this(TokenType type) { this(TokenType type) {
_type = type; _type = type;
@ -1025,9 +1027,19 @@ class KeywordToken : Token {
// do we need comment text? // do we need comment text?
class CommentToken : Token { class CommentToken : Token {
dchar[] _text; protected dchar[] _text;
public @property override dchar[] text() { return _text; } protected bool _isDocumentationComment;
public @property void text(dchar[] text) { _text = text; }
override @property bool isDocumentationComment() {
return _isDocumentationComment;
}
@property void isDocumentationComment(bool f) {
_isDocumentationComment = f;
}
@property override dchar[] text() { return _text; }
@property void text(dchar[] text) { _text = text; }
this() { this() {
super(TokenType.COMMENT); super(TokenType.COMMENT);
} }
@ -1258,35 +1270,50 @@ struct StringAppender {
class Tokenizer class Tokenizer
{ {
SourceLines _lineStream; protected SourceLines _lineStream;
dchar[] _lineText; protected dchar[] _lineText;
int _line; // current line number protected int _line; // current line number
int _len; // current line length protected int _len; // current line length
int _pos; // current line read position protected int _pos; // current line read position
int _prevLineLength; // previous line length protected int _prevLineLength; // previous line length
uint _state; // tokenizer state protected uint _state; // tokenizer state
enum : int { enum : int {
EOF_CHAR = 0x001A, EOF_CHAR = 0x001A,
EOL_CHAR = 0x000A EOL_CHAR = 0x000A
}; };
WhiteSpaceToken _sharedWhiteSpaceToken = new WhiteSpaceToken(); protected WhiteSpaceToken _sharedWhiteSpaceToken = new WhiteSpaceToken();
CommentToken _sharedCommentToken = new CommentToken(); protected CommentToken _sharedCommentToken = new CommentToken();
StringLiteralToken _sharedStringLiteralToken = new StringLiteralToken(); protected StringLiteralToken _sharedStringLiteralToken = new StringLiteralToken();
IdentToken _sharedIdentToken = new IdentToken(); protected IdentToken _sharedIdentToken = new IdentToken();
OpToken _sharedOpToken = new OpToken(); protected OpToken _sharedOpToken = new OpToken();
KeywordToken _sharedKeywordToken = new KeywordToken(); protected KeywordToken _sharedKeywordToken = new KeywordToken();
IntegerLiteralToken _sharedIntegerToken = new IntegerLiteralToken(); protected IntegerLiteralToken _sharedIntegerToken = new IntegerLiteralToken();
RealLiteralToken _sharedRealToken = new RealLiteralToken(); protected RealLiteralToken _sharedRealToken = new RealLiteralToken();
StringAppender _stringLiteralAppender; protected StringAppender _stringLiteralAppender;
StringAppender _commentAppender; protected StringAppender _commentAppender;
StringAppender _identAppender; protected StringAppender _identAppender;
bool _enableCommentText = true; protected bool _enableCommentText = true;
public void enableCommentText(bool enabled) { /// when false, does not put comment text into comment token - for less allocations
@property void enableCommentText(bool enabled) {
_enableCommentText = enabled; _enableCommentText = enabled;
} }
/// when false, does not put comment text into comment token - for less allocations
@property bool enableCommentText() {
return _enableCommentText;
}
protected bool _errorTolerant = false;
/// when true, returns BadToken instead of throwing exception
@property void errorTolerant(bool enabled) {
_errorTolerant = enabled;
}
/// when true, returns BadToken instead of throwing exception
@property bool errorTolerant() {
return _errorTolerant;
}
this(SourceLines lineStream) { this(SourceLines lineStream) {
init(lineStream); init(lineStream);
@ -1314,7 +1341,7 @@ class Tokenizer
} }
// fetch next line from source stream // fetch next line from source stream
bool nextLine() { protected bool nextLine() {
_prevLineLength = _lineText.length; _prevLineLength = _lineText.length;
_lineText = _lineStream.readLine(); _lineText = _lineStream.readLine();
if (!_lineText) { if (!_lineText) {
@ -1334,7 +1361,7 @@ class Tokenizer
return true; return true;
} }
dchar nextChar() { protected dchar nextChar() {
if (_lineText is null) { if (_lineText is null) {
if (!nextLine()) { if (!nextLine()) {
return EOF_CHAR; return EOF_CHAR;
@ -1348,7 +1375,7 @@ class Tokenizer
return _lineText[_pos++]; return _lineText[_pos++];
} }
dchar peekChar() { protected dchar peekChar() {
if (_lineText is null) { if (_lineText is null) {
if (!nextLine()) { if (!nextLine()) {
return EOF_CHAR; return EOF_CHAR;
@ -1359,12 +1386,12 @@ class Tokenizer
return _lineText[_pos++]; return _lineText[_pos++];
} }
Token emitEof() { protected Token emitEof() {
// TODO: check for current state // TODO: check for current state
return new EofToken(_lineStream.file, _line, _pos); return new EofToken(_lineStream.file, _line, _pos);
} }
Token processWhiteSpace(dchar firstChar) { protected Token processWhiteSpace(dchar firstChar) {
// reuse the same token instance, to avoid extra heap spamming // reuse the same token instance, to avoid extra heap spamming
if (_pos == 0) { if (_pos == 0) {
_sharedWhiteSpaceToken.setPos(_line - 1, _prevLineLength); _sharedWhiteSpaceToken.setPos(_line - 1, _prevLineLength);
@ -1388,8 +1415,9 @@ class Tokenizer
return _sharedWhiteSpaceToken; return _sharedWhiteSpaceToken;
} }
Token processOneLineComment() { protected Token processOneLineComment() {
_sharedCommentToken.setPos(_line, _pos - 1); _sharedCommentToken.setPos(_line, _pos - 1);
_sharedCommentToken.isDocumentationComment = _pos + 1 < _lineText.length && _lineText[_pos + 1] == '/';
if (_enableCommentText) { if (_enableCommentText) {
_sharedCommentToken.text = _lineText[_pos + 1 .. $]; _sharedCommentToken.text = _lineText[_pos + 1 .. $];
} }
@ -1397,7 +1425,7 @@ class Tokenizer
return _sharedCommentToken; return _sharedCommentToken;
} }
Token processOneLineSharpComment() { protected Token processOneLineSharpComment() {
_sharedCommentToken.setPos(_line, _pos - 1); _sharedCommentToken.setPos(_line, _pos - 1);
if (_enableCommentText) { if (_enableCommentText) {
_sharedCommentToken.text = _lineText[_pos .. $]; _sharedCommentToken.text = _lineText[_pos .. $];
@ -1407,8 +1435,9 @@ class Tokenizer
} }
// Comment /* */ // Comment /* */
Token processMultilineComment() { protected Token processMultilineComment() {
_sharedCommentToken.setPos(_line, _pos - 1); _sharedCommentToken.setPos(_line, _pos - 1);
_sharedCommentToken.isDocumentationComment = _pos + 1 < _lineText.length && _lineText[_pos + 1] == '*';
_commentAppender.reset(); _commentAppender.reset();
int textStart = _pos + 1; int textStart = _pos + 1;
for (;;) { for (;;) {
@ -1439,9 +1468,10 @@ class Tokenizer
return _sharedCommentToken; return _sharedCommentToken;
} }
// Comment /* */ // Comment /+ +/
Token processNestedComment() { protected Token processNestedComment() {
_sharedCommentToken.setPos(_line, _pos - 1); _sharedCommentToken.setPos(_line, _pos - 1);
_sharedCommentToken.isDocumentationComment = _pos + 1 < _lineText.length && _lineText[_pos + 1] == '+';
_commentAppender.reset(); _commentAppender.reset();
dchar[] text; dchar[] text;
int textStart = _pos + 1; int textStart = _pos + 1;
@ -1481,26 +1511,26 @@ class Tokenizer
return _sharedCommentToken; return _sharedCommentToken;
} }
Token processHexString() { protected Token processHexString() {
_pos++; _pos++;
// TODO: // TODO:
return null; return null;
} }
Token processDelimitedString() { protected Token processDelimitedString() {
_pos++; _pos++;
// TODO: // TODO:
return null; return null;
} }
// r"string" or `string` // r"string" or `string`
Token processWysiwygString(dchar ch) { protected Token processWysiwygString(dchar ch) {
_pos++; _pos++;
// TODO: // TODO:
return null; return null;
} }
Token processIdent() { protected Token processIdent() {
_sharedIdentToken.setPos(_line, _pos - 1); _sharedIdentToken.setPos(_line, _pos - 1);
_identAppender.reset(); _identAppender.reset();
int startPos = _pos - 1; int startPos = _pos - 1;
@ -1517,7 +1547,7 @@ class Tokenizer
return _sharedIdentToken; return _sharedIdentToken;
} }
Token processIntegerSuffix() { protected Token processIntegerSuffix() {
if (_pos >= _len) if (_pos >= _len)
return _sharedIntegerToken; return _sharedIntegerToken;
bool longFlag = false; bool longFlag = false;
@ -1546,7 +1576,7 @@ class Tokenizer
return _sharedIntegerToken; return _sharedIntegerToken;
} }
Token processBinaryNumber() { protected Token processBinaryNumber() {
_sharedIntegerToken.setPos(_line, _pos - 1); _sharedIntegerToken.setPos(_line, _pos - 1);
_pos++; _pos++;
if (_pos >= _len) if (_pos >= _len)
@ -1568,7 +1598,7 @@ class Tokenizer
return processIntegerSuffix(); return processIntegerSuffix();
} }
Token processHexNumber() { protected Token processHexNumber() {
_sharedIntegerToken.setPos(_line, _pos - 1); _sharedIntegerToken.setPos(_line, _pos - 1);
_sharedRealToken.setPos(_line, _pos - 1); _sharedRealToken.setPos(_line, _pos - 1);
_pos++; _pos++;
@ -1600,7 +1630,7 @@ class Tokenizer
return processIntegerSuffix(); return processIntegerSuffix();
} }
Token processOctNumber() { protected Token processOctNumber() {
_sharedIntegerToken.setPos(_line, _pos - 1); _sharedIntegerToken.setPos(_line, _pos - 1);
if (_pos >= _len) if (_pos >= _len)
parserError("Unexpected end of line in octal number"); parserError("Unexpected end of line in octal number");
@ -1635,14 +1665,14 @@ class Tokenizer
} }
// //
Token processDecFloatSuffix(real value) { protected Token processDecFloatSuffix(real value) {
_sharedRealToken.setValue(value); _sharedRealToken.setValue(value);
// TODO // TODO
return _sharedRealToken; return _sharedRealToken;
} }
// after E char // after E char
Token processDecFloatExponent(real value) { protected Token processDecFloatExponent(real value) {
dchar next = _pos < _len ? _lineText[_pos] : 0; dchar next = _pos < _len ? _lineText[_pos] : 0;
int sign = 1; int sign = 1;
if (next == '+') { if (next == '+') {
@ -1683,7 +1713,7 @@ class Tokenizer
return processDecFloatSuffix(value); return processDecFloatSuffix(value);
} }
Token processDecFloatSecondPart(ulong firstPart) { protected Token processDecFloatSecondPart(ulong firstPart) {
if (_pos >= _len) { if (_pos >= _len) {
_sharedRealToken.setValue(cast(real)firstPart); _sharedRealToken.setValue(cast(real)firstPart);
return _sharedRealToken; return _sharedRealToken;
@ -1722,7 +1752,7 @@ class Tokenizer
return processDecFloatSuffix(value); return processDecFloatSuffix(value);
} }
Token processDecNumber(dchar c) { protected Token processDecNumber(dchar c) {
_pos--; _pos--;
_sharedIntegerToken.setPos(_line, _pos); _sharedIntegerToken.setPos(_line, _pos);
_sharedRealToken.setPos(_line, _pos); _sharedRealToken.setPos(_line, _pos);
@ -1765,11 +1795,11 @@ class Tokenizer
return processIntegerSuffix(); return processIntegerSuffix();
} }
void parserError(string msg) { protected void parserError(string msg) {
throw new ParserException(msg, _lineStream.file.filename, _line, _pos); throw new ParserException(msg, _lineStream.file.filename, _line, _pos);
} }
Keyword detectKeyword(dchar ch) { protected Keyword detectKeyword(dchar ch) {
if (ch > 'z') if (ch > 'z')
return Keyword.NONE; return Keyword.NONE;
int len = _len - _pos; int len = _len - _pos;
@ -1928,7 +1958,7 @@ class Tokenizer
default: return Keyword.NONE; default: return Keyword.NONE;
} }
} }
OpCode detectOp(dchar ch) nothrow { protected OpCode detectOp(dchar ch) nothrow {
if (ch >= 128) if (ch >= 128)
return OpCode.NONE; return OpCode.NONE;
dchar ch2 = _pos < _len ? _lineText[_pos] : 0; dchar ch2 = _pos < _len ? _lineText[_pos] : 0;
@ -2195,7 +2225,7 @@ class Tokenizer
} }
} }
Token processDoubleQuotedOrWysiwygString(dchar delimiter) { protected Token processDoubleQuotedOrWysiwygString(dchar delimiter) {
bool wysiwyg = (delimiter == 'r' || delimiter == '`'); bool wysiwyg = (delimiter == 'r' || delimiter == '`');
//writeln("processDoubleQuotedString()"); //writeln("processDoubleQuotedString()");
_sharedStringLiteralToken.setPos(_line, _pos - 1); _sharedStringLiteralToken.setPos(_line, _pos - 1);
@ -2251,22 +2281,22 @@ class Tokenizer
return _sharedStringLiteralToken; return _sharedStringLiteralToken;
} }
SysTime buildTime; protected SysTime buildTime;
// string literal of the date of compilation "mmm dd yyyy" // string literal of the date of compilation "mmm dd yyyy"
dstring formatBuildDate() { protected dstring formatBuildDate() {
// TODO: provide proper format // TODO: provide proper format
return to!dstring(buildTime); return to!dstring(buildTime);
} }
// string literal of the time of compilation "hh:mm:ss" // string literal of the time of compilation "hh:mm:ss"
dstring formatBuildTime() { protected dstring formatBuildTime() {
// TODO: provide proper format // TODO: provide proper format
return to!dstring(buildTime); return to!dstring(buildTime);
} }
// string literal of the date and time of compilation "www mmm dd hh:mm:ss yyyy" // string literal of the date and time of compilation "www mmm dd hh:mm:ss yyyy"
dstring formatBuildTimestamp() { protected dstring formatBuildTimestamp() {
// TODO: provide proper format // TODO: provide proper format
return to!dstring(buildTime); return to!dstring(buildTime);
} }
@ -2274,13 +2304,13 @@ class Tokenizer
static immutable dstring VERSION = "0.1"; static immutable dstring VERSION = "0.1";
static immutable dstring VENDOR = "coolreader.org"; static immutable dstring VENDOR = "coolreader.org";
Token makeSpecialTokenString(dstring str, int pos) { protected Token makeSpecialTokenString(dstring str, int pos) {
_sharedStringLiteralToken.setPos(_line, pos); _sharedStringLiteralToken.setPos(_line, pos);
_sharedStringLiteralToken.setText(cast(dchar[])str, 0); _sharedStringLiteralToken.setText(cast(dchar[])str, 0);
return _sharedStringLiteralToken; return _sharedStringLiteralToken;
} }
Token processSpecialToken(Keyword keyword, int pos) { protected Token processSpecialToken(Keyword keyword, int pos) {
switch (keyword) { switch (keyword) {
//Special Token Replaced with //Special Token Replaced with
case Keyword.DATE: // string literal of the date of compilation "mmm dd yyyy" case Keyword.DATE: // string literal of the date of compilation "mmm dd yyyy"
@ -2300,7 +2330,7 @@ class Tokenizer
} }
// returns next token (clone it if you want to store for future usage, otherwise it may be overwritten by further nextToken() calls). // returns next token (clone it if you want to store for future usage, otherwise it may be overwritten by further nextToken() calls).
public Token nextToken() { Token nextToken() {
dchar ch = nextChar(); dchar ch = nextChar();
if (ch == EOF_CHAR) { if (ch == EOF_CHAR) {
return emitEof(); return emitEof();

7
src/dlangide/ui/frame.d
View File

@ -50,6 +50,7 @@ class SimpleDSyntaxHighlighter : SyntaxHighlighter {
int tokenPos = 0; int tokenPos = 0;
int tokenLine = 0; int tokenLine = 0;
ubyte category = 0; ubyte category = 0;
try {
for (;;) { for (;;) {
Token token = _tokenizer.nextToken(); Token token = _tokenizer.nextToken();
if (token is null) { if (token is null) {
@ -76,7 +77,7 @@ class SimpleDSyntaxHighlighter : SyntaxHighlighter {
// handle token - convert to category // handle token - convert to category
switch(token.type) { switch(token.type) {
case TokenType.COMMENT: case TokenType.COMMENT:
category = TokenCategory.Comment; category = token.isDocumentationComment ? TokenCategory.Comment_Documentation : TokenCategory.Comment;
break; break;
case TokenType.KEYWORD: case TokenType.KEYWORD:
category = TokenCategory.Keyword; category = TokenCategory.Keyword;
@ -95,6 +96,9 @@ class SimpleDSyntaxHighlighter : SyntaxHighlighter {
tokenLine= newLine; tokenLine= newLine;
} }
} catch (Exception e) {
log.e("exception while trying to parse D source", e);
}
_lines.close(); _lines.close();
_props = null; _props = null;
Log.d("updateHighlight took ", currentTimeMillis() - ms0, "ms"); Log.d("updateHighlight took ", currentTimeMillis() - ms0, "ms");
@ -110,6 +114,7 @@ class DSourceEdit : SourceEdit {
setTokenHightlightColor(TokenCategory.Comment, 0x008000); // green setTokenHightlightColor(TokenCategory.Comment, 0x008000); // green
setTokenHightlightColor(TokenCategory.Keyword, 0x0000FF); // blue setTokenHightlightColor(TokenCategory.Keyword, 0x0000FF); // blue
setTokenHightlightColor(TokenCategory.String, 0xA31515); // red setTokenHightlightColor(TokenCategory.String, 0xA31515); // red
setTokenHightlightColor(TokenCategory.Comment_Documentation, 0x206000);
//setTokenHightlightColor(TokenCategory.Identifier, 0x206000); // no colors //setTokenHightlightColor(TokenCategory.Identifier, 0x206000); // no colors
} }
this() { this() {