From 3eaea625d7c3a6ca31eb8286f48a9a1390e53832 Mon Sep 17 00:00:00 2001 From: Vadim Lopatin Date: Tue, 7 Apr 2015 11:46:03 +0300 Subject: [PATCH] refactoring of DML code --- dlanguilib.visualdproj | 6 +- examples/dmledit/src/dmledit.d | 2 +- src/dlangui/core/dmlhighlight.d | 668 ------------------------- src/dlangui/core/parser.d | 836 -------------------------------- src/dlangui/package.d | 2 +- 5 files changed, 6 insertions(+), 1508 deletions(-) delete mode 100644 src/dlangui/core/dmlhighlight.d delete mode 100644 src/dlangui/core/parser.d diff --git a/dlanguilib.visualdproj b/dlanguilib.visualdproj index e5c77654..ddb11da3 100644 --- a/dlanguilib.visualdproj +++ b/dlanguilib.visualdproj @@ -367,14 +367,12 @@ - - @@ -387,6 +385,10 @@ + + + + diff --git a/examples/dmledit/src/dmledit.d b/examples/dmledit/src/dmledit.d index 688c9b5e..be60ad80 100644 --- a/examples/dmledit/src/dmledit.d +++ b/examples/dmledit/src/dmledit.d @@ -3,7 +3,7 @@ module dmledit; import dlangui; import dlangui.dialogs.filedlg; import dlangui.dialogs.dialog; -import dlangui.core.dmlhighlight; +import dlangui.dml.dmlhighlight; import std.array : replaceFirst; mixin APP_ENTRY_POINT; diff --git a/src/dlangui/core/dmlhighlight.d b/src/dlangui/core/dmlhighlight.d deleted file mode 100644 index cfe9ae79..00000000 --- a/src/dlangui/core/dmlhighlight.d +++ /dev/null @@ -1,668 +0,0 @@ -module dlangui.core.dmlhighlight; - -import dlangui.core.parser; -import dlangui.core.editable; -import dlangui.core.linestream; -import dlangui.core.textsource; -import dlangui.core.logger; -import dlangui.widgets.metadata; - -class DMLSyntaxSupport : SyntaxSupport { - - EditableContent _content; - SourceFile _file; - this (string filename) { - _file = new SourceFile(filename); - } - - TokenPropString[] _props; - - /// returns editable content - @property EditableContent content() { return _content; } - /// set editable content - @property SyntaxSupport content(EditableContent content) { - _content = content; - return this; - } - - private enum BracketMatch { - CONTINUE, - FOUND, - ERROR - } - private static struct BracketStack { - dchar[] buf; - int pos; - bool reverse; - void init(bool reverse) { - this.reverse = reverse; - pos = 0; - } - void push(dchar ch) { - if (buf.length <= pos) - buf.length = pos + 16; - buf[pos++] = ch; - } - dchar pop() { - if (pos <= 0) - return 0; - return buf[--pos]; - } - BracketMatch process(dchar ch) { - if (reverse) { - if (isCloseBracket(ch)) { - push(ch); - return BracketMatch.CONTINUE; - } else { - if (pop() != pairedBracket(ch)) - return BracketMatch.ERROR; - if (pos == 0) - return BracketMatch.FOUND; - return BracketMatch.CONTINUE; - } - } else { - if (isOpenBracket(ch)) { - push(ch); - return BracketMatch.CONTINUE; - } else { - if (pop() != pairedBracket(ch)) - return BracketMatch.ERROR; - if (pos == 0) - return BracketMatch.FOUND; - return BracketMatch.CONTINUE; - } - } - } - } - BracketStack _bracketStack; - static bool isBracket(dchar ch) { - return pairedBracket(ch) != 0; - } - static dchar pairedBracket(dchar ch) { - switch (ch) { - case '(': - return ')'; - case ')': - return '('; - case '{': - return '}'; - case '}': - return '{'; - case '[': - return ']'; - case ']': - return '['; - default: - return 0; // not a bracket - } - } - static bool isOpenBracket(dchar ch) { - switch (ch) { - case '(': - case '{': - case '[': - return true; - default: - return false; - } - } - static bool isCloseBracket(dchar ch) { - switch (ch) { - case ')': - case '}': - case ']': - return true; - default: - return false; - } - } - - protected dchar nextBracket(int dir, ref TextPosition p) { - for (;;) { - TextPosition oldpos = p; - p = dir < 0 ? _content.prevCharPos(p) : _content.nextCharPos(p); - if (p == oldpos) - return 0; - auto prop = _content.tokenProp(p); - if (tokenCategory(prop) == TokenCategory.Op) { - dchar ch = _content[p]; - if (isBracket(ch)) - return ch; - } - } - } - - /// returns paired bracket {} () [] for char at position p, returns paired char position or p if not found or not bracket - override TextPosition findPairedBracket(TextPosition p) { - if (p.line < 0 || p.line >= content.length) - return p; - dstring s = content.line(p.line); - if (p.pos < 0 || p.pos >= s.length) - return p; - dchar ch = content[p]; - dchar paired = pairedBracket(ch); - if (!paired) - return p; - TextPosition startPos = p; - int dir = isOpenBracket(ch) ? 1 : -1; - _bracketStack.init(dir < 0); - _bracketStack.process(ch); - for (;;) { - ch = nextBracket(dir, p); - if (!ch) // no more brackets - return startPos; - auto match = _bracketStack.process(ch); - if (match == BracketMatch.FOUND) - return p; - if (match == BracketMatch.ERROR) - return startPos; - // continue - } - } - - - /// return true if toggle line comment is supported for file type - override @property bool supportsToggleLineComment() { - return true; - } - - /// return true if can toggle line comments for specified text range - override bool canToggleLineComment(TextRange range) { - TextRange r = content.fullLinesRange(range); - if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end)) - return false; - return true; - } - - protected bool isLineComment(dstring s) { - for (int i = 0; i < cast(int)s.length - 1; i++) { - if (s[i] == '/' && s[i + 1] == '/') - return true; - else if (s[i] != ' ' && s[i] != '\t') - return false; - } - return false; - } - - protected dstring commentLine(dstring s, int commentX) { - dchar[] res; - int x = 0; - bool commented = false; - for (int i = 0; i < s.length; i++) { - dchar ch = s[i]; - if (ch == '\t') { - int newX = (x + _content.tabSize) / _content.tabSize * _content.tabSize; - if (!commented && newX >= commentX) { - commented = true; - if (newX != commentX) { - // replace tab with space - for (; x <= commentX; x++) - res ~= ' '; - } else { - res ~= ch; - x = newX; - } - res ~= "//"d; - x += 2; - } else { - res ~= ch; - x = newX; - } - } else { - if (!commented && x == commentX) { - commented = true; - res ~= "//"d; - res ~= ch; - x += 3; - } else { - res ~= ch; - x++; - } - } - } - if (!commented) { - for (; x < commentX; x++) - res ~= ' '; - res ~= "//"d; - } - return cast(dstring)res; - } - - /// remove single line comment from beginning of line - protected dstring uncommentLine(dstring s) { - int p = -1; - for (int i = 0; i < cast(int)s.length - 1; i++) { - if (s[i] == '/' && s[i + 1] == '/') { - p = i; - break; - } - } - if (p < 0) - return s; - s = s[0..p] ~ s[p + 2 .. $]; - for (int i = 0; i < s.length; i++) { - if (s[i] != ' ' && s[i] != '\t') { - return s; - } - } - return null; - } - - /// searches for neares token start before or equal to position - protected TextPosition tokenStart(TextPosition pos) { - TextPosition p = pos; - for (;;) { - TextPosition prevPos = content.prevCharPos(p); - if (p == prevPos) - return p; // begin of file - TokenProp prop = content.tokenProp(p); - TokenProp prevProp = content.tokenProp(prevPos); - if (prop && prop != prevProp) - return p; - p = prevPos; - } - } - - static struct TokenWithRange { - Token token; - TextRange range; - @property string toString() { - return token.toString ~ range.toString; - } - } - - protected Token[] _tokens; - protected int _tokenIndex; - - protected bool initTokenizer() { - _tokens = tokenizeML(content.lines); - _tokenIndex = 0; - return true; - } - - protected TokenWithRange nextToken() { - TokenWithRange res; - if (_tokenIndex < _tokens.length) { - res.range.start = TextPosition(_tokens[_tokenIndex].line, _tokens[_tokenIndex].pos); - if (_tokenIndex + 1 < _tokens.length) - res.range.end = TextPosition(_tokens[_tokenIndex + 1].line, _tokens[_tokenIndex + 1].pos); - else - res.range.end = content.endOfFile(); - res.token = _tokens[_tokenIndex]; - _tokenIndex++; - } else { - res.range.end = res.range.start = content.endOfFile(); - } - return res; - } - - protected TokenWithRange getPositionToken(TextPosition pos) { - initTokenizer(); - for (;;) { - TokenWithRange tokenRange = nextToken(); - //Log.d("read token: ", tokenRange); - if (tokenRange.token.type == TokenType.eof) { - //Log.d("end of file"); - return tokenRange; - } - if (pos >= tokenRange.range.start && pos < tokenRange.range.end) { - //Log.d("found: ", pos, " in ", tokenRange); - return tokenRange; - } - } - } - - protected TokenWithRange[] getRangeTokens(TextRange range) { - TokenWithRange[] res; - initTokenizer(); - for (;;) { - TokenWithRange tokenRange = nextToken(); - //Log.d("read token: ", tokenRange); - if (tokenRange.token.type == TokenType.eof) { - return res; - } - if (tokenRange.range.intersects(range)) { - //Log.d("found: ", pos, " in ", tokenRange); - res ~= tokenRange; - } - } - } - - protected bool isInsideBlockComment(TextPosition pos) { - TokenWithRange tokenRange = getPositionToken(pos); - if (tokenRange.token.type == TokenType.comment && tokenRange.token.isMultilineComment) - return pos > tokenRange.range.start && pos < tokenRange.range.end; - return false; - } - - /// toggle line comments for specified text range - override void toggleLineComment(TextRange range, Object source) { - TextRange r = content.fullLinesRange(range); - if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end)) - return; - int lineCount = r.end.line - r.start.line; - bool noEolAtEndOfRange = false; - if (lineCount == 0 || r.end.pos > 0) { - noEolAtEndOfRange = true; - lineCount++; - } - int minLeftX = -1; - bool hasComments = false; - bool hasNoComments = false; - bool hasNonEmpty = false; - dstring[] srctext; - dstring[] dsttext; - for (int i = 0; i < lineCount; i++) { - int lineIndex = r.start.line + i; - dstring s = content.line(lineIndex); - srctext ~= s; - TextLineMeasure m = content.measureLine(lineIndex); - if (!m.empty) { - if (minLeftX < 0 || minLeftX > m.firstNonSpaceX) - minLeftX = m.firstNonSpaceX; - hasNonEmpty = true; - if (isLineComment(s)) - hasComments = true; - else - hasNoComments = true; - } - } - if (minLeftX < 0) - minLeftX = 0; - if (hasNoComments || !hasComments) { - // comment - for (int i = 0; i < lineCount; i++) { - dsttext ~= commentLine(srctext[i], minLeftX); - } - if (!noEolAtEndOfRange) - dsttext ~= ""d; - EditOperation op = new EditOperation(EditAction.Replace, r, dsttext); - _content.performOperation(op, source); - } else { - // uncomment - for (int i = 0; i < lineCount; i++) { - dsttext ~= uncommentLine(srctext[i]); - } - if (!noEolAtEndOfRange) - dsttext ~= ""d; - EditOperation op = new EditOperation(EditAction.Replace, r, dsttext); - _content.performOperation(op, source); - } - } - - /// return true if toggle block comment is supported for file type - override @property bool supportsToggleBlockComment() { - return true; - } - /// return true if can toggle block comments for specified text range - override bool canToggleBlockComment(TextRange range) { - TokenWithRange startToken = getPositionToken(range.start); - TokenWithRange endToken = getPositionToken(range.end); - //Log.d("canToggleBlockComment: startToken=", startToken, " endToken=", endToken); - if (startToken.range == endToken.range && startToken.token.isMultilineComment) { - //Log.d("canToggleBlockComment: can uncomment"); - return true; - } - if (range.empty) - return false; - TokenWithRange[] tokens = getRangeTokens(range); - foreach(ref t; tokens) { - if (t.token.type == TokenType.comment) { - if (t.token.isMultilineComment) { - // disable until nested comments support is implemented - return false; - } else { - // single line comment - if (t.range.isInside(range.start) || t.range.isInside(range.end)) - return false; - } - } - } - return true; - } - /// toggle block comments for specified text range - override void toggleBlockComment(TextRange srcrange, Object source) { - TokenWithRange startToken = getPositionToken(srcrange.start); - TokenWithRange endToken = getPositionToken(srcrange.end); - if (startToken.range == endToken.range && startToken.token.isMultilineComment) { - TextRange range = startToken.range; - dstring[] dsttext; - for (int i = range.start.line; i <= range.end.line; i++) { - dstring s = content.line(i); - int charsRemoved = 0; - int minp = 0; - if (i == range.start.line) { - int maxp = content.lineLength(range.start.line); - if (i == range.end.line) - maxp = range.end.pos - 2; - charsRemoved = 2; - for (int j = range.start.pos + charsRemoved; j < maxp; j++) { - if (s[j] != s[j - 1]) - break; - charsRemoved++; - } - //Log.d("line before removing start of comment:", s); - s = s[range.start.pos + charsRemoved .. $]; - //Log.d("line after removing start of comment:", s); - charsRemoved += range.start.pos; - } - if (i == range.end.line) { - int endp = range.end.pos; - if (charsRemoved > 0) - endp -= charsRemoved; - int endRemoved = 2; - for (int j = endp - endRemoved; j >= 0; j--) { - if (s[j] != s[j + 1]) - break; - endRemoved++; - } - //Log.d("line before removing end of comment:", s); - s = s[0 .. endp - endRemoved]; - //Log.d("line after removing end of comment:", s); - } - dsttext ~= s; - } - EditOperation op = new EditOperation(EditAction.Replace, range, dsttext); - _content.performOperation(op, source); - return; - } else { - if (srcrange.empty) - return; - TokenWithRange[] tokens = getRangeTokens(srcrange); - foreach(ref t; tokens) { - if (t.token.type == TokenType.comment) { - if (t.token.isMultilineComment) { - // disable until nested comments support is implemented - return; - } else { - // single line comment - if (t.range.isInside(srcrange.start) || t.range.isInside(srcrange.end)) - return; - } - } - } - dstring[] dsttext; - for (int i = srcrange.start.line; i <= srcrange.end.line; i++) { - dstring s = content.line(i); - int charsAdded = 0; - if (i == srcrange.start.line) { - int p = srcrange.start.pos; - if (p < s.length) { - s = s[p .. $]; - charsAdded = -p; - } else { - charsAdded = -(cast(int)s.length); - s = null; - } - s = "/*" ~ s; - charsAdded += 2; - } - if (i == srcrange.end.line) { - int p = srcrange.end.pos + charsAdded; - s = p > 0 ? s[0..p] : null; - s ~= "*/"; - } - dsttext ~= s; - } - EditOperation op = new EditOperation(EditAction.Replace, srcrange, dsttext); - _content.performOperation(op, source); - return; - } - - } - - /// categorize characters in content by token types - void updateHighlight(dstring[] lines, TokenPropString[] props, int changeStartLine, int changeEndLine) { - - initTokenizer(); - _props = props; - changeStartLine = 0; - changeEndLine = cast(int)lines.length; - int tokenPos = 0; - int tokenLine = 0; - ubyte category = 0; - try { - for (;;) { - TokenWithRange token = nextToken(); - if (token.token.type == TokenType.eof) { - break; - } - uint newPos = token.range.start.pos; - uint newLine = token.range.start.line; - - // fill with category - for (int i = tokenLine; i <= newLine; i++) { - int start = i > tokenLine ? 0 : tokenPos; - int end = i < newLine ? cast(int)lines[i].length : newPos; - for (int j = start; j < end; j++) { - if (j < _props[i].length) { - _props[i][j] = category; - } - } - } - - // handle token - convert to category - switch(token.token.type) { - case TokenType.comment: - category = TokenCategory.Comment; - break; - case TokenType.ident: - if (isWidgetClassName(token.token.text)) - category = TokenCategory.Identifier_Class; - else - category = TokenCategory.Identifier; - break; - case TokenType.str: - category = TokenCategory.String; - break; - case TokenType.integer: - category = TokenCategory.Integer; - break; - case TokenType.floating: - category = TokenCategory.Float; - break; - case TokenType.error: - category = TokenCategory.Error; - break; - default: - if (token.token.type >= TokenType.colon) - category = TokenCategory.Op; - else - category = 0; - break; - } - tokenPos = newPos; - tokenLine= newLine; - - } - } catch (Exception e) { - Log.e("exception while trying to parse DML source", e); - } - _props = null; - } - - - /// returns true if smart indent is supported - override bool supportsSmartIndents() { - return true; - } - - protected bool _opInProgress; - protected void applyNewLineSmartIndent(EditOperation op, Object source) { - int line = op.newRange.end.line; - if (line == 0) - return; // not for first line - int prevLine = line - 1; - dstring lineText = _content.line(line); - TextLineMeasure lineMeasurement = _content.measureLine(line); - TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine); - while (prevLineMeasurement.empty && prevLine > 0) { - prevLine--; - prevLineMeasurement = _content.measureLine(prevLine); - } - if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX < prevLineMeasurement.firstNonSpaceX) { - dstring prevLineText = _content.line(prevLine); - TokenPropString prevLineTokenProps = _content.lineTokenProps(prevLine); - dchar lastOpChar = 0; - for (int j = prevLineMeasurement.lastNonSpace; j >= 0; j--) { - auto cat = j < prevLineTokenProps.length ? tokenCategory(prevLineTokenProps[j]) : 0; - if (cat == TokenCategory.Op) { - lastOpChar = prevLineText[j]; - break; - } else if (cat != TokenCategory.Comment && cat != TokenCategory.WhiteSpace) { - break; - } - } - int spacex = prevLineMeasurement.firstNonSpaceX; - if (lastOpChar == '{') - spacex = _content.nextTab(spacex); - dstring txt = _content.fillSpace(spacex); - EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace : 0)), [txt]); - _opInProgress = true; - _content.performOperation(op2, source); - _opInProgress = false; - } - } - - protected void applyClosingCurlySmartIndent(EditOperation op, Object source) { - int line = op.newRange.end.line; - TextPosition p2 = findPairedBracket(op.newRange.start); - if (p2 == op.newRange.start || p2.line > op.newRange.start.line) - return; - int prevLine = p2.line; - TextLineMeasure lineMeasurement = _content.measureLine(line); - TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine); - if (lineMeasurement.firstNonSpace != op.newRange.start.pos) - return; // not in beginning of line - if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX != prevLineMeasurement.firstNonSpaceX) { - dstring prevLineText = _content.line(prevLine); - TokenPropString prevLineTokenProps = _content.lineTokenProps(prevLine); - int spacex = prevLineMeasurement.firstNonSpaceX; - if (spacex != lineMeasurement.firstNonSpaceX) { - dstring txt = _content.fillSpace(spacex); - txt = txt ~ "}"; - EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace + 1 : 0)), [txt]); - _opInProgress = true; - _content.performOperation(op2, source); - _opInProgress = false; - } - } - } - - /// apply smart indent, if supported - override void applySmartIndent(EditOperation op, Object source) { - if (_opInProgress) - return; - if (op.isInsertNewLine) { - // Enter key pressed - new line inserted or splitted - applyNewLineSmartIndent(op, source); - } else if (op.singleChar == '}') { - // } entered - probably need unindent - applyClosingCurlySmartIndent(op, source); - } else if (op.singleChar == '{') { - // { entered - probably need auto closing } - } - } - -} - diff --git a/src/dlangui/core/parser.d b/src/dlangui/core/parser.d deleted file mode 100644 index a53316ba..00000000 --- a/src/dlangui/core/parser.d +++ /dev/null @@ -1,836 +0,0 @@ -// Written in the D programming language. - -/** -This module is DML (DlangUI Markup Language) parser - similar to QML in QtQuick - -Synopsis: - ----- -// helloworld ----- - -Copyright: Vadim Lopatin, 2015 -License: Boost License 1.0 -Authors: Vadim Lopatin, coolreader.org@gmail.com - */ -module dlangui.core.parser; - -import dlangui.core.linestream; -import dlangui.core.collections; -import dlangui.core.types; -import dlangui.widgets.widget; -import dlangui.widgets.metadata; -import std.conv : to; -import std.algorithm : equal, min, max; -import std.utf : toUTF32, toUTF8; -import std.array : join; - -class ParserException : Exception { - protected string _msg; - protected string _file; - protected int _line; - protected int _pos; - - @property string file() { return _file; } - @property string msg() { return _msg; } - @property int line() { return _line; } - @property int pos() { return _pos; } - - this(string msg, string file, int line, int pos) { - super(msg ~ " at " ~ file ~ " line " ~ to!string(line) ~ " column " ~ to!string(pos)); - _msg = msg; - _file = file; - _line = line; - _pos = pos; - } -} - -/// parser exception - unknown (unregistered) widget name -class UnknownWidgetException : ParserException { - protected string _objectName; - - @property string objectName() { return _objectName; } - - this(string msg, string objectName, string file, int line, int pos) { - super(msg is null ? "Unknown widget name: " ~ objectName : msg, file, line, pos); - _objectName = objectName; - } -} - -/// parser exception - unknown property for widget -class UnknownPropertyException : UnknownWidgetException { - protected string _propName; - - @property string propName() { return _propName; } - - this(string msg, string objectName, string propName, string file, int line, int pos) { - super(msg is null ? "Unknown property " ~ objectName ~ "." ~ propName : msg, objectName, file, line, pos); - } -} - -enum TokenType : ushort { - /// end of file - eof, - /// end of line - eol, - /// whitespace - whitespace, - /// string literal - str, - /// integer literal - integer, - /// floating point literal - floating, - /// comment - comment, - /// ident - ident, - /// error - error, - // operators - /// : operator - colon, - /// . operator - dot, - /// ; operator - semicolon, - /// , operator - comma, - /// - operator - minus, - /// + operator - plus, - /// [ - curlyOpen, - /// ] - curlyClose, - /// ( - open, - /// ) - close, - /// [ - squareOpen, - /// ] - squareClose, -} - -struct Token { - TokenType type; - ushort line; - ushort pos; - bool multiline; - string text; - union { - int intvalue; - double floatvalue; - } - public @property string toString() { - if (type == TokenType.integer) - return "" ~ to!string(line) ~ ":" ~ to!string(pos) ~ " " ~ to!string(type) ~ " " ~ to!string(intvalue); - else if (type == TokenType.floating) - return "" ~ to!string(line) ~ ":" ~ to!string(pos) ~ " " ~ to!string(type) ~ " " ~ to!string(floatvalue); - else - return "" ~ to!string(line) ~ ":" ~ to!string(pos) ~ " " ~ to!string(type) ~ " \"" ~ text ~ "\""; - } - @property bool isMultilineComment() { - return type == TokenType.comment && multiline; - } -} - -/// simple tokenizer for DlangUI ML -class Tokenizer { - protected LineStream _lines; - - dchar[] _lineText; - ushort _line; - ushort _pos; - int _len; - dchar _prevChar; - string _filename; - - Token _token; - - enum : int { - EOF_CHAR = 0x001A, - EOL_CHAR = 0x000A - }; - - this(string source, string filename = "") { - _filename = filename; - _lines = LineStream.create(source, filename); - _lineText = _lines.readLine(); - _len = cast(int)_lineText.length; - _line = 0; - _pos = 0; - _prevChar = 0; - } - - ~this() { - destroy(_lines); - _lines = null; - } - - protected dchar peekChar() { - if (_pos < _len) - return _lineText[_pos]; - else if (_lineText is null) - return EOF_CHAR; - return EOL_CHAR; - } - - protected dchar peekNextChar() { - if (_pos < _len - 1) - return _lineText[_pos + 1]; - else if (_lineText is null) - return EOF_CHAR; - return EOL_CHAR; - } - - protected dchar nextChar() { - if (_pos < _len) - _prevChar = _lineText[_pos++]; - else if (_lineText is null) - _prevChar = EOF_CHAR; - else { - _lineText = _lines.readLine(); - _len = cast(int)_lineText.length; - _line++; - _pos = 0; - _prevChar = EOL_CHAR; - } - return _prevChar; - } - - protected dchar skipChar() { - nextChar(); - return peekChar(); - } - - protected void setTokenStart() { - _token.pos = _pos; - _token.line = _line; - _token.text = null; - _token.intvalue = 0; - } - - protected ref const(Token) parseEof() { - _token.type = TokenType.eof; - return _token; - } - - protected ref const(Token) parseEol() { - _token.type = TokenType.eol; - nextChar(); - return _token; - } - - protected ref const(Token) parseWhiteSpace() { - _token.type = TokenType.whitespace; - for(;;) { - dchar ch = skipChar(); - if (ch != ' ' && ch != '\t') - break; - } - return _token; - } - - static bool isAlpha(dchar ch) { - return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'; - } - - static bool isNum(dchar ch) { - return (ch >= '0' && ch <= '9'); - } - - static bool isAlphaNum(dchar ch) { - return isNum(ch) || isAlpha(ch); - } - - private char[] _stringbuf; - protected ref const(Token) parseString() { - _token.type = TokenType.str; - //skipChar(); // skip " - bool lastBackslash = false; - _stringbuf.length = 0; - for(;;) { - dchar ch = skipChar(); - if (ch == '\"') { - if (lastBackslash) { - _stringbuf ~= ch; - lastBackslash = false; - } else { - skipChar(); - break; - } - } else if (ch == '\\') { - if (lastBackslash) { - _stringbuf ~= ch; - lastBackslash = false; - } else { - lastBackslash = true; - } - } else if (ch == EOL_CHAR) { - skipChar(); - break; - } else if (lastBackslash) { - if (ch == 'n') - ch = '\n'; - else if (ch == 't') - ch = '\t'; - _stringbuf ~= ch; - lastBackslash = false; - } else { - _stringbuf ~= ch; - lastBackslash = false; - } - } - _token.text = _stringbuf.dup; - return _token; - } - - protected ref const(Token) parseIdent() { - _token.type = TokenType.ident; - _stringbuf.length = 0; - _stringbuf ~= peekChar(); - for(;;) { - dchar ch = skipChar(); - if (!isAlphaNum(ch)) - break; - _stringbuf ~= ch; - } - _token.text = _stringbuf.dup; - return _token; - } - - protected ref const(Token) parseFloating(int n) { - _token.type = TokenType.floating; - dchar ch = peekChar(); - // floating point - int div = 0; - int n2 = 0; - for (;;) { - ch = skipChar(); - if (!isNum(ch)) - break; - n2 = n2 * 10 + (ch - '0'); - div++; - } - _token.floatvalue = cast(double)n + (div > 0 ? cast(double)n2 / div : 0.0); - string suffix; - if (ch == '%') { - suffix ~= ch; - ch = skipChar(); - } else { - while (ch >= 'a' && ch <= 'z') { - suffix ~= ch; - ch = skipChar(); - } - } - if (isAlphaNum(ch) || ch == '.') - return parseError(); - _token.text = suffix; - return _token; - } - - protected ref const(Token) parseNumber() { - dchar ch = peekChar(); - uint n = ch - '0'; - for(;;) { - ch = skipChar(); - if (!isNum(ch)) - break; - n = n * 10 + (ch - '0'); - } - if (ch == '.') - return parseFloating(n); - string suffix; - if (ch == '%') { - suffix ~= ch; - ch = skipChar(); - } else { - while (ch >= 'a' && ch <= 'z') { - suffix ~= ch; - ch = skipChar(); - } - } - if (isAlphaNum(ch) || ch == '.') - return parseError(); - _token.type = TokenType.integer; - _token.intvalue = n; - _token.text = suffix; - return _token; - } - - protected ref const(Token) parseSingleLineComment() { - for(;;) { - dchar ch = skipChar(); - if (ch == EOL_CHAR || ch == EOF_CHAR) - break; - } - _token.type = TokenType.comment; - _token.multiline = false; - return _token; - } - - protected ref const(Token) parseMultiLineComment() { - skipChar(); - for(;;) { - dchar ch = skipChar(); - if (ch == '*' && peekNextChar() == '/') { - skipChar(); - skipChar(); - break; - } - if (ch == EOF_CHAR) - break; - } - _token.type = TokenType.comment; - _token.multiline = true; - return _token; - } - - protected ref const(Token) parseError() { - _token.type = TokenType.error; - for(;;) { - dchar ch = skipChar(); - if (ch == ' ' || ch == '\t' || ch == EOL_CHAR || ch == EOF_CHAR) - break; - } - return _token; - } - - protected ref const(Token) parseOp(TokenType op) { - _token.type = op; - skipChar(); - return _token; - } - - /// get next token - ref const(Token) nextToken() { - setTokenStart(); - dchar ch = peekChar(); - if (ch == EOF_CHAR) - return parseEof(); - if (ch == EOL_CHAR) - return parseEol(); - if (ch == ' ' || ch == '\t') - return parseWhiteSpace(); - if (ch == '\"') - return parseString(); - if (isAlpha(ch)) - return parseIdent(); - if (isNum(ch)) - return parseNumber(); - if (ch == '.' && isNum(peekNextChar())) - return parseFloating(0); - if (ch == '/' && peekNextChar() == '/') - return parseSingleLineComment(); - if (ch == '/' && peekNextChar() == '*') - return parseMultiLineComment(); - switch (ch) { - case '.': return parseOp(TokenType.dot); - case ':': return parseOp(TokenType.colon); - case ';': return parseOp(TokenType.semicolon); - case ',': return parseOp(TokenType.comma); - case '-': return parseOp(TokenType.minus); - case '+': return parseOp(TokenType.plus); - case '{': return parseOp(TokenType.curlyOpen); - case '}': return parseOp(TokenType.curlyClose); - case '(': return parseOp(TokenType.open); - case ')': return parseOp(TokenType.close); - case '[': return parseOp(TokenType.squareOpen); - case ']': return parseOp(TokenType.squareClose); - default: - return parseError(); - } - } - - string getContextSource() { - string s = toUTF8(_lineText); - if (_pos == 0) - return " near `^^^" ~ s[0..min($,30)] ~ "`"; - if (_pos >= _len) - return " near `" ~ s[max(_len - 30, 0) .. $] ~ "^^^`"; - return " near `" ~ s[max(_pos - 15, 0) .. _pos] ~ "^^^" ~ s[_pos .. min(_pos + 15, $)] ~ "`"; - } - - void emitError(string msg) { - throw new ParserException(msg ~ getContextSource(), _filename, _token.line, _token.pos); - } - - void emitUnknownPropertyError(string objectName, string propName) { - throw new UnknownPropertyException("Unknown property " ~ objectName ~ "." ~ propName ~ getContextSource(), objectName, propName, _filename, _token.line, _token.pos); - } - - void emitUnknownObjectError(string objectName) { - throw new UnknownWidgetException("Unknown widget type " ~ objectName ~ getContextSource(), objectName, _filename, _token.line, _token.pos); - } - - void emitError(string msg, ref const Token token) { - throw new ParserException(msg, _filename, token.line, token.pos); - } -} - -class MLParser { - protected string _code; - protected string _filename; - protected bool _ownContext; - protected Widget _context; - protected Widget _currentWidget; - protected Tokenizer _tokenizer; - protected Collection!Widget _treeStack; - - protected this(string code, string filename = "", Widget context = null) { - _code = code; - _filename = filename; - _context = context; - _tokenizer = new Tokenizer(code, filename); - } - - protected Token _token; - - /// move to next token - protected void nextToken() { - _token = _tokenizer.nextToken(); - Log.d("parsed token: ", _token.type, " ", _token.line, ":", _token.pos, " ", _token.text); - } - - /// throw exception if current token is eof - protected void checkNoEof() { - if (_token.type == TokenType.eof) - error("unexpected end of file"); - } - - /// move to next token, throw exception if eof - protected void nextTokenNoEof() { - nextToken(); - checkNoEof(); - } - - protected void skipWhitespaceAndEols() { - for (;;) { - if (_token.type != TokenType.eol && _token.type != TokenType.whitespace && _token.type != TokenType.comment) - break; - nextToken(); - } - if (_token.type == TokenType.error) - error("error while parsing ML code"); - } - - protected void skipWhitespaceAndEolsNoEof() { - skipWhitespaceAndEols(); - checkNoEof(); - } - - protected void skipWhitespaceNoEof() { - skipWhitespace(); - checkNoEof(); - } - - protected void skipWhitespace() { - for (;;) { - if (_token.type != TokenType.whitespace && _token.type != TokenType.comment) - break; - nextToken(); - } - if (_token.type == TokenType.error) - error("error while parsing ML code"); - } - - protected void error(string msg) { - _tokenizer.emitError(msg); - } - - protected void unknownObjectError(string objectName) { - _tokenizer.emitUnknownObjectError(objectName); - } - - protected void unknownPropertyError(string objectName, string propName) { - _tokenizer.emitUnknownPropertyError(objectName, propName); - } - - Widget createWidget(string name) { - auto metadata = findWidgetMetadata(name); - if (!metadata) - error("Cannot create widget " ~ name ~ " : unregistered widget class"); - return metadata.create(); - } - - protected void createContext(string name) { - if (_context) - error("Context widget is already specified, but identifier " ~ name ~ " is found"); - _context = createWidget(name); - _ownContext = true; - } - - protected int applySuffix(int value, string suffix) { - if (suffix.length > 0) { - if (suffix.equal("px")) { - // do nothing, value is in px by default - } else if (suffix.equal("pt")) { - value = makePointSize(value); - } else if (suffix.equal("%")) { - value = makePercentSize(value); - } else - error("unknown number suffix: " ~ suffix); - } - return value; - } - - protected void setIntProperty(string propName, int value, string suffix = null) { - value = applySuffix(value, suffix); - if (!_currentWidget.setIntProperty(propName, value)) - error("unknown int property " ~ propName); - } - - protected void setBoolProperty(string propName, bool value) { - if (!_currentWidget.setBoolProperty(propName, value)) - error("unknown int property " ~ propName); - } - - protected void setFloatProperty(string propName, double value) { - if (!_currentWidget.setDoubleProperty(propName, value)) - error("unknown double property " ~ propName); - } - - protected void setRectProperty(string propName, Rect value) { - if (!_currentWidget.setRectProperty(propName, value)) - error("unknown Rect property " ~ propName); - } - - protected void setStringProperty(string propName, string value) { - if (propName.equal("id") || propName.equal("styleId") || propName.equal("backgroundImageId")) { - if (!_currentWidget.setStringProperty(propName, value)) - error("cannot set " ~ propName ~ " property for widget"); - return; - } - - dstring v = toUTF32(value); - if (!_currentWidget.setDstringProperty(propName, v)) { - if (!_currentWidget.setStringProperty(propName, value)) - error("unknown string property " ~ propName); - } - } - - protected void setIdentProperty(string propName, string value) { - if (propName.equal("id") || propName.equal("styleId") || propName.equal("backgroundImageId")) { - if (!_currentWidget.setStringProperty(propName, value)) - error("cannot set id property for widget"); - return; - } - - if (value.equal("true")) - setBoolProperty(propName, true); - else if (value.equal("false")) - setBoolProperty(propName, false); - else if (value.equal("FILL") || value.equal("FILL_PARENT")) - setIntProperty(propName, FILL_PARENT); - else if (value.equal("WRAP") || value.equal("WRAP_CONTENT")) - setIntProperty(propName, WRAP_CONTENT); - else if (!_currentWidget.setStringProperty(propName, value)) - error("unknown ident property " ~ propName); - } - - protected void parseRectProperty(string propName) { - // current token is Rect - int[4] values = [0, 0, 0, 0]; - nextToken(); - skipWhitespaceAndEolsNoEof(); - if (_token.type != TokenType.curlyOpen) - error("{ expected after Rect"); - nextToken(); - skipWhitespaceAndEolsNoEof(); - int index = 0; - for (;;) { - if (_token.type == TokenType.curlyClose) - break; - if (_token.type == TokenType.integer) { - if (index >= 4) - error("too many values in Rect"); - int n = applySuffix(_token.intvalue, _token.text); - values[index++] = n; - nextToken(); - skipWhitespaceAndEolsNoEof(); - if (_token.type == TokenType.comma || _token.type == TokenType.semicolon) { - nextToken(); - skipWhitespaceAndEolsNoEof(); - } - } else if (_token.type == TokenType.ident) { - string name = _token.text; - nextToken(); - skipWhitespaceAndEolsNoEof(); - if (_token.type != TokenType.colon) - error(": expected after property name " ~ name ~ " in Rect definition"); - nextToken(); - skipWhitespaceNoEof(); - if (_token.type != TokenType.integer) - error("integer expected as Rect property value"); - int n = applySuffix(_token.intvalue, _token.text); - - if (name.equal("left")) - values[0] = n; - else if (name.equal("top")) - values[1] = n; - else if (name.equal("right")) - values[2] = n; - else if (name.equal("bottom")) - values[3] = n; - else - error("unknown property " ~ name ~ " in Rect"); - - nextToken(); - skipWhitespaceNoEof(); - if (_token.type == TokenType.comma || _token.type == TokenType.semicolon) { - nextToken(); - skipWhitespaceAndEolsNoEof(); - } - } else { - error("invalid Rect definition"); - } - - } - setRectProperty(propName, Rect(values[0], values[1], values[2], values[3])); - } - - protected void parseProperty() { - if (_token.type != TokenType.ident) - error("identifier expected"); - string propName = _token.text; - nextToken(); - skipWhitespaceNoEof(); - if (_token.type == TokenType.colon) { // : - nextTokenNoEof(); // skip : - skipWhitespaceNoEof(); - if (_token.type == TokenType.integer) - setIntProperty(propName, _token.intvalue, _token.text); - else if (_token.type == TokenType.minus || _token.type == TokenType.plus) { - int sign = _token.type == TokenType.minus ? -1 : 1; - nextTokenNoEof(); // skip : - skipWhitespaceNoEof(); - if (_token.type == TokenType.integer) { - setIntProperty(propName, _token.intvalue * sign, _token.text); - } else if (_token.type == TokenType.floating) { - setFloatProperty(propName, _token.floatvalue * sign); - } else - error("number expected after + and -"); - } else if (_token.type == TokenType.floating) - setFloatProperty(propName, _token.floatvalue); - else if (_token.type == TokenType.str) - setStringProperty(propName, _token.text); - else if (_token.type == TokenType.ident) { - if (_token.text.equal("Rect")) { - parseRectProperty(propName); - } else { - setIdentProperty(propName, _token.text); - } - } else - error("int, float, string or identifier are expected as property value"); - nextTokenNoEof(); - skipWhitespaceNoEof(); - if (_token.type == TokenType.semicolon) { - // separated by ; - nextTokenNoEof(); - skipWhitespaceAndEolsNoEof(); - return; - } else if (_token.type == TokenType.eol) { - nextTokenNoEof(); - skipWhitespaceAndEolsNoEof(); - return; - } else if (_token.type == TokenType.curlyClose) { - // it was last property in object - return; - } - error("; eol or } expected after property definition"); - } else if (_token.type == TokenType.curlyOpen) { // { -- start of object - Widget s = createWidget(propName); - parseWidgetProperties(s); - } else { - error(": or { expected after identifier"); - } - - } - - protected void parseWidgetProperties(Widget w) { - if (_token.type != TokenType.curlyOpen) // { - error("{ is expected"); - _treeStack.pushBack(w); - if (_currentWidget) - _currentWidget.addChild(w); - _currentWidget = w; - nextToken(); // skip { - skipWhitespaceAndEols(); - for (;;) { - checkNoEof(); - if (_token.type == TokenType.curlyClose) // end of object's internals - break; - parseProperty(); - } - if (_token.type != TokenType.curlyClose) // { - error("{ is expected"); - nextToken(); // skip } - skipWhitespaceAndEols(); - _treeStack.popBack(); - _currentWidget = _treeStack.peekBack(); - } - - protected Widget parse() { - try { - nextToken(); - skipWhitespaceAndEols(); - if (_token.type == TokenType.ident) { - createContext(_token.text); - nextToken(); - skipWhitespaceAndEols(); - } - if (_token.type != TokenType.curlyOpen) // { - error("{ is expected"); - if (!_context) - error("No context widget is specified!"); - parseWidgetProperties(_context); - - skipWhitespaceAndEols(); - if (_token.type != TokenType.eof) // { - error("end of file expected"); - return _context; - } catch (Exception e) { - Log.e("exception while parsing ML", e); - if (_context && _ownContext) - destroy(_context); - _context = null; - throw e; - } - } - - ~this() { - destroy(_tokenizer); - _tokenizer = null; - } - -} - - -/// Parse DlangUI ML code -public Widget parseML(string code, string filename = "", Widget context = null) { - MLParser parser = new MLParser(code, filename); - scope(exit) destroy(parser); - return parser.parse(); -} - -/// tokenize source into array of tokens (excluding EOF) -public Token[] tokenizeML(const(dstring[]) lines) { - string code = toUTF8(join(lines, "\n")); - return tokenizeML(code); -} - -/// tokenize source into array of tokens (excluding EOF) -public Token[] tokenizeML(string code) { - Token[] res; - auto tokenizer = new Tokenizer(code, ""); - for (;;) { - auto token = tokenizer.nextToken(); - if (token.type == TokenType.eof) - break; - res ~= token; - } - return res; -} diff --git a/src/dlangui/package.d b/src/dlangui/package.d index 46995fc6..a592a24e 100644 --- a/src/dlangui/package.d +++ b/src/dlangui/package.d @@ -73,7 +73,7 @@ public import dlangui.widgets.statusline; public import dlangui.widgets.docks; public import dlangui.widgets.toolbars; public import dlangui.platforms.common.platform; -public import dlangui.core.parser; +public import dlangui.dml.parser; // some useful imports from Phobos public import std.algorithm : equal;