From d13d680b7483d0418484c8a87434c189fb176076 Mon Sep 17 00:00:00 2001 From: Hackerpilot Date: Sun, 26 Jan 2014 22:47:21 -0800 Subject: [PATCH] Updated lexer docs. Implemented delete and fp operator rules. Fixed bug with AST traversal --- analysis/del.d | 29 +++ analysis/fish.d | 38 ++++ analysis/run.d | 10 +- main.d | 5 +- stdx/d/ast.d | 149 ++++++++++----- stdx/d/parser.d | 11 +- stdx/lexer.d | 355 +++++++++++++++++++++++++++++++---- stdx/lexer.html | 483 ++++++++++++++++++++++++++++++++++++++++++++++++ stdx/lexer.o | Bin 0 -> 54460 bytes 9 files changed, 995 insertions(+), 85 deletions(-) create mode 100644 analysis/del.d create mode 100644 analysis/fish.d create mode 100644 stdx/lexer.html create mode 100644 stdx/lexer.o diff --git a/analysis/del.d b/analysis/del.d new file mode 100644 index 0000000..8cd3646 --- /dev/null +++ b/analysis/del.d @@ -0,0 +1,29 @@ +// Copyright Brian Schott (Sir Alaran) 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +module analysis.del; + +import stdx.d.ast; +import stdx.d.lexer; +import analysis.base; + +/** + * Checks for use of the deprecated "delete" keyword + */ +class DeleteCheck : BaseAnalyzer +{ + alias visit = BaseAnalyzer.visit; + + this(string fileName) + { + super(fileName); + } + + override void visit(DeleteExpression d) + { + addErrorMessage(d.line, d.column, "Avoid using the deprecated delete keyword"); + d.accept(this); + } +} diff --git a/analysis/fish.d b/analysis/fish.d new file mode 100644 index 0000000..e1790d2 --- /dev/null +++ b/analysis/fish.d @@ -0,0 +1,38 @@ +// Copyright Brian Schott (Sir Alaran) 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +module analysis.fish; + +import stdx.d.ast; +import stdx.d.lexer; +import analysis.base; + +/** + * Checks for use of the deprecated floating point comparison operators. + */ +class FloatOperatorCheck : BaseAnalyzer +{ + alias visit = BaseAnalyzer.visit; + + this(string fileName) + { + super(fileName); + } + + override void visit(RelExpression r) + { + if (r.operator == tok!"<>" + || r.operator == tok!"!<>" + || r.operator == tok!"!>" + || r.operator == tok!"!<" + || r.operator == tok!"!<>=" + || r.operator == tok!"!>=" + || r.operator == tok!"!<=") + { + addErrorMessage(r.line, r.column, "Avoid using the deprecated floating-point operators"); + } + r.accept(this); + } +} diff --git a/analysis/run.d b/analysis/run.d index 19fa961..d55a7d1 100644 --- a/analysis/run.d +++ b/analysis/run.d @@ -15,6 +15,8 @@ import analysis.base; import analysis.style; import analysis.enumarrayliteral; import analysis.pokemon; +import analysis.del; +import analysis.fish; void messageFunction(string fileName, size_t line, size_t column, string message, bool isError) @@ -63,8 +65,14 @@ void analyze(File output, string[] fileNames, bool staticAnalyze = true) auto pokemon = new PokemonExceptionCheck(fileName); pokemon.visit(m); + auto del = new DeleteCheck(fileName); + del.visit(m); + + auto fish = new FloatOperatorCheck(fileName); + fish.visit(m); + foreach (message; sort(chain(enums.messages, style.messages, - pokemon.messages).array)) + pokemon.messages, del.messages, fish.messages).array)) { writeln(message); } diff --git a/main.d b/main.d index 4427959..3b3103a 100644 --- a/main.d +++ b/main.d @@ -109,10 +109,11 @@ int main(string[] args) } else if (tokenDump) { + writeln("text blank\tindex\tline\tcolumn\tcomment"); foreach (token; tokens) { - writeln("«", token.text is null ? str(token.type) : token.text, - "» ", token.text !is null, " ", token.index, " ", token.line, " ", token.column, " ", + writefln("<<%20s>>%b\t%d\t%d\t%d", token.text is null ? str(token.type) : token.text, + token.text !is null, token.index, token.line, token.column, token.comment); } return 0; diff --git a/stdx/d/ast.d b/stdx/d/ast.d index dc21ae3..7dd5e26 100644 --- a/stdx/d/ast.d +++ b/stdx/d/ast.d @@ -31,6 +31,58 @@ import std.string; abstract class ASTVisitor { public: + + void visit(ExpressionNode n) + { + if (cast(AddExpression) n) visit(cast(AddExpression) n); + else if (cast(AndAndExpression) n) visit(cast(AndAndExpression) n); + else if (cast(AndExpression) n) visit(cast(AndExpression) n); + else if (cast(AsmAddExp) n) visit(cast(AsmAddExp) n); + else if (cast(AsmAndExp) n) visit(cast(AsmAndExp) n); + else if (cast(AsmEqualExp) n) visit(cast(AsmEqualExp) n); + else if (cast(AsmLogAndExp) n) visit(cast(AsmLogAndExp) n); + else if (cast(AsmLogOrExp) n) visit(cast(AsmLogOrExp) n); + else if (cast(AsmMulExp) n) visit(cast(AsmMulExp) n); + else if (cast(AsmOrExp) n) visit(cast(AsmOrExp) n); + else if (cast(AsmRelExp) n) visit(cast(AsmRelExp) n); + else if (cast(AsmShiftExp) n) visit(cast(AsmShiftExp) n); + else if (cast(AssertExpression) n) visit(cast(AssertExpression) n); + else if (cast(AssignExpression) n) visit(cast(AssignExpression) n); + else if (cast(CmpExpression) n) visit(cast(CmpExpression) n); + else if (cast(DeleteExpression) n) visit(cast(DeleteExpression) n); + else if (cast(EqualExpression) n) visit(cast(EqualExpression) n); + else if (cast(Expression) n) visit(cast(Expression) n); + else if (cast(FunctionCallExpression) n) visit(cast(FunctionCallExpression) n); + else if (cast(FunctionLiteralExpression) n) visit(cast(FunctionLiteralExpression) n); + else if (cast(IdentityExpression) n) visit(cast(IdentityExpression) n); + else if (cast(ImportExpression) n) visit(cast(ImportExpression) n); + else if (cast(IndexExpression) n) visit(cast(IndexExpression) n); + else if (cast(InExpression) n) visit(cast(InExpression) n); + else if (cast(IsExpression) n) visit(cast(IsExpression) n); + else if (cast(LambdaExpression) n) visit(cast(LambdaExpression) n); + else if (cast(MixinExpression) n) visit(cast(MixinExpression) n); + else if (cast(MulExpression) n) visit(cast(MulExpression) n); + else if (cast(NewAnonClassExpression) n) visit(cast(NewAnonClassExpression) n); + else if (cast(NewExpression) n) visit(cast(NewExpression) n); + else if (cast(OrExpression) n) visit(cast(OrExpression) n); + else if (cast(OrOrExpression) n) visit(cast(OrOrExpression) n); + else if (cast(PostIncDecExpression) n) visit(cast(PostIncDecExpression) n); + else if (cast(PowExpression) n) visit(cast(PowExpression) n); + else if (cast(PragmaExpression) n) visit(cast(PragmaExpression) n); + else if (cast(PreIncDecExpression) n) visit(cast(PreIncDecExpression) n); + else if (cast(PrimaryExpression) n) visit(cast(PrimaryExpression) n); + else if (cast(RelExpression) n) visit(cast(RelExpression) n); + else if (cast(ShiftExpression) n) visit(cast(ShiftExpression) n); + else if (cast(SliceExpression) n) visit(cast(SliceExpression) n); + else if (cast(TemplateMixinExpression) n) visit(cast(TemplateMixinExpression) n); + else if (cast(TernaryExpression) n) visit(cast(TernaryExpression) n); + else if (cast(TraitsExpression) n) visit(cast(TraitsExpression) n); + else if (cast(TypeidExpression) n) visit(cast(TypeidExpression) n); + else if (cast(TypeofExpression) n) visit(cast(TypeofExpression) n); + else if (cast(UnaryExpression) n) visit(cast(UnaryExpression) n); + else if (cast(XorExpression) n) visit(cast(XorExpression) n); + } + /** */ void visit(AddExpression addExpression) { addExpression.accept(this); } /** */ void visit(AliasDeclaration aliasDeclaration) { aliasDeclaration.accept(this); } /** */ void visit(AliasInitializer aliasInitializer) { aliasInitializer.accept(this); } @@ -104,7 +156,6 @@ public: /** */ void visit(EponymousTemplateDeclaration eponymousTemplateDeclaration) { eponymousTemplateDeclaration.accept(this); } /** */ void visit(EqualExpression equalExpression) { equalExpression.accept(this); } /** */ void visit(Expression expression) { expression.accept(this); } - /** */ void visit(ExpressionNode expressionNode) { expressionNode.accept(this); } /** */ void visit(ExpressionStatement expressionStatement) { expressionStatement.accept(this); } /** */ void visit(FinalSwitchStatement finalSwitchStatement) { finalSwitchStatement.accept(this); } /** */ void visit(Finally finally_) { finally_.accept(this); } @@ -234,10 +285,11 @@ public: interface ASTNode { +public: /** */ void accept(ASTVisitor visitor); } -immutable string DEFAULT_ACCEPT = q{void accept(ASTVisitor visitor) {}}; +immutable string DEFAULT_ACCEPT = q{override void accept(ASTVisitor visitor) {}}; template visitIfNotNull(fields ...) { @@ -259,19 +311,28 @@ template visitIfNotNull(fields ...) } } -abstract class ExpressionNode : ASTNode {} +abstract class ExpressionNode : ASTNode +{ +public: + override void accept(ASTVisitor visitor) + { + assert (false); + } +} mixin template BinaryExpressionBody() { ExpressionNode left; ExpressionNode right; + size_t line; + size_t column; } /// class AddExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -283,7 +344,7 @@ public: class AliasDeclaration : ASTNode { public: - void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(type, name, initializers)); } @@ -332,7 +393,7 @@ public: class AndAndExpression : ExpressionNode { public: - void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -343,7 +404,7 @@ public: class AndExpression : ExpressionNode { public: - void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -566,7 +627,7 @@ public: class AssertExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(assertion, message)); } @@ -578,7 +639,7 @@ public: class AssignExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(ternaryExpression, assignExpression)); } @@ -816,7 +877,7 @@ public: class CmpExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(shiftExpression, equalExpression, identityExpression, relExpression, inExpression)); @@ -1031,11 +1092,13 @@ public: class DeleteExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression)); } /** */ UnaryExpression unaryExpression; + /** */ size_t line; + /** */ size_t column; } /// @@ -1151,7 +1214,7 @@ public: class EqualExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1163,7 +1226,7 @@ public: class Expression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(items)); } @@ -1293,7 +1356,7 @@ public: class FunctionCallExpression : ExpressionNode { public: - void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression, arguments, templateArguments)); } @@ -1306,7 +1369,7 @@ public: class FunctionCallStatement : ASTNode { public: - void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(functionCallExpression)); } @@ -1338,7 +1401,7 @@ public: class FunctionLiteralExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(type, parameters, functionAttributes, functionBody)); @@ -1413,7 +1476,7 @@ public: class IdentityExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1478,7 +1541,7 @@ public: class ImportExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(assignExpression)); } @@ -1489,7 +1552,7 @@ public: class IndexExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression, argumentList)); } @@ -1501,7 +1564,7 @@ public: class InExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1575,7 +1638,7 @@ public: class IsExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(type, identifier, typeSpecialization, templateParameterList)); @@ -1626,7 +1689,7 @@ public: class LambdaExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(identifier, parameters, functionAttributes, assignExpression)); @@ -1689,7 +1752,7 @@ public: class MixinExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(assignExpression)); } @@ -1748,7 +1811,7 @@ public: class MulExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1760,7 +1823,7 @@ public: class NewAnonClassExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(allocatorArguments, constructorArguments, baseClassList, structBody)); @@ -1775,7 +1838,7 @@ public: class NewExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(newAnonClassExpression, type, arguments, assignExpression)); @@ -1863,7 +1926,7 @@ public: class OrExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1874,7 +1937,7 @@ public: class OrOrExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1937,7 +2000,7 @@ public: class PostIncDecExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression)); } @@ -1949,7 +2012,7 @@ public: class PowExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -1971,7 +2034,7 @@ public: class PragmaExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(identifier, argumentList)); } @@ -1983,7 +2046,7 @@ public: class PreIncDecExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression)); } @@ -1995,7 +2058,7 @@ public: class PrimaryExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(basicType, primary, typeofExpression, typeidExpression, arrayLiteral, assocArrayLiteral, expression, @@ -2035,7 +2098,7 @@ public: class RelExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -2096,7 +2159,7 @@ public: class ShiftExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } @@ -2120,7 +2183,7 @@ public: class SliceExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(unaryExpression, lower, upper)); } @@ -2409,7 +2472,7 @@ public: class TemplateMixinExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(identifier, templateArguments, mixinTemplateName)); } @@ -2534,7 +2597,7 @@ public: class TernaryExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(orOrExpression, expression, ternaryExpression)); } @@ -2558,7 +2621,7 @@ public: class TraitsExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(identifier, templateArgumentList)); } @@ -2647,7 +2710,7 @@ public: class TypeidExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(type, expression)); } @@ -2659,7 +2722,7 @@ public: class TypeofExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(expression, return_)); } @@ -2671,7 +2734,7 @@ public: class UnaryExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { // TODO prefix, postfix, unary mixin (visitIfNotNull!(primaryExpression, newExpression, @@ -2803,7 +2866,7 @@ public: class XorExpression : ExpressionNode { public: - /+override+/ void accept(ASTVisitor visitor) + override void accept(ASTVisitor visitor) { mixin (visitIfNotNull!(left, right)); } diff --git a/stdx/d/parser.d b/stdx/d/parser.d index 0c6aea1..8164ef3 100644 --- a/stdx/d/parser.d +++ b/stdx/d/parser.d @@ -1874,6 +1874,8 @@ class ClassFour(A, B) if (someTest()) : Super {}}c; { mixin(traceEnterAndExit!(__FUNCTION__)); auto node = new DeleteExpression; + node.line = current.line; + node.column = current.column; if (expect(tok!"delete") is null) return null; node.unaryExpression = parseUnaryExpression(); return node; @@ -3990,6 +3992,7 @@ q{(int a, ...) */ PragmaDeclaration parsePragmaDeclaration() { + mixin (traceEnterAndExit!(__FUNCTION__)); auto node = new PragmaDeclaration; node.pragmaExpression = parsePragmaExpression(); expect(tok!";"); @@ -4005,6 +4008,7 @@ q{(int a, ...) */ PragmaExpression parsePragmaExpression() { + mixin (traceEnterAndExit!(__FUNCTION__)); auto node = new PragmaExpression; expect(tok!"pragma"); expect(tok!"("); @@ -4264,8 +4268,9 @@ q{(int a, ...) * | $(LITERAL '!<=') * ;) */ - ExpressionNode parseRelExpression(ExpressionNode shift = null) + ExpressionNode parseRelExpression(ExpressionNode shift) { + mixin (traceEnterAndExit!(__FUNCTION__)); return parseLeftAssocBinaryExpression!(RelExpression, ShiftExpression, tok!"<", tok!"<=", tok!">", tok!">=", tok!"!<>=", tok!"!<>", tok!"<>", tok!"<>=", tok!"!>", tok!"!>=", tok!"!>=", tok!"!<", @@ -6238,7 +6243,11 @@ protected: { auto n = new ExpressionType; static if (__traits(hasMember, ExpressionType, "operator")) + { + n.line = current.line; + n.column = current.column; n.operator = advance().type; + } else advance(); n.left = node; diff --git a/stdx/lexer.d b/stdx/lexer.d index 850253e..c013d2f 100644 --- a/stdx/lexer.d +++ b/stdx/lexer.d @@ -1,8 +1,99 @@ // Written in the D programming language /** + * $(H2 Summary) * This module contains a range-based _lexer generator. * + * $(H2 Overview) + * The _lexer generator consists of a template mixin, $(LREF Lexer), along with + * several helper templates for generating such things as token identifiers. + * + * To write a _lexer using this API: + * $(OL + * $(LI Create the string array costants for your language. + * $(UL + * $(LI $(LINK2 #.StringConstants, String Constants)) + * )) + * $(LI Create aliases for the various token and token identifier types + * specific to your language. + * $(UL + * $(LI $(LREF TokenIdType)) + * $(LI $(LREF tokenStringRepresentation)) + * $(LI $(LREF TokenStructure)) + * $(LI $(LREF TokenId)) + * )) + * $(LI Create a struct that mixes in the Lexer template mixin and + * implements the necessary functions. + * $(UL + * $(LI $(LREF Lexer)) + * )) + * ) + * Examples: + * $(UL + * $(LI A _lexer for D is available $(LINK2 https://github.com/Hackerpilot/Dscanner/blob/master/stdx/d/lexer.d, here).) + * $(LI A _lexer for Lua is available $(LINK2 https://github.com/Hackerpilot/lexer-demo/blob/master/lualexer.d, here).) + * ) + * $(DDOC_ANCHOR StringConstants) $(H2 String Constants) + * $(DL + * $(DT $(B staticTokens)) + * $(DD A listing of the tokens whose exact value never changes and which cannot + * possibly be a token handled by the default token lexing function. The + * most common example of this kind of token is an operator such as + * $(D_STRING "*"), or $(D_STRING "-") in a programming language.) + * $(DT $(B dynamicTokens)) + * $(DD A listing of tokens whose value is variable, such as whitespace, + * identifiers, number literals, and string literals.) + * $(DT $(B possibleDefaultTokens)) + * $(DD A listing of tokens that could posibly be one of the tokens handled by + * the default token handling function. An common example of this is + * a keyword such as $(D_STRING "for"), which looks like the beginning of + * the identifier $(D_STRING "fortunate"). isSeparating is called to + * determine if the character after the $(D_STRING 'r') separates the + * identifier, indicating that the token is $(D_STRING "for"), or if lexing + * should be turned over to the defaultTokenFunction.) + * $(DT $(B tokenHandlers)) + * $(DD A mapping of prefixes to custom token handling function names. The + * generated _lexer will search for the even-index elements of this array, + * and then call the function whose name is the element immedately after the + * even-indexed element. This is used for lexing complex tokens whose prefix + * is fixed.) + * ) + * + * Here are some example constants for a simple calculator _lexer: + * --- + * // There are a near infinite number of valid number literals, so numbers are + * // dynamic tokens. + * enum string[] dynamicTokens = ["numberLiteral", "whitespace"]; + * + * // The operators are always the same, and cannot start a numberLiteral, so + * // they are staticTokens + * enum string[] staticTokens = ["-", "+", "*", "/"]; + * + * // In this simple example there are no keywords or other tokens that could + * // look like dynamic tokens, so this is blank. + * enum string[] possibleDefaultTokens = []; + * + * // If any whitespace character or digit is encountered, pass lexing over to + * // our custom handler functions. These will be demonstrated in an example + * // later on. + * enum string[] tokenHandlers = [ + * "0", "lexNumber", + * "1", "lexNumber", + * "2", "lexNumber", + * "3", "lexNumber", + * "4", "lexNumber", + * "5", "lexNumber", + * "6", "lexNumber", + * "7", "lexNumber", + * "8", "lexNumber", + * "9", "lexNumber", + * " ", "lexWhitespace", + * "\n", "lexWhitespace", + * "\t", "lexWhitespace", + * "\r", "lexWhitespace" + * ]; + * --- + * * Copyright: Brian Schott 2013 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost, License 1.0) * Authors: Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu @@ -16,7 +107,12 @@ module stdx.lexer; * unsigned integral type that is able to hold the value * staticTokens.length + dynamicTokens.length. For example if there are 20 * static tokens, 30 dynamic tokens, and 10 possible default tokens, this - * template will alias itself to ubyte, as 20 + 30 + 10 < ubyte.max. + * template will alias itself to ubyte, as 20 + 30 + 10 < $(D_KEYWORD ubyte).max. + * Examples: + * --- + * // In our calculator example this means that IdType is an alias for ubyte. + * alias IdType = TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens); + * --- */ template TokenIdType(alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens) @@ -32,7 +128,15 @@ template TokenIdType(alias staticTokens, alias dynamicTokens, } /** - * Looks up the string representation of the given token type. + * Looks up the string representation of the given token type. This is the + * opposite of the function of the TokenId template. + * Params: type = the token type identifier + * Examples: + * --- + * alias str = tokenStringRepresentation(IdType, staticTokens, dynamicTokens, possibleDefaultTokens); + * assert (str(tok!"*") == "*"); + * --- + * See_also: $(LREF TokenId) */ string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type) @property { @@ -57,18 +161,18 @@ string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens * valid token type identifier) * ) * In all cases this template will alias itself to a constant of type IdType. + * This template will fail at compile time if $(D_PARAM symbol) is not one of + * the staticTokens, dynamicTokens, or possibleDefaultTokens. * Examples: * --- - * enum string[] staticTokens = ["+", "-", "*", "/"]; - * enum string[] dynamicTokens = ["number"]; - * enum string[] possibleDefaultTokens = []; - * alias IdType = TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens); * template tok(string symbol) * { * alias tok = TokenId!(IdType, staticTokens, dynamicTokens, * possibleDefaultTokens, symbol); * } + * // num and plus are of type ubyte. * IdType plus = tok!"+"; + * IdType num = tok!"numberLiteral"; * --- */ template TokenId(IdType, alias staticTokens, alias dynamicTokens, @@ -118,35 +222,49 @@ template TokenId(IdType, alias staticTokens, alias dynamicTokens, /** * The token that is returned by the lexer. * Params: - * IDType = The D type of the "type" token type field. + * IdType = The D type of the "type" token type field. * extraFields = A string containing D code for any extra fields that should * be included in the token structure body. This string is passed * directly to a mixin statement. + * Examples: + * --- + * // No extra struct fields are desired in this example, so leave it blank. + * alias Token = TokenStructure!(IdType, ""); + * Token minusToken = Token(tok!"-"); + * --- */ -struct TokenStructure(IDType, string extraFields = "") +struct TokenStructure(IdType, string extraFields = "") { public: /** * == overload for the the token type. */ - bool opEquals(IDType type) const pure nothrow @safe + bool opEquals(IdType type) const pure nothrow @safe { return this.type == type; } /** - * + * Constructs a token from a token type. + * Params: type = the token type */ - this(IDType type) + this(IdType type) { this.type = type; } /** - * + * Constructs a token. + * Params: + * type = the token type + * text = the text of the token, which may be null + * line = the line number at which this token occurs + * column = the column nmuber at which this token occurs + * index = the byte offset from the beginning of the input at which this + * token occurs */ - this(IDType type, string text, size_t line, size_t column, size_t index) + this(IdType type, string text, size_t line, size_t column, size_t index) { this.text = text; this.line = line; @@ -156,39 +274,105 @@ public: } /** - * + * The _text of the token. */ string text; /** - * + * The line number at which this token occurs. */ size_t line; /** - * + * The Column nmuber at which this token occurs. */ size_t column; /** - * + * The byte offset from the beginning of the input at which this token + * occurs. */ size_t index; /** - * + * The token type. */ - IDType type; + IdType type; mixin (extraFields); } +/** + * The implementation of the _lexer is contained within this mixin template. + * To use it, this template should be mixed in to a struct that represents the + * _lexer for your language. This struct should implement the following methods: + * $(UL + * $(LI popFront, which should call this mixin's _popFront() and + * additionally perform any token filtering or shuffling you deem + * necessary. For example, you can implement popFront to skip comment or + * tokens.) + * $(LI A function that serves as the default token lexing function. For + * most languages this will be the identifier lexing function.) + * $(LI A function that is able to determine if an identifier/keyword has + * come to an end. This function must retorn $(D_KEYWORD bool) and take + * a single $(D_KEYWORD size_t) argument representing the number of + * bytes to skip over before looking for a separating character.) + * $(LI Any functions referred to in the tokenHandlers template paramater. + * These functions must be marked $(D_KEYWORD pure nothrow), take no + * arguments, and return a token) + * $(LI A constructor that initializes the range field as well as calls + * popFront() exactly once (to initialize the _front field).) + * ) + * Examples: + * --- + * struct CalculatorLexer + * { + * mixin Lexer!(IdType, Token, defaultTokenFunction, isSeparating, + * staticTokens, dynamicTokens, tokenHandlers, possibleDefaultTokens); + * + * this (ubyte[] bytes) + * { + * this.range = LexerRange(bytes); + * popFront(); + * } + * + * void popFront() pure + * { + * _popFront(); + * } + * + * Token lexNumber() pure nothrow @safe + * { + * ... + * } + * + * Token lexWhitespace() pure nothrow @safe + * { + * ... + * } + * + * Token defaultTokenFunction() pure nothrow @safe + * { + * // There is no default token in the example calculator language, so + * // this is always an error. + * range.popFront(); + * return Token(tok!""); + * } + * + * bool isSeparating(size_t offset) pure nothrow @safe + * { + * // For this example language, always return true. + * return true; + * } + * } + * --- + */ mixin template Lexer(IDType, Token, alias defaultTokenFunction, alias tokenSeparatingFunction, alias staticTokens, alias dynamicTokens, - alias pseudoTokenHandlers, alias possibleDefaultTokens) + alias tokenHandlers, alias possibleDefaultTokens) { - static assert (pseudoTokenHandlers.length % 2 == 0, "Each pseudo-token must" + static assert (tokenHandlers.length % 2 == 0, "Each pseudo-token must" ~ " have a corresponding handler function name."); static string generateMask(const ubyte[] arr) @@ -214,7 +398,7 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, import std.string; import std.range; - string[] pseudoTokens = stupidToArray(pseudoTokenHandlers.stride(2)); + string[] pseudoTokens = stupidToArray(tokenHandlers.stride(2)); string[] allTokens = stupidToArray(sort(staticTokens ~ possibleDefaultTokens ~ pseudoTokens).uniq); string code; for (size_t i = 0; i < allTokens.length; i++) @@ -240,7 +424,7 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, if (pseudoTokens.countUntil(tokens[0]) >= 0) { return " return " - ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(tokens[0]) + 1] + ~ tokenHandlers[tokenHandlers.countUntil(tokens[0]) + 1] ~ "();\n"; } else if (staticTokens.countUntil(tokens[0]) >= 0) @@ -251,7 +435,7 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, else if (pseudoTokens.countUntil(tokens[0]) >= 0) { return " return " - ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(tokens[0]) + 1] + ~ tokenHandlers[tokenHandlers.countUntil(tokens[0]) + 1] ~ "();\n"; } } @@ -271,14 +455,14 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, if (token.length <= 8) { code ~= " return " - ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] + ~ tokenHandlers[tokenHandlers.countUntil(token) + 1] ~ "();\n"; } else { code ~= " if (range.peek(" ~ text(token.length - 1) ~ ") == \"" ~ escape(token) ~"\")\n"; code ~= " return " - ~ pseudoTokenHandlers[pseudoTokenHandlers.countUntil(token) + 1] + ~ tokenHandlers[tokenHandlers.countUntil(token) + 1] ~ "();\n"; } } @@ -325,16 +509,23 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, return code; } + /** + * Implements the range primitive front(). + */ ref const(Token) front() pure nothrow const @property { return _front; } + void _popFront() pure { _front = advance(); } + /** + * Implements the range primitive empty(). + */ bool empty() pure const nothrow @property { return _front.type == tok!"\0"; @@ -359,9 +550,7 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, return retVal; } - /** - * This only exists because the real array() can't be called at compile-time - */ + // This only exists because the real array() can't be called at compile-time static string[] stupidToArray(R)(R range) { string[] retVal; @@ -397,13 +586,30 @@ mixin template Lexer(IDType, Token, alias defaultTokenFunction, } } + /** + * The lexer input. + */ LexerRange range; + + /** + * The token that is currently at the front of the range. + */ Token _front; } +/** + * Range structure that wraps the _lexer's input. + */ struct LexerRange { + /** + * Params: + * bytes = the _lexer input + * index = the initial offset from the beginning of $(D_PARAM bytes) + * column = the initial column number + * line = the initial line number + */ this(const(ubyte)[] bytes, size_t index = 0, size_t column = 1, size_t line = 1) pure nothrow @safe { this.bytes = bytes; @@ -412,31 +618,52 @@ struct LexerRange this.line = line; } + /** + * Returns: a mark at the current position that can then be used with slice. + */ size_t mark() const nothrow pure @safe { return index; } + /** + * Sets the range to the given position + * Params: m = the position to seek to + */ void seek(size_t m) nothrow pure @safe { index = m; } + /** + * Returs a slice of the input byte array betwene the given mark and the + * current position. + * Params m = the beginning index of the slice to return + */ const(ubyte)[] slice(size_t m) const nothrow pure @safe { return bytes[m .. index]; } + /** + * Implements the range primitive _empty. + */ bool empty() const nothrow pure @safe { return index >= bytes.length; } + /** + * Implements the range primitive _front. + */ ubyte front() const nothrow pure @safe { return bytes[index]; } + /** + * Returns: the current item as well as the items $(D_PARAM p) items ahead. + */ const(ubyte)[] peek(size_t p) const nothrow pure @safe { return index + p + 1 > bytes.length @@ -444,48 +671,79 @@ struct LexerRange : bytes[index .. index + p + 1]; } + /** + * + */ ubyte peekAt(size_t offset) const nothrow pure @safe { return bytes[index + offset]; } + /** + * Returns: true if it is possible to peek $(D_PARAM p) bytes ahead. + */ bool canPeek(size_t p) const nothrow pure @safe { return index + p < bytes.length; } + /** + * Implements the range primitive _popFront. + */ void popFront() pure nothrow @safe { index++; column++; } + /** + * Implements the algorithm _popFrontN more efficiently. + */ void popFrontN(size_t n) pure nothrow @safe { index += n; + column += n; } + /** + * Increments the range's line number and resets the column counter. + */ void incrementLine() pure nothrow @safe { column = 1; line++; } + /** + * The input _bytes. + */ const(ubyte)[] bytes; + + /** + * The range's current position. + */ size_t index; + + /** + * The current _column number. + */ size_t column; + + /** + * The current _line number. + */ size_t line; } /** - * The string cache should be used within lexer implementations for several - * reasons: - * $(UL - * $(LI Reducing memory consumption.) - * $(LI Increasing performance in token comparisons) - * $(LI Correctly creating immutable token text if the lexing source is not - * immutable) - * ) + * The string cache implements a map/set for strings. Placing a string in the + * cache returns an identifier that can be used to instantly access the stored + * string. It is then possible to simply compare these indexes instead of + * performing full string comparisons when comparing the string content of + * dynamic tokens. The string cache also handles its own memory, so that mutable + * ubyte[] to lexers can still have immutable string fields in their tokens. + * Because the string cache also performs de-duplication it is possible to + * drastically reduce the memory usage of a lexer. */ struct StringCache { @@ -493,7 +751,10 @@ public: @disable this(); - this(size_t bucketCount = defaultBucketCount) + /** + * Params: bucketCount = the initial number of buckets. + */ + this(size_t bucketCount) { buckets = new Item*[bucketCount]; } @@ -512,6 +773,9 @@ public: return get(cache(bytes)); } + /** + * Equivalent to calling cache() and get(). + */ string cacheGet(const(ubyte[]) bytes, uint hash) pure nothrow @safe { return get(cache(bytes, hash)); @@ -536,6 +800,11 @@ public: return cache(bytes, hash); } + /** + * Caches a string as above, but uses the given has code instead of + * calculating one itself. Use this alongside hashStep() can reduce the + * amount of work necessary when lexing dynamic tokens. + */ size_t cache(const(ubyte)[] bytes, uint hash) pure nothrow @safe in { @@ -583,11 +852,21 @@ public: writeln("rehashes: ", rehashCount); } + /** + * Incremental hashing. + * Params: + * b = the byte to add to the hash + * h = the hash that has been calculated so far + * Returns: the new hash code for the string. + */ static uint hashStep(ubyte b, uint h) pure nothrow @safe { return (h ^ sbox[b]) * 3; } + /** + * The default bucket count for the string cache. + */ static enum defaultBucketCount = 2048; private: diff --git a/stdx/lexer.html b/stdx/lexer.html new file mode 100644 index 0000000..8881aa0 --- /dev/null +++ b/stdx/lexer.html @@ -0,0 +1,483 @@ +

stdx.lexer

+This module contains a range-based lexer generator. +

+The lexer generator consists of a template mixin, Lexer, along with several + helper templates for generating such things as token identifiers. +

+ + To generate a lexer using this API, several constants must be supplied: +
staticTokens
+
A listing of the tokens whose exact value never changes and which cannot + possibly be a token handled by the default token lexing function. The + most common example of this kind of token is an operator such as "*", or + "-" in a programming language.
+
dynamicTokens
+
A listing of tokens whose value is variable, such as whitespace, + identifiers, number literals, and string literals.
+
possibleDefaultTokens
+
A listing of tokens that could posibly be one of the tokens handled by + the default token handling function. An common example of this is + a keyword such as "for", which looks like the beginning of + the identifier "fortunate". isSeparating is called to + determine if the character after the 'r' separates the + identifier, indicating that the token is "for", or if lexing + should be turned over to the defaultTokenFunction.
+
tokenHandlers
+
A mapping of prefixes to custom token handling function names. The + generated lexer will search for the even-index elements of this array, + and then call the function whose name is the element immedately after the + even-indexed element. This is used for lexing complex tokens whose prefix + is fixed.
+
+

+ + Here are some example constants for a simple calculator lexer: +
// There are a near infinite number of valid number literals, so numbers are
+// dynamic tokens.
+enum string[] dynamicTokens = ["numberLiteral", "whitespace"];
+
+// The operators are always the same, and cannot start a numberLiteral, so
+// they are staticTokens
+enum string[] staticTokens = ["-", "+", "*", "/"];
+
+// In this simple example there are no keywords or other tokens that could
+// look like dynamic tokens, so this is blank.
+enum string[] possibleDefaultTokens = [];
+
+// If any whitespace character or digit is encountered, pass lexing over to
+// our custom handler functions. These will be demonstrated in an example
+// later on.
+enum string[] tokenHandlers = [
+    "0", "lexNumber",
+    "1", "lexNumber",
+    "2", "lexNumber",
+    "3", "lexNumber",
+    "4", "lexNumber",
+    "5", "lexNumber",
+    "6", "lexNumber",
+    "7", "lexNumber",
+    "8", "lexNumber",
+    "9", "lexNumber",
+    " ", "lexWhitespace",
+    "\n", "lexWhitespace",
+    "\t", "lexWhitespace",
+    "\r", "lexWhitespace"
+];
+
+ +

+Examples:
+

+License:
License 1.0 +

+Authors:
Brian Schott, with ideas shamelessly stolen from Andrei Alexandrescu +

+Source:
+std/lexer.d

+ +
template TokenIdType(alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)
+
Template for determining the type used for a token type. Selects the smallest + unsigned integral type that is able to hold the value + staticTokens.length + dynamicTokens.length. For example if there are 20 + static tokens, 30 dynamic tokens, and 10 possible default tokens, this + template will alias itself to ubyte, as 20 + 30 + 10 < ubyte.max. +

+Examples:
// In our calculator example this means that IdType is an alias for ubyte.
+alias IdType = TokenIdType!(staticTokens, dynamicTokens, possibleDefaultTokens);
+
+

+ +
+
@property string tokenStringRepresentation(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens)(IdType type); +
+
Looks up the string representation of the given token type. This is the + opposite of the function of the TokenId template. +

+Parameters: + +
IdType typethe token type identifier

+Examples:
alias str = tokenStringRepresentation(IdType, staticTokens, dynamicTokens, possibleDefaultTokens);
+assert (str(tok!"*") == "*");
+
+

+See Also:
TokenId

+ +
+
template TokenId(IdType, alias staticTokens, alias dynamicTokens, alias possibleDefaultTokens, string symbol)
+
Generates the token type identifier for the given symbol. There are two + special cases: +
  • If symbol is "", then the token identifier will be 0
    • +
  • If symbol is "\0", then the token identifier will be the maximum + valid token type identifier
    • +
+ In all cases this template will alias itself to a constant of type IdType. + This template will fail at compile time if symbol is not one of + the staticTokens, dynamicTokens, or possibleDefaultTokens. +

+Examples:
template tok(string symbol)
+{
+    alias tok = TokenId!(IdType, staticTokens, dynamicTokens,
+        possibleDefaultTokens, symbol);
+}
+// num and plus are of type ubyte.
+IdType plus = tok!"+";
+IdType num = tok!"numberLiteral";
+
+

+ +
+
struct TokenStructure(IdType, string extraFields = ""); +
+
The token that is returned by the lexer. +

+Parameters: + + + +
IdTypeThe D type of the "type" token type field.
extraFieldsA string containing D code for any extra fields that should + be included in the token structure body. This string is passed + directly to a mixin statement.

+Examples:
// No extra struct fields are desired in this example, so leave it blank.
+alias Token = TokenStructure!(IdType, "");
+Token minusToken = Token(tok!"-");
+
+

+ +
const pure nothrow @safe bool opEquals(IdType type); +
+
== overload for the the token type.

+ +
+
this(IdType type); +
+
Constructs a token from a token type. +

+Parameters: + +
IdType typethe token type

+ +
+
this(IdType type, string text, size_t line, size_t column, size_t index); +
+
Constructs a token. +

+Parameters: + + + + + + + + + +
IdType typethe token type
string textthe text of the token, which may be null
size_t linethe line number at which this token occurs
size_t columnthe column nmuber at which this token occurs
size_t indexthe byte offset from the beginning of the input at which this + token occurs

+ +
+
string text; +
+
The text of the token.

+ +
+
size_t line; +
+
The line number at which this token occurs.

+ +
+
size_t column; +
+
The Column nmuber at which this token occurs.

+ +
+
size_t index; +
+
The byte offset from the beginning of the input at which this token + occurs.

+ +
+
IdType type; +
+
The token type.

+ +
+
+
+
template Lexer(IDType, Token, alias defaultTokenFunction, alias tokenSeparatingFunction, alias staticTokens, alias dynamicTokens, alias tokenHandlers, alias possibleDefaultTokens)
+
The implementation of the lexer is contained within this mixin template. + To use it, this template should be mixed in to a struct that represents the + lexer for your language. This struct should implement the following methods: +
  • popFront, which should call this mixin's popFront() and + additionally perform any token filtering or shuffling you deem + necessary. For example, you can implement popFront to skip comment or + tokens.
    • +
  • A function that serves as the default token lexing function. For + most languages this will be the identifier lexing function.
    • +
  • A function that is able to determine if an identifier/keyword has + come to an end. This function must retorn bool and take + a single size_t argument representing the number of + bytes to skip over before looking for a separating character.
    • +
  • Any functions referred to in the tokenHandlers template paramater. + These functions must be marked pure nothrow, take no + arguments, and return a token
    • +
  • A constructor that initializes the range field as well as calls + popFront() exactly once (to initialize the front field).
    • +
+

+Examples:
struct CalculatorLexer
+{
+    mixin Lexer!(IdType, Token, defaultTokenFunction, isSeparating,
+        staticTokens, dynamicTokens, tokenHandlers, possibleDefaultTokens);
+
+    this (ubyte[] bytes)
+    {
+        this.range = LexerRange(bytes);
+        popFront();
+    }
+
+    void popFront() pure
+    {
+        _popFront();
+    }
+
+    Token lexNumber() pure nothrow @safe
+    {
+        ...
+    }
+
+    Token lexWhitespace() pure nothrow @safe
+    {
+        ...
+    }
+
+    Token defaultTokenFunction() pure nothrow @safe
+    {
+        // There is no default token in the example calculator language, so
+        // this is always an error.
+        range.popFront();
+        return Token(tok!"");
+    }
+
+    bool isSeparating(size_t offset) pure nothrow @safe
+    {
+        // For this example language, always return true.
+        return true;
+    }
+}
+
+

+ +
const pure nothrow @property const(Token) front(); +
+
Implements the range primitive front().

+ +
+
const pure nothrow @property bool empty(); +
+
Implements the range primitive empty().

+ +
+
LexerRange range; +
+
The lexer input.

+ +
+
Token _front; +
+
The token that is currently at the front of the range.

+ +
+
+
+
struct LexerRange; +
+
Range structure that wraps the lexer's input.

+ +
pure nothrow @safe this(const(ubyte)[] bytes, size_t index = 0, size_t column = 1, size_t line = 1); +
+
Parameters: + + + + + + + +
const(ubyte)[] bytesthe lexer input
size_t indexthe initial offset from the beginning of bytes
size_t columnthe initial column number
size_t linethe initial line number

+ +
+
const pure nothrow @safe size_t mark(); +
+
Returns:
a mark at the current position that can then be used with slice.

+ +
+
pure nothrow @safe void seek(size_t m); +
+
Sets the range to the given position +

+Parameters: + +
size_t mthe position to seek to

+ +
+
const pure nothrow @safe const(ubyte)[] slice(size_t m); +
+
Returs a slice of the input byte array betwene the given mark and the + current position. + Params m = the beginning index of the slice to return

+ +
+
const pure nothrow @safe bool empty(); +
+
Implements the range primitive empty.

+ +
+
const pure nothrow @safe ubyte front(); +
+
Implements the range primitive front.

+ +
+
const pure nothrow @safe const(ubyte)[] peek(size_t p); +
+
Returns:
the current item as well as the items p items ahead.

+ +
+
const pure nothrow @safe ubyte peekAt(size_t offset); +
+

+
+
const pure nothrow @safe bool canPeek(size_t p); +
+
Returns:
true if it is possible to peek p bytes ahead.

+ +
+
pure nothrow @safe void popFront(); +
+
Implements the range primitive popFront.

+ +
+
pure nothrow @safe void popFrontN(size_t n); +
+
Implements the algorithm popFrontN more efficiently.

+ +
+
pure nothrow @safe void incrementLine(); +
+
Increments the range's line number and resets the column counter.

+ +
+
const(ubyte)[] bytes; +
+
The input bytes.

+ +
+
size_t index; +
+
The range's current position.

+ +
+
size_t column; +
+
The current column number.

+ +
+
size_t line; +
+
The current line number.

+ +
+
+
+
struct StringCache; +
+
The string cache implements a map/set for strings. Placing a string in the + cache returns an identifier that can be used to instantly access the stored + string. It is then possible to simply compare these indexes instead of + performing full string comparisons when comparing the string content of + dynamic tokens. The string cache also handles its own memory, so that mutable + ubyte[] to lexers can still have immutable string fields in their tokens. + Because the string cache also performs de-duplication it is possible to + drastically reduce the memory usage of a lexer.

+ +
this(size_t bucketCount); +
+
Parameters: + +
size_t bucketCountthe initial number of buckets.

+ +
+
pure nothrow @safe string cacheGet(const(ubyte[]) bytes); +
+
Equivalent to calling cache() and get(). +
StringCache cache;
+ubyte[] str = ['a', 'b', 'c'];
+string s = cache.get(cache.cache(str));
+assert(s == "abc");
+
+

+ +
+
pure nothrow @safe string cacheGet(const(ubyte[]) bytes, uint hash); +
+
Equivalent to calling cache() and get().

+ +
+
pure nothrow @safe size_t cache(const(ubyte)[] bytes); +
+
Caches a string. +

+Parameters: + +
const(ubyte)[] bytesthe string to cache

+Returns:
A key that can be used to retrieve the cached string +

+Examples:
StringCache cache;
+ubyte[] bytes = ['a', 'b', 'c'];
+size_t first = cache.cache(bytes);
+size_t second = cache.cache(bytes);
+assert (first == second);
+
+

+ +
+
pure nothrow @safe size_t cache(const(ubyte)[] bytes, uint hash); +
+
Caches a string as above, but uses the given has code instead of + calculating one itself. Use this alongside hashStep() can reduce the + amount of work necessary when lexing dynamic tokens.

+ +
+
const pure nothrow @safe string get(size_t index); +
+
Gets a cached string based on its key. +

+Parameters: + +
size_t indexthe key

+Returns:
the cached string

+ +
+
static pure nothrow @safe uint hashStep(ubyte b, uint h); +
+
Incremental hashing. +

+Parameters: + + + +
ubyte bthe byte to add to the hash
uint hthe hash that has been calculated so far

+Returns:
the new hash code for the string.

+ +
+
static int defaultBucketCount; +
+
The default bucket count for the string cache.

+ +
+
+
+
+ +
[top]
diff --git a/stdx/lexer.o b/stdx/lexer.o new file mode 100644 index 0000000000000000000000000000000000000000..162acdb58776707508737e37dd555546507fffbe GIT binary patch literal 54460 zcmd^o33wD$)^>%C0f8}LQ3Eb?a0>z<8zC`(i6K~lghdj>D27hbA(3pPyEULd5YRMj zG@`hSqNph7=fXEC;xZCsWN{5TVq8X}G8zUP5oeTPRK9cWQg!=w-E=3>--wB^|KR(d-F4EuL0|6f)v-V3#QBH6>0P(kaq(Bl zm)I(XL~hu&erC$|=jMJnV&fZk4O^;R`hn-;Grn58wr=7Z&oy}ZeShrzv$FIx2M1+t z=<~_z`}dqzo_y^5@5X=o(ujVa{xG8Y^@V3{-rMr;^sTRcUHtp{PoCRzvwOqz+{K^S z{`lAVLnl1_-GO)8?rb@2;EAI~UzzCJn*4}s;&l_+PkwuN$)%ZVj*On)G-uA1N546k6tV55+m^ewMC@GiZSR6P zACBy3zd8AfEgub;8a+Jmg}LjtH1@lCY5wxOr=FO(SkulKFm}o4H}^Q6n?C)Ex{E7j zuE?$U=LJire0IgY`D;G8an1OTqR)vq^vq4SyqmxG#dClES=#<9M}!aP`|8G>vtnkK zrCdMer9r(5V~R`P+V(-rmyh54!uEfRezM`prt|k+vh>2cZyP(}-n|d}Jm|I?qn95m z%uen2^jPX$^NQ0Nd+i-_Af@+1!(U5Zy?NE(e$5+4SDid+l}$g=QMa|BcZMtgZ z#@macj(_sTbC-m5B%SoPf=OkA!*?CI;hC>oj))bvoZE4k@3pU2zxaIL_rH7oNZR{T zc7#vy9&T-X@Yb02`g_;4<(xNa$SoVQetf<0@xG(~oO=7>$k_XOrADMgw|1ny{^&)g zR~~h)o9 zKf4|oH|nA1rhSsM=ckgAjE}<>jveNBYTNw1HJ9Bn$s6(a2~jzW)_(fQJpn7#$0K5DsSXshk(w?BXCo&iaYt8Q$xpY_ro9=JT~*o${R zxxdf!7jue6KC*VphBH1{xgaTF!t#&*oxEN9@~*{cr6&zLxnknM59WQk>)n(cN8Y(M z;xD7e#t!`B1%G^c^Wnf_}XE{pBog$3@$K|-4Ric6REPB>)A=rs?*EuTOuDD6gDsZaO z8RMy_b{l7CGOnzquFBz#n@o1IYMmu9CFPDf;hac;nzOFjQ5ol{t|+M~br!iD#g$H1 zTz*}JGrQVdw9+ zqwM3ltq;ya1mORGB0%ioW>r>{IHzCjtjnn`g=8s>iG|Q!K6gaG1rj#YaG}OUP>V6K zU=;i^07C?>&Ty10KxNP z)j1ZCEH!iJEc1{YWx657YTb~Cc6W`vwx)J+T}`z+c7(mw;c`PFJMGTuQhQCA-BE2P z_VBo*rrPC>^%O61J4eX&I>`!&&OY<`fu8yNfh^R2?Ph=Tu+qsH`Zp7vxVKooaVE z7kZr4B~DZ56!VZA_0(VM{wVIE%9@e|Bn4>_W!Bb09yse_sONF~JOq)rNe;KeN-mbB z5vdYvB%Rq*$x`D|OvKZZ#S=J=mCl2mTFos{r;kFhes=Oe@lt{AJEX5T6hBM%7PRMj zvk&Kb^N#Ai0{CPf*1f%T@1=co@AN3$H@98)7DemcYwWuB=;eEMltm7MqnMUeSF+R)-HImDzyR%gQ+?Fz)v# zd}+6;({`gyS=^^IO^OK9ef{*b?GXPKV0NYI-npQ^DnQ-TRtu+jnqE4pd`PegxFzAgshz(7oT(CB}6hkU`_{(kgA5)C+eC^ySvG2b3mlp9qABB40IoEgFM$=Q4ZhdqcHf2&PtHTEnpLji3 z`13?J{qaqW$_1j_^nQ_Rg5}+_3XWiTeLO2lKb{p0MPPX(^$Ra6>;{RY9wzq-Qa`p+ zVYf4XU*uh~?t|XNdu6nKe6k%p_vS@z3=PZ z?MW@)J>p4$&KPR4HVS)7J-sf2T$0F z-T*tcL90z3uj8$RI>C5hWsyEIz|Q6IM;1Fx0%E1}V5h&dKf(Cq>nuN`$gs7IUuC^k z9*jhpaA_tCqr3gaIWiu>0P6t>=w4PVX(!`dFL;h*HIBAf^$~5nZ*0%?D zT*14KoO`n&NHAyd8SeNzZ9)Vhhc#gBu=>)mJ+^y29T0vV3YZr3-F7~BLhbX|oTu=u z#mcu9=9}aQ(9o*Tz1h3V1=_9q27w#|A{)xq zM^DRZg+VyZ@Y0vvihYe={X&7DC-U{y({_SvrYQ^g=o_G?6&wZlA`^zbW_vpfg1FoDdUJbk#>V+ncC}2oxr8B;#3{q^Eh1cM9jt^%47Qz{>IUkJ5cJ z;gbpTPbjOmphG|L6=0vt6(D2~pzL_gxr)tIicD2DNn!pG zIbZVXB+ucd@T-OR&B44Nu?uSfH=vF^EqXw+hHjoUYu`-hy2C)Sl~m@g>~>5tl8EVZ zTO(IILe4zxdb$Vet=e>mv{;B(c5ALL{56<~cVzl9qV=SA^dEqdOr4>}U$Fxs0+XG* z!=Sy;)P|(p{y9_`&Szkk+&9;k-I3$1wd3?A4O_+9tx$QORA(yTY^VugK5_Ae573Wi zw;#v_45BK`!^;EO2Ww_f)Xm`CRv4;sysf;`(YNt_nB_lI;LEWX3fOe-E{b{!Af8z6 zA-Z=csBVSQZzd7;PKHXK9pzin;awW-+od;M2J=psbgsG!y-j#0=ZC{N=@`JJW@+d( zj8~EohrxTPIT#gx_)s4L*FJ14*sc4hEQBTjpda+~_rX+yX(|u=63uxFHnFrM!!w+D z`|s0V;AI4kb~!0{2Ke$?pekB(j_;o0?f*(f!;;-vI^34&XYC~zd_|GFs3VbcwjYTsgaF&1yPz6v{uVhjxPurZ6-F=n9wrtifZ| ztA}la=^g9cy{l$G;DHJn`&v9CjF-mhzKelz)jYiDU0aU3U8`zghosPa@K8^0!t0SM zHW6$c#nvGPd3BEwo2KN;^`_*1QcY^hM=jfqzztPBnrfN`F@W|kw*l&>0aEAal%{aF z3(s!UeW&Q@1&wtVKp+}*?-1R0lAiv0-3fa7r*Nb4puys8gWF8dUQ6uxpkYBJ*a{l3 zK*x#e2PPrY%UT9{qG@6jw(DCt44lm{DY%QYX>Y-Ly~)$8H!az$KTQ4;vmb8IciPPs z0Ap)4E5crAK#-ljn9G4Pspv?jxl?_Rj8x;q9DvViNx>W8-ONqmi_^R za-&hmrf>*l_^ZT(Z!OiNCH3V-_0fH&gBg5afj5G4k3xJ*RZ_(`a(o97;o~G3|4DM} z%5I@naXNw17`ZEl4l6yc)m6?rb@1y6!!I9>;MuJitCNRjL~eP@yGKu_PeHt-qXpTg zOxbnVdUQTUvN7oQ37(b(c`Y_CbVAS^anCT#4mjL1Op)#xg#JI^eq~H&_bYHO z;(x#5e=mY||6gq^Ze*TG>NjUAr`CPbA#%{R@W~WZ8{U6Sq@53)q^xRj*NNgdAHqAX zEJ!ZYX@`#gDChyS6d>%iXpahtX#t9{pa@+FDt0hMfR+M;iu48_W`cfqZDq~eS-y|q z2W%fLod-MRW0ow}M+Vq=u=_QIngNp`$2h~ka(8X}CH(yvMpXXyuCB2MO3Tmtf9w0@ zKo2`dGNsG^ZT|na$N#_G|BL?1^Cv;_|3AI|94`5*>JKHcprb7%r8 zJHhJ zawk}z^W7#FOzCj;GXtg*@az}H`*__uqYupCg!vSV`s`kW|M{t`m$eTtR|xch{W*Z> z_T4rgR4KXfce_^Uc{6!l4v!CLc|TT^7bY};Ty#o9RQqmgH_Az#KhwW0^li-RXjszG z;F>_@HO9Tao;DFev)@#~us}#&r@@IQ!HLG;99RZf@jSk(lCgS8N5=Q>!{qt6_qZPU z$6X_Ld-D#h_*bvUdv`=``Dn$zA|mg9C34FPA8Yp%!G$!{LX%53p_8|!nNt6GXLkEtej-- z8=>1y)yYiwBzS94ZT~DHa!bL1<)7o|*zYx{@YUHJ zT1#U#6tV4x9jNDtTkbq`95UpG>_ZK3+hj=y_n~3EeA$QaRu{bLWhGS5wt7mh#ys$9 zd#7F%9NPXlc$Ifx`Detdz1{)~ucn|^$B9?kQ7f-}OAe8+B!>l-9B;$)6`IZT*k7zW|J0P#6rJd8vB z;WYpL0M0_!hBvDyf;XwOQxo;+-%b4LBmI5%3FP4%D9`}ob8~#YXVert21?IsaZP8T zZh^SQ31ir9sFoJ;2&EC88>Y`~hwEd6>)=s#`|AEfFUV`-R8VgRH*O9i+1Wq>rT98s z!hh@e-*CyFKGJ!xQyc)THkF-V^C#KXc@pqj$4hc2;B=&fsNx`r8K=K?JmvfC|I~Pz z^XQ+Y&dTrax6uD|^*VhQ0pq*@J`clZBYYl(kGSoI6TOLafir^buYz}Qe(XSPi}xJj zT#KsU-AXME-VCJsa(?7?*1NU1Qhfa{uGr<$;;L)hPA#sarmD(W4Oi-F=rx4nE}m8} z+U159T>*DiI^w7@Iiynf9al6dDJg$Ztuv>(tfpvIGQPbymb{Fe5O2Pool;a(QC;CK zWR|+4i0nY7CMa>&)a6Z{=9pGItz=qRW_@{8RaK$NA?64#Z|BC7s~mL;^6IJWvcf7M z9%_#Qm4cACoX!Qz|FWvWt3x9RQ0kmyEP0vPC`wf!gefE6Wn=~O7C{U|&n&eunAD#rEN6u2ObE*`i%5)24p_R!}uXCz#C64MD{sjxI znyeK&Crjlohx0TKG0QFWI;TqGRMR+Ds7*?t**RT8Qbl!1ofA9J+=^-^=M1rYooEN4 zO(-g=uc^&m=y6oKCQq$b^>*qj*eH2#oNV2VC(JTe#VGpmE@^VQwBU5fxY|a+*>!bB zoE&cu&+rSFSkO+GyQ9yk6AbFi?s0d=fJF|8MTPjXp|eC9FGVSJ-z`doYAc9KsU^4s z>td&yH>CQ^^1{qTFlkW@Afgj$?QYO6=ov3l4?;Yal$RG4#^lcC7-CT^6}w8dpxbOj zGYDPMe5giY*bKE6t@MIWrItHf<+I$*S~gQFUr-2h*C0wMbxu^{!SAfg79($=7L154 z>SIJssO69YpJJ^VgfO|RqPo-wJ>i?-KSYQHB0Zil7$gfFjk1#*OV%Wr{c{k1QXQ4B zhRNaP!=n=>QJqX|MCTN$K?u{ZVlRRvaW0k+*s_9W1c^jKXG5##8j+R;*nsOe%m#6m z1~X(@D&7e%73gfzEqbX6RY?7#RO z+I=(3kuH)mt+B#W5@La_1W!l>_Y>mH_2BuALh;nP5Ir^~7Nx6>Fx_z}{`HYc2PsEo zLbCs=^+c?pG-K&{e(~&z)Y%ouvn$dHP1L%azZQp3{9nrLo<_w@qB3rFMar)il^`75 ziArcDdsvTSAvV8WRA@dbIl2>-&`kC)Dk-xo#{Jq+QF3%YD#1*S6YlJ>qf|Fg>^jXJ z1p5_eKG3hg{~W;I0`N0OkkB|U6H$}H)}F7h)g3IkE=p7s|sD-^#6 z&M_-v&G{c0^@+uJta#&hIY@;=OE3WI686lJT&R#;V2a67x?wpI(y$!HFNfqWm|c-r zXr5aaqeq6MQiSXPn^j_S97}$FA|Vl6r;qCM%M0lNy~!$_R)$Y&%o`p-2}+c2Jj~69 zn=#{r-JngyJ2dG}AKlRp^fw9pCC~hkmmX2ll4;;F^2&b-5r{+74y5%j@gj#@V&_zN ztQB&JJJA=Yli*iT$gDVp{@O`CQm|tzGU1$BRd|j6<1{R87gQy}L<~_vJY~$Os;!(| zoIr?04A`-}l)Hxp65=c953|7UvgA!J%re|h1R%S<#0fv4Q&VkS#I8Fl-UoB0u38Dz zM}?yO!Qwh5y;(|$=e+_ zgg_+3mwBp7FijnmN!WhUiYuxeb&Do@s%Jw{94SR935oGZrAeiU@hORgw6``ypdZV> zrqEQOy;)g(`887Ko93vTUsG4%F0V?!QEM#xf{~-LkQ9X;=R@yDib7ABApkL+o%}ga ze1O!ghm9;2DK*7cIZNE6e5pm5_#0W-3$b|E-|_QKy}~Bh<6`oWT9|Q&ZySiEm7je2 zJ+U&M8fAw3=+w_ZHyq%;r~2QMv%z{CRC_A_F(*>f#+hQ0(E~fe9HT4rOig~W5!^)b zBzO(Qx)?n+)}^vYjxgQikV07xna(T7x-31h6SNrC^h_ROYCt?9^Nc(%tKNOP%F3ig@6Jzu}NQ2628}ji)xQw5Zfo!~T{; zj*IG?CG0OC^c>#?6X#|kS_&_6xu}gIJZ5r%F#g$^3*s>o41mSA?EH`MoxE`ETX>y4 zaI?ujOv*%T8Z0lc*lZekchBOA^&6HpEy6-p1MB@1cwYrh)>{W$vuTk5*h7GX0k551 ziDG+v7ZLFc3CB~LHh`i2JDWDhA7E!Tjl9hraLuNj8h{~S(}pN;`InKXm`yuFAwNuk z$0+c#6!o~6LE75HQYesKW4qc`Ti(5~6q0~EArQv&2Eo~yv8DexHz z{1OG8ufPiw_@xSbjsh=K;Fl}#xeEMB1wK!KI}~_{0xwnIWeR-00o< zZ9J)E@^;o9xIfo0e4d1(j!nA;cLAGCTNHpDV0M}AD)5IC_yz_3umazxz~z-48vH94_`B5RJgt#wj)VOGVcrp(T{+~ z2=g%>=aJCgVVvWc9N$j^JVuzOarx^BZqC!9nci~*k282V{x#qO4eRD_h{r*HG*C0= zX}o825Ioti!+W~51UKhtJpMZfo@&VNV@F2_&a*?){?71WCm^pmPn*W@X#}5O*a?Sv z!fi3Z&3W3_41bK^nN$eApJYcz0gn;pX*>>b5y+cm$Pa`1!0k$cXVVk--o%b>0UYaP z9y{mx{~VE@%urZE&5q84c^b+$vvbZn1Mq>`#fCi3pErm+R1U-mHy9?!_6^`M!n}^# zIlB+y=Dd#Mg#_nO)HIIY3iv>62D7SZFSDZ~MBbd&H8MO7I)WIDduQ6YlHlgNj?1qn z_@##Z40g1S;ByR|%a4Iu#2Bs6zb;cyEYWAqA|#^tXi_`Cr5 z-2`_SIG2ynv?L}mGYWZ^81fu{f#Bx6Xbj^`fuUu% zw%m~CJ^P&mzuv%k5Aqj+uQc$x+0o#E7|$CFoX0bV;NAfI27;UOqCQOGIfCC5Apb9d z-)!K#-p9ZY7^B@{;GDOT;I|q$x4(|yYYd#re@gJ%4V>FQc`)`rHZ6)J0*H|<5%7T; zFTJJ>Wq1z3&3O*TO9{S?Nog8iqkSXbP&EPDV+tG>&RfY3I>j1)LV-W0z-Ph~!ODKM z0$;AcA5!3(75GjC{-FZ@s{-E%6=&uDMwsGQ;T;M*11io+K2L#vrNDny;HN+bVrBm= z1^%G|?@-`-Ai=EcKMf7s3V%z1f2+WMcc!(SSOuP{z_S(j6$-psfqN9V@y|qcZPjvT z%Z}-~=FTM8>Z%YICKu)B&4}0VJ3&)g4Om>PmXucrDlLF)arJ6XwMu7;QG{jo(o6fo75Fngq2utwSGl1Ft2 zluHb|rb1T-s&%ZE_jN%U*O#fqEf(0Rg)NuXsl@|V$8{oWxwuU&-<>y2z?zUip}v)& z_Q85Rnp$45yh^PQ&}bBy{Ohb$Q~M>o_!3Mo;eLH`bE7y3G@qDk`G9VuO0Tf`W< ztkuJn6tG~xaxD^dN4)@{OADKdCUx7(&KKEq(aT`Qy6t7>i(b0uWiVqs^is75rCVOA zjs1FSS%Nv*)#W8clYY4sAk??+ul<02#O$`azUo7u4|SCWbi=|-mcHN}`uYr$Fw_MV zR3qf26nHn7NN)-g@vaaZWXdp=4QWXQF_Gx68``P{qM78c>CcL<>GI#(p|9!kWer3( zncU4Lj%6l7Ut_@I>MrZ*tM-&}oWR&s3n9L&-+dQDeRH0KFs&yg>G7-cw2;5z+v=2{ zr!R0r7uKB@$)W8Fy^ZQFHgMt_(tEwwGy>l1rNs^YX0vdkEl;e1EBn9t8Y-o%)q}T= zJEIg|7VdH>MP=Dvp=Xp{dhbk2taMn|mRD2isU$yWSP1P-xM;ov><9b5w(EDv`W0Ld zDXFP*rc^nrYU&mxUYwOuMSdHFbe>fU3TIr++IttLp3)j-|67P@WpJl}@#&e-U&IJ~ z92LsVg?||sdC*&uP~}WNLc8U#_9ir4?o(*e=virzyS%REni7@_A+~u-A4Ak-K87ew zqL*O)48)8Ya&rz1`f7J^CAJ;bA&c#XPOe~uqTf0_kjScpH{?O7)Of=kpI(^_+RBK4 zm9iJ?xl$LiT$;oC2eG?NM>5Ky+cVFMlhfJ5D_ocLu$ZMq)y`{5 zu;0LAy1=HW$~B)=O~@_8@{x#t@w(mxSQ_nW3$Zj}v=DUxL%jX(av4Lk8?=}q+M&5@ z{@IGG4Hd>0p3e!16|UKi%8JrSCFPDfwp0|?!4^$Q7;7$xbk})G+=6<7X*sK=+9h0K z?YyX%tfFR_Mw)&;+B93&e6w{i9Y<{~tS2kUpQ9ByO0yTjJvUzhHNOIWhkFse+|h@< zKZ38_z}soY7R!zhhgaYJ<{_{(z>;Unl9E3f*mOwTv zCi;0cJvH_m=kM7hvg;h5#Lm+sd^|hnZ!V(zC-C_Z{=o~}ek$xmKhodOc96Qs$Lot{=B5Xe=9$$|HJ(Gt@7vpH(pll zLHqd`Oi=9S@8-(F9{c~!#tZ(YuxQ78%hNQ3cHT5kjd$S?UoPPM5b@9W3GDg1bBJFr z;P4tS*$`jBj&TRCDUuEQ=P%*I`_FSItnq%E_vfP|ocHG!NI37$GbNn&=XnAy_UBFk zzY&hPKeYmmZ|idWxAL=G@TXDm$0gwZ34YEM{Am{aTrA+C|6)8}73Af42+xNM!4A%I zcs@*%aGno*MFGz%o(~Q|Ud)GU1-uE4dA&a*;P^H+$6pt4(SN@32YE$YoQM2Lup{~- z%8T_Y&g;Z@igCc$;>OF%!l{v0bkP86KbW1$og9 z@0ZXYln-Zed<722xg9=U$o>1&(FM4S7X*o?nP#47q)=pJ@~D zOW_zdS$+eB1MGjJz|Y5m!X}qDiQ#C!13uiJ6#|a=uo6DFJ*&Xql5oDl;8Ov|)d<{v zIsBb7wuFs z9CtS}UEEzK$cyoON5Dni-=QP0$^J~CZ~$H+;9|SDTEIp7yx!2h*e=94i*}e966gK= z1omQ_mkT&Q#Nw&O{X4jjut)jNgmcmUodV9+gP{Cf0xq`q1Az1Ty@<)k@(US`aS-Ks z|BUitJwB$8XK>JY7x$I>iT3w!Wnqu_7s7coJ4PJa9na}l0bj?CaVOe;O~7?_jCS@3 zI6n-=+s*_<`|ojOVIRzn9R*+$?eIRIUZLN?>=awr*)1Ai$HDA8Xkq8i-LUhyg`M_p z*f|Fi5;ieDA5+NhJ6O3cWp-+r7RrC+XMvssv$NR3&WC~>as0yk{4F^()#xUBE?o{G9^aFm9rJsep^}eEuWH zxn7VL?KB9uXlI>(i{*M)z(x6;0xsGY=bxhdN$4PKa=A_sa8Z7gfQ$B%1zeOLO3f4d z*j{dDc^)I-H!*y)fNz0gp69Y1-i~BDydA0RTp-vN+siZs-lV`^RN$=)$7*<+9pml; zc8p>A3w(IFz7}wd&!1Tw{vqLSFg!}AUu?77j@ZvUFW_Q3#kRxcJ%W7^XK>K@TRZcI zk5lN@?Oa*dBmRYO9?gys$MXKf>~N7_^7a#`q{#aQR~Gic?C8u6R|zKXw&3M;%5C5t z?d%YM-?3vEKZW6#4}XOZua{8d-|eD2<|Mb@8;)^9-mBol@kj(=L%bY596uF7*bpy; z568vlEyV&Z&MPVfJQ0p@!|MJUe7GINaKrdikt6>v749z;`q^n9iZ(I##4STS5sszw z)Mmjo856{f!#;3?Jh;b==Y0$uBhIEPc!_KoY|sDx$xaP@etsn!Sn($}Q3=k2yvQN* zWBu@JRD*uo`_la;5rkuY-^t;0I6(g~AH^eB|;E0L+TNIf3dAza{JrN z@-G9fiJQ!IE-1?w%IC8DKg^Vo7dd2pe4b~;f0*&JG-6f)xk?oLD_QyL1%Avmng1>Y zzuu232IJqY;Q!Hre~<u`#A5t$?2`HSD)^fjf1^oFj%5B`&~Ri71+QcLV)=&vF7xAi zGFIh}#>N8MQMxm}Wv&GZ{{FX8l}$_&d2y_h`5#j7N1a3!x&O>eKw0-G_@80?TpxMQ zu;4!l3S?FO^^Cuq6SKWs{)+&RF%+G&hAN2lk8>2c{MRb@`>+X6u=e+)g8wB8{um4X zFBSarLgYUU##gKIC*KxW{chph4!ehiCj+<$r&Sk_BH->UqB*ZS*YxzUzf{)ZI&@r7lYnOum z_B*JGSpF1%WPT0iX;$Um%=p_3W_lp=j|F`hL&@=XQhBbAys7Y!`D+yXjf1KFVY)NE zWv&Mm{DbfE*GFD_RweU)sNk>1$rfxg=+5|-`6FSTXBEHQOur9a6gF8u6ZB;a#Y^v| zO7%<^{hk0H+5dV4e=EChI~;`nNd^DMjGz0Dy!dW|%zr?^-x3YNux+M0<6AE41ehnH zi}LtWi<3Rr?Bv`0iZTz$(ITb~!+_t`%=KY@DsD3WQ=l(nC_m{Qs?|bwq}*9>ESLXp z3jWPFS%hs9-5K99R}9QkvHY_CbJ!ntc>Qw&vi_Y4{Z^(whbywZtdH|l8AH+Vd#M8F z$8u-GM=pP^N}uJ=JZgsMjknOh0raivKOQH$u-VDC`6bu?b_M@+%>Q_%ihfUlkL*AG zuBR3Mj1c)JfWB4y9=G7nwcy9!3AN&{4Uzvv1^>qu{L?J>e`5S{{xviH7SItlxvZs7 zAu@*Y{nt^gW~PjBp8+2^{;i;IRsI?5xfR!EW&)m{4TO1;75}}AUl;f_eEv93q2C&! z|Jy*{D*jj8M~TJwV_ziue-_L$to)y6r~2`9XMD@?zgnT+%=ClB|2@#R@_&Pc|CfUo zvj1r?PqFgf9z$IY=KoC!{TWO@nEwYs-^%|(7XDvp;s11)r&#&l9-{vbD)gg<)A9%N zKMdw6R{kg7PxZz8aaj281br+2*E9ZL@!zV@Zw=A^q3}@5%Kz&u{4cfeAAfJx%Kv%z zumQGU{=cu#Z)W~v>*M;S--0Ds@tNj_(Q&x z|F!tA2sYVY`Fs}Bm&gAGrf(-!%r80CjSBtsOuxZYB}cOU4$zk|6ufsm5jURyV7)Db zk6eE?m}g`8W&iExQvF)GGrnc6@t`kbC_L%`s?y9fk=F$unSZf@znSsJqjSP0b3Loz ze~Iz8FkR&Jz(?l)r-HxbJgV5i^!ZNaA9=cU`LBDBY8+;Y$a@WZWd1VHw~GI!SgM%O ziy2@$nSY}~zvFzWkm|1+R{V1qf3W;J73O(X{No>?{#!!A66*In@N7F;z|L_Y@{DAFUpl`{44;2v0 ze-S`3|1%2xE13T}<3YU}EcE}O@ZTOs6@&RdZn(Ao8yLUnzt@6)8R&-rF2`>(C!gmi^rIMYGo>@WW&K-0-^%}7=D*G~(eE|z zk^TQf!M~pI+nGM!$^1iMo`dz*iwm%Q3)7cigP-H)1kvaIa(VfDltO=Zi2UO~-zt8$ zZ=#I6o{{$s_;7xXXEFW=?jYN9e!GOrT=NwCk28K=4CK8VJ~IFH3jPkJ7>xfR1^>qu z{Me<+{CgDqeG)(zHkjV{ZF2dKDERw7O4Y>juLDTt$9b-76E#O!^q(s&^ov2?Dt}{{ z|7c6xI6uc0^ibc`L;dSOe*kwz*mM8cpOHx1DuwGjW?sf`lNxXaE5BS``TTCCA0g@M zOg~lN#6wyCG0^9}h@#_J{WmiZ^&S@VdHZW&{JkXph7kRKN5TIF