diff --git a/std/d/lexer.d b/std/d/lexer.d index e80fa88..eb11bf9 100644 --- a/std/d/lexer.d +++ b/std/d/lexer.d @@ -752,8 +752,7 @@ private: default: assert(false); } - static if (keep) keepNonNewlineChar(); - setTokenValue(); + setTokenValue(); } void lexHexString() @@ -815,12 +814,12 @@ private: void lexNumber() in { - assert(buffer[0] || buffer[0] == '.'); + assert((buffer[0] >= '0' && buffer[0] <= '9') || buffer[0] == '.'); } body { // hex and binary can start with zero, anything else is decimal - if (currentElement() != '0') + if (buffer[0] != '0' || isEoF()) lexDecimal(); else { @@ -930,18 +929,20 @@ private: { keepNonNewlineChar(); bool foundSign = false; + bool foundDigit = false; while (!isEoF()) { switch (currentElement()) { case '-': case '+': - if (foundSign) + if (foundSign || foundDigit) return; foundSign = true; keepNonNewlineChar(); case '0': .. case '9': case '_': + foundDigit = true; keepNonNewlineChar(); break; case 'L': @@ -1065,7 +1066,14 @@ private: case '_': keepNonNewlineChar(); break; + case 'u': + case 'U': + lexIntSuffix(); + return; case 'i': + if (foundDot) + lexFloatSuffix(); + return; case 'L': if (foundDot) { @@ -1606,6 +1614,7 @@ private: buffer[0] = 'q'; buffer[1] = '{'; buffer[2 .. bi + 2] = b[0 .. bi]; + bi++; buffer[bi++] = '}'; bufferIndex = bi; if (config.tokenStyle & TokenStyle.includeQuotes) @@ -2787,3 +2796,88 @@ immutable uint[] sbox = [ 0xA0B38F96, 0x51D39199, 0x37A6AD75, 0xDF84EE41, 0x3C034CBA, 0xACDA62FC, 0x11923B8B, 0x45EF170A, ]; + +unittest +{ + import std.stdio; + auto source = cast(ubyte[]) ( + " bool byte cdouble cent cfloat char creal dchar double float function" + ~ " idouble ifloat int ireal long real short ubyte ucent uint ulong" + ~ " ushort void wchar align deprecated extern pragma export package private" + ~ " protected public abstract auto const final __gshared immutable inout" + ~ " scope shared static synchronized alias asm assert body break case" + ~ " cast catch class continue debug default delegate delete do else" + ~ " enum false finally foreach foreach_reverse for goto if import in" + ~ " interface invariant is lazy macro mixin module new nothrow null" + ~ " out override pure ref return struct super switch template this" + ~ " throw true try typedef typeid typeof union unittest version volatile" + ~ " while with __traits __parameters __vector"); + auto expected = ["bool", "byte", "cdouble", + "cent", "cfloat", "char", "creal", + "dchar", "double", "float", "function", + "idouble", "ifloat", "int", "ireal", "long", + "real", "short", "ubyte", "ucent", "uint", + "ulong", "ushort", "void", "wchar", "align", + "deprecated", "extern", "pragma", "export", + "package", "private", "protected", "public", + "abstract", "auto", "const", "final", "__gshared", + "immutable", "inout", "scope", "shared", + "static", "synchronized", "alias", "asm", "assert", + "body", "break", "case", "cast", "catch", + "class", "continue", "debug", "default", "delegate", + "delete", "do", "else", "enum", "false", + "finally", "foreach", "foreach_reverse", "for", + "goto", "if", "import", "in", "interface", + "invariant", "is", "lazy","macro", "mixin", + "module", "new", "nothrow", "null", "out", + "override", "pure", "ref", "return", "struct", + "super", "switch", "template", "this", "throw", + "true", "try", "typedef", "typeid", "typeof", + "union", "unittest", "version", "volatile", + "while", "with", "__traits", "__parameters", "__vector"]; + LexerConfig config; + auto tokens = byToken(source, config); + //writeln(tokens.map!"a.value"().array()); + assert (equal(map!"a.value"(tokens), expected)); +} + +unittest +{ + //import std.stdio; + auto source = cast(ubyte[]) ("=@& &=| |=~=:,--/ /=$.===>> >=++{[< <=<>=<>&&||(- -=%%=*=!!=!>!>=!+ +=^^^^=}]);<< <<=>> >>=..*?~!<>=>>>>>>=...^ ^="); + auto expected = ["=", "@", "&", "&=", "|", "|=", "~=", + ":", ",", "--", "/", "/=", "$", ".", "==", + "=>", ">", ">=", "++", "{", "[", "<", + "<=", "<>=", "<>", "&&", "||", "(", "-", "-=", "%", + "%=", "*=", "!", "!=", "!>", "!>=", "!<", + "!<=", "!<>", "+", "+=", "^^", "^^=", + "}", "]", ")", ";", "<<", "<<=", ">>", + ">>=", "..", "*", "?", "~", "!<>=", + ">>>", ">>>=", "...", "^", "^="]; + LexerConfig config; + auto tokens = byToken(source, config); + //writeln(tokens.map!"a.value"().array()); + assert (equal(map!"a.value"(tokens), expected)); +} + +unittest +{ + import std.stdio; + auto source = cast(ubyte[]) (q{ + 1 1.2 1.2f 1u 1uL 0b11 0b1u 0b1 0x11001uL + }); + auto expected = ["1", "1.2", "1.2f", "1u", "1uL", "0b11", "0b1u", "0b1", + "0x11001uL"]; + LexerConfig config; + auto tokens = byToken(source, config); + writeln(tokens.map!"a.value"().array()); + assert (equal(map!"a.value"(tokens), expected)); +} + + +void main(string[] args) +{ + + + +}