mirror of
https://github.com/ldc-developers/ldc.git
synced 2025-05-08 11:56:12 +03:00
9f998a398d
Notably, the glue layer side of the changed multiple interface inheritance layout (DMD a54e89d) has not been implemented yet. This corresponds to DMD commit 3f6a763c0589dd03c1c206eafd434b593702564e.
1049 lines
28 KiB
D
1049 lines
28 KiB
D
// Compiler implementation of the D programming language
|
|
// Copyright (c) 1999-2015 by Digital Mars
|
|
// All Rights Reserved
|
|
// written by Walter Bright
|
|
// http://www.digitalmars.com
|
|
// Distributed under the Boost Software License, Version 1.0.
|
|
// http://www.boost.org/LICENSE_1_0.txt
|
|
|
|
module ddmd.tokens;
|
|
|
|
import core.stdc.ctype;
|
|
import core.stdc.stdio;
|
|
import core.stdc.string;
|
|
import ddmd.globals;
|
|
import ddmd.id;
|
|
import ddmd.identifier;
|
|
import ddmd.root.port;
|
|
import ddmd.root.outbuffer;
|
|
import ddmd.root.rmem;
|
|
import ddmd.utf;
|
|
|
|
enum TOK : int
|
|
{
|
|
TOKreserved,
|
|
|
|
// Other
|
|
TOKlparen,
|
|
TOKrparen,
|
|
TOKlbracket,
|
|
TOKrbracket,
|
|
TOKlcurly,
|
|
TOKrcurly,
|
|
TOKcolon,
|
|
TOKneg,
|
|
TOKsemicolon,
|
|
TOKdotdotdot,
|
|
TOKeof,
|
|
TOKcast,
|
|
TOKnull,
|
|
TOKassert,
|
|
TOKtrue,
|
|
TOKfalse,
|
|
TOKarray,
|
|
TOKcall,
|
|
TOKaddress,
|
|
TOKtype,
|
|
TOKthrow,
|
|
TOKnew,
|
|
TOKdelete,
|
|
TOKstar,
|
|
TOKsymoff,
|
|
TOKvar,
|
|
TOKdotvar,
|
|
TOKdotid,
|
|
TOKdotti,
|
|
TOKdottype,
|
|
TOKslice,
|
|
TOKarraylength,
|
|
TOKversion,
|
|
TOKmodule,
|
|
TOKdollar,
|
|
TOKtemplate,
|
|
TOKdottd,
|
|
TOKdeclaration,
|
|
TOKtypeof,
|
|
TOKpragma,
|
|
TOKdsymbol,
|
|
TOKtypeid,
|
|
TOKuadd,
|
|
TOKremove,
|
|
TOKnewanonclass,
|
|
TOKcomment,
|
|
TOKarrayliteral,
|
|
TOKassocarrayliteral,
|
|
TOKstructliteral,
|
|
TOKclassreference,
|
|
TOKthrownexception,
|
|
TOKdelegateptr,
|
|
TOKdelegatefuncptr,
|
|
|
|
// 54
|
|
// Operators
|
|
TOKlt,
|
|
TOKgt,
|
|
TOKle,
|
|
TOKge,
|
|
TOKequal,
|
|
TOKnotequal,
|
|
TOKidentity,
|
|
TOKnotidentity,
|
|
TOKindex,
|
|
TOKis,
|
|
TOKtobool,
|
|
|
|
// 65
|
|
// NCEG floating point compares
|
|
// !<>= <> <>= !> !>= !< !<= !<>
|
|
TOKunord,
|
|
TOKlg,
|
|
TOKleg,
|
|
TOKule,
|
|
TOKul,
|
|
TOKuge,
|
|
TOKug,
|
|
TOKue,
|
|
|
|
// 73
|
|
TOKshl,
|
|
TOKshr,
|
|
TOKshlass,
|
|
TOKshrass,
|
|
TOKushr,
|
|
TOKushrass,
|
|
TOKcat,
|
|
TOKcatass, // ~ ~=
|
|
TOKadd,
|
|
TOKmin,
|
|
TOKaddass,
|
|
TOKminass,
|
|
TOKmul,
|
|
TOKdiv,
|
|
TOKmod,
|
|
TOKmulass,
|
|
TOKdivass,
|
|
TOKmodass,
|
|
TOKand,
|
|
TOKor,
|
|
TOKxor,
|
|
TOKandass,
|
|
TOKorass,
|
|
TOKxorass,
|
|
TOKassign,
|
|
TOKnot,
|
|
TOKtilde,
|
|
TOKplusplus,
|
|
TOKminusminus,
|
|
TOKconstruct,
|
|
TOKblit,
|
|
TOKdot,
|
|
TOKarrow,
|
|
TOKcomma,
|
|
TOKquestion,
|
|
TOKandand,
|
|
TOKoror,
|
|
TOKpreplusplus,
|
|
TOKpreminusminus,
|
|
|
|
// 112
|
|
// Numeric literals
|
|
TOKint32v,
|
|
TOKuns32v,
|
|
TOKint64v,
|
|
TOKuns64v,
|
|
TOKint128v,
|
|
TOKuns128v,
|
|
TOKfloat32v,
|
|
TOKfloat64v,
|
|
TOKfloat80v,
|
|
TOKimaginary32v,
|
|
TOKimaginary64v,
|
|
TOKimaginary80v,
|
|
|
|
// Char constants
|
|
TOKcharv,
|
|
TOKwcharv,
|
|
TOKdcharv,
|
|
|
|
// Leaf operators
|
|
TOKidentifier,
|
|
TOKstring,
|
|
TOKxstring,
|
|
TOKthis,
|
|
TOKsuper,
|
|
TOKhalt,
|
|
TOKtuple,
|
|
TOKerror,
|
|
|
|
// Basic types
|
|
TOKvoid,
|
|
TOKint8,
|
|
TOKuns8,
|
|
TOKint16,
|
|
TOKuns16,
|
|
TOKint32,
|
|
TOKuns32,
|
|
TOKint64,
|
|
TOKuns64,
|
|
TOKint128,
|
|
TOKuns128,
|
|
TOKfloat32,
|
|
TOKfloat64,
|
|
TOKfloat80,
|
|
TOKimaginary32,
|
|
TOKimaginary64,
|
|
TOKimaginary80,
|
|
TOKcomplex32,
|
|
TOKcomplex64,
|
|
TOKcomplex80,
|
|
TOKchar,
|
|
TOKwchar,
|
|
TOKdchar,
|
|
TOKbool,
|
|
|
|
// 159
|
|
// Aggregates
|
|
TOKstruct,
|
|
TOKclass,
|
|
TOKinterface,
|
|
TOKunion,
|
|
TOKenum,
|
|
TOKimport,
|
|
TOKtypedef,
|
|
TOKalias,
|
|
TOKoverride,
|
|
TOKdelegate,
|
|
TOKfunction,
|
|
TOKmixin,
|
|
TOKalign,
|
|
TOKextern,
|
|
TOKprivate,
|
|
TOKprotected,
|
|
TOKpublic,
|
|
TOKexport,
|
|
TOKstatic,
|
|
TOKfinal,
|
|
TOKconst,
|
|
TOKabstract,
|
|
TOKvolatile,
|
|
TOKdebug,
|
|
TOKdeprecated,
|
|
TOKin,
|
|
TOKout,
|
|
TOKinout,
|
|
TOKlazy,
|
|
TOKauto,
|
|
TOKpackage,
|
|
TOKmanifest,
|
|
TOKimmutable,
|
|
|
|
// Statements
|
|
TOKif,
|
|
TOKelse,
|
|
TOKwhile,
|
|
TOKfor,
|
|
TOKdo,
|
|
TOKswitch,
|
|
TOKcase,
|
|
TOKdefault,
|
|
TOKbreak,
|
|
TOKcontinue,
|
|
TOKwith,
|
|
TOKsynchronized,
|
|
TOKreturn,
|
|
TOKgoto,
|
|
TOKtry,
|
|
TOKcatch,
|
|
TOKfinally,
|
|
TOKasm,
|
|
TOKforeach,
|
|
TOKforeach_reverse,
|
|
TOKscope,
|
|
TOKon_scope_exit,
|
|
TOKon_scope_failure,
|
|
TOKon_scope_success,
|
|
|
|
// Contracts
|
|
TOKbody,
|
|
TOKinvariant,
|
|
|
|
// Testing
|
|
TOKunittest,
|
|
|
|
// Added after 1.0
|
|
TOKargTypes,
|
|
TOKref,
|
|
TOKmacro,
|
|
|
|
TOKparameters,
|
|
TOKtraits,
|
|
TOKoverloadset,
|
|
TOKpure,
|
|
TOKnothrow,
|
|
TOKgshared,
|
|
TOKline,
|
|
TOKfile,
|
|
TOKmodulestring,
|
|
TOKfuncstring,
|
|
TOKprettyfunc,
|
|
TOKshared,
|
|
TOKat,
|
|
TOKpow,
|
|
TOKpowass,
|
|
TOKgoesto,
|
|
TOKvector,
|
|
TOKpound,
|
|
|
|
TOKinterval,
|
|
TOKvoidexp,
|
|
TOKcantexp,
|
|
|
|
TOKMAX,
|
|
}
|
|
|
|
alias TOKreserved = TOK.TOKreserved;
|
|
alias TOKlparen = TOK.TOKlparen;
|
|
alias TOKrparen = TOK.TOKrparen;
|
|
alias TOKlbracket = TOK.TOKlbracket;
|
|
alias TOKrbracket = TOK.TOKrbracket;
|
|
alias TOKlcurly = TOK.TOKlcurly;
|
|
alias TOKrcurly = TOK.TOKrcurly;
|
|
alias TOKcolon = TOK.TOKcolon;
|
|
alias TOKneg = TOK.TOKneg;
|
|
alias TOKsemicolon = TOK.TOKsemicolon;
|
|
alias TOKdotdotdot = TOK.TOKdotdotdot;
|
|
alias TOKeof = TOK.TOKeof;
|
|
alias TOKcast = TOK.TOKcast;
|
|
alias TOKnull = TOK.TOKnull;
|
|
alias TOKassert = TOK.TOKassert;
|
|
alias TOKtrue = TOK.TOKtrue;
|
|
alias TOKfalse = TOK.TOKfalse;
|
|
alias TOKarray = TOK.TOKarray;
|
|
alias TOKcall = TOK.TOKcall;
|
|
alias TOKaddress = TOK.TOKaddress;
|
|
alias TOKtype = TOK.TOKtype;
|
|
alias TOKthrow = TOK.TOKthrow;
|
|
alias TOKnew = TOK.TOKnew;
|
|
alias TOKdelete = TOK.TOKdelete;
|
|
alias TOKstar = TOK.TOKstar;
|
|
alias TOKsymoff = TOK.TOKsymoff;
|
|
alias TOKvar = TOK.TOKvar;
|
|
alias TOKdotvar = TOK.TOKdotvar;
|
|
alias TOKdotid = TOK.TOKdotid;
|
|
alias TOKdotti = TOK.TOKdotti;
|
|
alias TOKdottype = TOK.TOKdottype;
|
|
alias TOKslice = TOK.TOKslice;
|
|
alias TOKarraylength = TOK.TOKarraylength;
|
|
alias TOKversion = TOK.TOKversion;
|
|
alias TOKmodule = TOK.TOKmodule;
|
|
alias TOKdollar = TOK.TOKdollar;
|
|
alias TOKtemplate = TOK.TOKtemplate;
|
|
alias TOKdottd = TOK.TOKdottd;
|
|
alias TOKdeclaration = TOK.TOKdeclaration;
|
|
alias TOKtypeof = TOK.TOKtypeof;
|
|
alias TOKpragma = TOK.TOKpragma;
|
|
alias TOKdsymbol = TOK.TOKdsymbol;
|
|
alias TOKtypeid = TOK.TOKtypeid;
|
|
alias TOKuadd = TOK.TOKuadd;
|
|
alias TOKremove = TOK.TOKremove;
|
|
alias TOKnewanonclass = TOK.TOKnewanonclass;
|
|
alias TOKcomment = TOK.TOKcomment;
|
|
alias TOKarrayliteral = TOK.TOKarrayliteral;
|
|
alias TOKassocarrayliteral = TOK.TOKassocarrayliteral;
|
|
alias TOKstructliteral = TOK.TOKstructliteral;
|
|
alias TOKclassreference = TOK.TOKclassreference;
|
|
alias TOKthrownexception = TOK.TOKthrownexception;
|
|
alias TOKdelegateptr = TOK.TOKdelegateptr;
|
|
alias TOKdelegatefuncptr = TOK.TOKdelegatefuncptr;
|
|
alias TOKlt = TOK.TOKlt;
|
|
alias TOKgt = TOK.TOKgt;
|
|
alias TOKle = TOK.TOKle;
|
|
alias TOKge = TOK.TOKge;
|
|
alias TOKequal = TOK.TOKequal;
|
|
alias TOKnotequal = TOK.TOKnotequal;
|
|
alias TOKidentity = TOK.TOKidentity;
|
|
alias TOKnotidentity = TOK.TOKnotidentity;
|
|
alias TOKindex = TOK.TOKindex;
|
|
alias TOKis = TOK.TOKis;
|
|
alias TOKtobool = TOK.TOKtobool;
|
|
alias TOKunord = TOK.TOKunord;
|
|
alias TOKlg = TOK.TOKlg;
|
|
alias TOKleg = TOK.TOKleg;
|
|
alias TOKule = TOK.TOKule;
|
|
alias TOKul = TOK.TOKul;
|
|
alias TOKuge = TOK.TOKuge;
|
|
alias TOKug = TOK.TOKug;
|
|
alias TOKue = TOK.TOKue;
|
|
alias TOKshl = TOK.TOKshl;
|
|
alias TOKshr = TOK.TOKshr;
|
|
alias TOKshlass = TOK.TOKshlass;
|
|
alias TOKshrass = TOK.TOKshrass;
|
|
alias TOKushr = TOK.TOKushr;
|
|
alias TOKushrass = TOK.TOKushrass;
|
|
alias TOKcat = TOK.TOKcat;
|
|
alias TOKcatass = TOK.TOKcatass;
|
|
alias TOKadd = TOK.TOKadd;
|
|
alias TOKmin = TOK.TOKmin;
|
|
alias TOKaddass = TOK.TOKaddass;
|
|
alias TOKminass = TOK.TOKminass;
|
|
alias TOKmul = TOK.TOKmul;
|
|
alias TOKdiv = TOK.TOKdiv;
|
|
alias TOKmod = TOK.TOKmod;
|
|
alias TOKmulass = TOK.TOKmulass;
|
|
alias TOKdivass = TOK.TOKdivass;
|
|
alias TOKmodass = TOK.TOKmodass;
|
|
alias TOKand = TOK.TOKand;
|
|
alias TOKor = TOK.TOKor;
|
|
alias TOKxor = TOK.TOKxor;
|
|
alias TOKandass = TOK.TOKandass;
|
|
alias TOKorass = TOK.TOKorass;
|
|
alias TOKxorass = TOK.TOKxorass;
|
|
alias TOKassign = TOK.TOKassign;
|
|
alias TOKnot = TOK.TOKnot;
|
|
alias TOKtilde = TOK.TOKtilde;
|
|
alias TOKplusplus = TOK.TOKplusplus;
|
|
alias TOKminusminus = TOK.TOKminusminus;
|
|
alias TOKconstruct = TOK.TOKconstruct;
|
|
alias TOKblit = TOK.TOKblit;
|
|
alias TOKdot = TOK.TOKdot;
|
|
alias TOKarrow = TOK.TOKarrow;
|
|
alias TOKcomma = TOK.TOKcomma;
|
|
alias TOKquestion = TOK.TOKquestion;
|
|
alias TOKandand = TOK.TOKandand;
|
|
alias TOKoror = TOK.TOKoror;
|
|
alias TOKpreplusplus = TOK.TOKpreplusplus;
|
|
alias TOKpreminusminus = TOK.TOKpreminusminus;
|
|
alias TOKint32v = TOK.TOKint32v;
|
|
alias TOKuns32v = TOK.TOKuns32v;
|
|
alias TOKint64v = TOK.TOKint64v;
|
|
alias TOKuns64v = TOK.TOKuns64v;
|
|
alias TOKint128v = TOK.TOKint128v;
|
|
alias TOKuns128v = TOK.TOKuns128v;
|
|
alias TOKfloat32v = TOK.TOKfloat32v;
|
|
alias TOKfloat64v = TOK.TOKfloat64v;
|
|
alias TOKfloat80v = TOK.TOKfloat80v;
|
|
alias TOKimaginary32v = TOK.TOKimaginary32v;
|
|
alias TOKimaginary64v = TOK.TOKimaginary64v;
|
|
alias TOKimaginary80v = TOK.TOKimaginary80v;
|
|
alias TOKcharv = TOK.TOKcharv;
|
|
alias TOKwcharv = TOK.TOKwcharv;
|
|
alias TOKdcharv = TOK.TOKdcharv;
|
|
alias TOKidentifier = TOK.TOKidentifier;
|
|
alias TOKstring = TOK.TOKstring;
|
|
alias TOKxstring = TOK.TOKxstring;
|
|
alias TOKthis = TOK.TOKthis;
|
|
alias TOKsuper = TOK.TOKsuper;
|
|
alias TOKhalt = TOK.TOKhalt;
|
|
alias TOKtuple = TOK.TOKtuple;
|
|
alias TOKerror = TOK.TOKerror;
|
|
alias TOKvoid = TOK.TOKvoid;
|
|
alias TOKint8 = TOK.TOKint8;
|
|
alias TOKuns8 = TOK.TOKuns8;
|
|
alias TOKint16 = TOK.TOKint16;
|
|
alias TOKuns16 = TOK.TOKuns16;
|
|
alias TOKint32 = TOK.TOKint32;
|
|
alias TOKuns32 = TOK.TOKuns32;
|
|
alias TOKint64 = TOK.TOKint64;
|
|
alias TOKuns64 = TOK.TOKuns64;
|
|
alias TOKint128 = TOK.TOKint128;
|
|
alias TOKuns128 = TOK.TOKuns128;
|
|
alias TOKfloat32 = TOK.TOKfloat32;
|
|
alias TOKfloat64 = TOK.TOKfloat64;
|
|
alias TOKfloat80 = TOK.TOKfloat80;
|
|
alias TOKimaginary32 = TOK.TOKimaginary32;
|
|
alias TOKimaginary64 = TOK.TOKimaginary64;
|
|
alias TOKimaginary80 = TOK.TOKimaginary80;
|
|
alias TOKcomplex32 = TOK.TOKcomplex32;
|
|
alias TOKcomplex64 = TOK.TOKcomplex64;
|
|
alias TOKcomplex80 = TOK.TOKcomplex80;
|
|
alias TOKchar = TOK.TOKchar;
|
|
alias TOKwchar = TOK.TOKwchar;
|
|
alias TOKdchar = TOK.TOKdchar;
|
|
alias TOKbool = TOK.TOKbool;
|
|
alias TOKstruct = TOK.TOKstruct;
|
|
alias TOKclass = TOK.TOKclass;
|
|
alias TOKinterface = TOK.TOKinterface;
|
|
alias TOKunion = TOK.TOKunion;
|
|
alias TOKenum = TOK.TOKenum;
|
|
alias TOKimport = TOK.TOKimport;
|
|
alias TOKtypedef = TOK.TOKtypedef;
|
|
alias TOKalias = TOK.TOKalias;
|
|
alias TOKoverride = TOK.TOKoverride;
|
|
alias TOKdelegate = TOK.TOKdelegate;
|
|
alias TOKfunction = TOK.TOKfunction;
|
|
alias TOKmixin = TOK.TOKmixin;
|
|
alias TOKalign = TOK.TOKalign;
|
|
alias TOKextern = TOK.TOKextern;
|
|
alias TOKprivate = TOK.TOKprivate;
|
|
alias TOKprotected = TOK.TOKprotected;
|
|
alias TOKpublic = TOK.TOKpublic;
|
|
alias TOKexport = TOK.TOKexport;
|
|
alias TOKstatic = TOK.TOKstatic;
|
|
alias TOKfinal = TOK.TOKfinal;
|
|
alias TOKconst = TOK.TOKconst;
|
|
alias TOKabstract = TOK.TOKabstract;
|
|
alias TOKvolatile = TOK.TOKvolatile;
|
|
alias TOKdebug = TOK.TOKdebug;
|
|
alias TOKdeprecated = TOK.TOKdeprecated;
|
|
alias TOKin = TOK.TOKin;
|
|
alias TOKout = TOK.TOKout;
|
|
alias TOKinout = TOK.TOKinout;
|
|
alias TOKlazy = TOK.TOKlazy;
|
|
alias TOKauto = TOK.TOKauto;
|
|
alias TOKpackage = TOK.TOKpackage;
|
|
alias TOKmanifest = TOK.TOKmanifest;
|
|
alias TOKimmutable = TOK.TOKimmutable;
|
|
alias TOKif = TOK.TOKif;
|
|
alias TOKelse = TOK.TOKelse;
|
|
alias TOKwhile = TOK.TOKwhile;
|
|
alias TOKfor = TOK.TOKfor;
|
|
alias TOKdo = TOK.TOKdo;
|
|
alias TOKswitch = TOK.TOKswitch;
|
|
alias TOKcase = TOK.TOKcase;
|
|
alias TOKdefault = TOK.TOKdefault;
|
|
alias TOKbreak = TOK.TOKbreak;
|
|
alias TOKcontinue = TOK.TOKcontinue;
|
|
alias TOKwith = TOK.TOKwith;
|
|
alias TOKsynchronized = TOK.TOKsynchronized;
|
|
alias TOKreturn = TOK.TOKreturn;
|
|
alias TOKgoto = TOK.TOKgoto;
|
|
alias TOKtry = TOK.TOKtry;
|
|
alias TOKcatch = TOK.TOKcatch;
|
|
alias TOKfinally = TOK.TOKfinally;
|
|
alias TOKasm = TOK.TOKasm;
|
|
alias TOKforeach = TOK.TOKforeach;
|
|
alias TOKforeach_reverse = TOK.TOKforeach_reverse;
|
|
alias TOKscope = TOK.TOKscope;
|
|
alias TOKon_scope_exit = TOK.TOKon_scope_exit;
|
|
alias TOKon_scope_failure = TOK.TOKon_scope_failure;
|
|
alias TOKon_scope_success = TOK.TOKon_scope_success;
|
|
alias TOKbody = TOK.TOKbody;
|
|
alias TOKinvariant = TOK.TOKinvariant;
|
|
alias TOKunittest = TOK.TOKunittest;
|
|
alias TOKargTypes = TOK.TOKargTypes;
|
|
alias TOKref = TOK.TOKref;
|
|
alias TOKmacro = TOK.TOKmacro;
|
|
alias TOKparameters = TOK.TOKparameters;
|
|
alias TOKtraits = TOK.TOKtraits;
|
|
alias TOKoverloadset = TOK.TOKoverloadset;
|
|
alias TOKpure = TOK.TOKpure;
|
|
alias TOKnothrow = TOK.TOKnothrow;
|
|
alias TOKgshared = TOK.TOKgshared;
|
|
alias TOKline = TOK.TOKline;
|
|
alias TOKfile = TOK.TOKfile;
|
|
alias TOKmodulestring = TOK.TOKmodulestring;
|
|
alias TOKfuncstring = TOK.TOKfuncstring;
|
|
alias TOKprettyfunc = TOK.TOKprettyfunc;
|
|
alias TOKshared = TOK.TOKshared;
|
|
alias TOKat = TOK.TOKat;
|
|
alias TOKpow = TOK.TOKpow;
|
|
alias TOKpowass = TOK.TOKpowass;
|
|
alias TOKgoesto = TOK.TOKgoesto;
|
|
alias TOKvector = TOK.TOKvector;
|
|
alias TOKpound = TOK.TOKpound;
|
|
alias TOKinterval = TOK.TOKinterval;
|
|
alias TOKvoidexp = TOK.TOKvoidexp;
|
|
alias TOKcantexp = TOK.TOKcantexp;
|
|
alias TOKMAX = TOK.TOKMAX;
|
|
|
|
enum TOKwild = TOKinout;
|
|
|
|
/***********************************************************
|
|
*/
|
|
extern (C++) struct Token
|
|
{
|
|
Token* next;
|
|
Loc loc;
|
|
const(char)* ptr; // pointer to first character of this token within buffer
|
|
TOK value;
|
|
const(char)* blockComment; // doc comment string prior to this token
|
|
const(char)* lineComment; // doc comment for previous token
|
|
|
|
union
|
|
{
|
|
// Integers
|
|
d_int64 int64value;
|
|
d_uns64 uns64value;
|
|
// Floats
|
|
d_float80 float80value;
|
|
|
|
struct
|
|
{
|
|
const(char)* ustring; // UTF8 string
|
|
uint len;
|
|
ubyte postfix; // 'c', 'w', 'd'
|
|
}
|
|
|
|
Identifier ident;
|
|
}
|
|
|
|
static __gshared const(char)*[TOKMAX] tochars;
|
|
|
|
static this()
|
|
{
|
|
Identifier.initTable();
|
|
foreach (kw; keywords)
|
|
{
|
|
//printf("keyword[%d] = '%s'\n",u, keywords[u].name);
|
|
const(char)* s = kw.name;
|
|
TOK v = kw.value;
|
|
auto id = Identifier.idPool(s, strlen(s));
|
|
id.value = v;
|
|
//printf("tochars[%d] = '%s'\n",v, s);
|
|
Token.tochars[v] = s;
|
|
}
|
|
Token.tochars[TOKeof] = "EOF";
|
|
Token.tochars[TOKlcurly] = "{";
|
|
Token.tochars[TOKrcurly] = "}";
|
|
Token.tochars[TOKlparen] = "(";
|
|
Token.tochars[TOKrparen] = ")";
|
|
Token.tochars[TOKlbracket] = "[";
|
|
Token.tochars[TOKrbracket] = "]";
|
|
Token.tochars[TOKsemicolon] = ";";
|
|
Token.tochars[TOKcolon] = ":";
|
|
Token.tochars[TOKcomma] = ",";
|
|
Token.tochars[TOKdot] = ".";
|
|
Token.tochars[TOKxor] = "^";
|
|
Token.tochars[TOKxorass] = "^=";
|
|
Token.tochars[TOKassign] = "=";
|
|
Token.tochars[TOKconstruct] = "=";
|
|
Token.tochars[TOKblit] = "=";
|
|
Token.tochars[TOKlt] = "<";
|
|
Token.tochars[TOKgt] = ">";
|
|
Token.tochars[TOKle] = "<=";
|
|
Token.tochars[TOKge] = ">=";
|
|
Token.tochars[TOKequal] = "==";
|
|
Token.tochars[TOKnotequal] = "!=";
|
|
Token.tochars[TOKnotidentity] = "!is";
|
|
Token.tochars[TOKtobool] = "!!";
|
|
Token.tochars[TOKunord] = "!<>=";
|
|
Token.tochars[TOKue] = "!<>";
|
|
Token.tochars[TOKlg] = "<>";
|
|
Token.tochars[TOKleg] = "<>=";
|
|
Token.tochars[TOKule] = "!>";
|
|
Token.tochars[TOKul] = "!>=";
|
|
Token.tochars[TOKuge] = "!<";
|
|
Token.tochars[TOKug] = "!<=";
|
|
Token.tochars[TOKnot] = "!";
|
|
Token.tochars[TOKtobool] = "!!";
|
|
Token.tochars[TOKshl] = "<<";
|
|
Token.tochars[TOKshr] = ">>";
|
|
Token.tochars[TOKushr] = ">>>";
|
|
Token.tochars[TOKadd] = "+";
|
|
Token.tochars[TOKmin] = "-";
|
|
Token.tochars[TOKmul] = "*";
|
|
Token.tochars[TOKdiv] = "/";
|
|
Token.tochars[TOKmod] = "%";
|
|
Token.tochars[TOKslice] = "..";
|
|
Token.tochars[TOKdotdotdot] = "...";
|
|
Token.tochars[TOKand] = "&";
|
|
Token.tochars[TOKandand] = "&&";
|
|
Token.tochars[TOKor] = "|";
|
|
Token.tochars[TOKoror] = "||";
|
|
Token.tochars[TOKarray] = "[]";
|
|
Token.tochars[TOKindex] = "[i]";
|
|
Token.tochars[TOKaddress] = "&";
|
|
Token.tochars[TOKstar] = "*";
|
|
Token.tochars[TOKtilde] = "~";
|
|
Token.tochars[TOKdollar] = "$";
|
|
Token.tochars[TOKcast] = "cast";
|
|
Token.tochars[TOKplusplus] = "++";
|
|
Token.tochars[TOKminusminus] = "--";
|
|
Token.tochars[TOKpreplusplus] = "++";
|
|
Token.tochars[TOKpreminusminus] = "--";
|
|
Token.tochars[TOKtype] = "type";
|
|
Token.tochars[TOKquestion] = "?";
|
|
Token.tochars[TOKneg] = "-";
|
|
Token.tochars[TOKuadd] = "+";
|
|
Token.tochars[TOKvar] = "var";
|
|
Token.tochars[TOKaddass] = "+=";
|
|
Token.tochars[TOKminass] = "-=";
|
|
Token.tochars[TOKmulass] = "*=";
|
|
Token.tochars[TOKdivass] = "/=";
|
|
Token.tochars[TOKmodass] = "%=";
|
|
Token.tochars[TOKshlass] = "<<=";
|
|
Token.tochars[TOKshrass] = ">>=";
|
|
Token.tochars[TOKushrass] = ">>>=";
|
|
Token.tochars[TOKandass] = "&=";
|
|
Token.tochars[TOKorass] = "|=";
|
|
Token.tochars[TOKcatass] = "~=";
|
|
Token.tochars[TOKcat] = "~";
|
|
Token.tochars[TOKcall] = "call";
|
|
Token.tochars[TOKidentity] = "is";
|
|
Token.tochars[TOKnotidentity] = "!is";
|
|
Token.tochars[TOKidentifier] = "identifier";
|
|
Token.tochars[TOKat] = "@";
|
|
Token.tochars[TOKpow] = "^^";
|
|
Token.tochars[TOKpowass] = "^^=";
|
|
Token.tochars[TOKgoesto] = "=>";
|
|
Token.tochars[TOKpound] = "#";
|
|
|
|
// For debugging
|
|
Token.tochars[TOKerror] = "error";
|
|
Token.tochars[TOKdotid] = "dotid";
|
|
Token.tochars[TOKdottd] = "dottd";
|
|
Token.tochars[TOKdotti] = "dotti";
|
|
Token.tochars[TOKdotvar] = "dotvar";
|
|
Token.tochars[TOKdottype] = "dottype";
|
|
Token.tochars[TOKsymoff] = "symoff";
|
|
Token.tochars[TOKarraylength] = "arraylength";
|
|
Token.tochars[TOKarrayliteral] = "arrayliteral";
|
|
Token.tochars[TOKassocarrayliteral] = "assocarrayliteral";
|
|
Token.tochars[TOKstructliteral] = "structliteral";
|
|
Token.tochars[TOKstring] = "string";
|
|
Token.tochars[TOKdsymbol] = "symbol";
|
|
Token.tochars[TOKtuple] = "tuple";
|
|
Token.tochars[TOKdeclaration] = "declaration";
|
|
Token.tochars[TOKon_scope_exit] = "scope(exit)";
|
|
Token.tochars[TOKon_scope_success] = "scope(success)";
|
|
Token.tochars[TOKon_scope_failure] = "scope(failure)";
|
|
}
|
|
|
|
static __gshared Token* freelist = null;
|
|
|
|
static Token* alloc()
|
|
{
|
|
if (Token.freelist)
|
|
{
|
|
Token* t = freelist;
|
|
freelist = t.next;
|
|
t.next = null;
|
|
return t;
|
|
}
|
|
return new Token();
|
|
}
|
|
|
|
void free()
|
|
{
|
|
next = freelist;
|
|
freelist = &this;
|
|
}
|
|
|
|
int isKeyword() const
|
|
{
|
|
foreach (kw; keywords)
|
|
{
|
|
if (kw.value == value)
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
debug
|
|
{
|
|
void print()
|
|
{
|
|
fprintf(stderr, "%s\n", toChars());
|
|
}
|
|
}
|
|
|
|
/****
|
|
* Set to contents of ptr[0..length]
|
|
* Params:
|
|
* ptr = pointer to string
|
|
* length = length of string
|
|
*/
|
|
final void setString(const(char)* ptr, size_t length)
|
|
{
|
|
auto s = cast(char*)mem.xmalloc(length + 1);
|
|
memcpy(s, ptr, length);
|
|
s[length] = 0;
|
|
ustring = s;
|
|
len = cast(uint)length;
|
|
postfix = 0;
|
|
}
|
|
|
|
/****
|
|
* Set to contents of buf
|
|
* Params:
|
|
* buf = string (not zero terminated)
|
|
*/
|
|
final void setString(const ref OutBuffer buf)
|
|
{
|
|
setString(cast(const(char)*)buf.data, buf.offset);
|
|
}
|
|
|
|
/****
|
|
* Set to empty string
|
|
*/
|
|
final void setString()
|
|
{
|
|
ustring = "";
|
|
len = 0;
|
|
postfix = 0;
|
|
}
|
|
|
|
extern (C++) const(char)* toChars() const
|
|
{
|
|
__gshared char[3 + 3 * float80value.sizeof + 1] buffer;
|
|
const(char)* p = &buffer[0];
|
|
switch (value)
|
|
{
|
|
case TOKint32v:
|
|
sprintf(&buffer[0], "%d", cast(d_int32)int64value);
|
|
break;
|
|
case TOKuns32v:
|
|
case TOKcharv:
|
|
case TOKwcharv:
|
|
case TOKdcharv:
|
|
sprintf(&buffer[0], "%uU", cast(d_uns32)uns64value);
|
|
break;
|
|
case TOKint64v:
|
|
sprintf(&buffer[0], "%lldL", cast(long)int64value);
|
|
break;
|
|
case TOKuns64v:
|
|
sprintf(&buffer[0], "%lluUL", cast(ulong)uns64value);
|
|
break;
|
|
case TOKfloat32v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
strcat(&buffer[0], "f");
|
|
break;
|
|
case TOKfloat64v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
break;
|
|
case TOKfloat80v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
strcat(&buffer[0], "L");
|
|
break;
|
|
case TOKimaginary32v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
strcat(&buffer[0], "fi");
|
|
break;
|
|
case TOKimaginary64v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
strcat(&buffer[0], "i");
|
|
break;
|
|
case TOKimaginary80v:
|
|
Port.ld_sprint(&buffer[0], 'g', float80value);
|
|
strcat(&buffer[0], "Li");
|
|
break;
|
|
case TOKstring:
|
|
{
|
|
OutBuffer buf;
|
|
buf.writeByte('"');
|
|
for (size_t i = 0; i < len;)
|
|
{
|
|
dchar c;
|
|
utf_decodeChar(ustring, len, i, c);
|
|
switch (c)
|
|
{
|
|
case 0:
|
|
break;
|
|
case '"':
|
|
case '\\':
|
|
buf.writeByte('\\');
|
|
goto default;
|
|
default:
|
|
if (c <= 0x7F)
|
|
{
|
|
if (isprint(c))
|
|
buf.writeByte(c);
|
|
else
|
|
buf.printf("\\x%02x", c);
|
|
}
|
|
else if (c <= 0xFFFF)
|
|
buf.printf("\\u%04x", c);
|
|
else
|
|
buf.printf("\\U%08x", c);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
buf.writeByte('"');
|
|
if (postfix)
|
|
buf.writeByte(postfix);
|
|
p = buf.extractString();
|
|
}
|
|
break;
|
|
case TOKxstring:
|
|
{
|
|
OutBuffer buf;
|
|
buf.writeByte('x');
|
|
buf.writeByte('"');
|
|
foreach (size_t i; 0 .. len)
|
|
{
|
|
if (i)
|
|
buf.writeByte(' ');
|
|
buf.printf("%02x", ustring[i]);
|
|
}
|
|
buf.writeByte('"');
|
|
if (postfix)
|
|
buf.writeByte(postfix);
|
|
buf.writeByte(0);
|
|
p = buf.extractData();
|
|
break;
|
|
}
|
|
case TOKidentifier:
|
|
case TOKenum:
|
|
case TOKstruct:
|
|
case TOKimport:
|
|
case TOKwchar:
|
|
case TOKdchar:
|
|
case TOKbool:
|
|
case TOKchar:
|
|
case TOKint8:
|
|
case TOKuns8:
|
|
case TOKint16:
|
|
case TOKuns16:
|
|
case TOKint32:
|
|
case TOKuns32:
|
|
case TOKint64:
|
|
case TOKuns64:
|
|
case TOKint128:
|
|
case TOKuns128:
|
|
case TOKfloat32:
|
|
case TOKfloat64:
|
|
case TOKfloat80:
|
|
case TOKimaginary32:
|
|
case TOKimaginary64:
|
|
case TOKimaginary80:
|
|
case TOKcomplex32:
|
|
case TOKcomplex64:
|
|
case TOKcomplex80:
|
|
case TOKvoid:
|
|
p = ident.toChars();
|
|
break;
|
|
default:
|
|
p = toChars(value);
|
|
break;
|
|
}
|
|
return p;
|
|
}
|
|
|
|
static const(char)* toChars(TOK value)
|
|
{
|
|
static __gshared char[3 + 3 * value.sizeof + 1] buffer;
|
|
const(char)* p = tochars[value];
|
|
if (!p)
|
|
{
|
|
sprintf(&buffer[0], "TOK%d", value);
|
|
p = &buffer[0];
|
|
}
|
|
return p;
|
|
}
|
|
}
|
|
|
|
/****************************************
|
|
*/
|
|
struct Keyword
|
|
{
|
|
immutable(char)* name;
|
|
TOK value;
|
|
}
|
|
|
|
immutable Keyword[] keywords =
|
|
[
|
|
Keyword("this", TOKthis),
|
|
Keyword("super", TOKsuper),
|
|
Keyword("assert", TOKassert),
|
|
Keyword("null", TOKnull),
|
|
Keyword("true", TOKtrue),
|
|
Keyword("false", TOKfalse),
|
|
Keyword("cast", TOKcast),
|
|
Keyword("new", TOKnew),
|
|
Keyword("delete", TOKdelete),
|
|
Keyword("throw", TOKthrow),
|
|
Keyword("module", TOKmodule),
|
|
Keyword("pragma", TOKpragma),
|
|
Keyword("typeof", TOKtypeof),
|
|
Keyword("typeid", TOKtypeid),
|
|
Keyword("template", TOKtemplate),
|
|
Keyword("void", TOKvoid),
|
|
Keyword("byte", TOKint8),
|
|
Keyword("ubyte", TOKuns8),
|
|
Keyword("short", TOKint16),
|
|
Keyword("ushort", TOKuns16),
|
|
Keyword("int", TOKint32),
|
|
Keyword("uint", TOKuns32),
|
|
Keyword("long", TOKint64),
|
|
Keyword("ulong", TOKuns64),
|
|
Keyword("cent", TOKint128),
|
|
Keyword("ucent", TOKuns128),
|
|
Keyword("float", TOKfloat32),
|
|
Keyword("double", TOKfloat64),
|
|
Keyword("real", TOKfloat80),
|
|
Keyword("bool", TOKbool),
|
|
Keyword("char", TOKchar),
|
|
Keyword("wchar", TOKwchar),
|
|
Keyword("dchar", TOKdchar),
|
|
Keyword("ifloat", TOKimaginary32),
|
|
Keyword("idouble", TOKimaginary64),
|
|
Keyword("ireal", TOKimaginary80),
|
|
Keyword("cfloat", TOKcomplex32),
|
|
Keyword("cdouble", TOKcomplex64),
|
|
Keyword("creal", TOKcomplex80),
|
|
Keyword("delegate", TOKdelegate),
|
|
Keyword("function", TOKfunction),
|
|
Keyword("is", TOKis),
|
|
Keyword("if", TOKif),
|
|
Keyword("else", TOKelse),
|
|
Keyword("while", TOKwhile),
|
|
Keyword("for", TOKfor),
|
|
Keyword("do", TOKdo),
|
|
Keyword("switch", TOKswitch),
|
|
Keyword("case", TOKcase),
|
|
Keyword("default", TOKdefault),
|
|
Keyword("break", TOKbreak),
|
|
Keyword("continue", TOKcontinue),
|
|
Keyword("synchronized", TOKsynchronized),
|
|
Keyword("return", TOKreturn),
|
|
Keyword("goto", TOKgoto),
|
|
Keyword("try", TOKtry),
|
|
Keyword("catch", TOKcatch),
|
|
Keyword("finally", TOKfinally),
|
|
Keyword("with", TOKwith),
|
|
Keyword("asm", TOKasm),
|
|
Keyword("foreach", TOKforeach),
|
|
Keyword("foreach_reverse", TOKforeach_reverse),
|
|
Keyword("scope", TOKscope),
|
|
Keyword("struct", TOKstruct),
|
|
Keyword("class", TOKclass),
|
|
Keyword("interface", TOKinterface),
|
|
Keyword("union", TOKunion),
|
|
Keyword("enum", TOKenum),
|
|
Keyword("import", TOKimport),
|
|
Keyword("mixin", TOKmixin),
|
|
Keyword("static", TOKstatic),
|
|
Keyword("final", TOKfinal),
|
|
Keyword("const", TOKconst),
|
|
Keyword("typedef", TOKtypedef),
|
|
Keyword("alias", TOKalias),
|
|
Keyword("override", TOKoverride),
|
|
Keyword("abstract", TOKabstract),
|
|
Keyword("volatile", TOKvolatile),
|
|
Keyword("debug", TOKdebug),
|
|
Keyword("deprecated", TOKdeprecated),
|
|
Keyword("in", TOKin),
|
|
Keyword("out", TOKout),
|
|
Keyword("inout", TOKinout),
|
|
Keyword("lazy", TOKlazy),
|
|
Keyword("auto", TOKauto),
|
|
Keyword("align", TOKalign),
|
|
Keyword("extern", TOKextern),
|
|
Keyword("private", TOKprivate),
|
|
Keyword("package", TOKpackage),
|
|
Keyword("protected", TOKprotected),
|
|
Keyword("public", TOKpublic),
|
|
Keyword("export", TOKexport),
|
|
Keyword("body", TOKbody),
|
|
Keyword("invariant", TOKinvariant),
|
|
Keyword("unittest", TOKunittest),
|
|
Keyword("version", TOKversion),
|
|
Keyword("__argTypes", TOKargTypes),
|
|
Keyword("__parameters", TOKparameters),
|
|
Keyword("ref", TOKref),
|
|
Keyword("macro", TOKmacro),
|
|
Keyword("pure", TOKpure),
|
|
Keyword("nothrow", TOKnothrow),
|
|
Keyword("__gshared", TOKgshared),
|
|
Keyword("__traits", TOKtraits),
|
|
Keyword("__vector", TOKvector),
|
|
Keyword("__overloadset", TOKoverloadset),
|
|
Keyword("__FILE__", TOKfile),
|
|
Keyword("__LINE__", TOKline),
|
|
Keyword("__MODULE__", TOKmodulestring),
|
|
Keyword("__FUNCTION__", TOKfuncstring),
|
|
Keyword("__PRETTY_FUNCTION__", TOKprettyfunc),
|
|
Keyword("shared", TOKshared),
|
|
Keyword("immutable", TOKimmutable),
|
|
];
|