added support for token strings

src/ce_d2syn.pas

@@ -83,7 +83,7 @@ type
 
   TTokenKind = (tkCommt, tkIdent, tkKeywd, tkStrng, tkBlank, tkSymbl, tkNumbr, tkCurrI, tkDDocs);
 
-  TRangeKind = (rkNone, rkString1, rkString2, rkBlockCom1, rkBlockCom2, rkBlockDoc1, rkBlockDoc2, rkAsm);
+  TRangeKind = (rkNone, rkString1, rkString2, rkTokString, rkBlockCom1, rkBlockCom2, rkBlockDoc1, rkBlockDoc2, rkAsm);
 
   TFoldKind = (fkBrackets, fkComments1, fkComments2);
   TFoldKinds = set of TFoldKind;
@@ -451,7 +451,6 @@ end;
 //TODO-cnumber literals: stricter.
 //TODO-cnumber literals: binary.
 //TODO-cstring literals: delimited strings.
-//TODO-cstring literals: token strings.
 //TODO-ccomments: correct nested comments handling.
 //TODO-cfeature: something like pascal {$region} : /*folder blabla*/  /*endfolder*/
 
@@ -684,6 +683,32 @@ begin
     end;
   end;
 
+  //token string
+  if fRange = rkNone then if (readCurr = 'q') and (readNext = '{') then
+  begin
+    // go to end of string/eol
+    while ((readNext <> '}') and (not (readCurr = #10))) do (*!*);
+    if (readCurr = #10) then fRange := rkTokString else readNext;
+    fTokKind := tkStrng;
+    exit;
+  end else Dec(fTokStop);
+  if fRange = rkTokString then
+  begin
+    if (readCurr <> '}') then while ((readNext <> '}') and (not (readCurr = #10))) do (*!*);
+    if (readCurr = #10) then
+    begin
+      fTokKind := tkStrng;
+      exit;
+    end;
+    if (readCurr = '}') then
+    begin
+      fTokKind := tkStrng;
+      fRange := rkNone;
+      readNext;
+      exit;
+    end;
+  end;
+
   // char literals
   if fRange = rkNone then if (readCurr = #39) then
   begin

src/ce_dlang.pas

@@ -475,6 +475,22 @@ begin
       continue;
     end;
 
+    // token string
+    if (reader.head^ = 'q') and (reader.next^ = '{') then
+    begin
+      reader.next;
+      if isOutOfBound then exit;
+      while (reader.head^ <> '}') do
+      begin
+        identifier += reader.head^;
+        reader.next;
+        if isOutOfBound then exit;
+      end;
+      reader.next;
+      addToken(ltkString);
+      continue;
+    end else reader.previous;
+
     //chars, note: same escape error as in SynD2Syn
     if (reader.head^ = #39) then
     begin