fix lexer bug + add temp method to domp tokens to file

src/ce_dlang.pas

@@ -93,8 +93,15 @@ type
 
 const
   LexTokenKindString: array[TLexTokenKind] of string =
-    ('Illegal', 'Character', 'Comment', 'Identifier', 'Keyword',
+    ('Illegal  ',
-    'Number', 'Operator', 'String', 'Symbol');
+    'Character ',
+    'Comment   ',
+    'Identifier',
+    'Keyword   ',
+    'Number    ',
+    'Operator  ',
+    'String    ',
+    'Symbol    ');
 
 type
 
@@ -120,6 +127,7 @@ type
   public
     procedure Clear;
     procedure addToken(aValue: PLexToken);
+    procedure saveToFile(fname: string);
     property token[index: integer]: PLexToken read getToken; default;
   end;
 
@@ -328,6 +336,25 @@ begin
   add(Pointer(aValue));
 end;
 
+procedure TLexTokenList.saveToFile(fname: string);
+var
+  tok: PLexToken;
+  i: integer;
+begin
+  with TStringList.Create do
+  try
+    for i:= 0 to self.count-1 do
+    begin
+      tok := getToken(i);
+      add(format('line %.5d - col %.3d: (%s): %s', [tok^.position.Y, tok^.position.X,
+        LexTokenKindString[tok^.kind], tok^.Data]));
+    end;
+  finally
+    SaveToFile(fname);
+    free;
+  end;
+end;
+
 function TLexTokenEnumerator.GetCurrent: PLexToken;
 begin
   exit(fList.token[fIndex]);
@@ -484,6 +511,7 @@ begin
     end;
 
     // string 1, note: same escape error as in SynD2Syn
+    rstring := false;
     if (reader.head^ in ['r', 'x']) then
     begin
       rstring := reader.head^ = 'r';
@@ -500,6 +528,7 @@ begin
         if isStringPostfix(reader.Next^) then
           reader.Next;
         addToken(ltkString);
+        rstring := false;
         if callBackDoStop then
           exit;
         continue;
@@ -518,20 +547,30 @@ begin
           reader.Next;
           if isOutOfBound then
             exit;
-        end
+          continue;
-        else if (reader.head^ = '"') then
+        end;
+        if (reader.head^ = '"') then
+        begin
+          reader.Next;
+          if isOutOfBound then
+            exit;
           break;
+        end
+        else
+        begin
           identifier += reader.head^;
           reader.Next;
           if isOutOfBound then
             exit;
         end;
-      if isStringPostfix(reader.Next^) then
+      end;
+      if isStringPostfix(reader.head^) then
       begin
         identifier += reader.head^;
         reader.Next;
       end;
       addToken(ltkString);
+      rstring := false;
       if callBackDoStop then
         exit;
       continue;
@@ -585,20 +624,21 @@ begin
       begin
         if reader.head^ = '\' then
         begin
+          identifier += reader.head^;
           reader.Next;
+          identifier += reader.head^;
           if isOutOfBound then
             exit;
-          if reader.head^ = #10 then
-            exit;
           reader.Next;
           if isOutOfBound then
             exit;
         end;
-        if reader.head^ = #39 then
-          break;
-        reader.Next;
         if isOutOfBound then
           exit;
+        if reader.head^ = #39 then
+          break;
+        identifier += reader.head^;
+        reader.Next;
       end;
       reader.Next;
       if isOutOfBound then

src/ce_editor.lfm

@@ -120,5 +120,12 @@ inherited CEEditorWidget: TCEEditorWidget
       Caption = 'Show ddoc'
       OnClick = mnuedDdocClick
     end
+    object MenuItem3: TMenuItem
+      Caption = '-'
+    end
+    object MenuItem5: TMenuItem
+      Caption = 'Save lexical tokens to file...'
+      OnClick = MenuItem5Click
+    end
   end
 end

src/ce_editor.pas

@@ -6,7 +6,7 @@ interface
 
 uses
   Classes, SysUtils, FileUtil, Forms, Controls, lcltype, Graphics, SynEditKeyCmds,
-  ComCtrls, SynEditHighlighter, ExtCtrls, Menus, SynMacroRecorder,
+  ComCtrls, SynEditHighlighter, ExtCtrls, Menus, SynMacroRecorder, dialogs,
   SynPluginSyncroEdit, SynEdit, SynHighlighterMulti, ce_dialogs,
   ce_widget, ce_interfaces, ce_synmemo, ce_dlang, ce_common, ce_dcd, ce_observer,
   ce_sharedres, ce_controls;
@@ -18,6 +18,8 @@ type
   TCEEditorWidget = class(TCEWidget, ICEMultiDocObserver, ICEMultiDocHandler, ICEProjectObserver)
     MenuItem1: TMenuItem;
     MenuItem2: TMenuItem;
+    MenuItem3: TMenuItem;
+    MenuItem5: TMenuItem;
     mnEdInvAllNone: TMenuItem;
     mneEdComm: TMenuItem;
     mnuedPrev: TMenuItem;
@@ -35,6 +37,7 @@ type
     macRecorder: TSynMacroRecorder;
     editorStatus: TStatusBar;
     mnuEditor: TPopupMenu;
+    procedure MenuItem5Click(Sender: TObject);
     procedure mnEdInvAllNoneClick(Sender: TObject);
     procedure mneEdCommClick(Sender: TObject);
     procedure mnuedPrevClick(Sender: TObject);
@@ -595,6 +598,24 @@ begin
     fDoc.CommandProcessor(ecSwapVersionAllNone, '', nil);
 end;
 
+procedure TCEEditorWidget.MenuItem5Click(Sender: TObject);
+begin
+  if fDoc.isNil then
+    exit;
+  with TSaveDialog.Create(nil) do
+  try
+    if execute then
+    begin
+      fTokList.Clear;
+      lex(fDoc.Text, fTokList, nil);
+      fTokList.saveToFile(FileName);
+      fTokList.Clear;
+    end;
+  finally
+    free;
+  end;
+end;
+
 procedure TCEEditorWidget.mnuedCutClick(Sender: TObject);
 begin
   if fDoc.isNotNil then