replace cetodo by dastworx, #82

cetodo/cetodo.d

@@ -1,315 +0,0 @@
-/*******************************************************************************
-TODO source code analyzer for Coedit projects/files
-
-## Format: 
-
-`` 
-[D comment prefix] TODO|FIXME [fields] : description
-``
-
-- D comment prefix: todo comments are detected in all the D comments kind.
-In multi line comments, new lines must not be prefixed with '*' or '+'.
-For example the following multiline comment is not suitable for a TODO comment:
-
-/++
- + TODO:whatever.
- +/
-
-but this one is:
-
-/++
-  TODO:whatever.
- +/
-
-- TODO|FIXME: used to detect that the comment is a "TODO" comment.
-The keywords are not case sensitive.
-
-- fields: an optional list of properties with the format
-`-<char x><property for char x>
-the possible fields are:
-    - c: TODO category, e.g: _-cserialization_, -cerrorhandling_.
-    - a: TODO assignee, e.g: _-aMisterFreeze_, _-aFantomas_.
-    - p: TODO priority, as an integer literal, eg: _-p8_, _-p0_.
-    - s: TODO status, e.g _-sPartiallyFixed_, _-sDone_.
-  
-- description: what's to be done, e.g:  "set this property as const()".
-
-## Examples:
-
-``// TODO: set this property as const() to make it read-only.``
-
-``// TODO-cfeature: save this property in the inifile.``
-
-``// TODO-cannnotations-p8: annotate the members of the module with @safe and if possible nothrow.``
-
-``// FIXME-p8: This won't work if all the flags are OR-ed.``
-
-## Widget-to-tool IPC:
-
-The widget calls the tool with a file list as argument and reads the process
-output on exit. The widget expects to find some _TODO items_ in _LFM_ format, 
-according to the classes declarations of TTodoItems (the collection container) 
-and TTodoItem(the collection item).
-
-********************************************************************************/
-module cetodo;
-
-import std.stdio, std.getopt, std.string, std.algorithm;
-import std.array, std.conv, std.traits, std.ascii;
-import std.file, std.path, std.range;
-import dparse.lexer;
-
-/// Encapsulates the fields of a TODO comment_.
-private struct TodoItem 
-{
-    /** 
-     * Enumerates the possible fields of _a TODO comment_. 
-     * They must match the published member of the widget-side class TTodoItem.
-     */
-    private enum TodoField: ubyte {filename, line, text, category, assignee, priority, status}
-    private __gshared static string[TodoField] fFieldNames;
-    private string[TodoField.max+1] fFields;
-    
-    static this()
-    {
-        foreach(member; EnumMembers!TodoField)
-            fFieldNames[member] = to!string(member);
-    }
-     
-    /**
-     * Constructs a TODO item with its fields.
-     * Params:
-     * fname = the file where the _TODO comment is located. mandatory.
-     * line  = the line where the _TODO comment_is located. mandatory.
-     * text  = the _TODO comment main text. mandatory.
-     * cat   = the _TODO comment category, optional.
-     * ass   = the _TODO comment assignee, optional.
-     * prior = the _TODO comment priority, as an integer litteral, optional.
-     * status= the _TODO comment status, optional.
-     */
-    @safe this(string fname, string line, string text, string cat = "",
-        string ass = "", string prior = "", string status = "")
-    {
-        // priority must be convertible to int
-        if (prior.length) try to!long(prior);
-        catch(Exception e) prior = "";
-
-        // Pascal strings are not multi-line
-        version(Windows) immutable glue = "'#13#10'";
-        else immutable glue = "'#10'";
-        text = text.splitLines.join(glue);
-              
-        fFields[TodoField.filename] = fname;
-        fFields[TodoField.line]     = line;
-        fFields[TodoField.text]     = text;
-        fFields[TodoField.category] = cat;
-        fFields[TodoField.assignee] = ass;
-        fFields[TodoField.priority] = prior;
-        fFields[TodoField.status]   = status;
-    }
-    
-    /**
-     * The item writes itself as a TCollectionItem.
-     * Params:
-     * LfmString = the string containing the LFM script.
-     */
-    void serialize(ref Appender!string lfmApp)
-    {
-        lfmApp.put("  \r    item\r");
-        foreach(member; EnumMembers!TodoField)
-            if (fFields[member].length)
-                lfmApp.put(format("      %s = '%s'\r", fFieldNames[member], fFields[member]));   
-        lfmApp.put("    end");
-    }
-}
-
-private alias TodoItems = TodoItem* [];
-
-/**
- * Application main procedure.
- * Params:
- * args = a list of files to analyze. 
- * Called each time a document is focused. Args is set using:
- * - the symbolic string `<CFF>` (current file is not in a project).
- * - the symbolic string `<CPFS>` (current file is in a project).
- */
-void main(string[] args)
-{
-    string[] files = args[1..$];
-    Appender!string lfmApp;
-    TodoItems todoItems;
-    
-    // helper to test in Coedit with Compile file & run.
-    version(runnable_module)
-    {
-        if (!files.length)
-            files ~= __FILE__;
-    }
-   
-    foreach(f; files)
-    {
-        if (!f.exists) continue;
-        
-        // load and parse the file
-        auto src = cast(ubyte[]) read(f, size_t.max);
-        auto config = LexerConfig(f, StringBehavior.source);
-        StringCache cache = StringCache(StringCache.defaultBucketCount);
-        auto lexer = DLexer(src, config, &cache);
-        
-        // analyze the tokens
-        foreach(tok; lexer) token2TodoItem(tok, f, todoItems);                     
-    }
-    
-    // efficient appending if the item text ~ fields is about 100 chars
-    lfmApp.reserve(todoItems.length * 128 + 64);
-
-    // serialize the items using the pascal component streaming text format
-    lfmApp.put("object TTodoItems\r  items = <");
-    foreach(todoItem; todoItems) todoItem.serialize(lfmApp);
-    lfmApp.put(">\rend\r\n");
-
-    // the widget has the LFM script in the output
-    write(lfmApp.data);
-}
-
-// comments may include some "'", which in Pascal are string delim
-string patchPasStringLitteral(string str) @safe
-{
-    Appender!string app;
-    app.reserve(str.length);
-    bool skip;
-    foreach (immutable i; 0..str.length)
-    {
-        char c = str[i];
-        if (c > 0x7F)
-        {
-            app ~= str[i];
-            skip = true;
-        }
-        else if (c == '\'')
-        {
-            if (skip)
-                app ~= str[i];
-            else
-                app ~= "'#39'";
-            skip = false;
-        }
-        else
-        {
-            app ~= str[i];
-            skip = false;
-        }
-    }
-    return app.data;
-}
-
-/// Try to transforms a Token into a a TODO item
-@safe private void token2TodoItem(const(Token) atok, string fname, ref TodoItems todoItems)
-{
-    if (atok.type != (tok!"comment")) return;
-    string text = atok.text.strip.patchPasStringLitteral;
-    string identifier;
-
-    // always comment
-    text.popFrontN(2);
-    if (text.empty)
-        return;
-    // ddoc suffix
-    if (text.front.among('/', '*', '+'))
-    {
-        text.popFront;
-        if (text.empty)
-            return;
-    }
-    // leading whites
-    while (text.front.isWhite)
-    {
-        text.popFront;
-        if (text.empty)
-            return;
-    }
-
-    // "TODO|FIXME"
-    bool isTodoComment;
-    while (!text.empty)
-    {
-        identifier ~= std.ascii.toUpper(text.front);
-        text.popFront;   
-        if (identifier.among("TODO","FIXME"))
-        {
-            isTodoComment = true;
-            break;
-        }            
-    }
-    if (!isTodoComment) return;
-    identifier = "";
-
-    // splits "fields" and "description"
-    bool isWellFormed;
-    string fields;
-    while (!text.empty)
-    {
-        auto front = text.front;     
-        identifier ~= front;
-        text.popFront;
-        if (front == ':')
-        {
-            if (identifier.length) fields = identifier;
-            isWellFormed = text.length > 0;
-            break;
-        }
-    }
-    if (!isWellFormed) return;
-    identifier = ""; 
-
-    // parses "fields"
-    string a, c, p, s;
-    while (!fields.empty)
-    {
-        const dchar front = fields.front;
-        fields.popFront;
-        if ((front == '-' || fields.empty) && identifier.length > 2)
-        {
-            string fieldContent = identifier[2..$].strip;
-            switch(identifier[0..2].toUpper)
-            {
-                default: break;
-                case "-A": a = fieldContent; break;
-                case "-C": c = fieldContent; break;
-                case "-P": p = fieldContent; break;
-                case "-S": s = fieldContent; break;
-            }
-            identifier = "";
-        }
-        identifier ~= front;  
-    }
-
-    if (text.length > 1 && text[$-2..$].among("*/", "+/"))
-        text.length -=2;
-
-
-    string line;
-    try line = to!string(atok.line);
-    catch(ConvException e) line = "0";
-    todoItems ~= new TodoItem(fname, line, text, c, a, p, s);
-}
-
-// samples for testing the program as a runnable ('Compile file and run ...') with '<CFF>'
-
-// fixme-p8: èuèuuè``u`èuùè é ^çßßðđææ«€¶ 
-// fixme-p8: fixme also handled
-// TODO-cINVALID_because_no_content:
-/// TODO:set this property as const() to set it read only.
-// TODO-cfeature-sDone:save this property in the inifile.
-// TODO-cannnotations-p8: annotates the member of the module as @safe and if possible nothrow.
-// TODO-cfeature-sDone: save this property in the inifile.
-// TODO-aMe-cCat-p1-sjkjkj:todo body
-/**
- TODO-cd:
- - this
- - that
-*/
-/++ TODO-cx:a mqkjfmksmldkf
-+/
-// TODO: it's fixed or it's not (issue #40) [çéèà]
-

cetodo/cetodo_libman.coedit

@@ -1,29 +0,0 @@
-object CurrentProject: TCENativeProject
-  OptionsCollection = <  
-    item
-      name = 'testwith_CPFS'
-      messagesOptions.additionalWarnings = True
-      outputOptions.boundsCheck = onAlways
-      outputOptions.unittest = True
-      pathsOptions.outputFilename = '../lazproj/cetodo'
-      runOptions.options = [poUsePipes, poStderrToOutPut]
-      runOptions.parameters.Strings = (
-        '<CPFS>'
-      )
-      runOptions.showWindow = swoHIDE
-    end  
-    item
-      name = 'release'
-      outputOptions.boundsCheck = offAlways
-      outputOptions.optimizations = True
-      outputOptions.release = True
-      pathsOptions.outputFilename = '../bin/cetodo'
-    end>
-  Sources.Strings = (
-    'cetodo.d'
-  )
-  ConfigurationIndex = 1
-  LibraryAliases.Strings = (
-    'libdparse'
-  )
-end

cetodo/cetodo_submodule.coedit

@@ -1,32 +0,0 @@
-object CurrentProject: TCENativeProject
-  OptionsCollection = <  
-    item
-      name = 'testwith_CPFS'
-      messagesOptions.additionalWarnings = True
-      outputOptions.boundsCheck = onAlways
-      outputOptions.unittest = True
-      pathsOptions.outputFilename = '../lazproj/cetodo'
-      pathsOptions.extraSources.Strings = (
-        '../etc/libdparse/src/*'
-      )
-      runOptions.options = [poUsePipes, poStderrToOutPut]
-      runOptions.parameters.Strings = (
-        '<CPFS>'
-      )
-      runOptions.showWindow = swoHIDE
-    end  
-    item
-      name = 'release'
-      outputOptions.boundsCheck = offAlways
-      outputOptions.optimizations = True
-      outputOptions.release = True
-      pathsOptions.outputFilename = '../bin/cetodo'
-      pathsOptions.extraSources.Strings = (
-        '../etc/libdparse/src/*'
-      )
-    end>
-  Sources.Strings = (
-    'cetodo.d'
-  )
-  ConfigurationIndex = 1
-end

cetodo/readme.md

@@ -1,17 +0,0 @@
-ceTodo
-======
-
-Tool designed to analyze the _TODO comments_ in D source files.
-It's written in D using Coedit. 
-
-To build it, either [libdparse](https://github.com/Hackerpilot/libdparse)
-must be setup in the [libman](https://github.com/BBasile/Coedit/wiki#library-manager-widget) 
-as described in this [tutorial](https://github.com/BBasile/Coedit/wiki#lets-build-a-static-library),
-or *libdparse* submodule must be cloned with Coedit repository (`git submodule init` or `update`).
-
-Notice that *libdparse* can be easily build in Coedit using the [metad project](https://github.com/BBasile/metad).
-
-- `cetodo_submodule.coedit`: coedit project based on *libdparse* as a submodule.
-- `cetodo_libman.coedit`: coedit project based on *libdparse* as a *libman* entry.
-
-This tool is mandatory to enable the *todo list widget*.

dastworx/src/main.d

@@ -47,7 +47,10 @@ void main(string[] args)
     }
 
     if (args.length > 2)
+    {
         files = args[1].splitter(pathSeparator).array;
+        version(devel) writeln(files);
+    }
 
     config = LexerConfig("", StringBehavior.source, WhitespaceBehavior.skip);
     cache = construct!(StringCache)(StringCache.defaultBucketCount);
@@ -78,31 +81,7 @@ void handleSymListOption()
 void handleTodosOption()
 {
     mixin(logCall);
-    lex!true;
+    getTodos(files);
-    const(Token)[][] tokensArray;
-    if (tokens.length)
-        tokensArray ~= tokens;
-
-    import std.file: exists;
-    if (files.length)
-    {
-        StringCache cache = StringCache(StringCache.defaultBucketCount);
-        foreach(fname; files)
-            if (fname.exists)
-        {
-            try
-            {
-                File f = File(fname, "r");
-                ubyte[] src;
-                foreach(buffer; f.byChunk(4096))
-                    src ~= buffer;
-                tokensArray ~= getTokensForParser(src, config, &cache);
-                f.close;
-            }
-            catch (Exception e) continue;
-        }
-    }
-    getTodos(tokensArray);
 }
 
 void handleRunnableFlags()

dastworx/src/todos.d

@@ -10,21 +10,31 @@ import
 
 private __gshared Appender!string stream;
 
-void getTodos(const(Token)[][] tokensArray)
+//TODO: sdfsfd
+
+void getTodos(string[] files)
 {
     mixin(logCall);
-    stream.reserve(2^^16);
+    //stream.reserve(2^^16);
-    stream.put("object TTodoItems\ritems = <");
+    stream.put("object TTodoItems\r items = <");
-    assert(tokensArray.length);
+    foreach(fname; files)
-    assert(tokensArray[0].length);
+    {
-    foreach(tokens; tokensArray)
+        ubyte[] source;
-        foreach(token; tokens.filter!((a) => a.type == tok!"comment"))
+        StringCache cache = StringCache(StringCache.defaultBucketCount);
-            token.analyze;
+        LexerConfig config = LexerConfig(fname, StringBehavior.source);
-    stream.put(">\rend\r");
+        File f = File(fname, "r");
+        foreach (buffer; f.byChunk(4096))
+            source ~= buffer;
+        f.close;
+        foreach(token; DLexer(source, config, &cache).array
+            .filter!((a) => a.type == tok!"comment"))
+                analyze(token, fname);
+    }
+    stream.put(">\rend\r\n");
     writeln(stream.data);
 }
 
-private void analyze(const(Token) token)
+private void analyze(const(Token) token, string fname)
 {
     string text = token.text.strip.patchPascalString;
     string identifier;
@@ -110,9 +120,9 @@ private void analyze(const(Token) token)
 
 
 
-    stream.put("\ritem\r");
+    stream.put("\r item\r");
-    //stream.put(format("filename = '%s'\r", fname));
+    stream.put(format("filename = '%s'\r", fname));
-    stream.put(format("line = '%d'\r", token.line));
+    stream.put(format("line = '%s'\r", token.line));
     stream.put(format("text = '%s'\r", text));
     if (c.length)
         stream.put(format("category = '%s'\r", c));

src/ce_infos.lfm

@@ -1,21 +1,21 @@
 inherited CEInfoWidget: TCEInfoWidget
   Left = 713
-  Height = 502
+  Height = 471
   Top = 245
   Width = 411
   BorderIcons = [biSystemMenu, biMinimize, biMaximize]
   Caption = 'About'
-  ClientHeight = 502
+  ClientHeight = 471
   ClientWidth = 411
   inherited Back: TPanel
-    Height = 502
+    Height = 471
     Width = 411
-    ClientHeight = 502
+    ClientHeight = 471
     ClientWidth = 411
     inherited Content: TPanel
-      Height = 466
+      Height = 435
       Width = 411
-      ClientHeight = 466
+      ClientHeight = 435
       ClientWidth = 411
       object GroupBox1: TGroupBox[0]
         Left = 4
@@ -46,18 +46,18 @@ inherited CEInfoWidget: TCEInfoWidget
       end
       object GroupBox2: TGroupBox[1]
         Left = 4
-        Height = 349
+        Height = 318
         Top = 113
         Width = 403
         Align = alClient
         BorderSpacing.Around = 4
         Caption = 'tools status'
-        ClientHeight = 319
+        ClientHeight = 288
         ClientWidth = 399
         TabOrder = 1
         object boxTools: TScrollBox
           Left = 4
-          Height = 311
+          Height = 280
           Top = 4
           Width = 391
           HorzScrollBar.Page = 1

src/ce_infos.pas

@@ -211,11 +211,8 @@ begin
   toolItem.Parent := boxTools;
   toolItem.ReAlign;
   toolItem := TToolInfo.Construct(self, tikFindable, 'dastworx',
-    'background tool that works on the D modules AST');
+    'background tool that works on the D modules AST to extract informations' +
-  toolItem.Parent := boxTools;
+    LineEnding + 'such as the declarations, the imports, the "TODO" comments, etc.');
-  toolItem.ReAlign;
-  toolItem := TToolInfo.Construct(self, tikFindable, 'cetodo',
-    'background tool that collects information for the todo list widget');
   toolItem.Parent := boxTools;
   toolItem.ReAlign;
   toolItem := TToolInfo.Construct(self, tikOptional, 'dub',

src/ce_todolist.pas

@@ -46,7 +46,7 @@ type
   published
     property filename: string read fFile write fFile;
     property line: string read fLine write fLine;
-    property Text: string read fText write fText;
+    property text: string read fText write fText;
     property assignee: string read fAssignee write fAssignee;
     property category: string read fCategory write fCategory;
     property status: string read fStatus write fStatus;
@@ -144,7 +144,7 @@ implementation
 {$R *.lfm}
 
 const
-  ToolExeName = 'cetodo' + exeExt;
+  ToolExeName = 'dastworx' + exeExt;
   OptFname = 'todolist.txt';
 
 {$REGION TTodoItems ------------------------------------------------------------}
@@ -410,7 +410,9 @@ end;
 procedure TCETodoListWidget.callToolProcess;
 var
   ctxt: TTodoContext;
-  i: integer;
+  i,j: integer;
+  nme: string;
+  str: string = '';
 begin
   clearTodoList;
   if not exeInSysPath(ToolExeName) then
@@ -429,14 +431,23 @@ begin
   fToolProc.OnTerminate := @toolTerminated;
 
   // files passed to the tool argument
-  if ctxt = tcProject then
+  i := 0;
+  j := fProj.sourcesCount-1;
+  if ctxt = tcProject then for i := 0 to j do
   begin
-    for i := 0 to fProj.sourcesCount-1 do
+    nme := fProj.sourceAbsolute(i);
-      fToolProc.Parameters.Add(fProj.sourceAbsolute(i));
+    if not hasDlangSyntax(nme.extractFileExt) then
+      continue;
+    str += nme;
+    if i <> j then
+      str += PathSeparator;
   end
-  else fToolProc.Parameters.Add(fDoc.fileName);
+  else str := fDoc.fileName;
+  fToolProc.Parameters.Add(str);
+  fToolProc.Parameters.Add('-t');
   //
   fToolProc.Execute;
+  fToolProc.CloseInput;
 end;
 
 procedure TCETodoListWidget.procOutputDbg(Sender: TObject);
@@ -463,8 +474,6 @@ end;
 
 procedure TCETodoListWidget.toolTerminated(Sender: TObject);
 begin
-  //WASTODO-cbugfix: UTF chars in TODO comments bug either in the widget or the tool, symptom: empty todo list, conditions: to determine.
-  // seems to be fixed since the TODO scanner 's been rewritten using ranges (std.range.front() => autodecoding).
   fToolProc.OutputStack.Position := 0;
   fTodos.loadFromTxtStream(fToolProc.OutputStack);
   fillTodoList;