forks
/
dexed
mirror of https://gitlab.com/basile.b/dexed.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

replace cetodo by dastworx, #82

This commit is contained in:
Basile Burg 2016-07-02 08:23:05 +02:00
parent afa35e11d1
commit fcb76a4097
9 changed files with 56 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

315
cetodo/cetodo.d
View File

@ -1,315 +0,0 @@
/*******************************************************************************
TODO source code analyzer for Coedit projects/files
## Format:
``
[D comment prefix] TODO|FIXME [fields] : description
``
- D comment prefix: todo comments are detected in all the D comments kind.
In multi line comments, new lines must not be prefixed with '*' or '+'.
For example the following multiline comment is not suitable for a TODO comment:
/++
+ TODO:whatever.
+/
but this one is:
/++
TODO:whatever.
+/
- TODO|FIXME: used to detect that the comment is a "TODO" comment.
The keywords are not case sensitive.
- fields: an optional list of properties with the format
`-<char x><property for char x>
the possible fields are:
- c: TODO category, e.g: _-cserialization_, -cerrorhandling_.
- a: TODO assignee, e.g: _-aMisterFreeze_, _-aFantomas_.
- p: TODO priority, as an integer literal, eg: _-p8_, _-p0_.
- s: TODO status, e.g _-sPartiallyFixed_, _-sDone_.
- description: what's to be done, e.g: "set this property as const()".
## Examples:
``// TODO: set this property as const() to make it read-only.``
``// TODO-cfeature: save this property in the inifile.``
``// TODO-cannnotations-p8: annotate the members of the module with @safe and if possible nothrow.``
``// FIXME-p8: This won't work if all the flags are OR-ed.``
## Widget-to-tool IPC:
The widget calls the tool with a file list as argument and reads the process
output on exit. The widget expects to find some _TODO items_ in _LFM_ format,
according to the classes declarations of TTodoItems (the collection container)
and TTodoItem(the collection item).
********************************************************************************/
module cetodo;
import std.stdio, std.getopt, std.string, std.algorithm;
import std.array, std.conv, std.traits, std.ascii;
import std.file, std.path, std.range;
import dparse.lexer;
/// Encapsulates the fields of a TODO comment_.
private struct TodoItem
{
/**
* Enumerates the possible fields of _a TODO comment_.
* They must match the published member of the widget-side class TTodoItem.
*/
private enum TodoField: ubyte {filename, line, text, category, assignee, priority, status}
private __gshared static string[TodoField] fFieldNames;
private string[TodoField.max+1] fFields;
static this()
{
foreach(member; EnumMembers!TodoField)
fFieldNames[member] = to!string(member);
}
/**
* Constructs a TODO item with its fields.
* Params:
* fname = the file where the _TODO comment is located. mandatory.
* line = the line where the _TODO comment_is located. mandatory.
* text = the _TODO comment main text. mandatory.
* cat = the _TODO comment category, optional.
* ass = the _TODO comment assignee, optional.
* prior = the _TODO comment priority, as an integer litteral, optional.
* status= the _TODO comment status, optional.
*/
@safe this(string fname, string line, string text, string cat = "",
string ass = "", string prior = "", string status = "")
{
// priority must be convertible to int
if (prior.length) try to!long(prior);
catch(Exception e) prior = "";
// Pascal strings are not multi-line
version(Windows) immutable glue = "'#13#10'";
else immutable glue = "'#10'";
text = text.splitLines.join(glue);
fFields[TodoField.filename] = fname;
fFields[TodoField.line] = line;
fFields[TodoField.text] = text;
fFields[TodoField.category] = cat;
fFields[TodoField.assignee] = ass;
fFields[TodoField.priority] = prior;
fFields[TodoField.status] = status;
}
/**
* The item writes itself as a TCollectionItem.
* Params:
* LfmString = the string containing the LFM script.
*/
void serialize(ref Appender!string lfmApp)
{
lfmApp.put(" \r item\r");
foreach(member; EnumMembers!TodoField)
if (fFields[member].length)
lfmApp.put(format(" %s = '%s'\r", fFieldNames[member], fFields[member]));
lfmApp.put(" end");
}
}
private alias TodoItems = TodoItem* [];
/**
* Application main procedure.
* Params:
* args = a list of files to analyze.
* Called each time a document is focused. Args is set using:
* - the symbolic string `<CFF>` (current file is not in a project).
* - the symbolic string `<CPFS>` (current file is in a project).
*/
void main(string[] args)
{
string[] files = args[1..$];
Appender!string lfmApp;
TodoItems todoItems;
// helper to test in Coedit with Compile file & run.
version(runnable_module)
{
if (!files.length)
files ~= __FILE__;
}
foreach(f; files)
{
if (!f.exists) continue;
// load and parse the file
auto src = cast(ubyte[]) read(f, size_t.max);
auto config = LexerConfig(f, StringBehavior.source);
StringCache cache = StringCache(StringCache.defaultBucketCount);
auto lexer = DLexer(src, config, &cache);
// analyze the tokens
foreach(tok; lexer) token2TodoItem(tok, f, todoItems);
}
// efficient appending if the item text ~ fields is about 100 chars
lfmApp.reserve(todoItems.length * 128 + 64);
// serialize the items using the pascal component streaming text format
lfmApp.put("object TTodoItems\r items = <");
foreach(todoItem; todoItems) todoItem.serialize(lfmApp);
lfmApp.put(">\rend\r\n");
// the widget has the LFM script in the output
write(lfmApp.data);
}
// comments may include some "'", which in Pascal are string delim
string patchPasStringLitteral(string str) @safe
{
Appender!string app;
app.reserve(str.length);
bool skip;
foreach (immutable i; 0..str.length)
{
char c = str[i];
if (c > 0x7F)
{
app ~= str[i];
skip = true;
}
else if (c == '\'')
{
if (skip)
app ~= str[i];
else
app ~= "'#39'";
skip = false;
}
else
{
app ~= str[i];
skip = false;
}
}
return app.data;
}
/// Try to transforms a Token into a a TODO item
@safe private void token2TodoItem(const(Token) atok, string fname, ref TodoItems todoItems)
{
if (atok.type != (tok!"comment")) return;
string text = atok.text.strip.patchPasStringLitteral;
string identifier;
// always comment
text.popFrontN(2);
if (text.empty)
return;
// ddoc suffix
if (text.front.among('/', '*', '+'))
{
text.popFront;
if (text.empty)
return;
}
// leading whites
while (text.front.isWhite)
{
text.popFront;
if (text.empty)
return;
}
// "TODO|FIXME"
bool isTodoComment;
while (!text.empty)
{
identifier ~= std.ascii.toUpper(text.front);
text.popFront;
if (identifier.among("TODO","FIXME"))
{
isTodoComment = true;
break;
}
}
if (!isTodoComment) return;
identifier = "";
// splits "fields" and "description"
bool isWellFormed;
string fields;
while (!text.empty)
{
auto front = text.front;
identifier ~= front;
text.popFront;
if (front == ':')
{
if (identifier.length) fields = identifier;
isWellFormed = text.length > 0;
break;
}
}
if (!isWellFormed) return;
identifier = "";
// parses "fields"
string a, c, p, s;
while (!fields.empty)
{
const dchar front = fields.front;
fields.popFront;
if ((front == '-' || fields.empty) && identifier.length > 2)
{
string fieldContent = identifier[2..$].strip;
switch(identifier[0..2].toUpper)
{
default: break;
case "-A": a = fieldContent; break;
case "-C": c = fieldContent; break;
case "-P": p = fieldContent; break;
case "-S": s = fieldContent; break;
}
identifier = "";
}
identifier ~= front;
}
if (text.length > 1 && text[$-2..$].among("*/", "+/"))
text.length -=2;
string line;
try line = to!string(atok.line);
catch(ConvException e) line = "0";
todoItems ~= new TodoItem(fname, line, text, c, a, p, s);
}
// samples for testing the program as a runnable ('Compile file and run ...') with '<CFF>'
// fixme-p8: èuèuuè``u`èuùè é ^çßßðđææ«€¶
// fixme-p8: fixme also handled
// TODO-cINVALID_because_no_content:
/// TODO:set this property as const() to set it read only.
// TODO-cfeature-sDone:save this property in the inifile.
// TODO-cannnotations-p8: annotates the member of the module as @safe and if possible nothrow.
// TODO-cfeature-sDone: save this property in the inifile.
// TODO-aMe-cCat-p1-sjkjkj:todo body
/**
TODO-cd:
- this
- that
*/
/++ TODO-cx:a mqkjfmksmldkf
+/
// TODO: it's fixed or it's not (issue #40) [çéèà]

29
cetodo/cetodo_libman.coedit
View File

@ -1,29 +0,0 @@
object CurrentProject: TCENativeProject
OptionsCollection = <
item
name = 'testwith_CPFS'
messagesOptions.additionalWarnings = True
outputOptions.boundsCheck = onAlways
outputOptions.unittest = True
pathsOptions.outputFilename = '../lazproj/cetodo'
runOptions.options = [poUsePipes, poStderrToOutPut]
runOptions.parameters.Strings = (
'<CPFS>'
)
runOptions.showWindow = swoHIDE
end
item
name = 'release'
outputOptions.boundsCheck = offAlways
outputOptions.optimizations = True
outputOptions.release = True
pathsOptions.outputFilename = '../bin/cetodo'
end>
Sources.Strings = (
'cetodo.d'
)
ConfigurationIndex = 1
LibraryAliases.Strings = (
'libdparse'
)
end

32
cetodo/cetodo_submodule.coedit
View File

@ -1,32 +0,0 @@
object CurrentProject: TCENativeProject
OptionsCollection = <
item
name = 'testwith_CPFS'
messagesOptions.additionalWarnings = True
outputOptions.boundsCheck = onAlways
outputOptions.unittest = True
pathsOptions.outputFilename = '../lazproj/cetodo'
pathsOptions.extraSources.Strings = (
'../etc/libdparse/src/*'
)
runOptions.options = [poUsePipes, poStderrToOutPut]
runOptions.parameters.Strings = (
'<CPFS>'
)
runOptions.showWindow = swoHIDE
end
item
name = 'release'
outputOptions.boundsCheck = offAlways
outputOptions.optimizations = True
outputOptions.release = True
pathsOptions.outputFilename = '../bin/cetodo'
pathsOptions.extraSources.Strings = (
'../etc/libdparse/src/*'
)
end>
Sources.Strings = (
'cetodo.d'
)
ConfigurationIndex = 1
end

17
cetodo/readme.md
View File

@ -1,17 +0,0 @@
ceTodo
======
Tool designed to analyze the _TODO comments_ in D source files.
It's written in D using Coedit.
To build it, either [libdparse](https://github.com/Hackerpilot/libdparse)
must be setup in the [libman](https://github.com/BBasile/Coedit/wiki#library-manager-widget)
as described in this [tutorial](https://github.com/BBasile/Coedit/wiki#lets-build-a-static-library),
or *libdparse* submodule must be cloned with Coedit repository (`git submodule init` or `update`).
Notice that *libdparse* can be easily build in Coedit using the [metad project](https://github.com/BBasile/metad).
- `cetodo_submodule.coedit`: coedit project based on *libdparse* as a submodule.
- `cetodo_libman.coedit`: coedit project based on *libdparse* as a *libman* entry.
This tool is mandatory to enable the *todo list widget*.

29
dastworx/src/main.d
View File

@ -47,7 +47,10 @@ void main(string[] args)
}
if (args.length > 2)
{
files = args[1].splitter(pathSeparator).array;
version(devel) writeln(files);
}
config = LexerConfig("", StringBehavior.source, WhitespaceBehavior.skip);
cache = construct!(StringCache)(StringCache.defaultBucketCount);
@ -78,31 +81,7 @@ void handleSymListOption()
void handleTodosOption()
{
mixin(logCall);
lex!true;
const(Token)[][] tokensArray;
if (tokens.length)
tokensArray ~= tokens;
import std.file: exists;
if (files.length)
{
StringCache cache = StringCache(StringCache.defaultBucketCount);
foreach(fname; files)
if (fname.exists)
{
try
{
File f = File(fname, "r");
ubyte[] src;
foreach(buffer; f.byChunk(4096))
src ~= buffer;
tokensArray ~= getTokensForParser(src, config, &cache);
f.close;
}
catch (Exception e) continue;
}
}
getTodos(tokensArray);
getTodos(files);
}
void handleRunnableFlags()

36
dastworx/src/todos.d
View File

@ -10,21 +10,31 @@ import
private __gshared Appender!string stream;
void getTodos(const(Token)[][] tokensArray)
//TODO: sdfsfd
void getTodos(string[] files)
{
mixin(logCall);
stream.reserve(2^^16);
stream.put("object TTodoItems\ritems = <");
assert(tokensArray.length);
assert(tokensArray[0].length);
foreach(tokens; tokensArray)
foreach(token; tokens.filter!((a) => a.type == tok!"comment"))
token.analyze;
stream.put(">\rend\r");
//stream.reserve(2^^16);
stream.put("object TTodoItems\r items = <");
foreach(fname; files)
{
ubyte[] source;
StringCache cache = StringCache(StringCache.defaultBucketCount);
LexerConfig config = LexerConfig(fname, StringBehavior.source);
File f = File(fname, "r");
foreach (buffer; f.byChunk(4096))
source ~= buffer;
f.close;
foreach(token; DLexer(source, config, &cache).array
.filter!((a) => a.type == tok!"comment"))
analyze(token, fname);
}
stream.put(">\rend\r\n");
writeln(stream.data);
}
private void analyze(const(Token) token)
private void analyze(const(Token) token, string fname)
{
string text = token.text.strip.patchPascalString;
string identifier;
@ -110,9 +120,9 @@ private void analyze(const(Token) token)
stream.put("\ritem\r");
//stream.put(format("filename = '%s'\r", fname));
stream.put(format("line = '%d'\r", token.line));
stream.put("\r item\r");
stream.put(format("filename = '%s'\r", fname));
stream.put(format("line = '%s'\r", token.line));
stream.put(format("text = '%s'\r", text));
if (c.length)
stream.put(format("category = '%s'\r", c));

18
src/ce_infos.lfm
View File

@ -1,21 +1,21 @@
inherited CEInfoWidget: TCEInfoWidget
Left = 713
Height = 502
Height = 471
Top = 245
Width = 411
BorderIcons = [biSystemMenu, biMinimize, biMaximize]
Caption = 'About'
ClientHeight = 502
ClientHeight = 471
ClientWidth = 411
inherited Back: TPanel
Height = 502
Height = 471
Width = 411
ClientHeight = 502
ClientHeight = 471
ClientWidth = 411
inherited Content: TPanel
Height = 466
Height = 435
Width = 411
ClientHeight = 466
ClientHeight = 435
ClientWidth = 411
object GroupBox1: TGroupBox[0]
Left = 4
@ -46,18 +46,18 @@ inherited CEInfoWidget: TCEInfoWidget
end
object GroupBox2: TGroupBox[1]
Left = 4
Height = 349
Height = 318
Top = 113
Width = 403
Align = alClient
BorderSpacing.Around = 4
Caption = 'tools status'
ClientHeight = 319
ClientHeight = 288
ClientWidth = 399
TabOrder = 1
object boxTools: TScrollBox
Left = 4
Height = 311
Height = 280
Top = 4
Width = 391
HorzScrollBar.Page = 1

7
src/ce_infos.pas
View File

@ -211,11 +211,8 @@ begin
toolItem.Parent := boxTools;
toolItem.ReAlign;
toolItem := TToolInfo.Construct(self, tikFindable, 'dastworx',
'background tool that works on the D modules AST');
toolItem.Parent := boxTools;
toolItem.ReAlign;
toolItem := TToolInfo.Construct(self, tikFindable, 'cetodo',
'background tool that collects information for the todo list widget');
'background tool that works on the D modules AST to extract informations' +
LineEnding + 'such as the declarations, the imports, the "TODO" comments, etc.');
toolItem.Parent := boxTools;
toolItem.ReAlign;
toolItem := TToolInfo.Construct(self, tikOptional, 'dub',

27
src/ce_todolist.pas
View File

@ -46,7 +46,7 @@ type
published
property filename: string read fFile write fFile;
property line: string read fLine write fLine;
property Text: string read fText write fText;
property text: string read fText write fText;
property assignee: string read fAssignee write fAssignee;
property category: string read fCategory write fCategory;
property status: string read fStatus write fStatus;
@ -144,7 +144,7 @@ implementation
{$R *.lfm}
const
ToolExeName = 'cetodo' + exeExt;
ToolExeName = 'dastworx' + exeExt;
OptFname = 'todolist.txt';
{$REGION TTodoItems ------------------------------------------------------------}
@ -410,7 +410,9 @@ end;
procedure TCETodoListWidget.callToolProcess;
var
ctxt: TTodoContext;
i: integer;
i,j: integer;
nme: string;
str: string = '';
begin
clearTodoList;
if not exeInSysPath(ToolExeName) then
@ -429,14 +431,23 @@ begin
fToolProc.OnTerminate := @toolTerminated;
// files passed to the tool argument
if ctxt = tcProject then
i := 0;
j := fProj.sourcesCount-1;
if ctxt = tcProject then for i := 0 to j do
begin
for i := 0 to fProj.sourcesCount-1 do
fToolProc.Parameters.Add(fProj.sourceAbsolute(i));
nme := fProj.sourceAbsolute(i);
if not hasDlangSyntax(nme.extractFileExt) then
continue;
str += nme;
if i <> j then
str += PathSeparator;
end
else fToolProc.Parameters.Add(fDoc.fileName);
else str := fDoc.fileName;
fToolProc.Parameters.Add(str);
fToolProc.Parameters.Add('-t');
//
fToolProc.Execute;
fToolProc.CloseInput;
end;
procedure TCETodoListWidget.procOutputDbg(Sender: TObject);
@ -463,8 +474,6 @@ end;
procedure TCETodoListWidget.toolTerminated(Sender: TObject);
begin
//WASTODO-cbugfix: UTF chars in TODO comments bug either in the widget or the tool, symptom: empty todo list, conditions: to determine.
// seems to be fixed since the TODO scanner 's been rewritten using ranges (std.range.front() => autodecoding).
fToolProc.OutputStack.Position := 0;
fTodos.loadFromTxtStream(fToolProc.OutputStack);
fillTodoList;