rework background process handling

src/dlangide/builders/builder.d

@@ -8,10 +8,11 @@ import std.algorithm;
 import std.string;
 import std.conv;
 
-class Builder : BackgroundOperationWatcher, ProcessOutputTarget {
+class Builder : BackgroundOperationWatcher {
     protected Project _project;
     protected ExternalProcess _extprocess;
     protected OutputPanel _log;
+    protected ProtectedTextStorage _box;
 
     @property Project project() { return _project; }
     @property void project(Project p) { _project = p; }
@@ -21,9 +22,11 @@ class Builder : BackgroundOperationWatcher, ProcessOutputTarget {
         _project = project;
         _log = log;
         _extprocess = new ExternalProcess();
+        _box = new ProtectedTextStorage();
     }
     /// log lines
-    override void onText(dstring text) {
+    void pollText() {
+        dstring text = _box.readText();
         dstring[] lines = text.split('\n');
         _log.addLogLines(null, lines);
     }
@@ -32,11 +35,18 @@ class Builder : BackgroundOperationWatcher, ProcessOutputTarget {
     override @property string icon() { return "folder"; }
     /// update background operation status
     override void update() {
+        scope(exit)pollText();
         if (_extprocess.state == ExternalProcessState.None) {
-            onText("Running dub\n"d);
+            _box.writeText("Running dub\n"d);
-            _extprocess.run(cast(char[])"dub", cast(char[][])["build"], cast(char[])_project.dir, this, null);
+            char[] program = "dub".dup;
+            char[][] params;
+            char[] dir = _project.dir.dup;
+            params ~= "build".dup;
+            params ~= "-v".dup;
+            params ~= "--force".dup;
+            _extprocess.run(program, params, dir, _box, null);
             if (_extprocess.state != ExternalProcessState.Running) {
-                onText("Failed to run builder tool");
+                _box.writeText("Failed to run builder tool\n"d);
                 _finished = true;
                 destroy(_extprocess);
                 _extprocess = null;
@@ -45,7 +55,7 @@ class Builder : BackgroundOperationWatcher, ProcessOutputTarget {
         }
         ExternalProcessState state = _extprocess.poll();
         if (state == ExternalProcessState.Stopped) {
-            onText("Builder finished with result "d ~ to!dstring(_extprocess.result) ~ "\n"d);
+            _box.writeText("Builder finished with result "d ~ to!dstring(_extprocess.result) ~ "\n"d);
             _finished = true;
             return;
         }

src/dlangide/builders/extprocess.d

@@ -5,11 +5,69 @@ import dlangui.core.logger;
 import std.process;
 import std.file;
 import std.utf;
+import std.stdio;
+import core.thread;
+import core.sync.mutex;
 
 /// interface to forward process output to
-interface ProcessOutputTarget {
+interface TextWriter {
     /// log lines
-    void onText(dstring text);
+    void writeText(dstring text);
+}
+
+/// interface to read text
+interface TextReader {
+    /// log lines
+    dstring readText();
+}
+
+/// protected text storage box to read and write text from different threads
+class ProtectedTextStorage : TextReader, TextWriter {
+
+    private Mutex _mutex;
+    private shared bool _closed;
+    private dchar[] _buffer;
+
+    this() {
+        _mutex = new Mutex();
+    }
+
+    @property bool closed() { return _closed; }
+
+    void close() {
+        if (_closed)
+            return;
+        _closed = true;
+        _buffer = null;
+    }
+
+    /// log lines
+    override void writeText(dstring text) {
+        if (!_closed) {
+            // if not closed
+            _mutex.lock();
+            scope(exit) _mutex.unlock();
+            // append text
+            _buffer ~= text;
+        }
+    }
+
+    /// log lines
+    override dstring readText() {
+        if (!_closed) {
+            // if not closed
+            _mutex.lock();
+            scope(exit) _mutex.unlock();
+            if (!_buffer.length)
+                return null;
+            dstring res = _buffer.dup;
+            _buffer = null;
+            return res;
+        } else {
+            // reading from closed
+            return null;
+        }
+    }
 }
 
 enum ExternalProcessState : uint {
@@ -25,92 +83,70 @@ enum ExternalProcessState : uint {
     Error
 }
 
-/// runs external process, catches output, allows to stop
+/// base class for text reading from std.stdio.File in background thread
-class ExternalProcess {
+class BackgroundReaderBase : Thread {
+    private std.stdio.File _file;
+    private shared bool _finished;
+    private ubyte[1] _byteBuffer;
+    private ubyte[] _bytes;
+    dchar[] _textbuffer;
+    private int _len;
+    private bool _utfError;
 
-    protected char[][] _args;
+    this(std.stdio.File f) {
-    protected char[] _workDir;
+        super(&run);
-    protected char[] _program;
+        assert(f.isOpen());
-    protected string[string] _env;
+        _file = f;
-    protected ProcessOutputTarget _stdout;
+        _len = 0;
-    protected ProcessOutputTarget _stderr;
+        _finished = false;
-    protected ProcessPipes _pipes;
-    protected ExternalProcessState _state;
-
-    protected int _result;
-
-    @property ExternalProcessState state() { return _state; }
-    /// returns process result for stopped process
-    @property int result() { return _result; }
-
-    this() {
     }
 
-    ExternalProcessState run(char[] program, char[][]args, char[] dir, ProcessOutputTarget stdoutTarget, ProcessOutputTarget stderrTarget = null) {
+    @property bool finished() {
-        _state = ExternalProcessState.None;
+        return _finished;
-        _program = program;
+    }
-        _args = args;
+
-        _workDir = dir;
+    void addByte(ubyte data) {
-        _stdout = stdoutTarget;
+        if (_bytes.length < _len + 1)
-        _stdoutBuffer.clear();
+            _bytes.length = _bytes.length ? _bytes.length * 2 : 1024;
-        _stderrBuffer.clear();
+        ubyte prevchar = _len > 0 ? _bytes[_len - 1] : 0;
-        _result = 0;
+        _bytes[_len++] = data;
-        assert(_stdout);
+        bool eolchar = (data == '\r' || data == '\n');
-        _stderr = stderrTarget;
+        bool preveol = (prevchar == '\r' || prevchar == '\n');
-        Redirect redirect;
+        if (eolchar || (!eolchar && preveol))
-        char[][] params;
+            flush(_len);
-        params ~= _program;
+    }
-        params ~= _args;
+    void flush(int pos) {
-        if (!_stderr)
+        if (!_len)
-            redirect = Redirect.stdin | Redirect.stderrToStdout;
+            return;
-        else
+        if (_textbuffer.length < _len)
-            redirect = Redirect.all;
+            _textbuffer.length = _len + 256;
-        Log.i("Trying to run program ", _program, " with args ", _args);
+        size_t count = 0;
+        for(size_t i = 0; i < _len;) {
+            dchar ch = 0;
+            if (_utfError) {
+                ch = _bytes[i++];
+            } else {
                 try {
-            _pipes = pipeProcess(params, redirect, _env, Config.none, _workDir);
+                    ch = decode(cast(string)_bytes, i);
-            _state = ExternalProcessState.Running;
+                } catch (UTFException e) {
-            _stdoutBuffer.init();
+                    _utfError = true;
-            if (_stderr)
+                    ch = _bytes[i++];
-                _stderrBuffer.init();
+                    Log.d("non-unicode characters found in output of process");
-        } catch (ProcessException e) {
-            Log.e("Cannot run program ", _program, " ", e);
-        } catch (std.stdio.StdioException e) {
-            Log.e("Cannot redirect streams for program ", _program, " ", e);
                 }
-        return _state;
             }
+            _textbuffer[count++] = ch;
+        }
+        _len = 0;
 
-    static immutable READ_BUFFER_SIZE = 4096;
+        if (!count)
-    static class Buffer {
+            return;
-        ubyte[] buffer;
+
-        ubyte[] bytes;
+        // fix line endings - must be '\n'
-        dchar[] textbuffer;
+        count = convertLineEndings(_textbuffer[0..count]);
-        bool utfError;
+
-        size_t len;
+        // data is ready to send
-        void init() {
+        if (count)
-            buffer = new ubyte[READ_BUFFER_SIZE];
+            sendResult(_textbuffer[0..count].dup);
-            bytes = new ubyte[READ_BUFFER_SIZE * 2];
-            textbuffer = new dchar[textbuffer.length];
-            utfError = false;
-            len = 0;
-        }
-        void addBytes(ubyte[] data) {
-            if (bytes.length < len + data.length)
-                bytes.length = bytes.length * 2 + len + data.length;
-            for(size_t i = 0; i < data.length; i++)
-                bytes[len++] = data[i];
-        }
-        size_t read(std.file.File file) {
-            size_t bytesRead = 0;
-            for (;;) {
-                ubyte[] readData = file.rawRead(buffer);
-                if (!readData.length)
-                    break;
-                bytesRead += readData.length;
-                addBytes(readData);
-                bytes ~= readData;
-            }
-            return bytesRead;
     }
     /// inplace convert line endings to unix format (\n)
     size_t convertLineEndings(dchar[] text) {
@@ -132,66 +168,116 @@ class ExternalProcess {
         }
         return dst;
     }
-        dstring text() {
+    protected void sendResult(dstring text) {
-            if (textbuffer.length < len)
+        // override to deal with ready data
-                textbuffer.length = len;
+    }
-            size_t count = 0;
+
-            for(size_t i = 0; i < len;) {
+    protected void handleFinish() {
-                dchar ch = 0;
+        // override to do something when thread is finishing
-                if (utfError) {
+    }
-                    ch = bytes[i++];
+
-                } else {
+    private void run() {
+        // read file by bytes
         try {
-                        ch = decode(cast(string)bytes, i);
+            for (;;) {
-                    } catch (UTFException e) {
+                ubyte[] r = _file.rawRead(_byteBuffer);
-                        utfError = true;
+                if (!r.length)
-                        ch = bytes[i++];
+                    break;
-                        Log.d("non-unicode characters found in output of process");
+                addByte(r[0]);
             }
+            _file.close();
+        } catch (Exception e) {
+            Log.e("Exception occured while reading stream: ", e);
         }
-                textbuffer[count++] = ch;
+        handleFinish();
+        _finished = true;
     }
-            len = 0;
+}
-            if (!count)
+
-                return null;
+/// reader which sends output text to TextWriter (warning: call will be made from background thread)
-            count = convertLineEndings(textbuffer[0..count]);
+class BackgroundReader : BackgroundReaderBase {
-            return textbuffer[0 .. count].dup;
+    protected TextWriter _destination;
+    this(std.stdio.File f, TextWriter destination) {
+        super(f);
+        assert(destination);
+        _destination = destination;
     }
-        void clear() {
+    override protected void sendResult(dstring text) {
-            buffer = null;
+        // override to deal with ready data
-            bytes = null;
+        _destination.writeText(text);
-            textbuffer = null;
-            len = 0;
     }
+    override protected void handleFinish() {
+        // remove link to destination to help GC
+        _destination = null;
+    }
+}
+
+/// runs external process, catches output, allows to stop
+class ExternalProcess {
+
+    protected char[][] _args;
+    protected char[] _workDir;
+    protected char[] _program;
+    protected string[string] _env;
+    protected TextWriter _stdout;
+    protected TextWriter _stderr;
+    protected BackgroundReader _stdoutReader;
+    protected BackgroundReader _stderrReader;
+    protected ProcessPipes _pipes;
+    protected ExternalProcessState _state;
+
+    protected int _result;
+
+    @property ExternalProcessState state() { return _state; }
+    /// returns process result for stopped process
+    @property int result() { return _result; }
+
+    this() {
     }
 
-    
+    ExternalProcessState run(char[] program, char[][]args, char[] dir, TextWriter stdoutTarget, TextWriter stderrTarget = null) {
-    protected Buffer _stdoutBuffer;
+        _state = ExternalProcessState.None;
-    protected Buffer _stderrBuffer;
+        _program = program;
-
+        _args = args;
-    protected bool poll(ProcessOutputTarget dst, std.file.File src, ref Buffer buffer) {
+        _workDir = dir;
-        if (src.isOpen) {
+        _stdout = stdoutTarget;
-            buffer.read(src);
+        _stderr = stderrTarget;
-            dstring s = buffer.text;
+        _result = 0;
-            if (s)
+        assert(_stdout);
-                dst.onText(s);
+        Redirect redirect;
-            return true;
+        char[][] params;
-        } else {
+        params ~= _program;
-            return false;
+        params ~= _args;
-        }
+        if (!_stderr)
-    }
+            redirect = Redirect.stdin | Redirect.stderrToStdout;
-
+        else
-    protected bool pollStreams() {
+            redirect = Redirect.all;
-        bool res = true;
+        Log.i("Trying to run program ", _program, " with args ", _args);
         try {
-            res = poll(_stdout, _pipes.stdout, _stdoutBuffer) && res;
+            _pipes = pipeProcess(params, redirect, _env, Config.none, _workDir);
-            if (_stderr)
+            _state = ExternalProcessState.Running;
-                res = poll(_stderr, _pipes.stderr, _stderrBuffer) && res;
+            // start readers
-        } catch (Error e) {
+            _stdoutReader = new BackgroundReader(_pipes.stdout, _stdout);
-            Log.e("error occued while trying to poll streams for process ", _program);
+            _stdoutReader.start();
-            res = false;
+            if (_stderr) {
+                _stderrReader = new BackgroundReader(_pipes.stderr, _stderr);
+                _stderrReader.start();
             }
-        return res;
+        } catch (ProcessException e) {
+            Log.e("Cannot run program ", _program, " ", e);
+        } catch (std.stdio.StdioException e) {
+            Log.e("Cannot redirect streams for program ", _program, " ", e);
+        }
+        return _state;
+    }
+
+    protected void waitForReadingCompletion() {
+        if (_stdoutReader && !_stdoutReader.finished)
+            _stdoutReader.join(false);
+        if (_stderrReader && !_stderrReader.finished)
+            _stderrReader.join(false);
+        _stdoutReader = null;
+        _stderrReader = null;
     }
 
     /// polls all available output from process streams
@@ -199,16 +285,13 @@ class ExternalProcess {
         bool res = true;
         if (_state == ExternalProcessState.Error || _state == ExternalProcessState.None || _state == ExternalProcessState.Stopped)
             return _state;
-        if (_state == ExternalProcessState.Running) {
-            res = pollStreams();
-        }
         // check for process finishing
         try {
             auto pstate = std.process.tryWait(_pipes.pid);
             if (pstate.terminated) {
-                pollStreams();
                 _state = ExternalProcessState.Stopped;
                 _result = pstate.status;
+                waitForReadingCompletion();
             }
         } catch (Exception e) {
             Log.e("Exception while waiting for process ", _program);
@@ -224,6 +307,7 @@ class ExternalProcess {
         try {
             _result = std.process.wait(_pipes.pid);
             _state = ExternalProcessState.Stopped;
+            waitForReadingCompletion();
         } catch (Exception e) {
             Log.e("Exception while waiting for process ", _program);
             _state = ExternalProcessState.Error;