From 732cba5bff48b75fa2c5587b765e7a16540cf461 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 19 Dec 2018 09:05:07 -0500 Subject: [PATCH 01/44] moar dox --- terminal.d | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/terminal.d b/terminal.d index 49ab246..40bf7da 100644 --- a/terminal.d +++ b/terminal.d @@ -1,4 +1,3 @@ -// FIXME: have a simple function that integrates with sdpy event loop. it can be a template // for optional dependency /++ Module for interacting with the user's terminal, including color output, cursor manipulation, and full-featured real-time mouse and keyboard input. Also includes high-level convenience methods, like [Terminal.getline], which gives the user a line editor with history, completion, etc. See the [#examples]. @@ -68,7 +67,7 @@ module arsd.terminal; This example will demonstrate the high-level getline interface. - The user will be able to type a line and navigate around it with cursor keys and even the mouse on some systems, as well as perform editing as they expect (e.g. the backspace and delete keys work normally) until they press enter. Then, the final line will be returned to your program, which the example will simply print back to the user. + The user will be able to type a line and navigate around it with cursor keys and even the mouse on some systems, as well as perform editing as they expect (e.g. the backspace and delete keys work normally) until they press enter. Then, the final line will be returned to your program, which the example will simply print back to the user. +/ unittest { import arsd.terminal; @@ -78,6 +77,8 @@ unittest { string line = terminal.getline(); terminal.writeln("You wrote: ", line); } + + main; // exclude from docs } /++ @@ -88,6 +89,7 @@ unittest { +/ unittest { import arsd.terminal; + void main() { auto terminal = Terminal(ConsoleOutputType.linear); terminal.color(Color.green, Color.black); @@ -95,6 +97,8 @@ unittest { terminal.color(Color.DEFAULT, Color.DEFAULT); terminal.writeln("And back to normal."); } + + main; // exclude from docs } /++ @@ -105,6 +109,7 @@ unittest { +/ unittest { import arsd.terminal; + void main() { auto terminal = Terminal(ConsoleOutputType.linear); auto input = RealTimeConsoleInput(&terminal, ConsoleInputFlags.raw); @@ -113,6 +118,8 @@ unittest { auto ch = input.getch(); terminal.writeln("You pressed ", ch); } + + main; // exclude from docs } /* @@ -1587,6 +1594,16 @@ struct RealTimeConsoleInput { void delegate(InputEvent) userEventHandler; + /++ + If you are using [arsd.simpledisplay] and want terminal interop too, you can call + this function to add it to the sdpy event loop and get the callback called on new + input. + + Note that you will probably need to call `terminal.flush()` when you are doing doing + output, as the sdpy event loop doesn't know to do that (yet). I will probably change + that in a future version, but it doesn't hurt to call it twice anyway, so I recommend + calling flush yourself in any code you write using this. + +/ void integrateWithSimpleDisplayEventLoop()(void delegate(InputEvent) userEventHandler) { this.userEventHandler = userEventHandler; import arsd.simpledisplay; From 16b41baae7ba10d8bdb5c1238603534ebc98def3 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 31 Dec 2018 10:54:23 -0500 Subject: [PATCH 02/44] first part of add-on server stuff - event part --- cgi.d | 1393 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 1127 insertions(+), 266 deletions(-) diff --git a/cgi.d b/cgi.d index 66a9be5..eaf57ad 100644 --- a/cgi.d +++ b/cgi.d @@ -66,6 +66,34 @@ void main() { --- + Compile_versions: + + -version=plain_cgi + The default - a traditional, plain CGI executable will be generated. + -version=fastcgi + A FastCGI executable will be generated. + -version=scgi + A SCGI (SimpleCGI) executable will be generated. + -version=embedded_httpd + A HTTP server will be embedded in the generated executable. + -version=embedded_httpd_threads + The embedded HTTP server will use a single process with a thread pool. + -version=embedded_httpd_processes + The embedded HTTP server will use a prefork style process pool. + + -version=cgi_with_websocket + The CGI class has websocket server support. + + -version=with_openssl # not currently used + + -version=embedded_httpd_processes_accept_after_fork + It will call accept() in each child process, after forking. This is currently the only option, though I am experimenting with other ideas. + + -version=cgi_embedded_sessions + The session server will be embedded in the cgi.d server process + -version=cgi_session_server_process + The session will be provided in a separate process, provided by cgi.d. + Compile_and_run: For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory. @@ -253,7 +281,7 @@ void main() { Copyright: - cgi.d copyright 2008-2018, Adam D. Ruppe. Provided under the Boost Software License. + cgi.d copyright 2008-2019, Adam D. Ruppe. Provided under the Boost Software License. Yes, this file is almost ten years old, and yes, it is still actively maintained and used. +/ @@ -264,8 +292,9 @@ static import std.file; version(embedded_httpd) { version(linux) version=embedded_httpd_processes; - else + else { version=embedded_httpd_threads; + } /* version(with_openssl) { @@ -275,6 +304,31 @@ version(embedded_httpd) { */ } +version(embedded_httpd_processes) + version=embedded_httpd_processes_accept_after_fork; // I am getting much better average performance on this, so just keeping it. But the other way MIGHT help keep the variation down so i wanna keep the code to play with later + +version(embedded_httpd_threads) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} +version(scgi) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} + +// fall back if the other is not defined so we can cleanly version it below +version(cgi_embedded_sessions) {} +else version=cgi_session_server_process; + + +version=cgi_with_websocket; + enum long defaultMaxContentLength = 5_000_000; /* @@ -681,6 +735,14 @@ class Cgi { this.postJson = null; } + version(Posix) + int getOutputFileHandle() { + return _outputFileHandle; + } + + version(Posix) + int _outputFileHandle = -1; + /** Initializes it using a CGI or CGI-like interface */ this(long maxContentLength = defaultMaxContentLength, // use this to override the environment variable listing @@ -2832,315 +2894,402 @@ bool isCgiRequestMethod(string s) { /// If you want to use a subclass of Cgi with generic main, use this mixin. mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { // kinda hacky - the T... is passed to Cgi's constructor in standard cgi mode, and ignored elsewhere - mixin CustomCgiMainImpl!(CustomCgi, fun, maxContentLength) customCgiMainImpl_; - void main(string[] args) { - customCgiMainImpl_.cgiMainImpl(args); + cgiMainImpl!(fun, CustomCgi, maxContentLength)(args); } } version(embedded_httpd_processes) int processPoolSize = 8; -mixin template CustomCgiMainImpl(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { - void cgiMainImpl(string[] args) { - - - // we support command line thing for easy testing everywhere - // it needs to be called ./app method uri [other args...] - if(args.length >= 3 && isCgiRequestMethod(args[1])) { - Cgi cgi = new CustomCgi(args); - scope(exit) cgi.dispose(); - fun(cgi); - cgi.close(); - return; +void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) { + if(args.length > 1) { + // run the special separate processes if needed + switch(args[1]) { + case "--websocket-server": + runWebsocketServer(); + return; + case "--session-server": + runSessionServer(); + return; + case "--event-server": + runEventServer(); + return; + default: + // intentionally blank - do nothing and carry on to run normally } + } + + // we support command line thing for easy testing everywhere + // it needs to be called ./app method uri [other args...] + if(args.length >= 3 && isCgiRequestMethod(args[1])) { + Cgi cgi = new CustomCgi(args); + scope(exit) cgi.dispose(); + fun(cgi); + cgi.close(); + return; + } - ushort listeningPort(ushort def) { - bool found = false; - foreach(arg; args) { - if(found) - return to!ushort(arg); - if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") - found = true; + ushort listeningPort(ushort def) { + bool found = false; + foreach(arg; args) { + if(found) + return to!ushort(arg); + if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") + found = true; + } + return def; + } + + string listeningHost() { + bool found = false; + foreach(arg; args) { + if(found) + return arg; + if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") + found = true; + } + return ""; + } + version(netman_httpd) { + import arsd.httpd; + // what about forwarding the other constructor args? + // this probably needs a whole redoing... + serveHttp!CustomCgi(&fun, listeningPort(8080));//5005); + return; + } else + version(embedded_httpd_processes) { + import core.sys.posix.unistd; + import core.sys.posix.sys.socket; + import core.sys.posix.netinet.in_; + //import std.c.linux.socket; + + int sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket"); + + { + sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(listeningPort(8085)); + auto lh = listeningHost(); + if(lh.length) { + if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) + throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); + } else + addr.sin_addr.s_addr = INADDR_ANY; + + // HACKISH + int on = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); + // end hack + + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + close(sock); + throw new Exception("bind"); } - return def; - } - string listeningHost() { - bool found = false; - foreach(arg; args) { - if(found) - return arg; - if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") - found = true; + // FIXME: if this queue is full, it will just ignore it + // and wait for the client to retransmit it. This is an + // obnoxious timeout condition there. + if(sock.listen(128) == -1) { + close(sock); + throw new Exception("listen"); } - return ""; } - version(netman_httpd) { - import arsd.httpd; - // what about forwarding the other constructor args? - // this probably needs a whole redoing... - serveHttp!CustomCgi(&fun, listeningPort(8080));//5005); - return; - } else - version(embedded_httpd_processes) { - import core.sys.posix.unistd; - import core.sys.posix.sys.socket; - import core.sys.posix.netinet.in_; - //import std.c.linux.socket; - int sock = socket(AF_INET, SOCK_STREAM, 0); - if(sock == -1) - throw new Exception("socket"); + version(embedded_httpd_processes_accept_after_fork) {} else { + int pipeReadFd; + int pipeWriteFd; { - sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_port = htons(listeningPort(8085)); - auto lh = listeningHost(); - if(lh.length) { - if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) - throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); - } else - addr.sin_addr.s_addr = INADDR_ANY; - - // HACKISH - int on = 1; - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); - // end hack - - - if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { - close(sock); - throw new Exception("bind"); + int[2] pipeFd; + if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pipeFd)) { + import core.stdc.errno; + throw new Exception("pipe failed " ~ to!string(errno)); } - // FIXME: if this queue is full, it will just ignore it - // and wait for the client to retransmit it. This is an - // obnoxious timeout condition there. - if(sock.listen(128) == -1) { - close(sock); - throw new Exception("listen"); - } + pipeReadFd = pipeFd[0]; + pipeWriteFd = pipeFd[1]; } + } - int processCount; - pid_t newPid; - reopen: - while(processCount < processPoolSize) { - newPid = fork(); - if(newPid == 0) { - // start serving on the socket - //ubyte[4096] backingBuffer; - for(;;) { - bool closeConnection; + int processCount; + pid_t newPid; + reopen: + while(processCount < processPoolSize) { + newPid = fork(); + if(newPid == 0) { + // start serving on the socket + //ubyte[4096] backingBuffer; + for(;;) { + bool closeConnection; + uint i; + sockaddr addr; + i = addr.sizeof; + version(embedded_httpd_processes_accept_after_fork) + int s = accept(sock, &addr, &i); + else { + int s; + auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s); + if(readret != s.sizeof) { + import core.stdc.errno; + throw new Exception("pipe read failed " ~ to!string(errno)); + } + + //writeln("process ", getpid(), " got socket ", s); + } + + try { + + if(s == -1) + throw new Exception("accept"); + + scope(failure) close(s); + //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; + auto ir = new BufferedInputRange(s); + //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); + + while(!ir.empty) { + ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = s; + // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. + if(processPoolSize == 1) + closeConnection = true; + //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + stderr.writeln(t.toString()); + sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch(ConnectionException ce) { + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + stderr.writeln(t.toString); + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(closeConnection) { + ir.source.close(); + break; + } else { + if(!ir.empty) + ir.popFront(); // get the next + else if(ir.sourceClosed) { + ir.source.close(); + } + } + } + + ir.source.close(); + } catch(Throwable t) { + debug writeln(t); + // most likely cause is a timeout + } + } + } else { + processCount++; + } + } + + // the parent should wait for its children... + if(newPid) { + import core.sys.posix.sys.wait; + + version(embedded_httpd_processes_accept_after_fork) {} else { + import core.sys.posix.sys.select; + int[] fdQueue; + while(true) { + // writeln("select call"); + int nfds = pipeWriteFd; + if(sock > pipeWriteFd) + nfds = sock; + nfds += 1; + fd_set read_fds; + fd_set write_fds; + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_SET(sock, &read_fds); + if(fdQueue.length) + FD_SET(pipeWriteFd, &write_fds); + auto ret = select(nfds, &read_fds, &write_fds, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + goto try_wait; + else + throw new Exception("wtf select"); + } + + int s = -1; + if(FD_ISSET(sock, &read_fds)) { uint i; sockaddr addr; i = addr.sizeof; - int s = accept(sock, &addr, &i); + s = accept(sock, &addr, &i); + } - try { - - if(s == -1) - throw new Exception("accept"); - - scope(failure) close(s); - //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; - auto ir = new BufferedInputRange(s); - //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); - - while(!ir.empty) { - ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; - - Cgi cgi; - try { - cgi = new CustomCgi(ir, &closeConnection); - // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. - if(processPoolSize == 1) - closeConnection = true; - //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); - } catch(Throwable t) { - // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P - // anyway let's kill the connection - stderr.writeln(t.toString()); - sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); - closeConnection = true; - break; - } - assert(cgi !is null); - scope(exit) - cgi.dispose(); - - try { - fun(cgi); - cgi.close(); - } catch(ConnectionException ce) { - closeConnection = true; - } catch(Throwable t) { - // a processing error can be recovered from - stderr.writeln(t.toString); - if(!handleException(cgi, t)) - closeConnection = true; - } - - if(closeConnection) { - ir.source.close(); - break; - } else { - if(!ir.empty) - ir.popFront(); // get the next - else if(ir.sourceClosed) { - ir.source.close(); - } - } - } - - ir.source.close(); - } catch(Throwable t) { - debug writeln(t); - // most likely cause is a timeout + if(FD_ISSET(pipeWriteFd, &write_fds)) { + if(s == -1 && fdQueue.length) { + s = fdQueue[0]; + fdQueue = fdQueue[1 .. $]; // FIXME reuse buffer } - } - } else { - processCount++; + write_fd(pipeWriteFd, &s, s.sizeof, s); + close(s); // we are done with it, let the other process take ownership + } else + fdQueue ~= s; } } - // the parent should wait for its children... - if(newPid) { - import core.sys.posix.sys.wait; - int status; - // FIXME: maybe we should respawn if one dies unexpectedly - while(-1 != wait(&status)) { + try_wait: + + int status; + while(-1 != wait(&status)) { import std.stdio; writeln("Process died ", status); - processCount--; - goto reopen; - } - close(sock); + processCount--; + goto reopen; } - } else - version(embedded_httpd_threads) { - auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(8085), &doThreadHttpConnection!(CustomCgi, fun)); - manager.listen(); - } else - version(scgi) { - import std.exception; - import al = std.algorithm; - auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(4000), &doThreadScgiConnection!(CustomCgi, fun, maxContentLength)); - manager.listen(); - } else - version(fastcgi) { - // SetHandler fcgid-script - FCGX_Stream* input, output, error; - FCGX_ParamArray env; + close(sock); + } + } else + version(embedded_httpd_threads) { + auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(8085), &doThreadHttpConnection!(CustomCgi, fun)); + manager.listen(); + } else + version(scgi) { + import std.exception; + import al = std.algorithm; + auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(4000), &doThreadScgiConnection!(CustomCgi, fun, maxContentLength)); + manager.listen(); + } else + version(fastcgi) { + // SetHandler fcgid-script + FCGX_Stream* input, output, error; + FCGX_ParamArray env; - const(ubyte)[] getFcgiChunk() { - const(ubyte)[] ret; - while(FCGX_HasSeenEOF(input) != -1) - ret ~= cast(ubyte) FCGX_GetChar(input); - return ret; + const(ubyte)[] getFcgiChunk() { + const(ubyte)[] ret; + while(FCGX_HasSeenEOF(input) != -1) + ret ~= cast(ubyte) FCGX_GetChar(input); + return ret; + } + + void writeFcgi(const(ubyte)[] data) { + FCGX_PutStr(data.ptr, data.length, output); + } + + void doARequest() { + string[string] fcgienv; + + for(auto e = env; e !is null && *e !is null; e++) { + string cur = to!string(*e); + auto idx = cur.indexOf("="); + string name, value; + if(idx == -1) + name = cur; + else { + name = cur[0 .. idx]; + value = cur[idx + 1 .. $]; + } + + fcgienv[name] = value; } - void writeFcgi(const(ubyte)[] data) { - FCGX_PutStr(data.ptr, data.length, output); + void flushFcgi() { + FCGX_FFlush(output); } - void doARequest() { - string[string] fcgienv; - - for(auto e = env; e !is null && *e !is null; e++) { - string cur = to!string(*e); - auto idx = cur.indexOf("="); - string name, value; - if(idx == -1) - name = cur; - else { - name = cur[0 .. idx]; - value = cur[idx + 1 .. $]; - } - - fcgienv[name] = value; - } - - void flushFcgi() { - FCGX_FFlush(output); - } - - Cgi cgi; - try { - cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); - } catch(Throwable t) { - FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); - writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); - return; //continue; - } - assert(cgi !is null); - scope(exit) cgi.dispose(); - try { - fun(cgi); - cgi.close(); - } catch(Throwable t) { - // log it to the error stream - FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); - // handle it for the user, if we can - if(!handleException(cgi, t)) - return; // continue; - } - } - - auto lp = listeningPort(0); - FCGX_Request request; - if(lp) { - // if a listening port was specified on the command line, we want to spawn ourself - // (needed for nginx without spawn-fcgi, e.g. on Windows) - FCGX_Init(); - auto sock = FCGX_OpenSocket(toStringz(listeningHost() ~ ":" ~ to!string(lp)), 12); - if(sock < 0) - throw new Exception("Couldn't listen on the port"); - FCGX_InitRequest(&request, sock, 0); - while(FCGX_Accept_r(&request) >= 0) { - input = request.inStream; - output = request.outStream; - error = request.errStream; - env = request.envp; - doARequest(); - } - } else { - // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) - // using the version with a global variable since we are separate processes anyway - while(FCGX_Accept(&input, &output, &error, &env) >= 0) { - doARequest(); - } - } - } else { - // standard CGI is the default version Cgi cgi; try { - cgi = new CustomCgi(maxContentLength); + cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); } catch(Throwable t) { - stderr.writeln(t.msg); - // the real http server will probably handle this; - // most likely, this is a bug in Cgi. But, oh well. - stdout.write(plainHttpError(true, "400 Bad Request", t)); - return; + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); + return; //continue; } assert(cgi !is null); scope(exit) cgi.dispose(); - try { fun(cgi); cgi.close(); - } catch (Throwable t) { - stderr.writeln(t.msg); + } catch(Throwable t) { + // log it to the error stream + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + // handle it for the user, if we can if(!handleException(cgi, t)) - return; + return; // continue; } } + + auto lp = listeningPort(0); + FCGX_Request request; + if(lp) { + // if a listening port was specified on the command line, we want to spawn ourself + // (needed for nginx without spawn-fcgi, e.g. on Windows) + FCGX_Init(); + auto sock = FCGX_OpenSocket(toStringz(listeningHost() ~ ":" ~ to!string(lp)), 12); + if(sock < 0) + throw new Exception("Couldn't listen on the port"); + FCGX_InitRequest(&request, sock, 0); + while(FCGX_Accept_r(&request) >= 0) { + input = request.inStream; + output = request.outStream; + error = request.errStream; + env = request.envp; + doARequest(); + } + } else { + // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) + // using the version with a global variable since we are separate processes anyway + while(FCGX_Accept(&input, &output, &error, &env) >= 0) { + doARequest(); + } + } + } else { + // standard CGI is the default version + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength); + cgi._outputFileHandle = 1; // stdout + } catch(Throwable t) { + stderr.writeln(t.msg); + // the real http server will probably handle this; + // most likely, this is a bug in Cgi. But, oh well. + stdout.write(plainHttpError(true, "400 Bad Request", t)); + return; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch (Throwable t) { + stderr.writeln(t.msg); + if(!handleException(cgi, t)) + return; + } } } @@ -3161,6 +3310,7 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { Cgi cgi; try { cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = connection.handle; } catch(ConnectionException ce) { // broken pipe or something, just abort the connection closeConnection = true; @@ -3276,6 +3426,7 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket Cgi cgi; try { cgi = new CustomCgi(maxContentLength, headers, &getScgiChunk, &writeScgi, &flushScgi); + cgi._outputFileHandle = connection.handle; } catch(Throwable t) { sendAll(connection, plainHttpError(true, "400 Bad Request", t)); connection.close(); @@ -3438,7 +3589,7 @@ import std.socket; // it is a class primarily for reference semantics // I might change this interface -/// +/// This is NOT ACTUALLY an input range! It is too different. Historical mistake kinda. class BufferedInputRange { version(Posix) this(int source, ubyte[] buffer = null) { @@ -4051,10 +4202,11 @@ version(cgi_with_websocket) { cgi.header("Connection: upgrade"); string key = cgi.requestHeaders["sec-websocket-key"]; - key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; // the defined guid from the websocket spec - import arsd.sha; - auto accept = Base64.encode(SHA1(key)); + import std.digest.sha; + auto hash = sha1Of(key); + auto accept = Base64.encode(hash); cgi.header(("Sec-WebSocket-Accept: " ~ accept).idup); @@ -4266,12 +4418,721 @@ version(Windows) else static assert(0); } +version(Posix) +private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**); + + +// FIXME: these aren't quite public yet. +//private: + +// template for laziness +void startWebsocketServer()() { + version(linux) { + import core.sys.posix.unistd; + pid_t pid; + const(char)*[16] args; + args[0] = "ARSD_CGI_WEBSOCKET_SERVER"; + args[1] = "--websocket-server"; + posix_spawn(&pid, "/proc/self/exe", + null, + null, + args.ptr, + null // env + ); + } else version(Windows) { + wchar[2048] filename; + auto len = GetModuleFileNameW(null, filename.ptr, cast(DWORD) filename.length); + if(len == 0 || len == filename.length) + throw new Exception("could not get process name to start helper server"); + + STARTUPINFOW startupInfo; + startupInfo.cb = cast(DWORD) startupInfo.sizeof; + PROCESS_INFORMATION processInfo; + + // I *MIGHT* need to run it as a new job or a service... + auto ret = CreateProcessW( + filename.ptr, + "--websocket-server"w, + null, // process attributes + null, // thread attributes + false, // inherit handles + 0, // creation flags + null, // environment + null, // working directory + &startupInfo, + &processInfo + ); + + if(!ret) + throw new Exception("create process failed"); + + // when done with those, if we set them + /* + CloseHandle(hStdInput); + CloseHandle(hStdOutput); + CloseHandle(hStdError); + */ + + } else static assert(0, "Websocket server not implemented on this system yet (email me, i can prolly do it if you need it)"); +} + +// template for laziness /* -Copyright: Adam D. Ruppe, 2008 - 2016 + The websocket server is a single-process, single-thread, event + I/O thing. It is passed websockets from other CGI processes + and is then responsible for handling their messages and responses. + Note that the CGI process is responsible for websocket setup, + including authentication, etc. + + It also gets data sent to it by other processes and is responsible + for distributing that, as necessary. +*/ +void runWebsocketServer()() { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(WebSocket ws, string group) { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(string content, string group) { + assert(0, "not implemented"); +} + + +void runEventServer()() { + runAddonServer("/tmp/arsd_cgi_event_server"); +} + +// sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. +void sendConnectionToEventServer()(Cgi cgi, in char[] eventUrl) { + + cgi.setResponseContentType("text/event-stream"); + cgi.write(":\n"); // to initialize the chunking and send headers before keeping the fd for later + cgi.flush(); + + cgi.closed = true; + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + version(fastcgi) + static assert(0, "sending fcgi connections not supported"); + + int fd = cgi.getOutputFileHandle(); + if(fd == -1) + throw new Exception("bad fd from cgi!"); + + char[1024] buffer; + buffer[0] = cgi.responseChunked ? 1 : 0; + + buffer[1 .. eventUrl.length + 1] = eventUrl[]; + + auto res = write_fd(s, buffer.ptr, 1 + eventUrl.length, fd); + assert(res == 1 + eventUrl.length); +} + +void sendEventToEventServer()(string url, string event, string data, int lifetime) { + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + SendableEvent sev; + sev.populate(url, event, data, lifetime); + + auto ret = send(s, &sev, sev.sizeof, 0); + assert(ret == sev.sizeof); +} + +version(Posix) + alias LocalServerConnectionHandle = int; +else version(Windows) + alias LocalServerConnectionHandle = HANDLE; + +LocalServerConnectionHandle openLocalServerConnection(string name) { + version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.sys.un; + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + } + + if(connect(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("connect " ~ to!string(errno)); + + return sock; + } +} + +void closeLocalServerConnection(LocalServerConnectionHandle handle) { + version(Posix) { + import core.sys.posix.unistd; + close(handle); + } else version(Windows) + CloseHandle(handle); +} + +void runSessionServer()() { + /+ + The session server api should prolly be: + + setSessionValues + getSessionValues + changeSessionId + createSession + destroySesson + +/ + assert(0, "not implemented"); +} + +version(Posix) +private void makeNonBlocking(int fd) { + import core.sys.posix.fcntl; + auto flags = fcntl(fd, F_GETFL, 0); + if(flags == -1) + throw new Exception("fcntl get"); + flags |= O_NONBLOCK; + auto s = fcntl(fd, F_SETFL, flags); + if(s == -1) + throw new Exception("fcntl set"); +} + +import core.stdc.errno; + +struct IoOp { + @disable this(); + @disable this(this); + + enum Read = 1; + enum Write = 2; + enum Accept = 3; + enum ReadSocketHandle = 4; + + // Your handler may be called in a different thread than the one that initiated the IO request! + // It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution. + private void function(IoOp*, int) handler; + private void function(IoOp*) closeHandler; + private void function(IoOp*) completeHandler; + private int internalFd; + private int operation; + private int bufferLengthAllocated; + private int bufferLengthUsed; + private ubyte[1] internalBuffer; // it can be overallocated! + + ubyte[] allocatedBuffer() { + return internalBuffer.ptr[0 .. bufferLengthAllocated]; + } + + ubyte[] usedBuffer() { + return allocatedBuffer[0 .. bufferLengthUsed]; + } + + void reset() { + bufferLengthUsed = 0; + } + + int fd() { + return internalFd; + } +} + +IoOp* allocateIoOp(int fd, int operation, int bufferSize, void function(IoOp*, int) handler) { + import core.stdc.stdlib; + + auto ptr = malloc(IoOp.sizeof + bufferSize); + if(ptr is null) + assert(0); // out of memory! + + auto op = cast(IoOp*) ptr; + + op.handler = handler; + op.internalFd = fd; + op.operation = operation; + op.bufferLengthAllocated = bufferSize; + op.bufferLengthUsed = 0; + + return op; +} + +void freeIoOp(ref IoOp* ptr) { + import core.stdc.stdlib; + free(ptr); + ptr = null; +} + +/// +struct SendableEvent { + int urlLength; + char[256] urlBuffer = 0; + int typeLength; + char[32] typeBuffer = 0; + int messageLength; + char[2048] messageBuffer = 0; + int _lifetime; + + char[] message() { + return messageBuffer[0 .. messageLength]; + } + char[] type() { + return typeBuffer[0 .. typeLength]; + } + char[] url() { + return urlBuffer[0 .. urlLength]; + } + int lifetime() { + return _lifetime; + } + + /// + void populate(string url, string type, string message, int lifetime) + in { + assert(url.length < this.urlBuffer.length); + assert(type.length < this.typeBuffer.length); + assert(message.length < this.messageBuffer.length); + } + do { + this.urlLength = cast(int) url.length; + this.typeLength = cast(int) type.length; + this.messageLength = cast(int) message.length; + this._lifetime = lifetime; + + this.urlBuffer[0 .. url.length] = url[]; + this.typeBuffer[0 .. type.length] = type[]; + this.messageBuffer[0 .. message.length] = message[]; + } +} + +struct EventConnection { + int fd; + bool needsChunking; +} + +private EventConnection[][string] eventConnectionsByUrl; + +private void handleInputEvent(scope SendableEvent* event) { + static int eventId; + + static struct StoredEvent { + int id; + string type; + string message; + int lifetimeRemaining; + } + + StoredEvent[][string] byUrl; + + int thisId = ++eventId; + + if(event.lifetime) + byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); + + auto connectionsPtr = event.url in eventConnectionsByUrl; + EventConnection[] connections; + if(connectionsPtr is null) + return; + else + connections = *connectionsPtr; + + char[4096] buffer; + char[] formattedMessage; + + void append(const char[] a) { + // the 6's here are to leave room for a HTTP chunk header, if it proves necessary + buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; + formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; + } + + /* + rawDataOutput(cast(const(ubyte)[]) toHex(t.length)); + rawDataOutput(cast(const(ubyte)[]) "\r\n"); + rawDataOutput(cast(const(ubyte)[]) t); + rawDataOutput(cast(const(ubyte)[]) "\r\n"); + */ + import std.algorithm.iteration; + + if(connections.length) { + append("id: "); + append(to!string(thisId)); + append("\n"); + + append("event: "); + append(event.type); + append("\n"); + + foreach(line; event.message.splitter("\n")) { + append("data: "); + append(line); + append("\n"); + } + + append("\n"); + } + + // chunk it for HTTP! + auto len = toHex(formattedMessage.length); + buffer[4 .. 6] = "\r\n"[]; + buffer[4 - len.length .. 4] = len[]; + + auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length]; + // done + + // FIXME: send back requests when needed + // FIXME: send a single ":\n" every 15 seconds to keep alive + + foreach(connection; connections) { + if(connection.needsChunking) + nonBlockingWrite(connection.fd, chunkedMessage); + else + nonBlockingWrite(connection.fd, formattedMessage); + } +} + +void nonBlockingWrite(int connection, const void[] data) { + import core.sys.posix.unistd; + + auto ret = write(connection, data.ptr, data.length); + // FIXME: what if the file closed? + if(ret != data.length) { + if(ret == 0 || errno == EPIPE) { + // the file is closed, remove it + outer: foreach(url, ref connections; eventConnectionsByUrl) { + foreach(idx, conn; connections) { + if(conn.fd == connection) { + connections[idx] = connections[$-1]; + connections = connections[0 .. $ - 1]; + continue outer; + } + } + } + } else + throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME + } +} + +void runAddonServer()(string name) { + version(Posix) { + + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + import core.sys.posix.sys.un; + + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + + static void handleLocalConnectionData(IoOp* op, int receivedFd) { + if(receivedFd != -1) { + //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); + + //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); + + string url = (cast(char[]) op.usedBuffer[1 .. $]).idup; + eventConnectionsByUrl[url] ~= EventConnection(receivedFd, op.usedBuffer[0] > 0 ? true : false); + + // FIXME: catch up on past messages here + } else { + auto data = op.usedBuffer; + auto event = cast(SendableEvent*) data.ptr; + + handleInputEvent(event); + } + } + + static void handleLocalConnectionClose(IoOp* op) { + //writeln("CLOSED"); + } + static void handleLocalConnectionComplete(IoOp* op) { + //writeln("COMPLETED"); + } + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + } + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("bind " ~ to!string(errno)); + + if(listen(sock, 128) == -1) + throw new Exception("listen " ~ to!string(errno)); + + version(linux) { + + makeNonBlocking(sock); + + import core.sys.linux.epoll; + auto epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if(epoll_fd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(failure) + close(epoll_fd); + + auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); + scope(exit) + freeIoOp(acceptOp); + + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = acceptOp; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + epoll_event[64] events; + + while(true) { + int timeout_milliseconds = -1; // infinite + //writeln("waiting for ", name); + auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + + foreach(idx; 0 .. nfds) { + auto flags = events[idx].events; + auto ioop = cast(IoOp*) events[idx].data.ptr; + + //writeln(flags, " ", ioop.fd); + + if(ioop.fd == sock && (flags & EPOLLIN)) { + // on edge triggering, it is important that we get it all + while(true) { + auto size = addr.sizeof; + auto ns = accept(sock, cast(sockaddr*) &addr, &size); + if(ns == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + break; + } + throw new Exception("accept " ~ to!string(errno)); + } + + makeNonBlocking(ns); + epoll_event nev; + nev.events = EPOLLIN | EPOLLET; + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &handleLocalConnectionData); + niop.closeHandler = &handleLocalConnectionClose; + niop.completeHandler = &handleLocalConnectionComplete; + scope(failure) freeIoOp(niop); + nev.data.ptr = niop; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } + } else if(ioop.operation == IoOp.ReadSocketHandle) { + while(true) { + int in_fd; + auto got = read_fd(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, &in_fd); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = got; + ioop.handler(ioop, in_fd); + } + } else if(ioop.operation == IoOp.Read) { + while(true) { + auto got = recv(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, 0); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = got; + ioop.handler(ioop, -1); + } + } + + // EPOLLHUP? + } + } + } else { + // this isn't seriously implemented. + static assert(0); + } + + // then we need to run the event loop. a user-defined function may be called here to help + // the event loop needs to process the websocket messages + + + } else version(Windows) { + + // set up a named pipe + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724251(v=vs.85).aspx + // https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsaduplicatesocketw + // https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeserverprocessid + + } else static assert(0); +} + + +version(Posix) +// copied from the web and ported from C +// see https://stackoverflow.com/questions/2358684/can-i-share-a-file-descriptor-to-another-process-on-linux-or-are-they-local-to-t +ssize_t write_fd(int fd, void *ptr, size_t nbytes, int sendfd) { + msghdr msg; + iovec[1] iov; + + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + cmptr = CMSG_FIRSTHDR(&msg); + cmptr.cmsg_len = CMSG_LEN(int.sizeof); + cmptr.cmsg_level = SOL_SOCKET; + cmptr.cmsg_type = SCM_RIGHTS; + *(cast(int *) CMSG_DATA(cmptr)) = sendfd; + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + return sendmsg(fd, &msg, 0); +} + +version(Posix) +// copied from the web and ported from C +ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { + msghdr msg; + iovec[1] iov; + ssize_t n; + int newfd; + + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + if ( (n = recvmsg(fd, &msg, 0)) <= 0) + return n; + + if ( (cmptr = CMSG_FIRSTHDR(&msg)) != null && + cmptr.cmsg_len == CMSG_LEN(int.sizeof)) { + if (cmptr.cmsg_level != SOL_SOCKET) + throw new Exception("control level != SOL_SOCKET"); + if (cmptr.cmsg_type != SCM_RIGHTS) + throw new Exception("control type != SCM_RIGHTS"); + *recvfd = *(cast(int *) CMSG_DATA(cmptr)); + } else + *recvfd = -1; /* descriptor was not passed */ + + return n; +} +/* end read_fd */ + + +/* + Event source stuff + + The api is: + + sendEvent(string url, string type, string data, int timeout = 60*10); + + attachEventListener(string url, int fd, lastId) + + + It just sends to all attached listeners, and stores it until the timeout + for replaying via lastEventId. +*/ + +/* + Session process stuff + + it stores it all. the cgi object has a session object that can grab it + + session may be done in the same process if possible, there is a version + switch to choose if you want to override. +*/ + +/* +Copyright: Adam D. Ruppe, 2008 - 2019 License: Boost License 1.0. Authors: Adam D. Ruppe - Copyright Adam D. Ruppe 2008 - 2016. + Copyright Adam D. Ruppe 2008 - 2019. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 204c95760e62f247f86ab5a4f1d5eed4d483b1bc Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 31 Dec 2018 11:04:29 -0500 Subject: [PATCH 03/44] remove some unnecessary allocations --- cgi.d | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cgi.d b/cgi.d index eaf57ad..e10e0d1 100644 --- a/cgi.d +++ b/cgi.d @@ -354,6 +354,9 @@ public import std.string; public import std.stdio; public import std.conv; import std.uri; +import std.uni; +import std.algorithm.comparison; +import std.algorithm.searching; import std.exception; import std.base64; static import std.algorithm; @@ -667,7 +670,7 @@ class Cgi { lookingForMethod = false; lookingForUri = true; - if(arg.toLower() == "commandline") + if(arg.asLowerCase().equal("commandline")) requestMethod = RequestMethod.CommandLine; else requestMethod = to!RequestMethod(arg.toUpper()); @@ -869,7 +872,7 @@ class Cgi { lastEventId = getenv("HTTP_LAST_EVENT_ID"); auto ka = getenv("HTTP_CONNECTION"); - if(ka.length && ka.toLower().indexOf("keep-alive") != -1) + if(ka.length && ka.asLowerCase().canFind("keep-alive")) keepAliveRequested = true; auto or = getenv("HTTP_ORIGIN"); @@ -1641,7 +1644,7 @@ class Cgi { else if (name == "connection") { if(value == "close" && closeConnection) *closeConnection = true; - if(value.toLower().indexOf("keep-alive") != -1) { + if(value.asLowerCase().canFind("keep-alive")) { keepAliveRequested = true; // on http 1.0, the connection is closed by default, @@ -4187,10 +4190,10 @@ version(cgi_with_websocket) { "sec-websocket-key" in cgi.requestHeaders && "connection" in cgi.requestHeaders && - cgi.requestHeaders["connection"].toLower().indexOf("upgrade") != -1 + cgi.requestHeaders["connection"].asLowerCase().canFind("upgrade") && "upgrade" in cgi.requestHeaders && - cgi.requestHeaders["upgrade"].toLower() == "websocket" + cgi.requestHeaders["upgrade"].asLowerCase().equal("websocket") ; } From ab1f92fc93b71c46d3eb1e2176745708789a1400 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 2 Jan 2019 21:35:25 -0500 Subject: [PATCH 04/44] set utf8 mode on windows too --- terminal.d | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/terminal.d b/terminal.d index 40bf7da..7114ef7 100644 --- a/terminal.d +++ b/terminal.d @@ -804,11 +804,19 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as defaultForegroundColor = cast(Color) (originalSbi.wAttributes & 0x0f); defaultBackgroundColor = cast(Color) ((originalSbi.wAttributes >> 4) & 0x0f); + + oldCp = GetConsoleOutputCP(); + SetConsoleOutputCP(65001); // UTF-8 + + oldCpIn = GetConsoleCP(); + SetConsoleCP(65001); // UTF-8 } version(Windows) { private Color defaultBackgroundColor = Color.black; private Color defaultForegroundColor = Color.white; + UINT oldCp; + UINT oldCpIn; } // only use this if you are sure you know what you want, since the terminal is a shared resource you generally really want to reset it to normal when you leave... @@ -843,6 +851,10 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as if(lineGetter !is null) lineGetter.dispose(); + + SetConsoleOutputCP(oldCp); + SetConsoleCP(oldCpIn); + auto stdo = GetStdHandle(STD_OUTPUT_HANDLE); SetConsoleActiveScreenBuffer(stdo); if(hConsole !is stdo) From 87afa5056d5a1b0fd4befb3348b90e4f391a9b10 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 2 Jan 2019 21:40:02 -0500 Subject: [PATCH 05/44] spurious compiler warnings --- jsvar.d | 2 +- script.d | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jsvar.d b/jsvar.d index 5752aeb..696a490 100644 --- a/jsvar.d +++ b/jsvar.d @@ -709,7 +709,7 @@ struct var { case Type.String: case Type.Function: assert(0); // FIXME - break; + //break; case Type.Integral: return var(-this.get!long); case Type.Floating: diff --git a/script.d b/script.d index 1cd764d..e9e75ae 100644 --- a/script.d +++ b/script.d @@ -2538,7 +2538,7 @@ Expression parseStatement(MyTokenStreamHere)(ref MyTokenStreamHere tokens, strin return parseFunctionCall(tokens, new AssertKeyword(token)); - break; + //break; // declarations case "var": return parseVariableDeclaration(tokens, ";"); From 62bf62aa4d69d55cdaa7993d733da9338fa2d4da Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 4 Jan 2019 17:30:25 -0500 Subject: [PATCH 06/44] fix windows --- minigui.d | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/minigui.d b/minigui.d index a04b95e..b472d42 100644 --- a/minigui.d +++ b/minigui.d @@ -1,5 +1,16 @@ // http://msdn.microsoft.com/en-us/library/windows/desktop/bb775498%28v=vs.85%29.aspx +// So a window needs to have a selection, and that can be represented by a type. This is manipulated by various +// functions like cut, copy, paste. Widgets can have a selection and that would assert teh selection ownership for +// the window. + +// so what about context menus? + +// https://docs.microsoft.com/en-us/windows/desktop/Controls/about-custom-draw + +// FIXME: add a command search thingy built in and implement tip. +// FIXME: omg omg what if menu functions have arguments and it can pop up a gui or command line script them?! + // On Windows: // FIXME: various labels look broken in high contrast mode // FIXME: changing themes while the program is upen doesn't trigger a redraw @@ -3910,8 +3921,10 @@ class MainWindow : Window { .toolbar toolbar; bool separator; .accelerator accelerator; + .hotkey hotkey; .icon icon; string label; + string tip; foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) { static if(is(typeof(attr) == .menu)) menu = attr; @@ -3921,10 +3934,14 @@ class MainWindow : Window { separator = true; else static if(is(typeof(attr) == .accelerator)) accelerator = attr; + else static if(is(typeof(attr) == .hotkey)) + hotkey = attr; else static if(is(typeof(attr) == .icon)) icon = attr; else static if(is(typeof(attr) == .label)) label = attr.label; + else static if(is(typeof(attr) == .tip)) + tip = attr.tip; } if(menu !is .menu.init || toolbar !is .toolbar.init) { @@ -5504,6 +5521,15 @@ abstract class EditableTextWidget : EditableTextWidgetParent { super(parent); } + bool wordWrapEnabled_ = false; + void wordWrapEnabled(bool enabled) { + version(win32_widgets) { + SendMessageW(hwnd, EM_FMTLINES, enabled ? 1 : 0, 0); + } else version(custom_widgets) { + wordWrapEnabled_ = enabled; // FIXME + } else static assert(false); + } + override int minWidth() { return 16; } override int minHeight() { return Window.lineHeight + 0; } // the +0 is to leave room for the padding override int widthStretchiness() { return 7; } @@ -5563,6 +5589,7 @@ abstract class EditableTextWidget : EditableTextWidgetParent { void addText(string txt) { version(custom_widgets) { + textLayout.addText(txt); { @@ -5579,12 +5606,12 @@ abstract class EditableTextWidget : EditableTextWidgetParent { SendMessageW( hwnd, EM_GETSEL, cast(WPARAM)(&StartPos), cast(WPARAM)(&EndPos) ); // move the caret to the end of the text - int outLength = GetWindowTextLengthW( hwndOutput ); + int outLength = GetWindowTextLengthW(hwnd); SendMessageW( hwnd, EM_SETSEL, outLength, outLength ); // insert the text at the new caret position WCharzBuffer bfr = WCharzBuffer(txt, WindowsStringConversionFlags.convertNewLines); - SendMessageW( hwnd, EM_REPLACESEL, TRUE, txt ); + SendMessageW( hwnd, EM_REPLACESEL, TRUE, cast(int) bfr.ptr ); // restore the previous selection SendMessageW( hwnd, EM_SETSEL, StartPos, EndPos ); @@ -6721,6 +6748,12 @@ struct icon { ushort id; } /// /// Group: generating_from_code struct label { string label; } +/// +/// Group: generating_from_code +struct hotkey { dchar ch; } +/// +/// Group: generating_from_code +struct tip { string tip; } /++ From 09a44d8348a0f030c27f77b703636ebb547ba2f2 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 6 Jan 2019 14:35:43 -0500 Subject: [PATCH 07/44] better email conversion stuff --- htmltotext.d | 99 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 14 deletions(-) diff --git a/htmltotext.d b/htmltotext.d index 272dadc..dc877a1 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -28,7 +28,7 @@ class HtmlConverter { // The table stuff is removed right now because while it looks // ok for test tables, it isn't working well for the emails I have // - it handles data ok but not really nested layouts. - case "trfixme": + case "trlol": auto children = element.childElements; auto tdWidth = (width - cast(int)(children.length)*3) / cast(int)(children.length); @@ -91,6 +91,16 @@ class HtmlConverter { s ~= "\n"; } break; + case "tr": + startBlock(2); + sinkChildren(); + endBlock(); + break; + case "td": + startBlock(0); + sinkChildren(); + endBlock(); + break; case "a": sinkChildren(); if(element.href != element.innerText) { @@ -116,6 +126,8 @@ class HtmlConverter { if(csc.length) s ~= "\033[39m"; */ + + sinkChildren(); break; case "p": startBlock(); @@ -139,20 +151,28 @@ class HtmlConverter { break; case "ul": ulDepth++; + startBlock(2); sinkChildren(); + endBlock(); ulDepth--; break; case "ol": olDepth++; + startBlock(2); sinkChildren(); + endBlock(); olDepth--; break; case "li": startBlock(); //sink('\t', true); - sink(' ', true); - sink(' ', true); + /* + foreach(cnt; 0 .. olDepth + ulDepth) { + sink(' ', true); + sink(' ', true); + } + */ if(olDepth) sink('*', false); if(ulDepth) @@ -164,15 +184,33 @@ class HtmlConverter { endBlock(); break; - case "h1", "h2": + case "dl": + case "dt": + case "dd": + startBlock(element.tagName == "dd" ? 2 : 0); + sinkChildren(); + endBlock(); + break; + + case "h1": + startBlock(); + sink('#', true); + sink('#', true); + sink(' ', true); + sinkChildren(); + sink(' ', true); + sink('#', true); + sink('#', true); + endBlock(); + break; + case "h2", "h3": startBlock(); sinkChildren(); sink('\n', true); foreach(dchar ch; element.innerText) - sink(element.tagName == "h1" ? '=' : '-', false); + sink(element.tagName == "h2" ? '=' : '-', false); endBlock(); break; - case "hr": startBlock(); foreach(i; 0 .. width / 4) @@ -185,7 +223,6 @@ class HtmlConverter { case "br": sink('\n', true); break; - case "tr": case "div": startBlock(); @@ -207,7 +244,7 @@ class HtmlConverter { endBlock(); break; case "pre": - startBlock(); + startBlock(4); foreach(child; element.childNodes) htmlToText(child, true, width); endBlock(); @@ -237,6 +274,10 @@ class HtmlConverter { //auto stylesheet = new StyleSheet(readText("/var/www/dpldocs.info/experimental-docs/style.css")); //stylesheet.apply(document); + return convert(start, wantWordWrap, wrapAmount); + } + + string convert(Element start, bool wantWordWrap = true, int wrapAmount = 74) { htmlToText(start, false, wrapAmount); return s; } @@ -255,6 +296,12 @@ class HtmlConverter { int lineLength; void sink(dchar item, bool preformatted, int lineWidthOverride = int.min) { + + if(needsIndent && item != '\n') { + lineLength += doIndent(); + needsIndent = false; + } + int width = lineWidthOverride == int.min ? this.width : lineWidthOverride; if(!preformatted && isWhite(item)) { if(!justOutputWhitespace) { @@ -282,8 +329,9 @@ class HtmlConverter { auto os = s; s = os[0 .. idx]; s ~= '\n'; - s ~= os[idx + 1 .. $]; lineLength = cast(int)(os[idx+1..$].length); + lineLength += doIndent(); + s ~= os[idx + 1 .. $]; broken = true; break; } @@ -295,15 +343,17 @@ class HtmlConverter { if(!broken) { s ~= '\n'; lineLength = 0; + needsIndent = true; justOutputWhitespace = true; } } - if(item == '\n') + if(item == '\n') { lineLength = 0; - else + needsIndent = true; + } else lineLength ++; @@ -312,22 +362,45 @@ class HtmlConverter { justOutputMargin = false; } } - void startBlock() { + + int doIndent() { + int cnt = 0; + foreach(i; indentStack) + foreach(lol; 0 .. i) { + s ~= ' '; + cnt++; + } + return cnt; + } + + int[] indentStack; + bool needsIndent = false; + + void startBlock(int indent = 0) { + + indentStack ~= indent; + if(!justOutputBlock) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputBlock = true; } if(!justOutputMargin) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputMargin = true; } } void endBlock() { + if(indentStack.length) + indentStack = indentStack[0 .. $ - 1]; + if(!justOutputMargin) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputMargin = true; } } @@ -403,13 +476,11 @@ void penis() { ele.innerText = "^" ~ ele.innerText; ele.stripOut(); break; - /* case "img": string alt = ele.getAttribute("alt"); if(alt) result ~= ele.alt; break; - */ default: ele.stripOut(); goto again; From 08efa61c7d0cc218308298dfe2e75bb0f0ce79fb Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 6 Jan 2019 17:03:27 -0500 Subject: [PATCH 08/44] new writePng function for using existing array with more flexible format --- png.d | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/png.d b/png.d index e92e040..83ff420 100644 --- a/png.d +++ b/png.d @@ -22,6 +22,29 @@ void writePng(string filename, MemoryImage mi) { std.file.write(filename, writePng(png)); } +/// +enum PngType { + greyscale = 0, /// The data must be `depth` bits per pixel + truecolor = 2, /// The data will be RGB triples, so `depth * 3` bits per pixel + indexed = 3, /// The data must be `depth` bits per pixel, with a palette attached. Use [writePng] with [IndexedImage] for this mode + greyscale_with_alpha = 4, /// The data must be (grey, alpha) byte pairs for each pixel. Thus `depth * 2` bits per pixel + truecolor_with_alpha = 6 /// The data must be RGBA quads for each pixel. Thus, `depth * 4` bits per pixel. +} + +/// Saves an image from an existing array. Note that depth other than 8 may not be implemented yet. +void writePng(string filename, const ubyte[] data, int width, int height, PngType type, ubyte depth = 8) { + PngHeader h; + h.width = width; + h.height = height; + + auto png = blankPNG(h); + addImageDatastreamToPng(data, png); + + import std.file; + std.file.write(filename, writePng(png)); +} + + /* //Here's a simple test program that shows how to write a quick image viewer with simpledisplay: From e2e8a917e5cd8e1049caf3a43a398f9afee03785 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 6 Jan 2019 17:28:34 -0500 Subject: [PATCH 09/44] lol --- png.d | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/png.d b/png.d index 83ff420..4c417b6 100644 --- a/png.d +++ b/png.d @@ -25,17 +25,19 @@ void writePng(string filename, MemoryImage mi) { /// enum PngType { greyscale = 0, /// The data must be `depth` bits per pixel - truecolor = 2, /// The data will be RGB triples, so `depth * 3` bits per pixel - indexed = 3, /// The data must be `depth` bits per pixel, with a palette attached. Use [writePng] with [IndexedImage] for this mode - greyscale_with_alpha = 4, /// The data must be (grey, alpha) byte pairs for each pixel. Thus `depth * 2` bits per pixel - truecolor_with_alpha = 6 /// The data must be RGBA quads for each pixel. Thus, `depth * 4` bits per pixel. + truecolor = 2, /// The data will be RGB triples, so `depth * 3` bits per pixel. Depth must be 8 or 16. + indexed = 3, /// The data must be `depth` bits per pixel, with a palette attached. Use [writePng] with [IndexedImage] for this mode. Depth must be <= 8. + greyscale_with_alpha = 4, /// The data must be (grey, alpha) byte pairs for each pixel. Thus `depth * 2` bits per pixel. Depth must be 8 or 16. + truecolor_with_alpha = 6 /// The data must be RGBA quads for each pixel. Thus, `depth * 4` bits per pixel. Depth must be 8 or 16. } -/// Saves an image from an existing array. Note that depth other than 8 may not be implemented yet. +/// Saves an image from an existing array. Note that depth other than 8 may not be implemented yet. Also note depth of 16 must be stored big endian void writePng(string filename, const ubyte[] data, int width, int height, PngType type, ubyte depth = 8) { PngHeader h; h.width = width; h.height = height; + h.type = cast(ubyte) type; + h.depth = depth; auto png = blankPNG(h); addImageDatastreamToPng(data, png); @@ -652,9 +654,28 @@ void addImageDatastreamToPng(const(ubyte)[] data, PNG* png) { PngHeader h = getHeader(png); - auto bytesPerLine = h.width * 4; - if(h.type == 3) - bytesPerLine = h.width * h.depth / 8; + int bytesPerLine; + switch(h.type) { + case 0: + // FIXME: < 8 depth not supported here but should be + bytesPerLine = h.width * 1 * h.depth / 8; + break; + case 2: + bytesPerLine = h.width * 3 * h.depth / 8; + break; + case 3: + bytesPerLine = h.width * 1 * h.depth / 8; + break; + case 4: + // FIXME: < 8 depth not supported here but should be + bytesPerLine = h.width * 2 * h.depth / 8; + break; + case 6: + bytesPerLine = h.width * 4 * h.depth / 8; + break; + default: assert(0); + + } Chunk dat; dat.type = ['I', 'D', 'A', 'T']; int pos = 0; From 38c84d7fa23fcd988fc3fbb3571c685ad76d1210 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 6 Jan 2019 17:28:55 -0500 Subject: [PATCH 10/44] catchup --- cgi.d | 665 +++++++++++++++++++++++++++++++----------------- color.d | 73 ++++++ mssql.d | 12 +- nanovega.d | 3 + simpledisplay.d | 2 + 5 files changed, 512 insertions(+), 243 deletions(-) diff --git a/cgi.d b/cgi.d index e10e0d1..37c1aa0 100644 --- a/cgi.d +++ b/cgi.d @@ -738,13 +738,11 @@ class Cgi { this.postJson = null; } - version(Posix) - int getOutputFileHandle() { + CgiConnectionHandle getOutputFileHandle() { return _outputFileHandle; } - version(Posix) - int _outputFileHandle = -1; + CgiConnectionHandle _outputFileHandle = INVALID_CGI_CONNECTION_HANDLE; /** Initializes it using a CGI or CGI-like interface */ this(long maxContentLength = defaultMaxContentLength, @@ -1122,6 +1120,12 @@ class Cgi { } else if(pps.contentType == "multipart/form-data") { pps.isMultipart = true; enforce(pps.boundary.length, "no boundary"); + } else if(pps.contentType == "text/plain") { + pps.isMultipart = false; + pps.isJson = true; // FIXME: hack, it isn't actually this + } else if(pps.contentType == "text/xml") { // FIXME: what if we used this as a fallback? + pps.isMultipart = false; + pps.isJson = true; // FIXME: hack, it isn't actually this } else if(pps.contentType == "application/json") { pps.isJson = true; pps.isMultipart = false; @@ -4504,53 +4508,32 @@ void sendToWebsocketServer(string content, string group) { void runEventServer()() { - runAddonServer("/tmp/arsd_cgi_event_server"); + runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServer()); } -// sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. -void sendConnectionToEventServer()(Cgi cgi, in char[] eventUrl) { - - cgi.setResponseContentType("text/event-stream"); - cgi.write(":\n"); // to initialize the chunking and send headers before keeping the fd for later - cgi.flush(); - - cgi.closed = true; - auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); - scope(exit) - closeLocalServerConnection(s); - - version(fastcgi) - static assert(0, "sending fcgi connections not supported"); - - int fd = cgi.getOutputFileHandle(); - if(fd == -1) - throw new Exception("bad fd from cgi!"); - - char[1024] buffer; - buffer[0] = cgi.responseChunked ? 1 : 0; - - buffer[1 .. eventUrl.length + 1] = eventUrl[]; - - auto res = write_fd(s, buffer.ptr, 1 + eventUrl.length, fd); - assert(res == 1 + eventUrl.length); -} - -void sendEventToEventServer()(string url, string event, string data, int lifetime) { - auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); - scope(exit) - closeLocalServerConnection(s); - - SendableEvent sev; - sev.populate(url, event, data, lifetime); - - auto ret = send(s, &sev, sev.sizeof, 0); - assert(ret == sev.sizeof); -} - -version(Posix) +version(Posix) { alias LocalServerConnectionHandle = int; -else version(Windows) + alias CgiConnectionHandle = int; + alias SocketConnectionHandle = int; + + enum INVALID_CGI_CONNECTION_HANDLE = -1; +} else version(Windows) { alias LocalServerConnectionHandle = HANDLE; + version(embedded_httpd) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else version(fastcgi) { + alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point. + enum INVALID_CGI_CONNECTION_HANDLE = null; + } else version(scgi) { + alias CgiConnectionHandle = HANDLE; + enum INVALID_CGI_CONNECTION_HANDLE = null; + } else { /* version(plain_cgi) */ + alias CgiConnectionHandle = HANDLE; + enum INVALID_CGI_CONNECTION_HANDLE = null; + } + alias SocketConnectionHandle = SOCKET; +} LocalServerConnectionHandle openLocalServerConnection(string name) { version(Posix) { @@ -4582,6 +4565,8 @@ LocalServerConnectionHandle openLocalServerConnection(string name) { throw new Exception("connect " ~ to!string(errno)); return sock; + } else version(Windows) { + return null; // FIXME } } @@ -4594,15 +4579,6 @@ void closeLocalServerConnection(LocalServerConnectionHandle handle) { } void runSessionServer()() { - /+ - The session server api should prolly be: - - setSessionValues - getSessionValues - changeSessionId - createSession - destroySesson - +/ assert(0, "not implemented"); } @@ -4624,6 +4600,10 @@ struct IoOp { @disable this(); @disable this(this); + /* + So we want to be able to eventually handle generic sockets too. + */ + enum Read = 1; enum Write = 2; enum Accept = 3; @@ -4631,9 +4611,9 @@ struct IoOp { // Your handler may be called in a different thread than the one that initiated the IO request! // It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution. - private void function(IoOp*, int) handler; - private void function(IoOp*) closeHandler; - private void function(IoOp*) completeHandler; + private void delegate(IoOp*, int) handler; + private void delegate(IoOp*) closeHandler; + private void delegate(IoOp*) completeHandler; private int internalFd; private int operation; private int bufferLengthAllocated; @@ -4657,7 +4637,7 @@ struct IoOp { } } -IoOp* allocateIoOp(int fd, int operation, int bufferSize, void function(IoOp*, int) handler) { +IoOp* allocateIoOp(int fd, int operation, int bufferSize, void delegate(IoOp*, int) handler) { import core.stdc.stdlib; auto ptr = malloc(IoOp.sizeof + bufferSize); @@ -4681,156 +4661,389 @@ void freeIoOp(ref IoOp* ptr) { ptr = null; } -/// -struct SendableEvent { - int urlLength; - char[256] urlBuffer = 0; - int typeLength; - char[32] typeBuffer = 0; - int messageLength; - char[2048] messageBuffer = 0; - int _lifetime; - - char[] message() { - return messageBuffer[0 .. messageLength]; - } - char[] type() { - return typeBuffer[0 .. typeLength]; - } - char[] url() { - return urlBuffer[0 .. urlLength]; - } - int lifetime() { - return _lifetime; - } - - /// - void populate(string url, string type, string message, int lifetime) - in { - assert(url.length < this.urlBuffer.length); - assert(type.length < this.typeBuffer.length); - assert(message.length < this.messageBuffer.length); - } - do { - this.urlLength = cast(int) url.length; - this.typeLength = cast(int) type.length; - this.messageLength = cast(int) message.length; - this._lifetime = lifetime; - - this.urlBuffer[0 .. url.length] = url[]; - this.typeBuffer[0 .. type.length] = type[]; - this.messageBuffer[0 .. message.length] = message[]; - } -} - -struct EventConnection { - int fd; - bool needsChunking; -} - -private EventConnection[][string] eventConnectionsByUrl; - -private void handleInputEvent(scope SendableEvent* event) { - static int eventId; - - static struct StoredEvent { - int id; - string type; - string message; - int lifetimeRemaining; - } - - StoredEvent[][string] byUrl; - - int thisId = ++eventId; - - if(event.lifetime) - byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); - - auto connectionsPtr = event.url in eventConnectionsByUrl; - EventConnection[] connections; - if(connectionsPtr is null) - return; - else - connections = *connectionsPtr; - - char[4096] buffer; - char[] formattedMessage; - - void append(const char[] a) { - // the 6's here are to leave room for a HTTP chunk header, if it proves necessary - buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; - formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; - } - - /* - rawDataOutput(cast(const(ubyte)[]) toHex(t.length)); - rawDataOutput(cast(const(ubyte)[]) "\r\n"); - rawDataOutput(cast(const(ubyte)[]) t); - rawDataOutput(cast(const(ubyte)[]) "\r\n"); - */ - import std.algorithm.iteration; - - if(connections.length) { - append("id: "); - append(to!string(thisId)); - append("\n"); - - append("event: "); - append(event.type); - append("\n"); - - foreach(line; event.message.splitter("\n")) { - append("data: "); - append(line); - append("\n"); - } - - append("\n"); - } - - // chunk it for HTTP! - auto len = toHex(formattedMessage.length); - buffer[4 .. 6] = "\r\n"[]; - buffer[4 - len.length .. 4] = len[]; - - auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length]; - // done - - // FIXME: send back requests when needed - // FIXME: send a single ":\n" every 15 seconds to keep alive - - foreach(connection; connections) { - if(connection.needsChunking) - nonBlockingWrite(connection.fd, chunkedMessage); - else - nonBlockingWrite(connection.fd, formattedMessage); - } -} - -void nonBlockingWrite(int connection, const void[] data) { +version(Posix) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { import core.sys.posix.unistd; auto ret = write(connection, data.ptr, data.length); - // FIXME: what if the file closed? if(ret != data.length) { if(ret == 0 || errno == EPIPE) { // the file is closed, remove it - outer: foreach(url, ref connections; eventConnectionsByUrl) { - foreach(idx, conn; connections) { - if(conn.fd == connection) { - connections[idx] = connections[$-1]; - connections = connections[0 .. $ - 1]; - continue outer; - } - } - } + eis.fileClosed(connection); } else throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME } } +version(Windows) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + // FIXME +} -void runAddonServer()(string name) { +bool isInvalidHandle(CgiConnectionHandle h) { + return h == INVALID_CGI_CONNECTION_HANDLE; +} + +/++ + You can customize your server by subclassing the appropriate server. Then, register your + subclass at compile time with the [registerEventIoServer] template, or implement your own + main function and call it yourself. + + $(TIP If you make your subclass a `final class`, there is a slight performance improvement.) ++/ +interface EventIoServer { + void handleLocalConnectionData(IoOp* op, int receivedFd); + void handleLocalConnectionClose(IoOp* op); + void handleLocalConnectionComplete(IoOp* op); + void wait_timeout(); + void fileClosed(int fd); +} + +final class BasicDataServer : EventIoServer { + static struct ClientConnection { + /+ + The session server api should prolly be: + + setSessionValues + getSessionValues + changeSessionId + createSession + destroySesson + +/ + + } + + protected: + + void handleLocalConnectionData(IoOp* op, int receivedFd); + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) { assert(0); } + + private: + + static struct SendableDataRequest { + enum Operation : ubyte { + noop = 0, + // on data itself + get, + set, + append, + increment, + decrement, + + // on the session + createSession, + changeSessionId, + destroySession, + } + + char[16] sessionId; + Operation operation; + + int keyLength; + char[128] keyBuffer; + + int dataLength; + ubyte[1000] dataBuffer; + } +} + +/// +final class EventSourceServer : EventIoServer { + /++ + sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + See_Also: + [sendEvent] + +/ + public static void adoptConnection(Cgi cgi, in char[] eventUrl) { + /* + If lastEventId is missing or empty, you just get new events as they come. + + If it is set from something else, it sends all since then (that are still alive) + down the pipe immediately. + + The reason it can come from the header is that's what the standard defines for + browser reconnects. The reason it can come from a query string is just convenience + in catching up in a user-defined manner. + + The reason the header overrides the query string is if the browser tries to reconnect, + it will send the header AND the query (it reconnects to the same url), so we just + want to do the restart thing. + + Note that if you ask for "0" as the lastEventId, it will get ALL still living events. + */ + string lastEventId = cgi.lastEventId; + if(lastEventId.length == 0 && "lastEventId" in cgi.get) + lastEventId = cgi.get["lastEventId"]; + + cgi.setResponseContentType("text/event-stream"); + cgi.write(":\n", false); // to initialize the chunking and send headers before keeping the fd for later + cgi.flush(); + + cgi.closed = true; + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + version(fastcgi) + throw new Exception("sending fcgi connections not supported"); + + auto fd = cgi.getOutputFileHandle(); + if(isInvalidHandle(fd)) + throw new Exception("bad fd from cgi!"); + + SendableEventConnection sec; + sec.populate(cgi.responseChunked, eventUrl, lastEventId); + + version(Posix) { + auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); + assert(res == sec.sizeof); + } else version(Windows) { + // FIXME + } + } + + /++ + Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + Params: + url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request. + event = the event type string, which is used in the Javascript addEventListener API on EventSource + data = the event data. Available in JS as `event.data`. + lifetime = the amount of time to keep this event for replaying on the event server. + + See_Also: + [sendEventToEventServer] + +/ + public static void sendEvent(string url, string event, string data, int lifetime) { + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + SendableEvent sev; + sev.populate(url, event, data, lifetime); + + version(Posix) { + auto ret = send(s, &sev, sev.sizeof, 0); + assert(ret == sev.sizeof); + } else version(Windows) { + // FIXME + } + } + + + protected: + + + + void handleLocalConnectionData(IoOp* op, int receivedFd) { + if(receivedFd != -1) { + //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); + + //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); + + SendableEventConnection* got = cast(SendableEventConnection*) op.usedBuffer.ptr; + + auto url = got.url.idup; + eventConnectionsByUrl[url] ~= EventConnection(receivedFd, got.responseChunked > 0 ? true : false); + + // FIXME: catch up on past messages here + } else { + auto data = op.usedBuffer; + auto event = cast(SendableEvent*) data.ptr; + + handleInputEvent(event); + } + } + void handleLocalConnectionClose(IoOp* op) {} + void handleLocalConnectionComplete(IoOp* op) {} + + void wait_timeout() { + // just keeping alive + foreach(url, connections; eventConnectionsByUrl) + foreach(connection; connections) + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, "2\r\n:\n"); + else + nonBlockingWrite(this, connection.fd, ":\n"); + } + + void fileClosed(int fd) { + outer: foreach(url, ref connections; eventConnectionsByUrl) { + foreach(idx, conn; connections) { + if(fd == conn.fd) { + connections[idx] = connections[$-1]; + connections = connections[0 .. $ - 1]; + continue outer; + } + } + } + } + + + + private: + + + struct SendableEventConnection { + ubyte responseChunked; + + int urlLength; + char[256] urlBuffer = 0; + + int lastEventIdLength; + char[32] lastEventIdBuffer = 0; + + char[] url() { + return urlBuffer[0 .. urlLength]; + } + char[] lastEventId() { + return lastEventIdBuffer[0 .. lastEventIdLength]; + } + void populate(bool responseChunked, in char[] url, in char[] lastEventId) + in { + assert(url.length < this.urlBuffer.length); + assert(lastEventId.length < this.lastEventIdBuffer.length); + } + do { + this.responseChunked = responseChunked ? 1 : 0; + this.urlLength = cast(int) url.length; + this.lastEventIdLength = cast(int) lastEventId.length; + + this.urlBuffer[0 .. url.length] = url[]; + this.lastEventIdBuffer[0 .. lastEventId.length] = lastEventId[]; + } + } + + struct SendableEvent { + int urlLength; + char[256] urlBuffer = 0; + int typeLength; + char[32] typeBuffer = 0; + int messageLength; + char[2048] messageBuffer = 0; + int _lifetime; + + char[] message() { + return messageBuffer[0 .. messageLength]; + } + char[] type() { + return typeBuffer[0 .. typeLength]; + } + char[] url() { + return urlBuffer[0 .. urlLength]; + } + int lifetime() { + return _lifetime; + } + + /// + void populate(string url, string type, string message, int lifetime) + in { + assert(url.length < this.urlBuffer.length); + assert(type.length < this.typeBuffer.length); + assert(message.length < this.messageBuffer.length); + } + do { + this.urlLength = cast(int) url.length; + this.typeLength = cast(int) type.length; + this.messageLength = cast(int) message.length; + this._lifetime = lifetime; + + this.urlBuffer[0 .. url.length] = url[]; + this.typeBuffer[0 .. type.length] = type[]; + this.messageBuffer[0 .. message.length] = message[]; + } + } + + struct EventConnection { + int fd; + bool needsChunking; + } + + private EventConnection[][string] eventConnectionsByUrl; + + private void handleInputEvent(scope SendableEvent* event) { + static int eventId; + + static struct StoredEvent { + int id; + string type; + string message; + int lifetimeRemaining; + } + + StoredEvent[][string] byUrl; + + int thisId = ++eventId; + + if(event.lifetime) + byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); + + auto connectionsPtr = event.url in eventConnectionsByUrl; + EventConnection[] connections; + if(connectionsPtr is null) + return; + else + connections = *connectionsPtr; + + char[4096] buffer; + char[] formattedMessage; + + void append(const char[] a) { + // the 6's here are to leave room for a HTTP chunk header, if it proves necessary + buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; + formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; + } + + import std.algorithm.iteration; + + if(connections.length) { + append("id: "); + append(to!string(thisId)); + append("\n"); + + append("event: "); + append(event.type); + append("\n"); + + foreach(line; event.message.splitter("\n")) { + append("data: "); + append(line); + append("\n"); + } + + append("\n"); + } + + // chunk it for HTTP! + auto len = toHex(formattedMessage.length); + buffer[4 .. 6] = "\r\n"[]; + buffer[4 - len.length .. 4] = len[]; + + auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length]; + // done + + // FIXME: send back requests when needed + // FIXME: send a single ":\n" every 15 seconds to keep alive + + foreach(connection; connections) { + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, chunkedMessage); + else + nonBlockingWrite(this, connection.fd, formattedMessage); + } + } +} + +void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoServer)) { version(Posix) { import core.sys.posix.unistd; @@ -4840,30 +5053,6 @@ void runAddonServer()(string name) { import core.sys.posix.signal; signal(SIGPIPE, SIG_IGN); - static void handleLocalConnectionData(IoOp* op, int receivedFd) { - if(receivedFd != -1) { - //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); - - //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); - - string url = (cast(char[]) op.usedBuffer[1 .. $]).idup; - eventConnectionsByUrl[url] ~= EventConnection(receivedFd, op.usedBuffer[0] > 0 ? true : false); - - // FIXME: catch up on past messages here - } else { - auto data = op.usedBuffer; - auto event = cast(SendableEvent*) data.ptr; - - handleInputEvent(event); - } - } - - static void handleLocalConnectionClose(IoOp* op) { - //writeln("CLOSED"); - } - static void handleLocalConnectionComplete(IoOp* op) { - //writeln("COMPLETED"); - } int sock = socket(AF_UNIX, SOCK_STREAM, 0); if(sock == -1) throw new Exception("socket " ~ to!string(errno)); @@ -4879,10 +5068,10 @@ void runAddonServer()(string name) { version(linux) { // on linux, we will use the abstract namespace addr.sun_path[0] = 0; - addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + addr.sun_path[1 .. localListenerName.length + 1] = cast(typeof(addr.sun_path[])) localListenerName[]; } else { // but otherwise, just use a file cuz we must. - addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + addr.sun_path[0 .. localListenerName.length] = cast(typeof(addr.sun_path[])) localListenerName[]; } if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) @@ -4915,7 +5104,10 @@ void runAddonServer()(string name) { epoll_event[64] events; while(true) { - int timeout_milliseconds = -1; // infinite + + // FIXME: it should actually do a timerfd that runs on any thing that hasn't been run recently + + int timeout_milliseconds = 15000; // -1; // infinite //writeln("waiting for ", name); auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds); if(nfds == -1) { @@ -4924,6 +5116,10 @@ void runAddonServer()(string name) { throw new Exception("epoll_wait " ~ to!string(errno)); } + if(nfds == 0) { + eis.wait_timeout(); + } + foreach(idx; 0 .. nfds) { auto flags = events[idx].events; auto ioop = cast(IoOp*) events[idx].data.ptr; @@ -4933,7 +5129,7 @@ void runAddonServer()(string name) { if(ioop.fd == sock && (flags & EPOLLIN)) { // on edge triggering, it is important that we get it all while(true) { - auto size = addr.sizeof; + auto size = cast(uint) addr.sizeof; auto ns = accept(sock, cast(sockaddr*) &addr, &size); if(ns == -1) { if(errno == EAGAIN || errno == EWOULDBLOCK) { @@ -4946,9 +5142,9 @@ void runAddonServer()(string name) { makeNonBlocking(ns); epoll_event nev; nev.events = EPOLLIN | EPOLLET; - auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &handleLocalConnectionData); - niop.closeHandler = &handleLocalConnectionClose; - niop.completeHandler = &handleLocalConnectionComplete; + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &eis.handleLocalConnectionData); + niop.closeHandler = &eis.handleLocalConnectionClose; + niop.completeHandler = &eis.handleLocalConnectionComplete; scope(failure) freeIoOp(niop); nev.data.ptr = niop; if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1) @@ -4976,7 +5172,7 @@ void runAddonServer()(string name) { break; } - ioop.bufferLengthUsed = got; + ioop.bufferLengthUsed = cast(int) got; ioop.handler(ioop, in_fd); } } else if(ioop.operation == IoOp.Read) { @@ -5000,7 +5196,7 @@ void runAddonServer()(string name) { break; } - ioop.bufferLengthUsed = got; + ioop.bufferLengthUsed = cast(int) got; ioop.handler(ioop, -1); } } @@ -5012,11 +5208,6 @@ void runAddonServer()(string name) { // this isn't seriously implemented. static assert(0); } - - // then we need to run the event loop. a user-defined function may be called here to help - // the event loop needs to process the websocket messages - - } else version(Windows) { // set up a named pipe diff --git a/color.d b/color.d index 35c5aa9..39d7179 100644 --- a/color.d +++ b/color.d @@ -1081,6 +1081,79 @@ class TrueColorImage : MemoryImage { } } +/+ +/// An RGB array of image data. +class TrueColorImageWithoutAlpha : MemoryImage { + struct Data { + ubyte[] bytes; // the data as rgba bytes. Stored left to right, top to bottom, no padding. + } + + /// . + Data imageData; + + int _width; + int _height; + + override void clearInternal () nothrow @system {// @nogc { + import core.memory : GC; + // it is safe to call [GC.free] with `null` pointer. + GC.free(imageData.bytes.ptr); imageData.bytes = null; + _width = _height = 0; + } + + /// . + override TrueColorImageWithoutAlpha clone() const pure nothrow @trusted { + auto n = new TrueColorImageWithoutAlpha(width, height); + n.imageData.bytes[] = this.imageData.bytes[]; // copy into existing array ctor allocated + return n; + } + + /// . + override int width() const pure nothrow @trusted @nogc { return _width; } + ///. + override int height() const pure nothrow @trusted @nogc { return _height; } + + override Color getPixel(int x, int y) const pure nothrow @trusted @nogc { + if (x >= 0 && y >= 0 && x < _width && y < _height) { + uint pos = (y*_width+x) * 3; + return Color(imageData.bytes[0], imageData.bytes[1], imageData.bytes[2], 255); + } else { + return Color(0, 0, 0, 0); + } + } + + override void setPixel(int x, int y, in Color clr) nothrow @trusted { + if (x >= 0 && y >= 0 && x < _width && y < _height) { + uint pos = y*_width+x; + //if (pos < imageData.bytes.length/4) imageData.colors.ptr[pos] = clr; + // FIXME + } + } + + /// . + this(int w, int h) pure nothrow @safe { + _width = w; + _height = h; + imageData.bytes = new ubyte[w*h*3]; + } + + /// Creates with existing data. The data pointer is stored here. + this(int w, int h, ubyte[] data) pure nothrow @safe { + _width = w; + _height = h; + assert(data.length == w * h * 3); + imageData.bytes = data; + } + + /// + override TrueColorImage getAsTrueColorImage() pure nothrow @safe { + // FIXME + //return this; + } +} ++/ + + alias extern(C) int function(const void*, const void*) @system Comparator; @trusted void nonPhobosSort(T)(T[] obj, Comparator comparator) { import core.stdc.stdlib; diff --git a/mssql.d b/mssql.d index 8957bd8..1cbc9b3 100644 --- a/mssql.d +++ b/mssql.d @@ -155,16 +155,16 @@ class MsSqlResult : ResultSet { string a; more: - SQLCHAR[255] buf; - if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 255, &ptr) != SQL_SUCCESS) + SQLCHAR[1024] buf; + if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 1024, &ptr) != SQL_SUCCESS) throw new DatabaseException("get data: " ~ getSQLError(SQL_HANDLE_STMT, statement)); assert(ptr != SQL_NO_TOTAL); if(ptr == SQL_NULL_DATA) a = null; else { - a ~= cast(string) buf[0 .. ptr > 255 ? 255 : ptr].idup; - ptr -= ptr > 255 ? 255 : ptr; + a ~= cast(string) buf[0 .. ptr > 1024 ? 1024 : ptr].idup; + ptr -= ptr > 1024 ? 1024 : ptr; if(ptr) goto more; } @@ -181,11 +181,11 @@ class MsSqlResult : ResultSet { void makeFieldMapping() { for(int i = 0; i < numFields; i++) { SQLSMALLINT len; - SQLCHAR[255] buf; + SQLCHAR[1024] buf; auto ret = SQLDescribeCol(statement, cast(ushort)(i+1), cast(ubyte*)buf.ptr, - 255, + 1024, &len, null, null, null, null); if (ret != SQL_SUCCESS) diff --git a/nanovega.d b/nanovega.d index fc7e4d1..d3f22e6 100644 --- a/nanovega.d +++ b/nanovega.d @@ -1,4 +1,7 @@ // +// Would be nice: way to take output of the canvas to an image file (raster and/or svg) +// +// // Copyright (c) 2013 Mikko Mononen memon@inside.org // // This software is provided 'as-is', without any express or implied diff --git a/simpledisplay.d b/simpledisplay.d index 7ab6158..001abdb 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -1165,6 +1165,8 @@ TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int wid auto display = XDisplayConnection.get; auto image = XGetImage(display, handle, 0, 0, width, height, (cast(c_ulong) ~0) /*AllPlanes*/, ZPixmap); + // https://github.com/adamdruppe/arsd/issues/98 + // FIXME: copy that shit XDestroyImage(image); From baeadebbc4e68e19c445d8808e363e0c894163b4 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Tue, 8 Jan 2019 11:03:53 -0500 Subject: [PATCH 11/44] add dpi function --- simpledisplay.d | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/simpledisplay.d b/simpledisplay.d index 001abdb..0e26bce 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -1158,6 +1158,27 @@ string sdpyWindowClass () { return null; } +/++ + Returns the DPI of the default monitor. [0] is width, [1] is height (they are usually the same though). You may wish to round the numbers off. ++/ +float[2] getDpi() { + float[2] dpi; + version(Windows) { + HDC screen = GetDC(null); + dpi[0] = GetDeviceCaps(screen, LOGPIXELSX); + dpi[1] = GetDeviceCaps(screen, LOGPIXELSY); + } else version(X11) { + auto display = XDisplayConnection.get; + auto screen = DefaultScreen(display); + + // 25.4 millimeters in an inch... + dpi[0] = cast(float) DisplayWidth(display, screen) / DisplayWidthMM(display, screen) * 25.4; + dpi[1] = cast(float) DisplayHeight(display, screen) / DisplayHeightMM(display, screen) * 25.4; + } + + return dpi; +} + TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int width, int height) { throw new Exception("not implemented"); version(none) { @@ -11897,6 +11918,8 @@ struct Visual int DefaultDepth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).root_depth; } int DisplayWidth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).width; } int DisplayHeight(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).height; } + int DisplayWidthMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mwidth; } + int DisplayHeightMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mheight; } auto DefaultColormap(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).cmap; } int ConnectionNumber(Display* dpy) { return dpy.fd; } From 589df3f3dad9465142e211e5a43e9566c66e869d Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Tue, 8 Jan 2019 17:33:44 -0500 Subject: [PATCH 12/44] new dpi check too --- simpledisplay.d | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/simpledisplay.d b/simpledisplay.d index 0e26bce..e6e5585 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -1171,14 +1171,51 @@ float[2] getDpi() { auto display = XDisplayConnection.get; auto screen = DefaultScreen(display); - // 25.4 millimeters in an inch... - dpi[0] = cast(float) DisplayWidth(display, screen) / DisplayWidthMM(display, screen) * 25.4; - dpi[1] = cast(float) DisplayHeight(display, screen) / DisplayHeightMM(display, screen) * 25.4; + void fallback() { + // 25.4 millimeters in an inch... + dpi[0] = cast(float) DisplayWidth(display, screen) / DisplayWidthMM(display, screen) * 25.4; + dpi[1] = cast(float) DisplayHeight(display, screen) / DisplayHeightMM(display, screen) * 25.4; + } + + char* resourceString = XResourceManagerString(display); + XrmInitialize(); + + auto db = XrmGetStringDatabase(resourceString); + + if (resourceString) { + XrmValue value; + char* type; + if (XrmGetResource(db, "Xft.dpi", "String", &type, &value) == true) { + if (value.addr) { + import core.stdc.stdlib; + dpi[0] = atof(cast(char*) value.addr); + dpi[1] = dpi[0]; + } else { + fallback(); + } + } else { + fallback(); + } + } else { + fallback(); + } } return dpi; } +version(X11) { + extern(C) char* XResourceManagerString(Display*); + extern(C) void XrmInitialize(); + extern(C) XrmDatabase XrmGetStringDatabase(char* data); + extern(C) bool XrmGetResource(XrmDatabase, const char*, const char*, char**, XrmValue*); + alias XrmDatabase = void*; + struct XrmValue { + uint size; + void* addr; + } +} + TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int width, int height) { throw new Exception("not implemented"); version(none) { From 661df66a5d191ff56c2732e18d401081e75f0cfa Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 9 Jan 2019 10:32:06 -0500 Subject: [PATCH 13/44] deprecated loop --- dom.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dom.d b/dom.d index 988b089..98a134a 100644 --- a/dom.d +++ b/dom.d @@ -5042,7 +5042,7 @@ class Table : Element { return position; } - foreach(int i, rowElement; rows) { + foreach(i, rowElement; rows) { auto row = cast(TableRow) rowElement; assert(row !is null); assert(i < ret.length); @@ -5059,7 +5059,7 @@ class Table : Element { foreach(int j; 0 .. cell.colspan) { foreach(int k; 0 .. cell.rowspan) // if the first row, always append. - insertCell(k + i, k == 0 ? -1 : position, cell); + insertCell(k + cast(int) i, k == 0 ? -1 : position, cell); position++; } } From d8720892c84c902e9fc0140b09135e2f421b08a4 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 10 Jan 2019 10:25:28 -0500 Subject: [PATCH 14/44] add db libs (dub sux btw, dmd knows from the source itself!) --- dub.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dub.json b/dub.json index e91aa02..364dda4 100644 --- a/dub.json +++ b/dub.json @@ -95,6 +95,8 @@ "description": "MySQL client library. Wraps the official C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs-posix": ["mysqlclient"], + "libs-windows": ["libmysql"], "sourceFiles": ["mysql.d"] }, { @@ -102,6 +104,7 @@ "description": "Postgresql client library. Wraps the libpq C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs": ["pq"], "sourceFiles": ["postgres.d"] }, @@ -110,6 +113,8 @@ "description": "sqlite wrapper. Wraps the official C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs": ["sqlite3"], + "libs-posix": ["dl"], "sourceFiles": ["sqlite.d"] }, @@ -118,6 +123,7 @@ "description": "Microsoft SQL Server client library. Wraps the official ODBC library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs-windows": ["odbc32"], "sourceFiles": ["mssql.d"] }, From 8c67623cf6817b6ac1edabfff8ba1b928e051706 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 12 Jan 2019 08:57:05 -0500 Subject: [PATCH 15/44] fix compile --- cgi.d | 103 +++++++++++++++++++- characterencodings.d | 18 ++-- dom.d | 7 +- htmltotext.d | 218 ++++++++++++++----------------------------- terminal.d | 86 ++++++++++++++++- 5 files changed, 271 insertions(+), 161 deletions(-) diff --git a/cgi.d b/cgi.d index 37c1aa0..3cfa1ae 100644 --- a/cgi.d +++ b/cgi.d @@ -4698,6 +4698,99 @@ interface EventIoServer { void fileClosed(int fd); } +// the sink should buffer it +private void serialize(T)(scope void delegate(ubyte[]) sink, T t) { + static if(is(T == struct)) { + foreach(member; __traits(allMembers, T)) + serialize(sink, __traits(getMember, t, member)); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + sink((cast(ubyte*) &t)[0 .. t.sizeof]); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc. + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + sink(cast(ubyte[]) t[]); + } else static if(is(T : A[], A)) { + // generic array is less optimal but still prolly ok + static assert(0, T.stringof); + int len = cast(int) t.length; + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + foreach(item; t) + serialize(item); + } else static assert(0, T.stringof); +} + +private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) { + static if(is(T == struct)) { + T t; + foreach(member; __traits(allMembers, T)) + deserialize(get, (T mbr) { __traits(getMember, t, member) = mbr; }); + dg(t); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + T t; + auto data = get(t.sizeof); + t = (cast(T[]) data)[0]; + dg(t); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + + /* + typeof(T[0])[2000] stackBuffer; + T buffer; + + if(len < stackBuffer.length) + buffer = stackBuffer[0 .. len]; + else + buffer = new T(len); + + data = get(len * typeof(T[0]).sizeof); + */ + + T t = cast(T) get(len * typeof(T[0]).sizeof); + + dg(t); + + } else static assert(0, T.stringof); +} + +unittest { + serialize((ubyte[] b) { assert(b == [0, 0, 0, 1]); }, 1); +} + + +interface MyLocalServer {} + +MyLocalServer connectToLocalServer() { return null; } // return an auto-generated version that RPCs to the interface + +class MyLocalServerImpl : MyLocalServer {} // handle the calls. + + +/* + FIXME: + add a version command line arg + version data in the library + management gui as external program + + at server with event_fd for each run + use .mangleof in the at function name + + i think the at server will have to: + pipe args to the child + collect child output for logging + get child return value for logging + + on windows timers work differently. idk how to best combine with the io stuff. + + will have to have dump and restore too, so i can restart without losing stuff. +*/ + final class BasicDataServer : EventIoServer { static struct ClientConnection { /+ @@ -4714,7 +4807,7 @@ final class BasicDataServer : EventIoServer { protected: - void handleLocalConnectionData(IoOp* op, int receivedFd); + void handleLocalConnectionData(IoOp* op, int receivedFd) {} void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant @@ -4742,6 +4835,14 @@ final class BasicDataServer : EventIoServer { char[16] sessionId; Operation operation; + ushort dataType; + /* + int + float + string + ubyte[] + */ + int keyLength; char[128] keyBuffer; diff --git a/characterencodings.d b/characterencodings.d index 19eb06a..1a2aac4 100644 --- a/characterencodings.d +++ b/characterencodings.d @@ -18,22 +18,26 @@ ought to just work. Example: + --- auto data = cast(immutable(ubyte)[]) std.file.read("my-windows-file.txt"); string utf8String = convertToUtf8(data, "windows-1252"); // utf8String can now be used + --- The encodings currently implemented for decoding are: - UTF-8 (a no-op; it simply casts the array to string) - UTF-16, - UTF-32, - Windows-1252, - ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16. + $(LIST + * UTF-8 (a no-op; it simply casts the array to string) + * UTF-16, + * UTF-32, + * Windows-1252, + * ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16. + * KOI8-R + ) It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since - those labels are pretty much de-facto the same thing in wild documents. - + those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic). This module currently makes no attempt to look at control characters. */ diff --git a/dom.d b/dom.d index 98a134a..1362c48 100644 --- a/dom.d +++ b/dom.d @@ -71,7 +71,7 @@ bool isConvenientAttribute(string name) { /// The main document interface, including a html parser. class Document : FileResource { /// Convenience method for web scraping. Requires [arsd.http2] to be - /// included in the build. + /// included in the build as well as [arsd.characterencodings]. static Document fromUrl()(string url) { import arsd.http2; auto client = new HttpClient(); @@ -79,7 +79,10 @@ class Document : FileResource { auto req = client.navigateTo(Uri(url), HttpVerb.GET); auto res = req.waitForCompletion(); - return new Document(cast(string) res.content); + auto document = new Document(); + document.parseGarbage(cast(string) res.content); + + return document; } ///. diff --git a/htmltotext.d b/htmltotext.d index dc877a1..781e3b5 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -7,10 +7,47 @@ import std.string; import std.uni : isWhite; +/// class HtmlConverter { int width; + /++ + Will enable color output using VT codes. Determines color through dom.d's css support, which means you need to apply a stylesheet first. + + --- + import arsd.dom; + + auto document = new Document(source_code_for_html); + auto stylesheet = new Stylesheet(source_code_for_css); + stylesheet.apply(document); + --- + +/ + bool enableVtOutput; + + + string color; + string backgroundColor; + + /// void htmlToText(Element element, bool preformatted, int width) { + string color, backgroundColor; + if(enableVtOutput) { + color = element.computedStyle.getValue("color"); + backgroundColor = element.computedStyle.getValue("background-color"); + } + + string originalColor = this.color, originalBackgroundColor = this.backgroundColor; + + this.color = color.length ? color : this.color; + this.backgroundColor = backgroundColor.length ? backgroundColor : this.backgroundColor; + + scope(exit) { + // the idea is as we pop working back up the tree, it restores what it was here + this.color = originalColor; + this.backgroundColor = originalBackgroundColor; + } + + this.width = width; if(auto tn = cast(TextNode) element) { foreach(dchar ch; tn.nodeValue) { @@ -115,19 +152,27 @@ class HtmlConverter { } break; case "span": - /* - auto csc = element.computedStyle.getValue("color"); - if(csc.length) { - auto c = Color.fromString(csc); - s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b); + if(enableVtOutput) { + auto csc = color; // element.computedStyle.getValue("color"); + if(csc.length) { + auto c = Color.fromString(csc); + s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b); + } + + bool bold = element.computedStyle.getValue("font-weight") == "bold"; + + if(bold) + s ~= "\033[1m"; + + sinkChildren(); + + if(bold) + s ~= "\033[0m"; + if(csc.length) + s ~= "\033[39m"; + } else { + sinkChildren(); } - sinkChildren(); - - if(csc.length) - s ~= "\033[39m"; - */ - - sinkChildren(); break; case "p": startBlock(); @@ -138,9 +183,15 @@ class HtmlConverter { case "em", "i": if(element.innerText.length == 0) break; - sink('*', false); - sinkChildren(); - sink('*', false); + if(enableVtOutput) { + s ~= "\033[1m"; + sinkChildren(); + s ~= "\033[0m"; + } else { + sink('*', false); + sinkChildren(); + sink('*', false); + } break; case "u": if(element.innerText.length == 0) @@ -258,6 +309,7 @@ class HtmlConverter { int olDepth; int ulDepth; + /// string convert(string html, bool wantWordWrap = true, int wrapAmount = 74) { Document document = new Document; @@ -277,11 +329,13 @@ class HtmlConverter { return convert(start, wantWordWrap, wrapAmount); } + /// string convert(Element start, bool wantWordWrap = true, int wrapAmount = 74) { htmlToText(start, false, wrapAmount); return s; } + /// void reset() { s = null; justOutputWhitespace = true; @@ -289,6 +343,7 @@ class HtmlConverter { justOutputMargin = true; } + /// string s; bool justOutputWhitespace = true; bool justOutputBlock = true; @@ -406,140 +461,9 @@ class HtmlConverter { } } +/// string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { auto converter = new HtmlConverter(); return converter.convert(html, true, wrapAmount); } -string repeat(string s, ulong num) { - string ret; - foreach(i; 0 .. num) - ret ~= s; - return ret; -} - -import std.stdio; -version(none) -void penis() { - - again: - string result = ""; - foreach(ele; start.tree) { - if(ele is start) continue; - if(ele.nodeType != 1) continue; - - switch(ele.tagName) { - goto again; - case "h1": - ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("=", ele.innerText.length) ~ "\r"; - ele.stripOut(); - goto again; - case "h2": - ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("-", ele.innerText.length) ~ "\r"; - ele.stripOut(); - goto again; - case "h3": - ele.innerText = "\r" ~ ele.innerText.toUpper ~ "\r"; - ele.stripOut(); - goto again; - case "td": - case "p": - /* - if(ele.innerHTML.length > 1) - ele.innerHTML = "\r" ~ wrap(ele.innerHTML) ~ "\r"; - ele.stripOut(); - goto again; - */ - break; - case "a": - string href = ele.getAttribute("href"); - if(href && !ele.hasClass("no-brackets")) { - if(ele.hasClass("href-text")) - ele.innerText = href; - else { - if(ele.innerText != href) - ele.innerText = ele.innerText ~ " <" ~ href ~ "> "; - } - } - ele.stripOut(); - goto again; - case "ol": - case "ul": - ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r"; - break; - case "li": - if(!ele.innerHTML.startsWith("* ")) - ele.innerHTML = "* " ~ ele.innerHTML ~ "\r"; - // ele.stripOut(); - break; - case "sup": - ele.innerText = "^" ~ ele.innerText; - ele.stripOut(); - break; - case "img": - string alt = ele.getAttribute("alt"); - if(alt) - result ~= ele.alt; - break; - default: - ele.stripOut(); - goto again; - } - } - - again2: - //start.innerHTML = start.innerHTML().replace("\u0001", "\n"); - - foreach(ele; start.tree) { - if(ele.tagName == "td") { - if(ele.directText().strip().length) { - ele.prependText("\r"); - ele.appendText("\r"); - } - ele.stripOut(); - goto again2; - } else if(ele.tagName == "p") { - if(strip(ele.innerText()).length > 1) { - string res = ""; - string all = ele.innerText().replace("\n \n", "\n\n"); - foreach(part; all.split("\n\n")) - res ~= "\r" ~ strip( wantWordWrap ? wrap(part, /*74*/ wrapAmount) : part ) ~ "\r"; - ele.innerText = res; - } else - ele.innerText = strip(ele.innerText); - ele.stripOut(); - goto again2; - } else if(ele.tagName == "li") { - auto part = ele.innerText; - part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part ); - part = " " ~ part.replace("\n", "\n\v") ~ "\r"; - ele.innerText = part; - ele.stripOut(); - goto again2; - } - } - - result = start.innerText(); - result = squeeze(result, " "); - - result = result.replace("\r ", "\r"); - result = result.replace(" \r", "\r"); - - //result = result.replace("\u00a0", " "); - - - result = squeeze(result, "\r"); - result = result.replace("\r", "\n\n"); - - result = result.replace("\v", " "); - - result = result.replace("舗", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere - result = result.replace(""", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere - //result = htmlEntitiesDecode(result); // for special chars mainly - - result = result.replace("\u0001 ", "\n"); - result = result.replace("\u0001", "\n"); - - //a = std.regex.replace(a, std.regex.regex("(\n\t)+", "g"), "\n"); //\t"); - return result.strip; -} diff --git a/terminal.d b/terminal.d index 7114ef7..47543b8 100644 --- a/terminal.d +++ b/terminal.d @@ -3606,7 +3606,7 @@ struct ScrollbackBuffer { } void clear() { - lines = null; + lines.clear(); clickRegions = null; scrollbackPosition = 0; } @@ -3708,9 +3708,87 @@ struct ScrollbackBuffer { } } - // FIXME: limit scrollback lines.length + static struct CircularBuffer(T) { + T[] backing; - Line[] lines; + enum maxScrollback = 8192; // as a power of 2, i hope the compiler optimizes the % below to a simple bit mask... + + int start; + int length_; + + void clear() { + backing = null; + start = 0; + length_ = 0; + } + + size_t length() { + return length_; + } + + void opOpAssign(string op : "~")(T line) { + if(length_ < maxScrollback) { + backing.assumeSafeAppend(); + backing ~= line; + length_++; + } else { + backing[start] = line; + start++; + if(start == maxScrollback) + start = 0; + } + } + + T opIndex(int idx) { + return backing[(start + idx) % maxScrollback]; + } + T opIndex(Dollar idx) { + return backing[(start + (length + idx.offsetFromEnd)) % maxScrollback]; + } + + CircularBufferRange opSlice(int startOfIteration, Dollar end) { + return CircularBufferRange(&this, startOfIteration, cast(int) length - startOfIteration + end.offsetFromEnd); + } + CircularBufferRange opSlice(int startOfIteration, int end) { + return CircularBufferRange(&this, startOfIteration, end - startOfIteration); + } + CircularBufferRange opSlice() { + return CircularBufferRange(&this, 0, cast(int) length); + } + + static struct CircularBufferRange { + CircularBuffer* item; + int position; + int remaining; + this(CircularBuffer* item, int startOfIteration, int count) { + this.item = item; + position = startOfIteration; + remaining = count; + } + + T front() { return (*item)[position]; } + bool empty() { return remaining <= 0; } + void popFront() { + position++; + remaining--; + } + + T back() { return (*item)[remaining - 1 - position]; } + void popBack() { + remaining--; + } + } + + static struct Dollar { + int offsetFromEnd; + Dollar opBinary(string op : "-")(int rhs) { + return Dollar(offsetFromEnd - rhs); + } + } + Dollar opDollar() { return Dollar(0); } + } + + CircularBuffer!Line lines; string name; int x, y, width, height; @@ -3840,7 +3918,7 @@ struct ScrollbackBuffer { // second pass: actually draw it int linePos = remaining; - foreach(idx, line; lines[start .. start + howMany]) { + foreach(line; lines[start .. start + howMany]) { int written = 0; if(linePos < 0) { From e2074fa2ccf71b1c623c3389bf34b61a705846ee Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 20 Jan 2019 16:29:09 -0500 Subject: [PATCH 16/44] compiles again --- cgi.d | 381 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 317 insertions(+), 64 deletions(-) diff --git a/cgi.d b/cgi.d index 3cfa1ae..6c597f3 100644 --- a/cgi.d +++ b/cgi.d @@ -3278,7 +3278,11 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC Cgi cgi; try { cgi = new CustomCgi(maxContentLength); - cgi._outputFileHandle = 1; // stdout + version(Posix) + cgi._outputFileHandle = 1; // stdout + else version(Windows) + cgi._outputFileHandle = GetStdHandle(STD_OUTPUT_HANDLE); + else static assert(0); } catch(Throwable t) { stderr.writeln(t.msg); // the real http server will probably handle this; @@ -4508,7 +4512,7 @@ void sendToWebsocketServer(string content, string group) { void runEventServer()() { - runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServer()); + runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServerImplementation()); } version(Posix) { @@ -4579,7 +4583,7 @@ void closeLocalServerConnection(LocalServerConnectionHandle handle) { } void runSessionServer()() { - assert(0, "not implemented"); + runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation()); } version(Posix) @@ -4722,11 +4726,12 @@ private void serialize(T)(scope void delegate(ubyte[]) sink, T t) { } else static assert(0, T.stringof); } +// all may be stack buffers, so use cautio private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) { static if(is(T == struct)) { T t; foreach(member; __traits(allMembers, T)) - deserialize(get, (T mbr) { __traits(getMember, t, member) = mbr; }); + deserialize!(typeof(__traits(getMember, T, member)))(get, (mbr) { __traits(getMember, t, member) = mbr; }); dg(t); } else static if(is(T : int)) { // no need to think of endianness just because this is only used @@ -4753,7 +4758,7 @@ private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void deleg data = get(len * typeof(T[0]).sizeof); */ - T t = cast(T) get(len * typeof(T[0]).sizeof); + T t = cast(T) get(len * cast(int) typeof(T.init[0]).sizeof); dg(t); @@ -4761,16 +4766,202 @@ private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void deleg } unittest { - serialize((ubyte[] b) { assert(b == [0, 0, 0, 1]); }, 1); + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 1); }); + }, 1); + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 56674); }); + }, 56674); + ubyte[1000] buffer; + int bufferPoint; + void add(ubyte[] b) { + buffer[bufferPoint .. bufferPoint + b.length] = b[]; + bufferPoint += b.length; + } + ubyte[] get(int sz) { + auto b = buffer[bufferPoint .. bufferPoint + sz]; + bufferPoint += sz; + return b; + } + serialize(&add, "test here"); + bufferPoint = 0; + deserialize!string(&get, (t) { assert(t == "test here"); }); + bufferPoint = 0; + + struct Foo { + int a; + ubyte c; + string d; + } + serialize(&add, Foo(403, 37, "amazing")); + bufferPoint = 0; + deserialize!Foo(&get, (t) { + assert(t.a == 403); + assert(t.c == 37); + assert(t.d == "amazing"); + }); + bufferPoint = 0; +} + +/* + Here's the way the RPC interface works: + + You define the interface that lists the functions you can call on the remote process. + The interface may also have static methods for convenience. These forward to a singleton + instance of an auto-generated class, which actually sends the args over the pipe. + + An impl class actually implements it. A receiving server deserializes down the pipe and + calls methods on the class. + + I went with the interface to get some nice compiler checking and documentation stuff. + + I could have skipped the interface and just implemented it all from the server class definition + itself, but then the usage may call the method instead of rpcing it; I just like having the user + interface and the implementation separate so you aren't tempted to `new impl` to call the methods. + + + I fiddled with newlines in the mixin string to ensure the assert line numbers matched up to the source code line number. Idk why dmd didn't do this automatically, but it was important to me. + + Realistically though the bodies would just be + connection.call(this.mangleof, args...) sooooo. + + FIXME: overloads aren't supported +*/ + +mixin template ImplementRpcClientInterface(T, string serverPath) { + static import std.traits; + + // derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering. + static foreach(idx, member; __traits(derivedMembers, T)) { + static if(__traits(isVirtualFunction, __traits(getMember, T, member))) + mixin( q{ + std.traits.ReturnType!(__traits(getMember, T, member)) + } ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params) + { + SerializationBuffer buffer; + auto i = cast(ushort) idx; + serialize(&buffer.sink, i); + serialize(&buffer.sink, __traits(getMember, T, member).mangleof); + foreach(param; params) + serialize(&buffer.sink, param); + + auto sendable = buffer.sendable; + + version(Posix) {{ + auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0); + assert(ret == sendable.length); + }} // FIXME Windows impl + + static if(!is(typeof(return) == void)) { + // there is a return value; we need to wait for it too + version(Posix) { + ubyte[3000] revBuffer; + auto ret = recv(connectionHandle, revBuffer.ptr, revBuffer.length, 0); + auto got = revBuffer[0 .. ret]; + + int dataLocation; + ubyte[] grab(int sz) { + auto d = got[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + typeof(return) retu; + deserialize!(typeof(return))(&grab, (a) { retu = a; }); + return retu; + } else { + // FIXME Windows impl + return typeof(return).init; + } + + } + }}); + } + + private static typeof(this) singletonInstance; + private LocalServerConnectionHandle connectionHandle; + + static typeof(this) connection() { + if(singletonInstance is null) { + singletonInstance = new typeof(this)(); + singletonInstance.connect(); + } + return singletonInstance; + } + + void connect() { + connectionHandle = openLocalServerConnection(serverPath); + } + + void disconnect() { + closeLocalServerConnection(connectionHandle); + } +} + +void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(is(Class : Interface)) { + ushort calledIdx; + string calledFunction; + + int dataLocation; + ubyte[] grab(int sz) { + auto d = data[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + again: + + deserialize!ushort(&grab, (a) { calledIdx = a; }); + deserialize!string(&grab, (a) { calledFunction = a; }); + + import std.traits; + + sw: switch(calledIdx) { + static foreach(idx, memberName; __traits(derivedMembers, Interface)) + static if(__traits(isVirtualFunction, __traits(getMember, Interface, memberName))) { + case idx: + assert(calledFunction == __traits(getMember, Interface, memberName).mangleof); + + Parameters!(__traits(getMember, Interface, memberName)) params; + foreach(ref param; params) + deserialize!(typeof(param))(&grab, (a) { param = a; }); + + static if(is(ReturnType!(__traits(getMember, Interface, memberName)) == void)) { + __traits(getMember, this_, memberName)(params); + } else { + auto ret = __traits(getMember, this_, memberName)(params); + SerializationBuffer buffer; + serialize(&buffer.sink, ret); + + auto sendable = buffer.sendable; + + version(Posix) { + auto r = send(fd, sendable.ptr, sendable.length, 0); + assert(r == sendable.length); + } // FIXME Windows impl + } + break sw; + } + default: assert(0); + } + + if(dataLocation != data.length) + goto again; } -interface MyLocalServer {} - -MyLocalServer connectToLocalServer() { return null; } // return an auto-generated version that RPCs to the interface - -class MyLocalServerImpl : MyLocalServer {} // handle the calls. +private struct SerializationBuffer { + ubyte[2048] bufferBacking; + int bufferLocation; + void sink(scope ubyte[] data) { + bufferBacking[bufferLocation .. bufferLocation + data.length] = data[]; + bufferLocation += data.length; + } + ubyte[] sendable() { + return bufferBacking[0 .. bufferLocation]; + } +} /* FIXME: @@ -4791,68 +4982,80 @@ class MyLocalServerImpl : MyLocalServer {} // handle the calls. will have to have dump and restore too, so i can restart without losing stuff. */ -final class BasicDataServer : EventIoServer { - static struct ClientConnection { - /+ - The session server api should prolly be: +/++ - setSessionValues - getSessionValues - changeSessionId - createSession - destroySesson - +/ ++/ +interface BasicDataServer { + /// + void createSession(string sessionId, int lifetime); + /// + void renewSession(string sessionId, int lifetime); + /// + void destroySession(string sessionId); + /// + void renameSession(string oldSessionId, string newSessionId); + /// + void setSessionData(string sessionId, string dataKey, string dataValue); + /// + string getSessionData(string sessionId, string dataKey); + + /// + static BasicDataServerConnection connection() { + return BasicDataServerConnection.connection(); } +} + +class BasicDataServerConnection : BasicDataServer { + mixin ImplementRpcClientInterface!(BasicDataServer, "/tmp/arsd_session_server"); +} + +final class BasicDataServerImplementation : BasicDataServer, EventIoServer { + + void createSession(string sessionId, int lifetime) { + sessions[sessionId.idup] = Session(lifetime); + } + void destroySession(string sessionId) { + sessions.remove(sessionId); + } + void renewSession(string sessionId, int lifetime) { + sessions[sessionId].lifetime = lifetime; + } + void renameSession(string oldSessionId, string newSessionId) { + sessions[newSessionId.idup] = sessions[oldSessionId]; + sessions.remove(oldSessionId); + } + void setSessionData(string sessionId, string dataKey, string dataValue) { + sessions[sessionId].values[dataKey.idup] = dataValue.idup; + } + string getSessionData(string sessionId, string dataKey) { + return sessions[sessionId].values[dataKey]; + } + protected: - void handleLocalConnectionData(IoOp* op, int receivedFd) {} + struct Session { + int lifetime; + + string[string] values; + } + + Session[string] sessions; + + void handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!BasicDataServer(this, data, op.fd); + } void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant void wait_timeout() {} - void fileClosed(int fd) { assert(0); } - - private: - - static struct SendableDataRequest { - enum Operation : ubyte { - noop = 0, - // on data itself - get, - set, - append, - increment, - decrement, - - // on the session - createSession, - changeSessionId, - destroySession, - } - - char[16] sessionId; - Operation operation; - - ushort dataType; - /* - int - float - string - ubyte[] - */ - - int keyLength; - char[128] keyBuffer; - - int dataLength; - ubyte[1000] dataBuffer; - } + void fileClosed(int fd) {} // stateless so irrelevant } /// -final class EventSourceServer : EventIoServer { +interface EventSourceServer { /++ sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. @@ -4898,7 +5101,7 @@ final class EventSourceServer : EventIoServer { if(isInvalidHandle(fd)) throw new Exception("bad fd from cgi!"); - SendableEventConnection sec; + EventSourceServerImplementation.SendableEventConnection sec; sec.populate(cgi.responseChunked, eventUrl, lastEventId); version(Posix) { @@ -4928,7 +5131,7 @@ final class EventSourceServer : EventIoServer { scope(exit) closeLocalServerConnection(s); - SendableEvent sev; + EventSourceServerImplementation.SendableEvent sev; sev.populate(url, event, data, lifetime); version(Posix) { @@ -4939,10 +5142,40 @@ final class EventSourceServer : EventIoServer { } } + /++ + Messages sent to `url` will also be sent to anyone listening on `forwardUrl`. + + See_Also: [disconnect] + +/ + void connect(string url, string forwardUrl); + + /++ + Disconnects `forwardUrl` from `url` + + See_Also: [connect] + +/ + void disconnect(string url, string forwardUrl); +} + +/// +final class EventSourceServerImplementation : EventSourceServer, EventIoServer { protected: - + void connect(string url, string forwardUrl) { + pipes[url] ~= forwardUrl; + } + void disconnect(string url, string forwardUrl) { + auto t = url in pipes; + if(t is null) + return; + foreach(idx, n; (*t)) + if(n == forwardUrl) { + (*t)[idx] = (*t)[$-1]; + (*t) = (*t)[0 .. $-1]; + break; + } + } void handleLocalConnectionData(IoOp* op, int receivedFd) { if(receivedFd != -1) { @@ -4960,7 +5193,17 @@ final class EventSourceServer : EventIoServer { auto data = op.usedBuffer; auto event = cast(SendableEvent*) data.ptr; - handleInputEvent(event); + if(event.magic == 0xdeadbeef) { + handleInputEvent(event); + + if(event.url in pipes) + foreach(pipe; pipes[event.url]) { + event.url = pipe; + handleInputEvent(event); + } + } else { + dispatchRpcServer!EventSourceServer(this, data, op.fd); + } } } void handleLocalConnectionClose(IoOp* op) {} @@ -5005,6 +5248,10 @@ final class EventSourceServer : EventIoServer { char[] url() { return urlBuffer[0 .. urlLength]; } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } char[] lastEventId() { return lastEventIdBuffer[0 .. lastEventIdLength]; } @@ -5024,6 +5271,7 @@ final class EventSourceServer : EventIoServer { } struct SendableEvent { + int magic = 0xdeadbeef; int urlLength; char[256] urlBuffer = 0; int typeLength; @@ -5041,6 +5289,10 @@ final class EventSourceServer : EventIoServer { char[] url() { return urlBuffer[0 .. urlLength]; } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } int lifetime() { return _lifetime; } @@ -5070,6 +5322,7 @@ final class EventSourceServer : EventIoServer { } private EventConnection[][string] eventConnectionsByUrl; + private string[][string] pipes; private void handleInputEvent(scope SendableEvent* event) { static int eventId; From fdd8eec60e0b41bd3dcff9ea6014ce657765cc2f Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 24 Jan 2019 21:14:23 -0500 Subject: [PATCH 17/44] wip --- cgi.d | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/cgi.d b/cgi.d index 6c597f3..b074f26 100644 --- a/cgi.d +++ b/cgi.d @@ -4718,11 +4718,10 @@ private void serialize(T)(scope void delegate(ubyte[]) sink, T t) { sink(cast(ubyte[]) t[]); } else static if(is(T : A[], A)) { // generic array is less optimal but still prolly ok - static assert(0, T.stringof); int len = cast(int) t.length; sink((cast(ubyte*) &len)[0 .. int.sizeof]); foreach(item; t) - serialize(item); + serialize(sink, item); } else static assert(0, T.stringof); } @@ -5054,6 +5053,66 @@ final class BasicDataServerImplementation : BasicDataServer, EventIoServer { void fileClosed(int fd) {} // stateless so irrelevant } +/++ + ++/ +struct ScheduledJobHelper { + private string func; + private string[] args; + + /++ + Schedules the job to be run at the given time. + +/ + void at(DateTime when, immutable TimeZone timezone = UTC()) { + + } + + /++ + Schedules the job to run at least after the specified delay. + +/ + void delay(Duration delay) { + + } + + /++ + Runs the job in the background ASAP. + + $(NOTE It may run in a background thread. Don't segfault!) + +/ + void runNowInBackground() { + //delay(0); + } + + /++ + Schedules the job to recur on the given pattern. + +/ + version(none) + void recur(string spec) { + + } +} + +/++ + First step to schedule a job on the scheduled job server. + + You MUST set details on the returned object to actually do anything! ++/ +ScheduledJobHelper schedule(alias fn, T...)(T args) { + return ScheduledJobHelper(); +} + +/// +interface ScheduledJobServer { + /// + int scheduleJob(int whenIs, int when, string executable, string func, string[] args); + /// + void cancelJob(int jobId); +} + +class ScheduledJobServerConnection : ScheduledJobServer { + mixin ImplementRpcClientInterface!(ScheduledJobServer, "/tmp/arsd_scheduled_job_server"); +} + /// interface EventSourceServer { /++ From e9b1f5c650971003bd53d58399904b0dbad84692 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Tue, 29 Jan 2019 14:21:18 -0500 Subject: [PATCH 18/44] wow i write awful, useless, untested code --- csv.d | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/csv.d b/csv.d index 72fe093..c770952 100644 --- a/csv.d +++ b/csv.d @@ -6,9 +6,10 @@ import std.array; /// string[][] readCsv(string data) { + data = data.replace("\r\n", "\n"); data = data.replace("\r", ""); - auto idx = data.indexOf("\n"); + //auto idx = data.indexOf("\n"); //data = data[idx + 1 .. $]; // skip headers string[] fields; @@ -39,9 +40,9 @@ string[][] readCsv(string data) { field ~= c; break; case 1: // in quote - if(c == '"') + if(c == '"') { state = 2; - else + } else field ~= c; break; case 2: // is it a closing quote or an escaped one? @@ -55,6 +56,11 @@ string[][] readCsv(string data) { } } + if(field !is null) + current ~= field; + if(current !is null) + records ~= current; + return records; } From ee12ef0b6e9cf188d9d174b59e954b4f2e814bab Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 6 Feb 2019 21:45:36 -0500 Subject: [PATCH 19/44] remove cruft --- cgi.d | 301 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 220 insertions(+), 81 deletions(-) diff --git a/cgi.d b/cgi.d index b074f26..b604236 100644 --- a/cgi.d +++ b/cgi.d @@ -289,6 +289,9 @@ module arsd.cgi; static import std.file; +// for a single thread, linear request thing, use: +// -version=embedded_httpd_threads -version=cgi_no_threads + version(embedded_httpd) { version(linux) version=embedded_httpd_processes; @@ -839,9 +842,9 @@ class Cgi { } - get = getGetVariables(queryString); auto ugh = decodeVariables(queryString); getArray = assumeUnique(ugh); + get = keepLastOf(getArray); // NOTE: on shitpache, you need to specifically forward this @@ -1617,7 +1620,6 @@ class Cgi { pathInfo = requestUri[pathInfoStarts..question]; } - get = cast(string[string]) getGetVariables(queryString); auto ugh = decodeVariables(queryString); getArray = cast(string[][string]) assumeUnique(ugh); @@ -1753,7 +1755,7 @@ class Cgi { this.queryString = queryString; this.scriptName = scriptName; - this.get = cast(immutable) get; + this.get = keepLastOf(getArray); this.getArray = cast(immutable) getArray; this.keepAliveRequested = keepAliveRequested; this.acceptsGzip = acceptsGzip; @@ -1788,43 +1790,6 @@ class Cgi { return assumeUnique(forTheLoveOfGod); } - // this function only exists because of the with_cgi_packed thing, which is - // a filthy hack I put in here for a work app. Which still depends on it, so it - // stays for now. But I want to remove it. - private immutable(string[string]) getGetVariables(in string queryString) { - if(queryString.length) { - auto _get = decodeVariablesSingle(queryString); - - // Some sites are shit and don't let you handle multiple parameters. - // If so, compile this in and encode it as a single parameter - version(with_cgi_packed) { - auto idx = pathInfo.indexOf("PACKED"); - if(idx != -1) { - auto pi = pathInfo[idx + "PACKED".length .. $]; - - auto _unpacked = decodeVariables( - cast(string) base64UrlDecode(pi)); - - foreach(k, v; _unpacked) - _get[k] = v[$-1]; - // possible problem: it used to cut PACKED off the path info - // but it doesn't now. I want to kill this crap anyway though. - } - - if("arsd_packed_data" in getArray) { - auto _unpacked = decodeVariables( - cast(string) base64UrlDecode(getArray["arsd_packed_data"][0])); - - foreach(k, v; _unpacked) - _get[k] = v[$-1]; - } - } - - return assumeUnique(_get); - } - - return null; - } /// Very simple method to require a basic auth username and password. /// If the http request doesn't include the required credentials, it throws a /// HTTP 401 error, and an exception. @@ -3068,7 +3033,7 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC cgi = new CustomCgi(ir, &closeConnection); cgi._outputFileHandle = s; // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. - if(processPoolSize == 1) + if(processPoolSize <= 1) closeConnection = true; //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); } catch(Throwable t) { @@ -3471,46 +3436,6 @@ string printDate(DateTime date) { } -version(with_cgi_packed) { -// This is temporary until Phobos supports base64 -immutable(ubyte)[] base64UrlDecode(string e) { - string encoded = e.idup; - while (encoded.length % 4) { - encoded ~= "="; // add padding - } - - // convert base64 URL to standard base 64 - encoded = encoded.replace("-", "+"); - encoded = encoded.replace("_", "/"); - - return cast(immutable(ubyte)[]) Base64.decode(encoded); -} - // should be set as arsd_packed_data - string packedDataEncode(in string[string] variables) { - string result; - - bool outputted = false; - foreach(k, v; variables) { - if(outputted) - result ~= "&"; - else - outputted = true; - - result ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); - } - - result = cast(string) Base64.encode(cast(ubyte[]) result); - - // url variant - result.replace("=", ""); - result.replace("+", "-"); - result.replace("/", "_"); - - return result; - } -} - - // Referencing this gigantic typeid seems to remind the compiler // to actually put the symbol in the object file. I guess the immutable // assoc array array isn't actually included in druntime @@ -5734,6 +5659,220 @@ ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { switch to choose if you want to override. */ +/+ +struct StaticFile { + string path; + string file; + // the following will be guessed automatically from the file type + string contentType; + bool gzip; + bool cache; +} + + +with(cgi.urlDispatcher()) { + +} ++/ +struct DispatcherDefinition(alias dispatchHandler) {// if(is(typeof(dispatchHandler("str", Cgi.init) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler; + alias handler = dispatchHandler; + string urlPrefix; + bool rejectFurther; +} + +struct CallableFromWeb { + Cgi.RequestMethod httpMethod; + string path; + void delegate(Cgi cgi) callFromCgi; + string[] parameters; +} + +private string urlify(string name) { + return name; +} + +/+ + Argument conversions: for the most part, it is to!Thing(string). + + But arrays and structs are a bit different. Arrays come from the cgi array. Thus + they are passed + + arr=foo&arr=bar <-- notice the same name. + + Structs are first declared with an empty thing, then have their members set individually, + with dot notation. The members are not required, just the initial declaration. + + struct Foo { + int a; + string b; + } + void test(Foo foo){} + + foo&foo.a=5&foo.b=str <-- the first foo declares the arg, the others set the members + + Arrays of structs use this declaration. + + void test(Foo[] foo) {} + + foo&foo.a=5&foo.b=bar&foo&foo.a=9 + + You can use a hidden input field in HTML forms to achieve this. The value of the naked name + declaration is ignored. + + Mind that order matters! The declaration MUST come first in the string. + + Arrays of struct members follow this rule recursively. + + struct Foo { + int[] a; + } + + foo&foo.a=1&foo.a=2&foo&foo.a=1 ++/ +void callFromCgi(alias method, T)(T dg, Cgi cgi) { + import std.traits; + + Parameters!method params; + static if(is(ReturnType!method == void)) { + dg(params); + } else { + auto ret = dg(params); + cgi.write(ret, true); + } +} + +class WebObject { + Cgi cgi; + void initialize(Cgi cgi) { + this.cgi = cgi; + } +} + +/++ + Serves a class' methods. To be used with [dispatcher]. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + FIXME: explain this better ++/ +auto serveApi(T)(string urlPrefix) { + static bool handler(string urlPrefix, Cgi cgi) { + + auto obj = new T(); + obj.initialize(cgi); + + switch(cgi.pathInfo[urlPrefix.length .. $]) { + static foreach(methodName; __traits(derivedMembers, T)) + //static if(is({ + { + case urlify(methodName): + callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + return true; + } + default: + return false; + } + + assert(0); + } + return DispatcherDefinition!handler(urlPrefix, false); +} + +/++ + Serves a REST object. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. ++/ +auto serveRestObject(T)(string urlPrefix) { + static bool handler(string urlPrefix, Cgi cgi) { + string url = cgi.pathInfo[urlPrefix.length .. $]; + + return true; + } + return DispatcherDefinition!handler(urlPrefix, false); +} + +/++ + Serves a static file. To be used with [dispatcher]. ++/ +auto serveStaticFile(string urlPrefix, string filename = null, string contentType = null) { + if(filename is null) + filename = urlPrefix[1 .. $]; + if(contentType is null) { + + } + static bool handler(string urlPrefix, Cgi cgi) { + //cgi.setResponseContentType(contentType); + //cgi.write(std.file.read(filename), true); + cgi.write(std.file.read(urlPrefix[1 .. $]), true); + return true; + } + return DispatcherDefinition!handler(urlPrefix, true); +} + +auto serveRedirect(string urlPrefix, string redirectTo) { + +} + +/+ +/++ + See [serveStaticFile] if you want to serve a file off disk. ++/ +auto serveStaticData(string urlPrefix, const(void)[] data, string contentType) { + +} ++/ + +/++ + A URL dispatcher. + + --- + if(cgi.dispatcher!( + "/api/".serveApi!MyApiClass, + "/objects/lol".serveRestObject!MyRestObject, + "/file.js".serveStaticFile, + )) return; + --- ++/ +bool dispatcher(definitions...)(Cgi cgi) { + // I can prolly make this more efficient later but meh. + foreach(definition; definitions) { + if(definition.rejectFurther) { + if(cgi.pathInfo == definition.urlPrefix) { + auto ret = definition.handler(definition.urlPrefix, cgi); + if(ret) + return true; + } + } else if(cgi.pathInfo.startsWith(definition.urlPrefix)) { + auto ret = definition.handler(definition.urlPrefix, cgi); + if(ret) + return true; + } + } + return false; +} + +/+ +/++ + This is the beginnings of my web.d 2.0 - it dispatches web requests to a class object. + + It relies on jsvar.d and dom.d. + + + You can get javascript out of it to call. The generated functions need to look + like + + function name(a,b,c,d,e) { + return _call("name", {"realName":a,"sds":b}); + } + + And _call returns an object you can call or set up or whatever. ++/ +bool apiDispatcher()(Cgi cgi) { + import arsd.jsvar; + import arsd.dom; +} ++/ /* Copyright: Adam D. Ruppe, 2008 - 2019 License: Boost License 1.0. From ec4a7187303b7e44309ab191275a6a92514fc894 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 7 Feb 2019 19:51:59 -0500 Subject: [PATCH 20/44] fixup garbage with no > --- dom.d | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/dom.d b/dom.d index 1362c48..c80d9f8 100644 --- a/dom.d +++ b/dom.d @@ -1,4 +1,8 @@ // FIXME: add classList +// FIXME: add matchesSelector +// FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML +// FIXME: appendChild should not fail if the thing already has a parent; it should just automatically remove it per standard. + /++ This is an html DOM implementation, started with cloning what the browser offers in Javascript, but going well beyond @@ -824,6 +828,12 @@ class Document : FileResource { while(pos < data.length && data[pos] != '>') pos++; + + if(pos >= data.length) { + // the tag never closed + assert(data.length != 0); + pos = data.length - 1; // rewinding so it hits the end at the bottom.. + } } auto whereThisTagStarted = pos; // for better error messages @@ -3497,8 +3507,14 @@ struct ElementCollection { return !elements.length; } - /// Collects strings from the collection, concatenating them together - /// Kinda like running reduce and ~= on it. + /++ + Collects strings from the collection, concatenating them together + Kinda like running reduce and ~= on it. + + --- + document["p"].collect!"innerText"; + --- + +/ string collect(string method)(string separator = "") { string text; foreach(e; elements) { @@ -3517,6 +3533,17 @@ struct ElementCollection { return this; } + /++ + Calls [Element.wrapIn] on each member of the collection, but clones the argument `what` for each one. + +/ + ElementCollection wrapIn(Element what) { + foreach(e; elements) { + e.wrapIn(what.cloneNode(false)); + } + + return this; + } + /// Concatenates two ElementCollection together. ElementCollection opBinary(string op : "~")(ElementCollection rhs) { return ElementCollection(this.elements ~ rhs.elements); From 7bcd3eca8da6bdb2d23905cf65eb49e1c68bb6d1 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 11 Feb 2019 14:38:53 -0500 Subject: [PATCH 21/44] prototype of web.d 2.0 in cgi.d --- cgi.d | 720 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 677 insertions(+), 43 deletions(-) diff --git a/cgi.d b/cgi.d index b604236..fa2e5ae 100644 --- a/cgi.d +++ b/cgi.d @@ -612,6 +612,9 @@ class Cgi { environmentVariables = cast(const) environment.toAA; + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + foreach(arg; args[1 .. $]) { if(arg.startsWith("--")) { nextArgIs = arg[2 .. $]; @@ -694,6 +697,8 @@ class Cgi { if(requestMethod == Cgi.RequestMethod.POST) { auto parts = breakUp(arg); _post[parts[0]] ~= parts[1]; + allPostNamesInOrder ~= parts[0]; + allPostValuesInOrder ~= parts[1]; } else { if(_queryString.length) _queryString ~= "&"; @@ -712,7 +717,7 @@ class Cgi { cookies = keepLastOf(cookiesArray); queryString = _queryString; - getArray = cast(immutable) decodeVariables(queryString); + getArray = cast(immutable) decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); get = keepLastOf(getArray); postArray = cast(immutable) _post; @@ -741,6 +746,13 @@ class Cgi { this.postJson = null; } + private { + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + string[] allGetNamesInOrder; + string[] allGetValuesInOrder; + } + CgiConnectionHandle getOutputFileHandle() { return _outputFileHandle; } @@ -842,7 +854,7 @@ class Cgi { } - auto ugh = decodeVariables(queryString); + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); getArray = assumeUnique(ugh); get = keepLastOf(getArray); @@ -1208,9 +1220,16 @@ class Cgi { // I used to not do it, but I think I should, since it is there... pps._post[pps.piece.name] ~= pps.piece.filename; pps._files[pps.piece.name] ~= pps.piece; - } else + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= pps.piece.filename; + } else { pps._post[pps.piece.name] ~= cast(string) pps.piece.content; + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= cast(string) pps.piece.content; + } + /* stderr.writeln("RECEIVED: ", pps.piece.name, "=", pps.piece.content.length < 1000 @@ -1455,7 +1474,7 @@ class Cgi { if(pps.isJson) pps.postJson = cast(string) pps.buffer; else - pps._post = decodeVariables(cast(string) pps.buffer); + pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder); version(preserveData) originalPostData = pps.buffer; } else { @@ -1620,7 +1639,7 @@ class Cgi { pathInfo = requestUri[pathInfoStarts..question]; } - auto ugh = decodeVariables(queryString); + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); getArray = cast(string[][string]) assumeUnique(ugh); if(header.indexOf("HTTP/1.0") != -1) { @@ -2671,18 +2690,28 @@ struct Uri { */ /// breaks down a url encoded string -string[][string] decodeVariables(string data, string separator = "&") { +string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) { auto vars = data.split(separator); string[][string] _get; foreach(var; vars) { auto equal = var.indexOf("="); + string name; + string value; if(equal == -1) { - _get[decodeComponent(var)] ~= ""; + name = decodeComponent(var); + value = ""; } else { //_get[decodeComponent(var[0..equal])] ~= decodeComponent(var[equal + 1 .. $].replace("+", " ")); // stupid + -> space conversion. - _get[decodeComponent(var[0..equal].replace("+", " "))] ~= decodeComponent(var[equal + 1 .. $].replace("+", " ")); + name = decodeComponent(var[0..equal].replace("+", " ")); + value = decodeComponent(var[equal + 1 .. $].replace("+", " ")); } + + _get[name] ~= value; + if(namesInOrder) + (*namesInOrder) ~= name; + if(valuesInOrder) + (*valuesInOrder) ~= value; } return _get; } @@ -5659,38 +5688,50 @@ ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { switch to choose if you want to override. */ -/+ -struct StaticFile { - string path; - string file; - // the following will be guessed automatically from the file type - string contentType; - bool gzip; - bool cache; -} - - -with(cgi.urlDispatcher()) { - -} -+/ struct DispatcherDefinition(alias dispatchHandler) {// if(is(typeof(dispatchHandler("str", Cgi.init) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler; alias handler = dispatchHandler; string urlPrefix; bool rejectFurther; } -struct CallableFromWeb { - Cgi.RequestMethod httpMethod; - string path; - void delegate(Cgi cgi) callFromCgi; - string[] parameters; -} - private string urlify(string name) { return name; } +private string beautify(string name) { + char[160] buffer; + int bufferIndex = 0; + bool shouldCap = true; + bool shouldSpace; + bool lastWasCap; + foreach(idx, char ch; name) { + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + + if(ch >= 'A' && ch <= 'Z') { + if(lastWasCap) { + // two caps in a row, don't change. Prolly acronym. + } else { + if(idx) + shouldSpace = true; // new word, add space + } + + lastWasCap = true; + } + + if(shouldSpace) { + buffer[bufferIndex++] = ' '; + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + } + if(shouldCap) { + if(ch >= 'a' && ch <= 'z') + ch -= 32; + shouldCap = false; + } + buffer[bufferIndex++] = ch; + } + return buffer[0 .. bufferIndex].idup; +} + /+ Argument conversions: for the most part, it is to!Thing(string). @@ -5728,47 +5769,522 @@ private string urlify(string name) { } foo&foo.a=1&foo.a=2&foo&foo.a=1 + + + Associative arrays are formatted with brackets, after a declaration, like structs: + + foo&foo[key]=value&foo[other_key]=value + + + Note: for maximum compatibility with outside code, keep your types simple. Some libraries + do not support the strict ordering requirements to work with these struct protocols. + + FIXME: also perhaps accept application/json to better work with outside trash. + + + Return values are also auto-formatted according to user-requested type: + for json, it loops over and converts. + for html, basic types are strings. Arrays are
    . Structs are
    . Arrays of structs are tables! +/ -void callFromCgi(alias method, T)(T dg, Cgi cgi) { + +// actually returns an arsd.dom.Form +auto createAutomaticFormForFunction(alias method, T)(T dg) { + import arsd.dom; + + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, method))); + import std.traits; - Parameters!method params; - static if(is(ReturnType!method == void)) { - dg(params); - } else { - auto ret = dg(params); - cgi.write(ret, true); + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static Element elementFor(T)(string displayName, string name) { + auto div = Element.make("div"); + div.addClass("form-field"); + + static if(is(T == struct)) { + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto fieldset = div.addChild("fieldset"); + fieldset.addChild("legend", beautify(T.stringof)); // FIXME + fieldset.addChild("input", name); + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName)); + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + static if(isSomeString!T) + i.attrs.type = "text"; + else + i.attrs.type = "number"; + i.attrs.value = to!string(T.init); + } else static if(is(T == K[], K)) { + auto templ = div.addChild("template"); + templ.appendChild(elementFor!(K)(null, name)); + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto btn = div.addChild("button"); + btn.addClass("add-array-button"); + btn.attrs.type = "button"; + btn.innerText = "Add"; + btn.attrs.onclick = q{ + var a = document.importNode(this.parentNode.firstChild.content, true); + this.parentNode.insertBefore(a, this); + }; + } else static if(is(T == V[K], K, V)) { + div.innerText = "assoc array not implemented for automatic form at this time"; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + + return div; + } + + static if(is(typeof(method) P == __parameters)) + static foreach(idx, _; P) {{ + alias param = P[idx .. idx + 1]; + string displayName = beautify(__traits(identifier, param)); + static foreach(attr; __traits(getAttributes, param)) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + form.appendChild(elementFor!(param)(displayName, __traits(identifier, param))); + }} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; +} + +/* +string urlFor(alias func)() { + return __traits(identifier, func); +} +*/ + +/++ + UDA: The name displayed to the user in auto-generated HTML. + + Default is `beautify(identifier)`. ++/ +struct DisplayName { + string name; +} + +/++ + UDA: The name used in the URL or web parameter. + + Default is `urlify(identifier)` for functions and `identifier` for parameters and data members. ++/ +struct UrlName { + string name; +} + +class MissingArgumentException : Exception { + string functionName; + string argumentName; + string argumentType; + + this(string functionName, string argumentName, string argumentType, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.functionName = functionName; + this.argumentName = argumentName; + this.argumentType = argumentType; + + super("Missing Argument", file, line, next); } } -class WebObject { +auto callFromCgi(alias method, T)(T dg, Cgi cgi) { + + // FIXME: think more about checkboxes and bools. + + import std.traits; + + Parameters!method params; + alias idents = ParameterIdentifierTuple!method; + alias defaults = ParameterDefaults!method; + + const(string)[] names; + const(string)[] values; + + // first, check for missing arguments and initialize to defaults if necessary + static foreach(idx, param; params) {{ + auto ident = idents[idx]; + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + if(ident !in cgi.post) { + static if(is(defaults[idx] == void)) + throw new MissingArgumentException(__traits(identifier, method), ident, typeof(param).stringof); + else + params[idx] = defaults[idx]; + } + } else { + if(ident !in cgi.get) { + static if(is(defaults[idx] == void)) + throw new MissingArgumentException(__traits(identifier, method), ident, typeof(param).stringof); + else + params[idx] = defaults[idx]; + } + } + }} + + // second, parse the arguments in order to build up arrays, etc. + + static bool setVariable(T)(string name, string paramName, T* what, string value) { + static if(is(T == struct)) { + if(name == paramName) { + *what = T.init; + return true; + } else { + // could be a child + if(name[paramName.length] == '.') { + paramName = name[paramName.length + 1 .. $]; + name = paramName; + int p = 0; + foreach(ch; paramName) { + if(ch == '.' || ch == '[') + break; + p++; + } + + // set the child member + switch(paramName) { + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + // data member! + case memberName: + return setVariable(name, paramName, &(__traits(getMember, *what, memberName)), value); + } + default: + // ok, not a member + } + } + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + *what = to!T(value); + return true; + } else static if(is(T == K[], K)) { + K tmp; + if(name == paramName) { + // direct - set and append + if(setVariable(name, paramName, &tmp, value)) { + (*what) ~= tmp; + return true; + } else { + return false; + } + } else { + // child, append to last element + // FIXME: what about range violations??? + auto ptr = &(*what)[(*what).length - 1]; + return setVariable(name, paramName, ptr, value); + + } + } else static if(is(T == V[K], K, V)) { + // assoc array, name[key] is valid + if(name == paramName) { + // no action necessary + return true; + } else if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + auto afterName = name[idx + 1 .. $]; + + auto k = to!K(insideBrackets); + V v; + + name = name[0 .. paramName.length]; + writeln(name, afterName, " ", paramName); + + auto ret = setVariable(name ~ afterName, paramName, &v, value); + if(ret) { + (*what)[k] = v; + return true; + } + } + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + return false; + } + + void setArgument(string name, string value) { + int p; + foreach(ch; name) { + if(ch == '.' || ch == '[') + break; + p++; + } + + auto paramName = name[0 .. p]; + + sw: switch(paramName) { + static foreach(idx, param; params) { + case idents[idx]: + setVariable(name, paramName, ¶ms[idx], value); + break sw; + } + default: + // ignore; not relevant argument + } + } + + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + names = cgi.allPostNamesInOrder; + values = cgi.allPostValuesInOrder; + } else { + names = cgi.allGetNamesInOrder; + values = cgi.allGetValuesInOrder; + } + + foreach(idx, name; names) { + setArgument(name, values[idx]); + } + + static if(is(ReturnType!method == void)) { + typeof(null) ret; + dg(params); + } else { + auto ret = dg(params); + } + + // FIXME: format return values + // options are: json, html, csv. + // also may need to wrap in envelope format: none, html, or json. + return ret; +} + +auto formatReturnValueAsHtml(T)(T t) { + import arsd.dom; + import std.traits; + + static if(is(T == typeof(null))) { + return Element.make("span"); + } else static if(isIntegral!T || isSomeString!T || isFloatingPoint!T) { + return Element.make("span", to!string(t), "automatic-data-display"); + } else static if(is(T == V[K], K, V)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display"); + foreach(k, v; t) { + dl.addChild("dt", to!string(k)); + dl.addChild("dd", formatReturnValueAsHtml(v)); + } + return dl; + } else static if(is(T == struct)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display"); + + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + dl.addChild("dt", memberName); + dl.addChild("dt", formatReturnValueAsHtml(__traits(getMember, t, memberName))); + } + + return dl; + } else static if(is(T == E[], E)) { + static if(is(E == struct)) { + // an array of structs is kinda special in that I like + // having those formatted as tables. + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + static foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + static foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + + return table; + } else { + // otherwise, I will just make a list. + auto ol = Element.make("ol"); + ol.addClass("automatic-data-display"); + foreach(e; t) + ol.addChild("li", formatReturnValueAsHtml(e)); + return ol; + } + } else static assert(0, "bad return value for cgi call " ~ T.stringof); + + assert(0); +} + +/++ + The base class for the [dispatcher] object support. ++/ +class WebObject() { Cgi cgi; void initialize(Cgi cgi) { this.cgi = cgi; } + + string script() { + return ` + `; + } + + string style() { + return ` + :root { + --mild-border: #ccc; + --middle-border: #999; + } + table.automatic-data-display { + border-collapse: collapse; + border: solid 1px var(--mild-border); + } + + table.automatic-data-display td { + vertical-align: top; + border: solid 1px var(--mild-border); + padding: 2px 4px; + } + + table.automatic-data-display th { + border: solid 1px var(--mild-border); + border-bottom: solid 1px var(--middle-border); + padding: 2px 4px; + } + + ol.automatic-data-display { + margin: 0px; + list-style-position: inside; + padding: 0px; + } + + .automatic-form { + max-width: 600px; + } + + .form-field { + margin: 0.5em; + padding-left: 0.5em; + } + + .label-text { + display: block; + font-weight: bold; + margin-left: -0.5em; + } + + .add-array-button { + + } + `; + } + + import arsd.dom; + Element htmlContainer() { + auto document = new Document(` + + + D Application + + + +
    + + +`, true, true); + + return document.requireElementById("container"); + } } /++ - Serves a class' methods. To be used with [dispatcher]. + Serves a class' methods, as a kind of low-state RPC over the web. To be used with [dispatcher]. Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. FIXME: explain this better +/ auto serveApi(T)(string urlPrefix) { + import arsd.dom; + import arsd.jsvar; + static bool handler(string urlPrefix, Cgi cgi) { auto obj = new T(); obj.initialize(cgi); switch(cgi.pathInfo[urlPrefix.length .. $]) { - static foreach(methodName; __traits(derivedMembers, T)) - //static if(is({ + static foreach(methodName; __traits(derivedMembers, T)){{ + static if(is(typeof(__traits(getMember, T, methodName)) P == __parameters)) { case urlify(methodName): - callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + switch(cgi.request("format", "html")) { + case "html": + auto container = obj.htmlContainer(); + try { + auto ret = callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + container.appendChild(formatReturnValueAsHtml(ret)); + } catch(MissingArgumentException mae) { + container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing")); + container.appendChild(createAutomaticFormForFunction!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName))); + } + cgi.write(container.parentDocument.toString(), true); + break; + case "json": + auto ret = callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + var json = ret; + var envelope = var.emptyObject; + envelope.success = true; + envelope.result = json; + envelope.error = null; + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + + break; + default: + } return true; } + }} + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(obj.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(obj.style(), true); + return true; default: return false; } @@ -5779,9 +6295,127 @@ auto serveApi(T)(string urlPrefix) { } /++ - Serves a REST object. + Serves a REST object, similar to a Ruby on Rails resource. + + You put data members in your class. cgi.d will automatically make something out of those. + + It will call your constructor with the ID from the URL. This may be null. + It will then populate the data members from the request. + It will then call a method, if present, telling what happened. You don't need to write these! + It finally returns a reply. + + Your methods are passed a list of fields it actually set. + + The URL mapping - despite my general skepticism of the wisdom - matches up with what most REST + APIs I have used seem to follow. (I REALLY want to put trailing slashes on it though. Works better + with relative linking. But meh.) + + GET /items -> index. all values not set. + GET /items/id -> get. only ID will be set, other params ignored. + POST /items -> create. values set as given + PUT /items/id -> replace. values set as given + or POST /items/id with cgi.post["_method"] (thus urlencoded or multipart content-type) set to "PUT" to work around browser/html limitation + a GET with cgi.get["_method"] (in the url) set to "PUT" will render a form. + PATCH /items/id -> update. values set as given, list of changed fields passed + or POST /items/id with cgi.post["_method"] == "PATCH" + DELETE /items/id -> destroy. only ID guaranteed to be set + or POST /items/id with cgi.post["_method"] == "DELETE" + + Following the stupid convention, there will never be a trailing slash here, and if it is there, it will + redirect you away from it. + + API clients should set the `Accept` HTTP header to application/json or the cgi.get["_format"] = "json" var. + + I will also let you change the default, if you must. + + // One add-on is validation. You can issue a HTTP GET to a resource with _method = VALIDATE to check potential changes. + + You can define sub-resources on your object inside the object. These sub-resources are also REST objects + that follow the same thing. They may be individual resources or collections themselves. + + Your class is expected to have at least the following methods: + + FIXME: i kinda wanna add a routes object to the initialize call + + create + Create returns the new address on success, some code on failure. + show + index + update + remove + + You will want to be able to customize the HTTP, HTML, and JSON returns but generally shouldn't have to - the defaults + should usually work. The returned JSON will include a field "href" on all returned objects along with "id". Or omething like that. Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + NOT IMPLEMENTED + + + Really, a collection is a resource with a bunch of subresources. + + GET /items + index because it is GET on the top resource + + GET /items/foo + item but different than items? + + class Items { + + } + + ... but meh, a collection can be automated. not worth making it + a separate thing, let's look at a real example. Users has many + items and a virtual one, /users/current. + + the individual users have properties and two sub-resources: + session, which is just one, and comments, a collection. + + class User : RestObject!() { // no parent + int id; + string name; + + void show() {} // automated! GET of this specific thing + void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + void replace() {} // this is the PUT; really, it just updates all fields. + void update() {} // PATCH, it updates some fields. + void remove() {} // DELETE + + void load(string urlId) {} // the default implementation of show() populates the id, then + + this() {} + + mixin Subresource!Session; + mixin Subresource!Comment; + } + + class Session : RestObject!() { + // the parent object may not be fully constructed/loaded + this(User parent) {} + + } + + class Comment : CollectionOf!Comment { + this(User parent) {} + } + + class Users : CollectionOf!User { + // but you don't strictly need ANYTHING on a collection; it will just... collect. Implement the subobjects. + void index() {} // GET on this specific thing; just like show really, just different name for the different semantics. + User create() {} // You MAY implement this, but the default is to create a new object, populate it from args, and then call create() on the child + } + + // so CollectionOf will mixin the stuff to forward it + + OK, the underlying functions are actually really low level + + GET(string url) + POST(string url) + PUT(string url) + + The url starts with the initial thing passed. + + It is the mixins that actually do the work. +/ auto serveRestObject(T)(string urlPrefix) { static bool handler(string urlPrefix, Cgi cgi) { From 13f3709acb98e9545f46d5483e18b7f57364687e Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 11 Feb 2019 14:38:59 -0500 Subject: [PATCH 22/44] catchup --- http.d | 2 +- http2.d | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/http.d b/http.d index 624c2dc..28cc3f1 100644 --- a/http.d +++ b/http.d @@ -3,7 +3,7 @@ I no longer work on this, use http2.d instead. +/ -module arsd.http; +deprecated module arsd.http; import std.socket; diff --git a/http2.d b/http2.d index 614781b..1f97919 100644 --- a/http2.d +++ b/http2.d @@ -75,6 +75,20 @@ HttpRequest get(string url) { return request; } +HttpRequest post(string url, string[string] req) { + auto client = new HttpClient(); + ubyte[] bdata; + foreach(k, v; req) { + if(bdata.length) + bdata ~= cast(ubyte[]) "&"; + bdata ~= cast(ubyte[]) encodeComponent(k); + bdata ~= cast(ubyte[]) "="; + bdata ~= cast(ubyte[]) encodeComponent(v); + } + auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); + return request; +} + /// gets the text off a url. basic operation only. string getText(string url) { auto request = get(url); @@ -1090,7 +1104,6 @@ class HttpRequest { ubyte[] sendBuffer; HttpResponse responseData; - HttpRequestParameters parameters; private HttpClient parentClient; size_t bodyBytesSent; From ece8c2002e7c81e8bbd8aba83c155959aaa5ef21 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 11 Feb 2019 20:34:17 -0500 Subject: [PATCH 23/44] change alloca to static buffer to avoid spurious exception handling error --- jpeg.d | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jpeg.d b/jpeg.d index f1c51c7..f2a5e8b 100644 --- a/jpeg.d +++ b/jpeg.d @@ -2971,9 +2971,10 @@ public bool detect_jpeg_image_from_file (const(char)[] filename, out int width, bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + if (filename.length < 512) { + char[513] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); From ba017f906505632df101d3947b1fef1b18b9c8dc Mon Sep 17 00:00:00 2001 From: Murilo Miranda Date: Tue, 12 Feb 2019 00:38:19 -0200 Subject: [PATCH 24/44] Update jpeg.d I did what you had already done before to all instances of core.stdc.stdlib.alloca() calls to solve the error with -m64. --- jpeg.d | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jpeg.d b/jpeg.d index f2a5e8b..5ec08c4 100644 --- a/jpeg.d +++ b/jpeg.d @@ -3149,8 +3149,9 @@ public ubyte[] decompress_jpeg_image_from_file(bool useMalloc=false) (const(char if (filename.length == 0) throw new Exception("cannot open unnamed file"); if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + char[2049] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); @@ -3340,8 +3341,9 @@ public MemoryImage readJpeg (const(char)[] filename) { if (filename.length == 0) throw new Exception("cannot open unnamed file"); if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + char[2049] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); From 3f15f41ddc3a9e47d9ad02993e7881a63ed6ea78 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 11 Feb 2019 21:43:54 -0500 Subject: [PATCH 25/44] smaller buffer tbh no need for something so huge --- cgi.d | 190 +++++++++++++++++++++++++++++++++++++++++++++++++++------ jpeg.d | 8 +-- 2 files changed, 177 insertions(+), 21 deletions(-) diff --git a/cgi.d b/cgi.d index fa2e5ae..ef33624 100644 --- a/cgi.d +++ b/cgi.d @@ -6147,7 +6147,7 @@ auto formatReturnValueAsHtml(T)(T t) { /++ The base class for the [dispatcher] object support. +/ -class WebObject() { +class WebObject(Helper = void) { Cgi cgi; void initialize(Cgi cgi) { this.cgi = cgi; @@ -6234,6 +6234,8 @@ class WebObject() { FIXME: explain this better +/ auto serveApi(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] == '/'); + import arsd.dom; import arsd.jsvar; @@ -6294,6 +6296,124 @@ auto serveApi(T)(string urlPrefix) { return DispatcherDefinition!handler(urlPrefix, false); } +/++ + The base of all REST objects. ++/ +class RestObject(Helper = void) { + + import arsd.dom; + import arsd.jsvar; + + /// Show + void show() {} + /// ditto + void show(string urlId) { + load(urlId); + show(); + } + + enum AccessCheck { + allowed, + denied, + nonExistant, + } + + enum Operation { + show, + create, + replace, + remove, + update + } + + enum UpdateResult { + accessDenied, + noSuchResource, + success, + failure, + unnecessary + } + + enum ValidationResult { + valid, + invalid + } + + ValidationResult delegate(typeof(this)) validateFromReflection; + Element delegate(typeof(this)) toHtmlFromReflection; + var delegate(typeof(this)) toJsonFromReflection; + + /// Override this to provide access control to this object. + AccessCheck accessCheck(string urlId, Operation operation) { + return AccessCheck.allowed; + } + + ValidationResult validate() { + if(validateFromReflection !is null) + return validateFromReflection(this); + return ValidationResult.valid; + } + + // The functions with more arguments are the low-level ones, + // they forward to the ones with fewer arguments by default. + + void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + + void replace() {} + void replace(string urlId, scope void delegate() applyChanges) { + load(urlId); + applyChanges(); + replace(); + } + + void update(string[] fieldList) {} + void update(string urlId, scope void delegate() applyChanges, string[] fieldList) { + load(urlId); + applyChanges(); + update(fieldList); + } + + void remove() {} + + void remove(string urlId) { + load(urlId); + remove(); + } + + abstract void load(string urlId) {} + abstract void save() {} + + Element toHtml() { + if(toHtmlFromReflection) + return toHtmlFromReflection(this); + else + assert(0); + } + + var toJson() { + if(toJsonFromReflection) + return toJsonFromReflection(this); + else + assert(0); + } +} + +/++ + Base class for REST collections. ++/ +class CollectionOf(Obj, Helper = void) : RestObject!(Helper) { + void index() {} + override void create() {} + override void load(string urlId) { assert(0); } + override void save() { assert(0); } + override void show() { + index(); + } + override void show(string urlId) { + show(); + } +} + /++ Serves a REST object, similar to a Ruby on Rails resource. @@ -6375,11 +6495,17 @@ auto serveApi(T)(string urlPrefix) { int id; string name; - void show() {} // automated! GET of this specific thing + // the default implementations of the urlId ones is to call load(that_id) then call the arg-less one. + // but you can override them to do it differently. + + // any member which is of type RestObject can be linked automatically via href btw. + + void show() {} + void show(string urlId) {} // automated! GET of this specific thing void create() {} // POST on a parent collection - this is called from a collection class after the members are updated - void replace() {} // this is the PUT; really, it just updates all fields. - void update() {} // PATCH, it updates some fields. - void remove() {} // DELETE + void replace(string urlId) {} // this is the PUT; really, it just updates all fields. + void update(string urlId, string[] fieldList) {} // PATCH, it updates some fields. + void remove(string urlId) {} // DELETE void load(string urlId) {} // the default implementation of show() populates the id, then @@ -6405,22 +6531,52 @@ auto serveApi(T)(string urlPrefix) { User create() {} // You MAY implement this, but the default is to create a new object, populate it from args, and then call create() on the child } - // so CollectionOf will mixin the stuff to forward it - - OK, the underlying functions are actually really low level - - GET(string url) - POST(string url) - PUT(string url) - - The url starts with the initial thing passed. - - It is the mixins that actually do the work. +/ auto serveRestObject(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] != '/', "Do NOT use a trailing slash on REST objects."); static bool handler(string urlPrefix, Cgi cgi) { string url = cgi.pathInfo[urlPrefix.length .. $]; + if(url.length && url[$ - 1] == '/') { + // remove the final slash... + cgi.setResponseLocation(".."); + return true; + } + + string urlId = null; + if(url.length && url[0] == '/') { + // asking for a subobject + urlId = url[1 .. $]; + foreach(idx, ch; urlId) { + if(ch == '/') { + urlId = urlId[0 .. idx]; + break; + } + } + } + + // FIXME: support precondition failed, if-modified-since, expectation failed, etc. + + auto obj = new T(); + obj.toHtmlFromReflection = delegate(t) { + import arsd.dom; + return Element.make("div", T.stringof ~ "/" ~ urlId); + }; + // FIXME: populate reflection info delegates + + switch(cgi.requestMethod) { + case Cgi.RequestMethod.GET: + obj.show(urlId); + cgi.write(obj.toHtml().toString, true); + break; + case Cgi.RequestMethod.POST: + case Cgi.RequestMethod.PUT: + case Cgi.RequestMethod.PATCH: + case Cgi.RequestMethod.DELETE: + default: + // FIXME: OPTIONS, HEAD + } + return true; } return DispatcherDefinition!handler(urlPrefix, false); @@ -6445,7 +6601,7 @@ auto serveStaticFile(string urlPrefix, string filename = null, string contentTyp } auto serveRedirect(string urlPrefix, string redirectTo) { - + // FIXME } /+ diff --git a/jpeg.d b/jpeg.d index 5ec08c4..1bd106e 100644 --- a/jpeg.d +++ b/jpeg.d @@ -3148,8 +3148,8 @@ public ubyte[] decompress_jpeg_image_from_file(bool useMalloc=false) (const(char bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - char[2049] buffer; + if (filename.length < 512) { + char[513] buffer; //import core.stdc.stdlib : alloca; auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; @@ -3340,8 +3340,8 @@ public MemoryImage readJpeg (const(char)[] filename) { bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - char[2049] buffer; + if (filename.length < 512) { + char[513] buffer; //import core.stdc.stdlib : alloca; auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; From 076d28dba11ff82fdc916f94f6ef7f8cdba8ab04 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 18 Feb 2019 15:25:10 -0500 Subject: [PATCH 26/44] for now rest sample --- cgi.d | 626 +++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 513 insertions(+), 113 deletions(-) diff --git a/cgi.d b/cgi.d index ef33624..751ad53 100644 --- a/cgi.d +++ b/cgi.d @@ -1,6 +1,9 @@ // FIXME: if an exception is thrown, we shouldn't necessarily cache... // FIXME: there's some annoying duplication of code in the various versioned mains +// FIXME: I might make a cgi proxy class which can change things; the underlying one is still immutable +// but the later one can edit and simplify the api. You'd have to use the subclass tho! + /* void foo(int f, @("test") string s) {} @@ -3031,9 +3034,14 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC uint i; sockaddr addr; i = addr.sizeof; - version(embedded_httpd_processes_accept_after_fork) + version(embedded_httpd_processes_accept_after_fork) { int s = accept(sock, &addr, &i); - else { + int opt = 1; + import core.sys.posix.netinet.tcp; + // the Cgi class does internal buffering, so disabling this + // helps with latency in many cases... + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } else { int s; auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s); if(readret != s.sizeof) { @@ -3147,6 +3155,9 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC sockaddr addr; i = addr.sizeof; s = accept(sock, &addr, &i); + import core.sys.posix.netinet.tcp; + int opt = 1; + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); } if(FD_ISSET(pipeWriteFd, &write_fds)) { @@ -5787,6 +5798,63 @@ private string beautify(string name) { for html, basic types are strings. Arrays are
      . Structs are
      . Arrays of structs are tables! +/ +// returns an arsd.dom.Element +static auto elementFor(T)(string displayName, string name) { + import arsd.dom; + import std.traits; + + auto div = Element.make("div"); + div.addClass("form-field"); + + static if(is(T == struct)) { + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto fieldset = div.addChild("fieldset"); + fieldset.addChild("legend", beautify(T.stringof)); // FIXME + fieldset.addChild("input", name); + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName)); + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + static if(isSomeString!T) + i.attrs.type = "text"; + else + i.attrs.type = "number"; + i.attrs.value = to!string(T.init); + } else static if(is(T == K[], K)) { + auto templ = div.addChild("template"); + templ.appendChild(elementFor!(K)(null, name)); + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto btn = div.addChild("button"); + btn.addClass("add-array-button"); + btn.attrs.type = "button"; + btn.innerText = "Add"; + btn.attrs.onclick = q{ + var a = document.importNode(this.parentNode.firstChild.content, true); + this.parentNode.insertBefore(a, this); + }; + } else static if(is(T == V[K], K, V)) { + div.innerText = "assoc array not implemented for automatic form at this time"; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + + return div; +} + // actually returns an arsd.dom.Form auto createAutomaticFormForFunction(alias method, T)(T dg) { import arsd.dom; @@ -5803,59 +5871,6 @@ auto createAutomaticFormForFunction(alias method, T)(T dg) { //alias idents = ParameterIdentifierTuple!method; //alias defaults = ParameterDefaults!method; - static Element elementFor(T)(string displayName, string name) { - auto div = Element.make("div"); - div.addClass("form-field"); - - static if(is(T == struct)) { - if(displayName !is null) - div.addChild("span", displayName, "label-text"); - auto fieldset = div.addChild("fieldset"); - fieldset.addChild("legend", beautify(T.stringof)); // FIXME - fieldset.addChild("input", name); - static foreach(idx, memberName; __traits(allMembers, T)) - static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { - fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName)); - } - } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { - Element lbl; - if(displayName !is null) { - lbl = div.addChild("label"); - lbl.addChild("span", displayName, "label-text"); - lbl.appendText(" "); - } else { - lbl = div; - } - auto i = lbl.addChild("input", name); - i.attrs.name = name; - static if(isSomeString!T) - i.attrs.type = "text"; - else - i.attrs.type = "number"; - i.attrs.value = to!string(T.init); - } else static if(is(T == K[], K)) { - auto templ = div.addChild("template"); - templ.appendChild(elementFor!(K)(null, name)); - if(displayName !is null) - div.addChild("span", displayName, "label-text"); - auto btn = div.addChild("button"); - btn.addClass("add-array-button"); - btn.attrs.type = "button"; - btn.innerText = "Add"; - btn.attrs.onclick = q{ - var a = document.importNode(this.parentNode.firstChild.content, true); - this.parentNode.insertBefore(a, this); - }; - } else static if(is(T == V[K], K, V)) { - div.innerText = "assoc array not implemented for automatic form at this time"; - } else { - static assert(0, "unsupported type for cgi call " ~ T.stringof); - } - - - return div; - } - static if(is(typeof(method) P == __parameters)) static foreach(idx, _; P) {{ alias param = P[idx .. idx + 1]; @@ -5871,6 +5886,38 @@ auto createAutomaticFormForFunction(alias method, T)(T dg) { return form; } +// actually returns an arsd.dom.Form +auto createAutomaticFormForObject(T)(T obj) { + import arsd.dom; + + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, T))); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static foreach(idx, memberName; __traits(derivedMembers, T)) {{ + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + string displayName = beautify(memberName); + static foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + form.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(displayName, memberName)); + + form.setValue(memberName, to!string(__traits(getMember, obj, memberName))); + }}} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; +} + /* string urlFor(alias func)() { return __traits(identifier, func); @@ -5911,6 +5958,8 @@ class MissingArgumentException : Exception { auto callFromCgi(alias method, T)(T dg, Cgi cgi) { + // FIXME: any array of structs should also be settable or gettable from csv as well. + // FIXME: think more about checkboxes and bools. import std.traits; @@ -6110,7 +6159,32 @@ auto formatReturnValueAsHtml(T)(T t) { return dl; } else static if(is(T == E[], E)) { - static if(is(E == struct)) { + static if(is(E : RestObject!Proxy, Proxy)) { + // treat RestObject similar to struct + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + static foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + static foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + static if(memberName == "id") { + string val = to!string(__traits(getMember, l, memberName)); + tr.addChild("td", Element.make("a", val, E.stringof.toLower ~ "s/" ~ val)); // FIXME + } else { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + } + + return table; + } else static if(is(E == struct)) { // an array of structs is kinda special in that I like // having those formatted as tables. auto table = cast(Table) Element.make("table"); @@ -6145,7 +6219,19 @@ auto formatReturnValueAsHtml(T)(T t) { } /++ - The base class for the [dispatcher] object support. + A web presenter is responsible for rendering things to HTML to be usable + in a web browser. + + They are passed as template arguments to the base classes of [WebObject] + + FIXME ++/ +class WebPresenter() { + +} + +/++ + The base class for the [dispatcher] function and object support. +/ class WebObject(Helper = void) { Cgi cgi; @@ -6163,7 +6249,14 @@ class WebObject(Helper = void) { :root { --mild-border: #ccc; --middle-border: #999; + --accent-color: #e8e8e8; + --sidebar-color: #f2f2f2; } + ` ~ genericFormStyling() ~ genericSiteStyling(); + } + + string genericFormStyling() { + return ` table.automatic-data-display { border-collapse: collapse; border: solid 1px var(--mild-border); @@ -6208,6 +6301,39 @@ class WebObject(Helper = void) { `; } + string genericSiteStyling() { + return ` + * { box-sizing: border-box; } + html, body { margin: 0px; } + body { + font-family: sans-serif; + } + #header { + background: var(--accent-color); + height: 64px; + } + #footer { + background: var(--accent-color); + height: 64px; + } + #main-site { + display: flex; + } + #container { + flex: 1 1 auto; + order: 2; + min-height: calc(100vh - 64px - 64px); + padding: 4px; + padding-left: 1em; + } + #sidebar { + flex: 0 0 16em; + order: 1; + background: var(--sidebar-color); + } + `; + } + import arsd.dom; Element htmlContainer() { auto document = new Document(` @@ -6217,7 +6343,12 @@ class WebObject(Helper = void) { -
      + +
      +
      + +
      + `, true, true); @@ -6296,21 +6427,6 @@ auto serveApi(T)(string urlPrefix) { return DispatcherDefinition!handler(urlPrefix, false); } -/++ - The base of all REST objects. -+/ -class RestObject(Helper = void) { - - import arsd.dom; - import arsd.jsvar; - - /// Show - void show() {} - /// ditto - void show(string urlId) { - load(urlId); - show(); - } enum AccessCheck { allowed, @@ -6339,6 +6455,23 @@ class RestObject(Helper = void) { invalid } + +/++ + The base of all REST objects, to be used with [serveRestObject] and [serveRestCollectionOf]. ++/ +class RestObject(Helper = void) : WebObject!Helper { + + import arsd.dom; + import arsd.jsvar; + + /// Prepare the object to be shown. + void show() {} + /// ditto + void show(string urlId) { + load(urlId); + show(); + } + ValidationResult delegate(typeof(this)) validateFromReflection; Element delegate(typeof(this)) toHtmlFromReflection; var delegate(typeof(this)) toJsonFromReflection; @@ -6357,7 +6490,14 @@ class RestObject(Helper = void) { // The functions with more arguments are the low-level ones, // they forward to the ones with fewer arguments by default. - void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + // POST on a parent collection - this is called from a collection class after the members are updated + /++ + Given a populated object, this creates a new entry. Returns the url identifier + of the new object. + +/ + string create(scope void delegate() applyChanges) { + return null; + } void replace() {} void replace(string urlId, scope void delegate() applyChanges) { @@ -6380,8 +6520,8 @@ class RestObject(Helper = void) { remove(); } - abstract void load(string urlId) {} - abstract void save() {} + abstract void load(string urlId); + abstract void save(); Element toHtml() { if(toHtmlFromReflection) @@ -6396,14 +6536,55 @@ class RestObject(Helper = void) { else assert(0); } + + /+ + auto structOf(this This) { + + } + +/ +} + +/++ + Responsible for displaying stuff as HTML. You can put this into your own aggregate + and override it. Use forwarding and specialization to customize it. ++/ +mixin template Presenter() { + } /++ Base class for REST collections. +/ class CollectionOf(Obj, Helper = void) : RestObject!(Helper) { - void index() {} - override void create() {} + /// You might subclass this and use the cgi object's query params + /// to implement a search filter, for example. + /// + /// FIXME: design a way to auto-generate that form + /// (other than using the WebObject thing above lol + // it'll prolly just be some searchParams UDA or maybe an enum. + // + // pagination too perhaps. + // + // and sorting too + IndexResult index() { return IndexResult.init; } + + string[] sortableFields() { return null; } + string[] searchableFields() { return null; } + + struct IndexResult { + Obj[] results; + + string[] sortableFields; + + string previousPageIdentifier; + string nextPageIdentifier; + string firstPageIdentifier; + string lastPageIdentifier; + + int numberOfPages; + } + + override string create(scope void delegate() applyChanges) { assert(0); } override void load(string urlId) { assert(0); } override void save() { assert(0); } override void show() { @@ -6412,6 +6593,9 @@ class CollectionOf(Obj, Helper = void) : RestObject!(Helper) { override void show(string urlId) { show(); } + + /// Proxy POST requests (create calls) to the child collection + alias PostProxy = Obj; } /++ @@ -6539,49 +6723,265 @@ auto serveRestObject(T)(string urlPrefix) { if(url.length && url[$ - 1] == '/') { // remove the final slash... - cgi.setResponseLocation(".."); + cgi.setResponseLocation(cgi.pathInfo[0 .. $ - 1]); return true; } - string urlId = null; - if(url.length && url[0] == '/') { - // asking for a subobject - urlId = url[1 .. $]; - foreach(idx, ch; urlId) { - if(ch == '/') { - urlId = urlId[0 .. idx]; - break; - } - } - } + return restObjectServeHandler!T(cgi, url); - // FIXME: support precondition failed, if-modified-since, expectation failed, etc. - - auto obj = new T(); - obj.toHtmlFromReflection = delegate(t) { - import arsd.dom; - return Element.make("div", T.stringof ~ "/" ~ urlId); - }; - // FIXME: populate reflection info delegates - - switch(cgi.requestMethod) { - case Cgi.RequestMethod.GET: - obj.show(urlId); - cgi.write(obj.toHtml().toString, true); - break; - case Cgi.RequestMethod.POST: - case Cgi.RequestMethod.PUT: - case Cgi.RequestMethod.PATCH: - case Cgi.RequestMethod.DELETE: - default: - // FIXME: OPTIONS, HEAD - } - - return true; } return DispatcherDefinition!handler(urlPrefix, false); } +/// Convenience method for serving a collection. It will be named the same +/// as type T, just with an s at the end. If you need any further, just +/// write the class yourself. +auto serveRestCollectionOf(T)(string urlPrefix) { + mixin(`static class `~T.stringof~`s : CollectionOf!(T) {}`); + return serveRestObject!(mixin(T.stringof ~ "s"))(urlPrefix); +} + +bool restObjectServeHandler(T)(Cgi cgi, string url) { + string urlId = null; + if(url.length && url[0] == '/') { + // asking for a subobject + urlId = url[1 .. $]; + foreach(idx, ch; urlId) { + if(ch == '/') { + urlId = urlId[0 .. idx]; + break; + } + } + } + + // FIXME handle other subresources + + static if(is(T : CollectionOf!(C, P), C, P)) { + if(urlId !is null) { + return restObjectServeHandler!C(cgi, url); // FIXME? urlId); + } + } + + // FIXME: support precondition failed, if-modified-since, expectation failed, etc. + + auto obj = new T(); + obj.toHtmlFromReflection = delegate(t) { + import arsd.dom; + auto div = Element.make("div"); + div.addClass("Dclass_" ~ T.stringof); + div.dataset.url = urlId; + static foreach(idx, memberName; __traits(derivedMembers, T)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + div.appendChild(formatReturnValueAsHtml(__traits(getMember, obj, memberName))); + } + return div; + }; + obj.toJsonFromReflection = delegate(t) { + import arsd.jsvar; + var v = var.emptyObject(); + static foreach(idx, memberName; __traits(derivedMembers, T)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + v[memberName] = __traits(getMember, obj, memberName); + } + return v; + }; + obj.validateFromReflection = delegate(t) { + // FIXME + return ValidationResult.valid; + }; + obj.initialize(cgi); + // FIXME: populate reflection info delegates + + + // FIXME: I am not happy with this. + switch(urlId) { + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(obj.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(obj.style(), true); + return true; + default: + // intentionally blank + } + + + + + static void applyChangesTemplate(Obj)(Cgi cgi, Obj obj) { + static foreach(idx, memberName; __traits(derivedMembers, Obj)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + __traits(getMember, obj, memberName) = cgi.request(memberName, __traits(getMember, obj, memberName)); + } + } + void applyChanges() { + applyChangesTemplate(cgi, obj); + } + + string[] modifiedList; + + void writeObject(bool addFormLinks) { + if(cgi.request("format") == "json") { + cgi.setResponseContentType("application/json"); + cgi.write(obj.toJson().toString, true); + } else { + auto container = obj.htmlContainer(); + if(addFormLinks) { + static if(is(T : CollectionOf!(C, P), C, P)) + container.appendHtml(` +
      + +
      + `); + else + container.appendHtml(` +
      + + +
      + `); + } + container.appendChild(obj.toHtml()); + cgi.write(container.parentDocument.toString, true); + } + } + + // FIXME: I think I need a set type in here.... + // it will be nice to pass sets of members. + + switch(cgi.requestMethod) { + case Cgi.RequestMethod.GET: + // I could prolly use template this parameters in the implementation above for some reflection stuff. + // sure, it doesn't automatically work in subclasses... but I instantiate here anyway... + + // automatic forms here for usable basic auto site from browser. + // even if the format is json, it could actually send out the links and formats, but really there i'ma be meh. + switch(cgi.request("_method", "GET")) { + case "GET": + static if(is(T : CollectionOf!(C, P), C, P)) { + auto results = obj.index(); + if(cgi.request("format", "html") == "html") { + auto container = obj.htmlContainer(); + auto html = formatReturnValueAsHtml(results.results); + container.appendHtml(` +
      + +
      + `); + + container.appendChild(html); + cgi.write(container.parentDocument.toString, true); + } else { + cgi.setResponseContentType("application/json"); + import arsd.jsvar; + var json = var.emptyArray; + foreach(r; results.results) { + var o = var.emptyObject; + static foreach(idx, memberName; __traits(derivedMembers, typeof(r))) + static if(__traits(compiles, __traits(getMember, r, memberName).offsetof)) { + o[memberName] = __traits(getMember, r, memberName); + } + + json ~= o; + } + cgi.write(json.toJson(), true); + } + } else { + obj.show(urlId); + writeObject(true); + } + break; + case "PATCH": + obj.load(urlId); + goto case; + case "PUT": + case "POST": + // an editing form for the object + auto container = obj.htmlContainer(); + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto form = (cgi.request("_method") == "POST") ? createAutomaticFormForObject(new obj.PostProxy()) : createAutomaticFormForObject(obj); + } else { + auto form = createAutomaticFormForObject(obj); + } + form.attrs.method = "POST"; + form.setValue("_method", cgi.request("_method", "GET")); + container.appendChild(form); + cgi.write(container.parentDocument.toString(), true); + break; + case "DELETE": + // FIXME: a delete form for the object (can be phrased "are you sure?") + auto container = obj.htmlContainer(); + container.appendHtml(` +
      + Are you sure you want to delete this item? + + +
      + + `); + cgi.write(container.parentDocument.toString(), true); + break; + default: + cgi.write("bad method\n", true); + } + break; + case Cgi.RequestMethod.POST: + // this is to allow compatibility with HTML forms + switch(cgi.request("_method", "POST")) { + case "PUT": + goto PUT; + case "PATCH": + goto PATCH; + case "DELETE": + goto DELETE; + case "POST": + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto p = new obj.PostProxy(); + void specialApplyChanges() { + applyChangesTemplate(cgi, p); + } + string n = p.create(&specialApplyChanges); + } else { + string n = obj.create(&applyChanges); + } + + auto newUrl = cgi.pathInfo ~ "/" ~ n; + cgi.setResponseLocation(newUrl); + cgi.setResponseStatus("201 Created"); + cgi.write(`The object has been created.`); + break; + default: + cgi.write("bad method\n", true); + } + // FIXME this should be valid on the collection, but not the child.... + // 303 See Other + break; + case Cgi.RequestMethod.PUT: + PUT: + obj.replace(urlId, &applyChanges); + writeObject(false); + break; + case Cgi.RequestMethod.PATCH: + PATCH: + obj.update(urlId, &applyChanges, modifiedList); + writeObject(false); + break; + case Cgi.RequestMethod.DELETE: + DELETE: + obj.remove(urlId); + cgi.setResponseStatus("204 No Content"); + break; + default: + // FIXME: OPTIONS, HEAD + } + + return true; +} + /++ Serves a static file. To be used with [dispatcher]. +/ From 15e648f43e91980c3c4c620cf5b18f0d029d5c74 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 18 Feb 2019 15:59:06 -0500 Subject: [PATCH 27/44] catchup --- cgi.d | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cgi.d b/cgi.d index 751ad53..b1f1cee 100644 --- a/cgi.d +++ b/cgi.d @@ -697,7 +697,7 @@ class Cgi { } } else { // it is an argument of some sort - if(requestMethod == Cgi.RequestMethod.POST) { + if(requestMethod == Cgi.RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) { auto parts = breakUp(arg); _post[parts[0]] ~= parts[1]; allPostNamesInOrder ~= parts[0]; @@ -906,7 +906,7 @@ class Cgi { // FIXME: DOCUMENT_ROOT? // FIXME: what about PUT? - if(requestMethod == RequestMethod.POST) { + if(requestMethod == RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) { version(preserveData) // a hack to make forwarding simpler immutable(ubyte)[] data; size_t amountReceived = 0; @@ -6499,14 +6499,18 @@ class RestObject(Helper = void) : WebObject!Helper { return null; } - void replace() {} + void replace() { + save(); + } void replace(string urlId, scope void delegate() applyChanges) { load(urlId); applyChanges(); replace(); } - void update(string[] fieldList) {} + void update(string[] fieldList) { + save(); + } void update(string urlId, scope void delegate() applyChanges, string[] fieldList) { load(urlId); applyChanges(); @@ -6723,7 +6727,7 @@ auto serveRestObject(T)(string urlPrefix) { if(url.length && url[$ - 1] == '/') { // remove the final slash... - cgi.setResponseLocation(cgi.pathInfo[0 .. $ - 1]); + cgi.setResponseLocation(cgi.scriptName ~ cgi.pathInfo[0 .. $ - 1]); return true; } @@ -6949,7 +6953,7 @@ bool restObjectServeHandler(T)(Cgi cgi, string url) { string n = obj.create(&applyChanges); } - auto newUrl = cgi.pathInfo ~ "/" ~ n; + auto newUrl = cgi.scriptName ~ cgi.pathInfo ~ "/" ~ n; cgi.setResponseLocation(newUrl); cgi.setResponseStatus("201 Created"); cgi.write(`The object has been created.`); From 26868a75268d5d1f81bcb7dd2152e3c9cd6da7bd Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 22 Feb 2019 11:37:03 -0500 Subject: [PATCH 28/44] speculative bug fix --- cgi.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgi.d b/cgi.d index b1f1cee..99633c8 100644 --- a/cgi.d +++ b/cgi.d @@ -3648,7 +3648,7 @@ class BufferedInputRange { return; } - view = underlyingBuffer[underlyingBuffer.ptr - view.ptr .. view.length + ret]; + view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; } while(view.length < minBytesToSettleFor); } From 2fa8f7cfa978b74f2a0ce44d5a2b784489716fc1 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 25 Feb 2019 09:42:47 -0500 Subject: [PATCH 29/44] catchup --- cgi.d | 20 ++++++++++++++++++++ script.d | 6 +++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/cgi.d b/cgi.d index 99633c8..daee713 100644 --- a/cgi.d +++ b/cgi.d @@ -6774,8 +6774,10 @@ bool restObjectServeHandler(T)(Cgi cgi, string url) { auto div = Element.make("div"); div.addClass("Dclass_" ~ T.stringof); div.dataset.url = urlId; + bool first = true; static foreach(idx, memberName; __traits(derivedMembers, T)) static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + if(!first) div.addChild("br"); else first = false; div.appendChild(formatReturnValueAsHtml(__traits(getMember, obj, memberName))); } return div; @@ -6986,6 +6988,24 @@ bool restObjectServeHandler(T)(Cgi cgi, string url) { return true; } +/+ +struct SetOfFields(T) { + private void[0][string] storage; + void set(string what) { + //storage[what] = + } + void unset(string what) {} + void setAll() {} + void unsetAll() {} + bool isPresent(string what) { return false; } +} ++/ + +/+ +enum readonly; +enum hideonindex; ++/ + /++ Serves a static file. To be used with [dispatcher]. +/ diff --git a/script.d b/script.d index e9e75ae..ae106c9 100644 --- a/script.d +++ b/script.d @@ -90,6 +90,8 @@ obj.__prop("name", value); // bypasses operator overloading, useful for use inside the opIndexAssign especially Note: if opIndex is not overloaded, getting a non-existent member will actually add it to the member. This might be a bug but is needed right now in the D impl for nice chaining. Or is it? FIXME + + FIXME: it doesn't do opIndex with multiple args. * if/else * array slicing, but note that slices are rvalues currently * variables must start with A-Z, a-z, _, or $, then must be [A-Za-z0-9_]*. @@ -2304,7 +2306,9 @@ Expression parseExpression(MyTokenStreamHere)(ref MyTokenStreamHere tokens, bool auto ident = tokens.requireNextToken(ScriptToken.Type.identifier); tokens.requireNextToken(ScriptToken.Type.symbol, "("); - auto args = parseVariableDeclaration(tokens, ")"); + VariableDeclaration args; + if(!tokens.peekNextToken(ScriptToken.Type.symbol, ")")) + args = parseVariableDeclaration(tokens, ")"); tokens.requireNextToken(ScriptToken.Type.symbol, ")"); auto bod = parseExpression(tokens); From 3bcce8b478848867bf2316847a58a6ec08456b01 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 1 Mar 2019 12:29:43 -0500 Subject: [PATCH 30/44] new method to check if valid to create Terminal; --- terminal.d | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/terminal.d b/terminal.d index 47543b8..bec8ef3 100644 --- a/terminal.d +++ b/terminal.d @@ -434,6 +434,25 @@ struct Terminal { @disable this(this); private ConsoleOutputType type; + /++ + Terminal is only valid to use on an actual console device or terminal + handle. You should not attempt to construct a Terminal instance if this + returns false; + +/ + static bool stdoutIsTerminal() { + version(Posix) { + import core.sys.posix.unistd; + return cast(bool) isatty(1); + } else version(Windows) { + auto hConsole = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO originalSbi; + if(GetConsoleScreenBufferInfo(hConsole, &originalSbi) == 0) + return false; + else + return true; + } else static assert(0); + } + version(Posix) { private int fdOut; private int fdIn; From 10468ee9f4bb5013eba075506b9242059d6a46f3 Mon Sep 17 00:00:00 2001 From: Adam Ruppe Date: Sat, 2 Mar 2019 21:30:47 -0500 Subject: [PATCH 31/44] fixing some of the old cocoa code --- simpledisplay.d | 52 ++++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/simpledisplay.d b/simpledisplay.d index 7ab6158..882e789 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -1990,11 +1990,15 @@ class SimpleWindow : CapableOfHandlingNativeEvent, CapableOfBeingDrawnUpon { +/ @property void cursor(MouseCursor cursor) { - auto ch = cursor.cursorHandle; + version(OSXCocoa) + featureNotImplemented(); + else if(this.impl.curHidden <= 0) { static if(UsingSimpledisplayX11) { + auto ch = cursor.cursorHandle; XDefineCursor(XDisplayConnection.get(), this.impl.window, ch); } else version(Windows) { + auto ch = cursor.cursorHandle; impl.currentCursor = ch; SetCursor(ch); // redraw without waiting for mouse movement to update } else featureNotImplemented(); @@ -12189,7 +12193,7 @@ private: alias const(void)* CGContextRef; alias const(void)* CGColorSpaceRef; alias const(void)* CGImageRef; - alias uint CGBitmapInfo; + alias ulong CGBitmapInfo; struct objc_super { id self; @@ -12197,18 +12201,18 @@ private: } struct CFRange { - int location, length; + long location, length; } struct NSPoint { - float x, y; + double x, y; static fromTuple(T)(T tupl) { return NSPoint(tupl.tupleof); } } struct NSSize { - float width, height; + double width, height; } struct NSRect { NSPoint origin; @@ -12219,7 +12223,7 @@ private: alias NSRect CGRect; struct CGAffineTransform { - float a, b, c, d, tx, ty; + double a, b, c, d, tx, ty; } enum NSApplicationActivationPolicyRegular = 0; @@ -12235,7 +12239,7 @@ private: NSTexturedBackgroundWindowMask = 1 << 8 } - enum : uint { + enum : ulong { kCGImageAlphaNone, kCGImageAlphaPremultipliedLast, kCGImageAlphaPremultipliedFirst, @@ -12244,7 +12248,7 @@ private: kCGImageAlphaNoneSkipLast, kCGImageAlphaNoneSkipFirst } - enum : uint { + enum : ulong { kCGBitmapAlphaInfoMask = 0x1F, kCGBitmapFloatComponents = (1 << 8), kCGBitmapByteOrderMask = 0x7000, @@ -12293,12 +12297,12 @@ private: CFStringRef CFStringCreateWithBytes(CFAllocatorRef allocator, const(char)* bytes, long numBytes, - int encoding, + long encoding, BOOL isExternalRepresentation); - int CFStringGetBytes(CFStringRef theString, CFRange range, int encoding, + long CFStringGetBytes(CFStringRef theString, CFRange range, long encoding, char lossByte, bool isExternalRepresentation, char* buffer, long maxBufLen, long* usedBufLen); - int CFStringGetLength(CFStringRef theString); + long CFStringGetLength(CFStringRef theString); CGContextRef CGBitmapContextCreate(void* data, size_t width, size_t height, @@ -12316,13 +12320,13 @@ private: void CGColorSpaceRelease(CGColorSpaceRef cs); void CGContextSetRGBStrokeColor(CGContextRef c, - float red, float green, float blue, - float alpha); + double red, double green, double blue, + double alpha); void CGContextSetRGBFillColor(CGContextRef c, - float red, float green, float blue, - float alpha); + double red, double green, double blue, + double alpha); void CGContextDrawImage(CGContextRef c, CGRect rect, CGImageRef image); - void CGContextShowTextAtPoint(CGContextRef c, float x, float y, + void CGContextShowTextAtPoint(CGContextRef c, double x, double y, const(char)* str, size_t length); void CGContextStrokeLineSegments(CGContextRef c, const(CGPoint)* points, size_t count); @@ -12330,15 +12334,15 @@ private: void CGContextBeginPath(CGContextRef c); void CGContextDrawPath(CGContextRef c, CGPathDrawingMode mode); void CGContextAddEllipseInRect(CGContextRef c, CGRect rect); - void CGContextAddArc(CGContextRef c, float x, float y, float radius, - float startAngle, float endAngle, int clockwise); + void CGContextAddArc(CGContextRef c, double x, double y, double radius, + double startAngle, double endAngle, long clockwise); void CGContextAddRect(CGContextRef c, CGRect rect); void CGContextAddLines(CGContextRef c, const(CGPoint)* points, size_t count); void CGContextSaveGState(CGContextRef c); void CGContextRestoreGState(CGContextRef c); - void CGContextSelectFont(CGContextRef c, const(char)* name, float size, - uint textEncoding); + void CGContextSelectFont(CGContextRef c, const(char)* name, double size, + ulong textEncoding); CGAffineTransform CGContextGetTextMatrix(CGContextRef c); void CGContextSetTextMatrix(CGContextRef c, CGAffineTransform t); @@ -12348,7 +12352,7 @@ private: private: // A convenient method to create a CFString (=NSString) from a D string. CFStringRef createCFString(string str) { - return CFStringCreateWithBytes(null, str.ptr, cast(int) str.length, + return CFStringCreateWithBytes(null, str.ptr, cast(long) str.length, kCFStringEncodingUTF8, false); } @@ -12435,7 +12439,7 @@ version(OSXCocoa) { // if rawData had a length.... //assert(rawData.length == where.length); - for(int idx = 0; idx < where.length; idx += 4) { + for(long idx = 0; idx < where.length; idx += 4) { auto alpha = rawData[idx + 3]; if(alpha == 255) { where[idx + 0] = rawData[idx + 0]; // r @@ -12458,7 +12462,7 @@ version(OSXCocoa) { // if rawData had a length.... //assert(rawData.length == where.length); - for(int idx = 0; idx < where.length; idx += 4) { + for(long idx = 0; idx < where.length; idx += 4) { auto alpha = rawData[idx + 3]; if(alpha == 255) { rawData[idx + 0] = where[idx + 0]; // r @@ -12530,7 +12534,7 @@ version(OSXCocoa) { // end @property void outlineColor(Color color) { - float alphaComponent = color.a/255.0f; + double alphaComponent = color.a/255.0f; CGContextSetRGBStrokeColor(context, color.r/255.0f, color.g/255.0f, color.b/255.0f, alphaComponent); From 08e525e2de0fe667635204f6a6ef5d9e4f374772 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 2 Mar 2019 21:34:46 -0500 Subject: [PATCH 32/44] more stuff --- simpledisplay.d | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/simpledisplay.d b/simpledisplay.d index e6e5585..04c35d3 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -2662,10 +2662,13 @@ static struct GenericCursor { struct EventLoop { @disable this(); + /// Gets a reference to an existing event loop static EventLoop get() { return EventLoop(0, null); } + /// Construct an application-global event loop for yourself + /// See_Also: [SimpleWindow.setEventHandlers] this(long pulseTimeout, void delegate() handlePulse) { if(impl is null) impl = new EventLoopImpl(pulseTimeout, handlePulse); @@ -2693,12 +2696,14 @@ struct EventLoop { impl.refcount++; } + /// Runs the event loop until the whileCondition, if present, returns false int run(bool delegate() whileCondition = null) { assert(impl !is null); impl.notExited = true; return impl.run(whileCondition); } + /// Exits the event loop void exit() { assert(impl !is null); impl.notExited = false; From fe943409d16a17e4d5afdaf170b14733ed8133db Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 4 Mar 2019 10:19:29 -0500 Subject: [PATCH 33/44] add more context to exceptions --- dom.d | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dom.d b/dom.d index c80d9f8..1f1cbe0 100644 --- a/dom.d +++ b/dom.d @@ -1416,7 +1416,7 @@ class Element { body { auto e = cast(SomeElementType) getElementById(id); if(e is null) - throw new ElementNotFoundException(SomeElementType.stringof, "id=" ~ id, file, line); + throw new ElementNotFoundException(SomeElementType.stringof, "id=" ~ id, this, file, line); return e; } @@ -1431,7 +1431,7 @@ class Element { body { auto e = cast(SomeElementType) querySelector(selector); if(e is null) - throw new ElementNotFoundException(SomeElementType.stringof, selector, file, line); + throw new ElementNotFoundException(SomeElementType.stringof, selector, this, file, line); return e; } @@ -2140,7 +2140,7 @@ class Element { static if(!is(T == Element)) { auto t = cast(T) par; if(t is null) - throw new ElementNotFoundException("", tagName ~ " parent not found"); + throw new ElementNotFoundException("", tagName ~ " parent not found", this); } else auto t = par; @@ -3806,7 +3806,7 @@ T require(T = Element, string file = __FILE__, int line = __LINE__)(Element e) i body { auto ret = cast(T) e; if(ret is null) - throw new ElementNotFoundException(T.stringof, "passed value", file, line); + throw new ElementNotFoundException(T.stringof, "passed value", e, file, line); return ret; } @@ -5169,9 +5169,12 @@ class MarkupException : Exception { class ElementNotFoundException : Exception { /// type == kind of element you were looking for and search == a selector describing the search. - this(string type, string search, string file = __FILE__, size_t line = __LINE__) { + this(string type, string search, Element searchContext, string file = __FILE__, size_t line = __LINE__) { + this.searchContext = searchContext; super("Element of type '"~type~"' matching {"~search~"} not found.", file, line); } + + Element searchContext; } /// The html struct is used to differentiate between regular text nodes and html in certain functions From 0c328aa3b06cdfca7c3c022b71aa59f10f5af821 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 7 Mar 2019 22:23:00 -0500 Subject: [PATCH 34/44] notes to self --- cgi.d | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cgi.d b/cgi.d index daee713..3c35c64 100644 --- a/cgi.d +++ b/cgi.d @@ -4652,6 +4652,20 @@ bool isInvalidHandle(CgiConnectionHandle h) { return h == INVALID_CGI_CONNECTION_HANDLE; } +/+ +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv +https://support.microsoft.com/en-gb/help/181611/socket-overlapped-i-o-versus-blocking-nonblocking-mode +https://stackoverflow.com/questions/18018489/should-i-use-iocps-or-overlapped-wsasend-receive +https://docs.microsoft.com/en-us/windows/desktop/fileio/i-o-completion-ports +https://docs.microsoft.com/en-us/windows/desktop/fileio/createiocompletionport +https://docs.microsoft.com/en-us/windows/desktop/api/mswsock/nf-mswsock-acceptex +https://docs.microsoft.com/en-us/windows/desktop/Sync/waitable-timer-objects +https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-setwaitabletimer +https://docs.microsoft.com/en-us/windows/desktop/Sync/using-a-waitable-timer-with-an-asynchronous-procedure-call +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsagetoverlappedresult + ++/ + /++ You can customize your server by subclassing the appropriate server. Then, register your subclass at compile time with the [registerEventIoServer] template, or implement your own From b3df2871d280514f3dbad8b6d7ad59a7a35e527e Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 7 Mar 2019 22:23:51 -0500 Subject: [PATCH 35/44] stupid dmd warnings are useless --- cgi.d | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/cgi.d b/cgi.d index 3c35c64..17c91d4 100644 --- a/cgi.d +++ b/cgi.d @@ -5134,19 +5134,20 @@ interface EventSourceServer { version(fastcgi) throw new Exception("sending fcgi connections not supported"); + else { + auto fd = cgi.getOutputFileHandle(); + if(isInvalidHandle(fd)) + throw new Exception("bad fd from cgi!"); - auto fd = cgi.getOutputFileHandle(); - if(isInvalidHandle(fd)) - throw new Exception("bad fd from cgi!"); + EventSourceServerImplementation.SendableEventConnection sec; + sec.populate(cgi.responseChunked, eventUrl, lastEventId); - EventSourceServerImplementation.SendableEventConnection sec; - sec.populate(cgi.responseChunked, eventUrl, lastEventId); - - version(Posix) { - auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); - assert(res == sec.sizeof); - } else version(Windows) { - // FIXME + version(Posix) { + auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); + assert(res == sec.sizeof); + } else version(Windows) { + // FIXME + } } } From b106b2cc5c6eb0b7bbe7b02ee6c4ace3ef8f5d04 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 8 Mar 2019 09:16:35 -0500 Subject: [PATCH 36/44] wrong version on windows --- cgi.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgi.d b/cgi.d index 17c91d4..56f547f 100644 --- a/cgi.d +++ b/cgi.d @@ -4488,7 +4488,7 @@ version(Posix) { enum INVALID_CGI_CONNECTION_HANDLE = -1; } else version(Windows) { alias LocalServerConnectionHandle = HANDLE; - version(embedded_httpd) { + version(embedded_httpd_threads) { alias CgiConnectionHandle = SOCKET; enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; } else version(fastcgi) { From b426390abe8159fc368c77272425ffb791270dd9 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 9 Mar 2019 12:00:41 -0500 Subject: [PATCH 37/44] sanity check on header size jic --- cgi.d | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cgi.d b/cgi.d index 56f547f..cedb5ad 100644 --- a/cgi.d +++ b/cgi.d @@ -384,6 +384,12 @@ int locationOf(T)(T[] data, string item) { const(ubyte[]) d = cast(const(ubyte[])) data; const(ubyte[]) i = cast(const(ubyte[])) item; + // this is a vague sanity check to ensure we aren't getting insanely + // sized input that will infinite loop below. it should never happen; + // even huge file uploads ought to come in smaller individual pieces. + if(d.length > (int.max/2)) + throw new Exception("excessive block of input"); + for(int a = 0; a < d.length; a++) { if(a + i.length > d.length) return -1; From 6acfc5cb6e28d463fe24c7fc000de7fd27397179 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 9 Mar 2019 17:40:13 -0500 Subject: [PATCH 38/44] add unix domain socket support for scgi (and httpd but idk if that is useful) --- cgi.d | 55 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/cgi.d b/cgi.d index cedb5ad..5917cdd 100644 --- a/cgi.d +++ b/cgi.d @@ -3752,10 +3752,12 @@ class ListeningConnectionManager { while(!loopBroken && running) { auto sn = listener.accept(); - // disable Nagle's algorithm to avoid a 40ms delay when we send/recv - // on the socket because we do some buffering internally. I think this helps, - // certainly does for small requests, and I think it does for larger ones too - sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + if(tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + } while(queueLength >= queue.length) Thread.sleep(1.msecs); synchronized(this) { @@ -3771,14 +3773,49 @@ class ListeningConnectionManager { } } } + + // FIXME: i typically stop this with ctrl+c which never + // actually gets here. i need to do a sigint handler. + if(cleanup) + cleanup(); } } + bool tcp; + void delegate() cleanup; + this(string host, ushort port, void function(Socket) handler) { this.handler = handler; - listener = new TcpSocket(); - listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); - listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port)); + + if(host.startsWith("unix:")) { + version(Posix) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + string filename = host["unix:".length .. $].idup; + listener.bind(new UnixAddress(filename)); + cleanup = delegate() { + import std.file; + remove(filename); + }; + tcp = false; + } else { + throw new Exception("unix sockets not supported on this system"); + } + } else if(host.startsWith("abstract:")) { + version(linux) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + string filename = "\0" ~ host["abstract:".length .. $]; + listener.bind(new UnixAddress(filename)); + tcp = false; + } else { + throw new Exception("abstract unix sockets not supported on this system"); + } + } else { + listener = new TcpSocket(); + listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); + listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port)); + tcp = true; + } + listener.listen(128); } @@ -4501,8 +4538,8 @@ version(Posix) { alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point. enum INVALID_CGI_CONNECTION_HANDLE = null; } else version(scgi) { - alias CgiConnectionHandle = HANDLE; - enum INVALID_CGI_CONNECTION_HANDLE = null; + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; } else { /* version(plain_cgi) */ alias CgiConnectionHandle = HANDLE; enum INVALID_CGI_CONNECTION_HANDLE = null; From c266c59b1f096363c6b4396219eddb0c2460179a Mon Sep 17 00:00:00 2001 From: Adam Ruppe Date: Sun, 10 Mar 2019 17:18:57 -0400 Subject: [PATCH 39/44] (very slightly) better mac support --- simpledisplay.d | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/simpledisplay.d b/simpledisplay.d index 0e9d232..925ad42 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -12574,17 +12574,19 @@ version(OSXCocoa) { mixin template NativeScreenPainterImplementation() { CGContextRef context; ubyte[4] _outlineComponents; + id view; void create(NativeWindowHandle window) { context = window.drawingContext; + view = window.view; } void dispose() { + setNeedsDisplay(view, true); } // NotYetImplementedException Size textSize(in char[] txt) { return Size(32, 16); throw new NotYetImplementedException(); } - void pen(Pen p) {} void rasterOp(RasterOp op) {} Pen _activePen; Color _fillColor; @@ -12595,7 +12597,9 @@ version(OSXCocoa) { // end - @property void outlineColor(Color color) { + void pen(Pen pen) { + _activePen = pen; + auto color = pen.color; // FIXME double alphaComponent = color.a/255.0f; CGContextSetRGBStrokeColor(context, color.r/255.0f, color.g/255.0f, color.b/255.0f, alphaComponent); @@ -12647,7 +12651,7 @@ version(OSXCocoa) { _outlineComponents[1]*invAlpha, _outlineComponents[2]*invAlpha, _outlineComponents[3]/255.0f); - CGContextShowTextAtPoint(context, x, y, text.ptr, text.length); + CGContextShowTextAtPoint(context, x, y + 12 /* this is cuz this picks baseline but i want bounding box */, text.ptr, text.length); // auto cfstr = cast(id)createCFString(text); // objc_msgSend(cfstr, sel_registerName("drawAtPoint:withAttributes:"), // NSPoint(x, y), null); @@ -12814,9 +12818,13 @@ version(OSXCocoa) { // will like it. Let's leave it to the native handler. // perform the default action. - auto superData = objc_super(self, superclass(self)); - alias extern(C) void function(objc_super*, SEL, id) T; - (cast(T)&objc_msgSendSuper)(&superData, _cmd, event); + + // so the default action is to make a bomp sound and i dont want that + // sooooooooo yeah not gonna do that. + + //auto superData = objc_super(self, superclass(self)); + //alias extern(C) void function(objc_super*, SEL, id) T; + //(cast(T)&objc_msgSendSuper)(&superData, _cmd, event); } } From 49341201fe7e3e76068f72049b9a3a94d5820c69 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 14 Mar 2019 16:51:34 -0400 Subject: [PATCH 40/44] real sux --- color.d | 97 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/color.d b/color.d index 39d7179..ab59287 100644 --- a/color.d +++ b/color.d @@ -6,8 +6,8 @@ module arsd.color; // importing phobos explodes the size of this code 10x, so not doing it. private { - real toInternal(T)(scope const(char)[] s) { - real accumulator = 0.0; + double toInternal(T)(scope const(char)[] s) { + double accumulator = 0.0; size_t i = s.length; foreach(idx, c; s) { if(c >= '0' && c <= '9') { @@ -17,21 +17,21 @@ private { i = idx + 1; break; } else { - string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from "; + string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make double from "; wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s; throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute); } } - real accumulator2 = 0.0; - real count = 1; + double accumulator2 = 0.0; + double count = 1; foreach(c; s[i .. $]) { if(c >= '0' && c <= '9') { accumulator2 *= 10; accumulator2 += c - '0'; count *= 10; } else { - string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from "; + string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make double from "; wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s; throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute); } @@ -64,8 +64,8 @@ private { return cast(string) ret; } - string toInternal(T)(real a) { - // a simplifying assumption here is the fact that we only use this in one place: toInternal!string(cast(real) a / 255) + string toInternal(T)(double a) { + // a simplifying assumption here is the fact that we only use this in one place: toInternal!string(cast(double) a / 255) // thus we know this will always be between 0.0 and 1.0, inclusive. if(a <= 0.0) return "0.0"; @@ -78,16 +78,16 @@ private { } nothrow @safe @nogc pure - real absInternal(real a) { return a < 0 ? -a : a; } + double absInternal(double a) { return a < 0 ? -a : a; } nothrow @safe @nogc pure - real minInternal(real a, real b, real c) { + double minInternal(double a, double b, double c) { auto m = a; if(b < m) m = b; if(c < m) m = c; return m; } nothrow @safe @nogc pure - real maxInternal(real a, real b, real c) { + double maxInternal(double a, double b, double c) { auto m = a; if(b > m) m = b; if(c > m) m = c; @@ -226,7 +226,7 @@ struct Color { if(a == 255) return "#" ~ toHexInternal(r) ~ toHexInternal(g) ~ toHexInternal(b); else { - return "rgba("~toInternal!string(r)~", "~toInternal!string(g)~", "~toInternal!string(b)~", "~toInternal!string(cast(real)a / 255.0)~")"; + return "rgba("~toInternal!string(r)~", "~toInternal!string(g)~", "~toInternal!string(b)~", "~toInternal!string(cast(double)a / 255.0)~")"; } } @@ -277,15 +277,15 @@ struct Color { assert(s[$-1] == ')'); s = s[s.startsWithInternal("hsl(") ? 4 : 5 .. $ - 1]; // the closing paren - real[3] hsl; + double[3] hsl; ubyte a = 255; auto parts = s.splitInternal(','); foreach(i, part; parts) { if(i < 3) - hsl[i] = toInternal!real(part.stripInternal); + hsl[i] = toInternal!double(part.stripInternal); else - a = clampToByte(cast(int) (toInternal!real(part.stripInternal) * 255)); + a = clampToByte(cast(int) (toInternal!double(part.stripInternal) * 255)); } c = .fromHsl(hsl); @@ -302,7 +302,7 @@ struct Color { auto parts = s.splitInternal(','); foreach(i, part; parts) { // lol the loop-switch pattern - auto v = toInternal!real(part.stripInternal); + auto v = toInternal!double(part.stripInternal); switch(i) { case 0: // red c.r = clampToByte(cast(int) v); @@ -353,7 +353,7 @@ struct Color { } /// from hsl - static Color fromHsl(real h, real s, real l) { + static Color fromHsl(double h, double s, double l) { return .fromHsl(h, s, l); } @@ -460,22 +460,26 @@ private ubyte fromHexInternal(in char[] s) { /// Converts hsl to rgb Color fromHsl(real[3] hsl) nothrow pure @safe @nogc { + return fromHsl(cast(double) hsl[0], cast(double) hsl[1], cast(double) hsl[2]); +} + +Color fromHsl(double[3] hsl) nothrow pure @safe @nogc { return fromHsl(hsl[0], hsl[1], hsl[2]); } /// Converts hsl to rgb -Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { +Color fromHsl(double h, double s, double l, double a = 255) nothrow pure @safe @nogc { h = h % 360; - real C = (1 - absInternal(2 * l - 1)) * s; + double C = (1 - absInternal(2 * l - 1)) * s; - real hPrime = h / 60; + double hPrime = h / 60; - real X = C * (1 - absInternal(hPrime % 2 - 1)); + double X = C * (1 - absInternal(hPrime % 2 - 1)); - real r, g, b; + double r, g, b; - if(h is real.nan) + if(h is double.nan) r = g = b = 0; else if (hPrime >= 0 && hPrime < 1) { r = C; @@ -503,7 +507,7 @@ Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { b = X; } - real m = l - C / 2; + double m = l - C / 2; r += m; g += m; @@ -517,22 +521,23 @@ Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { } /// Converts an RGB color into an HSL triplet. useWeightedLightness will try to get a better value for luminosity for the human eye, which is more sensitive to green than red and more to red than blue. If it is false, it just does average of the rgb. -real[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted @nogc { - real r1 = cast(real) c.r / 255; - real g1 = cast(real) c.g / 255; - real b1 = cast(real) c.b / 255; +double[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted @nogc { + double r1 = cast(double) c.r / 255; + double g1 = cast(double) c.g / 255; + double b1 = cast(double) c.b / 255; - real maxColor = maxInternal(r1, g1, b1); - real minColor = minInternal(r1, g1, b1); + double maxColor = maxInternal(r1, g1, b1); + double minColor = minInternal(r1, g1, b1); - real L = (maxColor + minColor) / 2 ; + double L = (maxColor + minColor) / 2 ; if(useWeightedLightness) { // the colors don't affect the eye equally // this is a little more accurate than plain HSL numbers L = 0.2126*r1 + 0.7152*g1 + 0.0722*b1; + // maybe a better number is 299, 587, 114 } - real S = 0; - real H = 0; + double S = 0; + double H = 0; if(maxColor != minColor) { if(L < 0.5) { S = (maxColor - minColor) / (maxColor + minColor); @@ -557,7 +562,7 @@ real[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted } /// . -Color lighten(Color c, real percentage) nothrow pure @safe @nogc { +Color lighten(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] *= (1 + percentage); if(hsl[2] > 1) @@ -566,7 +571,7 @@ Color lighten(Color c, real percentage) nothrow pure @safe @nogc { } /// . -Color darken(Color c, real percentage) nothrow pure @safe @nogc { +Color darken(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] *= (1 - percentage); return fromHsl(hsl); @@ -574,7 +579,7 @@ Color darken(Color c, real percentage) nothrow pure @safe @nogc { /// for light colors, call darken. for dark colors, call lighten. /// The goal: get toward center grey. -Color moderate(Color c, real percentage) nothrow pure @safe @nogc { +Color moderate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); if(hsl[2] > 0.5) hsl[2] *= (1 - percentage); @@ -590,7 +595,7 @@ Color moderate(Color c, real percentage) nothrow pure @safe @nogc { } /// the opposite of moderate. Make darks darker and lights lighter -Color extremify(Color c, real percentage) nothrow pure @safe @nogc { +Color extremify(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c, true); if(hsl[2] < 0.5) hsl[2] *= (1 - percentage); @@ -626,7 +631,7 @@ Color makeTextColor(Color c) nothrow pure @safe @nogc { // These provide functional access to hsl manipulation; useful if you need a delegate -Color setLightness(Color c, real lightness) nothrow pure @safe @nogc { +Color setLightness(Color c, double lightness) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] = lightness; return fromHsl(hsl); @@ -634,28 +639,28 @@ Color setLightness(Color c, real lightness) nothrow pure @safe @nogc { /// -Color rotateHue(Color c, real degrees) nothrow pure @safe @nogc { +Color rotateHue(Color c, double degrees) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[0] += degrees; return fromHsl(hsl); } /// -Color setHue(Color c, real hue) nothrow pure @safe @nogc { +Color setHue(Color c, double hue) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[0] = hue; return fromHsl(hsl); } /// -Color desaturate(Color c, real percentage) nothrow pure @safe @nogc { +Color desaturate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] *= (1 - percentage); return fromHsl(hsl); } /// -Color saturate(Color c, real percentage) nothrow pure @safe @nogc { +Color saturate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] *= (1 + percentage); if(hsl[1] > 1) @@ -664,7 +669,7 @@ Color saturate(Color c, real percentage) nothrow pure @safe @nogc { } /// -Color setSaturation(Color c, real saturation) nothrow pure @safe @nogc { +Color setSaturation(Color c, double saturation) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] = saturation; return fromHsl(hsl); @@ -785,9 +790,9 @@ void main() { import browser.document; foreach(ele; document.querySelectorAll("input")) { ele.addEventListener("change", { - auto h = toInternal!real(document.querySelector("input[name=h]").value); - auto s = toInternal!real(document.querySelector("input[name=s]").value); - auto l = toInternal!real(document.querySelector("input[name=l]").value); + auto h = toInternal!double(document.querySelector("input[name=h]").value); + auto s = toInternal!double(document.querySelector("input[name=s]").value); + auto l = toInternal!double(document.querySelector("input[name=l]").value); Color c = Color.fromHsl(h, s, l); From 95755a1512258750a37f0df3cfd46f094c58f996 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 23 Mar 2019 14:44:53 -0400 Subject: [PATCH 41/44] making it suck less --- minigui_addons/color_dialog.d | 119 +++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 25 deletions(-) diff --git a/minigui_addons/color_dialog.d b/minigui_addons/color_dialog.d index b29ed0e..18cce3f 100644 --- a/minigui_addons/color_dialog.d +++ b/minigui_addons/color_dialog.d @@ -11,7 +11,7 @@ static if(UsingWin32Widgets) /++ +/ -void showColorDialog(Window owner, Color current, void delegate(Color choice) onOK, void delegate() onCancel = null) { +auto showColorDialog(Window owner, Color current, void delegate(Color choice) onOK, void delegate() onCancel = null) { static if(UsingWin32Widgets) { import core.sys.windows.windows; static COLORREF[16] customColors; @@ -30,6 +30,7 @@ void showColorDialog(Window owner, Color current, void delegate(Color choice) on } else static if(UsingCustomWidgets) { auto cpd = new ColorPickerDialog(current, onOK, owner); cpd.show(); + return cpd; } else static assert(0); } @@ -53,7 +54,7 @@ class ColorPickerDialog : Dialog { void delegate(Color) onOK; this(Color current, void delegate(Color) onOK, Window owner) { - super(360, 350, "Color picker"); + super(360, 460, "Color picker"); this.onOK = onOK; @@ -79,7 +80,7 @@ class ColorPickerDialog : Dialog { canUseImage = true; if(hslImage is null && canUseImage) { - auto img = new TrueColorImage(180, 128); + auto img = new TrueColorImage(360, 255); double h = 0.0, s = 1.0, l = 0.5; foreach(y; 0 .. img.height) { foreach(x; 0 .. img.width) { @@ -99,60 +100,112 @@ class ColorPickerDialog : Dialog { this() { super(t); } override int minHeight() { return hslImage ? hslImage.height : 4; } override int maxHeight() { return hslImage ? hslImage.height : 4; } + override int marginBottom() { return 4; } override void paint(ScreenPainter painter) { if(hslImage) hslImage.drawAt(painter, Point(0, 0)); } }; + auto hr = new HorizontalLayout(t); + auto vlRgb = new class VerticalLayout { this() { - super(t); + super(hr); } override int maxWidth() { return 150; }; }; + auto vlHsl = new class VerticalLayout { + this() { + super(hr); + } + override int maxWidth() { return 150; }; + }; + + h = new LabeledLineEdit("Hue:", vlHsl); + s = new LabeledLineEdit("Saturation:", vlHsl); + l = new LabeledLineEdit("Lightness:", vlHsl); + + css = new LabeledLineEdit("CSS:", vlHsl); + r = new LabeledLineEdit("Red:", vlRgb); g = new LabeledLineEdit("Green:", vlRgb); b = new LabeledLineEdit("Blue:", vlRgb); a = new LabeledLineEdit("Alpha:", vlRgb); import std.conv; + import std.format; - r.content = to!string(current.r); - g.content = to!string(current.g); - b.content = to!string(current.b); - a.content = to!string(current.a); + void updateCurrent() { + r.content = to!string(current.r); + g.content = to!string(current.g); + b.content = to!string(current.b); + a.content = to!string(current.a); + + auto hsl = current.toHsl; + + h.content = format("%0.2f", hsl[0]); + s.content = format("%0.2f", hsl[1]); + l.content = format("%0.2f", hsl[2]); + + css.content = current.toCssString(); + } + + updateCurrent(); r.addEventListener("focus", &r.selectAll); g.addEventListener("focus", &g.selectAll); b.addEventListener("focus", &b.selectAll); a.addEventListener("focus", &a.selectAll); + h.addEventListener("focus", &h.selectAll); + s.addEventListener("focus", &s.selectAll); + l.addEventListener("focus", &l.selectAll); - if(hslImage !is null) - wid.addEventListener("mousedown", (Event event) { - auto h = cast(double) event.clientX / hslImage.width * 360.0; - auto s = 1.0 - (cast(double) event.clientY / hslImage.height * 1.0); - auto l = 0.5; + css.addEventListener("focus", &css.selectAll); - auto color = Color.fromHsl(h, s, l); - - r.content = to!string(color.r); - g.content = to!string(color.g); - b.content = to!string(color.b); - a.content = to!string(color.a); - - }); - - Color currentColor() { + void convertFromHsl() { try { - return Color(to!int(r.content), to!int(g.content), to!int(b.content), to!int(a.content)); + auto c = Color.fromHsl(h.content.to!double, s.content.to!double, l.content.to!double); + c.a = a.content.to!ubyte; + current = c; + updateCurrent(); } catch(Exception e) { - return Color.transparent; } } + h.addEventListener("change", &convertFromHsl); + s.addEventListener("change", &convertFromHsl); + l.addEventListener("change", &convertFromHsl); + + css.addEventListener("change", () { + current = Color.fromString(css.content); + updateCurrent(); + }); + + void helper(Event event) { + auto h = cast(double) event.clientX / hslImage.width * 360.0; + auto s = 1.0 - (cast(double) event.clientY / hslImage.height * 1.0); + auto l = this.l.content.to!double; + + current = Color.fromHsl(h, s, l); + current.a = a.content.to!ubyte; + + updateCurrent(); + + auto e2 = new Event("change", this); + e2.dispatch(); + } + + if(hslImage !is null) + wid.addEventListener("mousedown", &helper); + if(hslImage !is null) + wid.addEventListener("mousemove", (Event event) { + if(event.state & ModifierState.leftButtonDown) + helper(event); + }); + this.addEventListener("keydown", (Event event) { if(event.key == Key.Enter || event.key == Key.PadEnter) OK(); @@ -205,6 +258,22 @@ class ColorPickerDialog : Dialog { LabeledLineEdit b; LabeledLineEdit a; + LabeledLineEdit h; + LabeledLineEdit s; + LabeledLineEdit l; + + LabeledLineEdit css; + + Color currentColor() { + import std.conv; + try { + return Color(to!int(r.content), to!int(g.content), to!int(b.content), to!int(a.content)); + } catch(Exception e) { + return Color.transparent; + } + } + + override void OK() { import std.conv; try { From dc8c03b7c7aec4fa65467c1e5af3065050c3fecd Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 23 Mar 2019 14:45:15 -0400 Subject: [PATCH 42/44] sucking a wee bit less --- color.d | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/color.d b/color.d index ab59287..dd863d0 100644 --- a/color.d +++ b/color.d @@ -217,6 +217,7 @@ struct Color { /// Return black-and-white color Color toBW() () nothrow pure @safe @nogc { + // FIXME: gamma? int intens = clampToByte(cast(int)(0.2126*r+0.7152*g+0.0722*b)); return Color(intens, intens, intens, a); } @@ -520,6 +521,15 @@ Color fromHsl(double h, double s, double l, double a = 255) nothrow pure @safe @ cast(int)(a)); } +/// Assumes the input `u` is already between 0 and 1 fyi. +nothrow pure @safe @nogc +double srgbToLinearRgb(double u) { + if(u < 0.4045) + return u / 12.92; + else + return ((u + 0.055) / 1.055) ^^ 2.4; +} + /// Converts an RGB color into an HSL triplet. useWeightedLightness will try to get a better value for luminosity for the human eye, which is more sensitive to green than red and more to red than blue. If it is false, it just does average of the rgb. double[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted @nogc { double r1 = cast(double) c.r / 255; @@ -533,7 +543,7 @@ double[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @truste if(useWeightedLightness) { // the colors don't affect the eye equally // this is a little more accurate than plain HSL numbers - L = 0.2126*r1 + 0.7152*g1 + 0.0722*b1; + L = 0.2126*srgbToLinearRgb(r1) + 0.7152*srgbToLinearRgb(g1) + 0.0722*srgbToLinearRgb(b1); // maybe a better number is 299, 587, 114 } double S = 0; From 8131c414408766c221565eca7eb43636f9f0311d Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 23 Mar 2019 14:45:24 -0400 Subject: [PATCH 43/44] dox sux less --- cgi.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgi.d b/cgi.d index 5917cdd..23c605b 100644 --- a/cgi.d +++ b/cgi.d @@ -568,7 +568,7 @@ class Cgi { Non-simulation arguments: --port xxx listening port for non-cgi things (valid for the cgi interfaces) - --listening-host the ip address the application should listen on + --listening-host the ip address the application should listen on, or if you want to use unix domain sockets, it is here you can set them: `--listening-host unix:filename` or, on Linux, `--listening-host abstract:name`. */ From 3132e05f0f00ef30da31a7dbafa4d7f5203ae4a0 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 23 Mar 2019 18:05:42 -0400 Subject: [PATCH 44/44] dox --- gamehelpers.d | 14 ++++++++++++-- joystick.d | 17 ++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/gamehelpers.d b/gamehelpers.d index dd86c02..c2c8d42 100644 --- a/gamehelpers.d +++ b/gamehelpers.d @@ -31,9 +31,10 @@ } int x, y; - override void update(Duration deltaTime) { + override bool update(Duration deltaTime) { x += 1; y += 1; + return true; } override SimpleWindow getWindow() { @@ -100,7 +101,8 @@ class GameHelperBase { abstract void drawFrame(); /// Implement this to update. The deltaTime tells how much real time has passed since the last update. - abstract void update(Duration deltaTime); + /// Returns true if anything changed, which will queue up a redraw + abstract bool update(Duration deltaTime); //abstract void fillAudioBuffer(short[] buffer); /// Returns the main game window. This function will only be @@ -155,6 +157,14 @@ void runGame(T : GameHelperBase)(T game, int maxUpdateRate = 20, int maxRedrawRa delegate (KeyEvent ke) { game.keyboardState[ke.hardwareCode] = ke.pressed; + /* + switch(ke.key) { + case Key.UpArrow: + game.joysticks[0] + break; + default: + } + */ // FIXME } ); diff --git a/joystick.d b/joystick.d index 27c4a09..7a76069 100644 --- a/joystick.d +++ b/joystick.d @@ -167,6 +167,8 @@ version(linux) { // I'd just use my xbox controller. ); + /// For Linux only, reads the latest joystick events into the change buffer, if available. + /// It is non-blocking void readJoystickEvents(int fd) { js_event event; @@ -312,6 +314,7 @@ int enableJoystickInput( // return 0; } +/// void closeJoysticks() { version(linux) { foreach(ref fd; joystickFds) { @@ -330,33 +333,36 @@ void closeJoysticks() { } else static assert(0); } +/// struct JoystickUpdate { + /// int player; JoystickState old; JoystickState current; - // changes from last update + /// changes from last update bool buttonWasJustPressed(Button button) { return buttonIsPressed(button) && !oldButtonIsPressed(button); } + /// ditto bool buttonWasJustReleased(Button button) { return !buttonIsPressed(button) && oldButtonIsPressed(button); } - // this is normalized down to a 16 step change - // and ignores a dead zone near the middle + /// this is normalized down to a 16 step change + /// and ignores a dead zone near the middle short axisChange(Axis axis) { return cast(short) (axisPosition(axis) - oldAxisPosition(axis)); } - // current state + /// current state bool buttonIsPressed(Button button) { return buttonIsPressedHelper(button, ¤t); } - // Note: UP is negative! + /// Note: UP is negative! short axisPosition(Axis axis, short digitalFallbackValue = short.max) { return axisPositionHelper(axis, ¤t, digitalFallbackValue); } @@ -521,6 +527,7 @@ struct JoystickUpdate { } } +/// JoystickUpdate getJoystickUpdate(int player) { static JoystickState[4] previous;