diff --git a/cgi.d b/cgi.d index 66a9be5..23c605b 100644 --- a/cgi.d +++ b/cgi.d @@ -1,6 +1,9 @@ // FIXME: if an exception is thrown, we shouldn't necessarily cache... // FIXME: there's some annoying duplication of code in the various versioned mains +// FIXME: I might make a cgi proxy class which can change things; the underlying one is still immutable +// but the later one can edit and simplify the api. You'd have to use the subclass tho! + /* void foo(int f, @("test") string s) {} @@ -66,6 +69,34 @@ void main() { --- + Compile_versions: + + -version=plain_cgi + The default - a traditional, plain CGI executable will be generated. + -version=fastcgi + A FastCGI executable will be generated. + -version=scgi + A SCGI (SimpleCGI) executable will be generated. + -version=embedded_httpd + A HTTP server will be embedded in the generated executable. + -version=embedded_httpd_threads + The embedded HTTP server will use a single process with a thread pool. + -version=embedded_httpd_processes + The embedded HTTP server will use a prefork style process pool. + + -version=cgi_with_websocket + The CGI class has websocket server support. + + -version=with_openssl # not currently used + + -version=embedded_httpd_processes_accept_after_fork + It will call accept() in each child process, after forking. This is currently the only option, though I am experimenting with other ideas. + + -version=cgi_embedded_sessions + The session server will be embedded in the cgi.d server process + -version=cgi_session_server_process + The session will be provided in a separate process, provided by cgi.d. + Compile_and_run: For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory. @@ -253,7 +284,7 @@ void main() { Copyright: - cgi.d copyright 2008-2018, Adam D. Ruppe. Provided under the Boost Software License. + cgi.d copyright 2008-2019, Adam D. Ruppe. Provided under the Boost Software License. Yes, this file is almost ten years old, and yes, it is still actively maintained and used. +/ @@ -261,11 +292,15 @@ module arsd.cgi; static import std.file; +// for a single thread, linear request thing, use: +// -version=embedded_httpd_threads -version=cgi_no_threads + version(embedded_httpd) { version(linux) version=embedded_httpd_processes; - else + else { version=embedded_httpd_threads; + } /* version(with_openssl) { @@ -275,6 +310,31 @@ version(embedded_httpd) { */ } +version(embedded_httpd_processes) + version=embedded_httpd_processes_accept_after_fork; // I am getting much better average performance on this, so just keeping it. But the other way MIGHT help keep the variation down so i wanna keep the code to play with later + +version(embedded_httpd_threads) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} +version(scgi) { + // unless the user overrides the default.. + version(cgi_session_server_process) + {} + else + version=cgi_embedded_sessions; +} + +// fall back if the other is not defined so we can cleanly version it below +version(cgi_embedded_sessions) {} +else version=cgi_session_server_process; + + +version=cgi_with_websocket; + enum long defaultMaxContentLength = 5_000_000; /* @@ -300,6 +360,9 @@ public import std.string; public import std.stdio; public import std.conv; import std.uri; +import std.uni; +import std.algorithm.comparison; +import std.algorithm.searching; import std.exception; import std.base64; static import std.algorithm; @@ -321,6 +384,12 @@ int locationOf(T)(T[] data, string item) { const(ubyte[]) d = cast(const(ubyte[])) data; const(ubyte[]) i = cast(const(ubyte[])) item; + // this is a vague sanity check to ensure we aren't getting insanely + // sized input that will infinite loop below. it should never happen; + // even huge file uploads ought to come in smaller individual pieces. + if(d.length > (int.max/2)) + throw new Exception("excessive block of input"); + for(int a = 0; a < d.length; a++) { if(a + i.length > d.length) return -1; @@ -499,7 +568,7 @@ class Cgi { Non-simulation arguments: --port xxx listening port for non-cgi things (valid for the cgi interfaces) - --listening-host the ip address the application should listen on + --listening-host the ip address the application should listen on, or if you want to use unix domain sockets, it is here you can set them: `--listening-host unix:filename` or, on Linux, `--listening-host abstract:name`. */ @@ -552,6 +621,9 @@ class Cgi { environmentVariables = cast(const) environment.toAA; + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + foreach(arg; args[1 .. $]) { if(arg.startsWith("--")) { nextArgIs = arg[2 .. $]; @@ -613,7 +685,7 @@ class Cgi { lookingForMethod = false; lookingForUri = true; - if(arg.toLower() == "commandline") + if(arg.asLowerCase().equal("commandline")) requestMethod = RequestMethod.CommandLine; else requestMethod = to!RequestMethod(arg.toUpper()); @@ -631,9 +703,11 @@ class Cgi { } } else { // it is an argument of some sort - if(requestMethod == Cgi.RequestMethod.POST) { + if(requestMethod == Cgi.RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) { auto parts = breakUp(arg); _post[parts[0]] ~= parts[1]; + allPostNamesInOrder ~= parts[0]; + allPostValuesInOrder ~= parts[1]; } else { if(_queryString.length) _queryString ~= "&"; @@ -652,7 +726,7 @@ class Cgi { cookies = keepLastOf(cookiesArray); queryString = _queryString; - getArray = cast(immutable) decodeVariables(queryString); + getArray = cast(immutable) decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); get = keepLastOf(getArray); postArray = cast(immutable) _post; @@ -681,6 +755,19 @@ class Cgi { this.postJson = null; } + private { + string[] allPostNamesInOrder; + string[] allPostValuesInOrder; + string[] allGetNamesInOrder; + string[] allGetValuesInOrder; + } + + CgiConnectionHandle getOutputFileHandle() { + return _outputFileHandle; + } + + CgiConnectionHandle _outputFileHandle = INVALID_CGI_CONNECTION_HANDLE; + /** Initializes it using a CGI or CGI-like interface */ this(long maxContentLength = defaultMaxContentLength, // use this to override the environment variable listing @@ -776,9 +863,9 @@ class Cgi { } - get = getGetVariables(queryString); - auto ugh = decodeVariables(queryString); + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); getArray = assumeUnique(ugh); + get = keepLastOf(getArray); // NOTE: on shitpache, you need to specifically forward this @@ -807,7 +894,7 @@ class Cgi { lastEventId = getenv("HTTP_LAST_EVENT_ID"); auto ka = getenv("HTTP_CONNECTION"); - if(ka.length && ka.toLower().indexOf("keep-alive") != -1) + if(ka.length && ka.asLowerCase().canFind("keep-alive")) keepAliveRequested = true; auto or = getenv("HTTP_ORIGIN"); @@ -825,7 +912,7 @@ class Cgi { // FIXME: DOCUMENT_ROOT? // FIXME: what about PUT? - if(requestMethod == RequestMethod.POST) { + if(requestMethod == RequestMethod.POST || requestMethod == Cgi.RequestMethod.PATCH || requestMethod == Cgi.RequestMethod.PUT) { version(preserveData) // a hack to make forwarding simpler immutable(ubyte)[] data; size_t amountReceived = 0; @@ -1057,6 +1144,12 @@ class Cgi { } else if(pps.contentType == "multipart/form-data") { pps.isMultipart = true; enforce(pps.boundary.length, "no boundary"); + } else if(pps.contentType == "text/plain") { + pps.isMultipart = false; + pps.isJson = true; // FIXME: hack, it isn't actually this + } else if(pps.contentType == "text/xml") { // FIXME: what if we used this as a fallback? + pps.isMultipart = false; + pps.isJson = true; // FIXME: hack, it isn't actually this } else if(pps.contentType == "application/json") { pps.isJson = true; pps.isMultipart = false; @@ -1136,9 +1229,16 @@ class Cgi { // I used to not do it, but I think I should, since it is there... pps._post[pps.piece.name] ~= pps.piece.filename; pps._files[pps.piece.name] ~= pps.piece; - } else + + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= pps.piece.filename; + } else { pps._post[pps.piece.name] ~= cast(string) pps.piece.content; + allPostNamesInOrder ~= pps.piece.name; + allPostValuesInOrder ~= cast(string) pps.piece.content; + } + /* stderr.writeln("RECEIVED: ", pps.piece.name, "=", pps.piece.content.length < 1000 @@ -1383,7 +1483,7 @@ class Cgi { if(pps.isJson) pps.postJson = cast(string) pps.buffer; else - pps._post = decodeVariables(cast(string) pps.buffer); + pps._post = decodeVariables(cast(string) pps.buffer, "&", &allPostNamesInOrder, &allPostValuesInOrder); version(preserveData) originalPostData = pps.buffer; } else { @@ -1548,8 +1648,7 @@ class Cgi { pathInfo = requestUri[pathInfoStarts..question]; } - get = cast(string[string]) getGetVariables(queryString); - auto ugh = decodeVariables(queryString); + auto ugh = decodeVariables(queryString, "&", &allGetNamesInOrder, &allGetValuesInOrder); getArray = cast(string[][string]) assumeUnique(ugh); if(header.indexOf("HTTP/1.0") != -1) { @@ -1579,7 +1678,7 @@ class Cgi { else if (name == "connection") { if(value == "close" && closeConnection) *closeConnection = true; - if(value.toLower().indexOf("keep-alive") != -1) { + if(value.asLowerCase().canFind("keep-alive")) { keepAliveRequested = true; // on http 1.0, the connection is closed by default, @@ -1684,7 +1783,7 @@ class Cgi { this.queryString = queryString; this.scriptName = scriptName; - this.get = cast(immutable) get; + this.get = keepLastOf(getArray); this.getArray = cast(immutable) getArray; this.keepAliveRequested = keepAliveRequested; this.acceptsGzip = acceptsGzip; @@ -1719,43 +1818,6 @@ class Cgi { return assumeUnique(forTheLoveOfGod); } - // this function only exists because of the with_cgi_packed thing, which is - // a filthy hack I put in here for a work app. Which still depends on it, so it - // stays for now. But I want to remove it. - private immutable(string[string]) getGetVariables(in string queryString) { - if(queryString.length) { - auto _get = decodeVariablesSingle(queryString); - - // Some sites are shit and don't let you handle multiple parameters. - // If so, compile this in and encode it as a single parameter - version(with_cgi_packed) { - auto idx = pathInfo.indexOf("PACKED"); - if(idx != -1) { - auto pi = pathInfo[idx + "PACKED".length .. $]; - - auto _unpacked = decodeVariables( - cast(string) base64UrlDecode(pi)); - - foreach(k, v; _unpacked) - _get[k] = v[$-1]; - // possible problem: it used to cut PACKED off the path info - // but it doesn't now. I want to kill this crap anyway though. - } - - if("arsd_packed_data" in getArray) { - auto _unpacked = decodeVariables( - cast(string) base64UrlDecode(getArray["arsd_packed_data"][0])); - - foreach(k, v; _unpacked) - _get[k] = v[$-1]; - } - } - - return assumeUnique(_get); - } - - return null; - } /// Very simple method to require a basic auth username and password. /// If the http request doesn't include the required credentials, it throws a /// HTTP 401 error, and an exception. @@ -2637,18 +2699,28 @@ struct Uri { */ /// breaks down a url encoded string -string[][string] decodeVariables(string data, string separator = "&") { +string[][string] decodeVariables(string data, string separator = "&", string[]* namesInOrder = null, string[]* valuesInOrder = null) { auto vars = data.split(separator); string[][string] _get; foreach(var; vars) { auto equal = var.indexOf("="); + string name; + string value; if(equal == -1) { - _get[decodeComponent(var)] ~= ""; + name = decodeComponent(var); + value = ""; } else { //_get[decodeComponent(var[0..equal])] ~= decodeComponent(var[equal + 1 .. $].replace("+", " ")); // stupid + -> space conversion. - _get[decodeComponent(var[0..equal].replace("+", " "))] ~= decodeComponent(var[equal + 1 .. $].replace("+", " ")); + name = decodeComponent(var[0..equal].replace("+", " ")); + value = decodeComponent(var[equal + 1 .. $].replace("+", " ")); } + + _get[name] ~= value; + if(namesInOrder) + (*namesInOrder) ~= name; + if(valuesInOrder) + (*valuesInOrder) ~= value; } return _get; } @@ -2832,315 +2904,414 @@ bool isCgiRequestMethod(string s) { /// If you want to use a subclass of Cgi with generic main, use this mixin. mixin template CustomCgiMain(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { // kinda hacky - the T... is passed to Cgi's constructor in standard cgi mode, and ignored elsewhere - mixin CustomCgiMainImpl!(CustomCgi, fun, maxContentLength) customCgiMainImpl_; - void main(string[] args) { - customCgiMainImpl_.cgiMainImpl(args); + cgiMainImpl!(fun, CustomCgi, maxContentLength)(args); } } version(embedded_httpd_processes) int processPoolSize = 8; -mixin template CustomCgiMainImpl(CustomCgi, alias fun, long maxContentLength = defaultMaxContentLength) if(is(CustomCgi : Cgi)) { - void cgiMainImpl(string[] args) { - - - // we support command line thing for easy testing everywhere - // it needs to be called ./app method uri [other args...] - if(args.length >= 3 && isCgiRequestMethod(args[1])) { - Cgi cgi = new CustomCgi(args); - scope(exit) cgi.dispose(); - fun(cgi); - cgi.close(); - return; +void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) { + if(args.length > 1) { + // run the special separate processes if needed + switch(args[1]) { + case "--websocket-server": + runWebsocketServer(); + return; + case "--session-server": + runSessionServer(); + return; + case "--event-server": + runEventServer(); + return; + default: + // intentionally blank - do nothing and carry on to run normally } + } + + // we support command line thing for easy testing everywhere + // it needs to be called ./app method uri [other args...] + if(args.length >= 3 && isCgiRequestMethod(args[1])) { + Cgi cgi = new CustomCgi(args); + scope(exit) cgi.dispose(); + fun(cgi); + cgi.close(); + return; + } - ushort listeningPort(ushort def) { - bool found = false; - foreach(arg; args) { - if(found) - return to!ushort(arg); - if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") - found = true; + ushort listeningPort(ushort def) { + bool found = false; + foreach(arg; args) { + if(found) + return to!ushort(arg); + if(arg == "--port" || arg == "-p" || arg == "/port" || arg == "--listening-port") + found = true; + } + return def; + } + + string listeningHost() { + bool found = false; + foreach(arg; args) { + if(found) + return arg; + if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") + found = true; + } + return ""; + } + version(netman_httpd) { + import arsd.httpd; + // what about forwarding the other constructor args? + // this probably needs a whole redoing... + serveHttp!CustomCgi(&fun, listeningPort(8080));//5005); + return; + } else + version(embedded_httpd_processes) { + import core.sys.posix.unistd; + import core.sys.posix.sys.socket; + import core.sys.posix.netinet.in_; + //import std.c.linux.socket; + + int sock = socket(AF_INET, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket"); + + { + sockaddr_in addr; + addr.sin_family = AF_INET; + addr.sin_port = htons(listeningPort(8085)); + auto lh = listeningHost(); + if(lh.length) { + if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) + throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); + } else + addr.sin_addr.s_addr = INADDR_ANY; + + // HACKISH + int on = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); + // end hack + + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { + close(sock); + throw new Exception("bind"); } - return def; - } - string listeningHost() { - bool found = false; - foreach(arg; args) { - if(found) - return arg; - if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") - found = true; + // FIXME: if this queue is full, it will just ignore it + // and wait for the client to retransmit it. This is an + // obnoxious timeout condition there. + if(sock.listen(128) == -1) { + close(sock); + throw new Exception("listen"); } - return ""; } - version(netman_httpd) { - import arsd.httpd; - // what about forwarding the other constructor args? - // this probably needs a whole redoing... - serveHttp!CustomCgi(&fun, listeningPort(8080));//5005); - return; - } else - version(embedded_httpd_processes) { - import core.sys.posix.unistd; - import core.sys.posix.sys.socket; - import core.sys.posix.netinet.in_; - //import std.c.linux.socket; - int sock = socket(AF_INET, SOCK_STREAM, 0); - if(sock == -1) - throw new Exception("socket"); + version(embedded_httpd_processes_accept_after_fork) {} else { + int pipeReadFd; + int pipeWriteFd; { - sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_port = htons(listeningPort(8085)); - auto lh = listeningHost(); - if(lh.length) { - if(inet_pton(AF_INET, lh.toStringz(), &addr.sin_addr.s_addr) != 1) - throw new Exception("bad listening host given, please use an IP address.\nExample: --listening-host 127.0.0.1 means listen only on Localhost.\nExample: --listening-host 0.0.0.0 means listen on all interfaces.\nOr you can pass any other single numeric IPv4 address."); - } else - addr.sin_addr.s_addr = INADDR_ANY; - - // HACKISH - int on = 1; - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, on.sizeof); - // end hack - - - if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) { - close(sock); - throw new Exception("bind"); + int[2] pipeFd; + if(socketpair(AF_UNIX, SOCK_DGRAM, 0, pipeFd)) { + import core.stdc.errno; + throw new Exception("pipe failed " ~ to!string(errno)); } - // FIXME: if this queue is full, it will just ignore it - // and wait for the client to retransmit it. This is an - // obnoxious timeout condition there. - if(sock.listen(128) == -1) { - close(sock); - throw new Exception("listen"); - } + pipeReadFd = pipeFd[0]; + pipeWriteFd = pipeFd[1]; } + } - int processCount; - pid_t newPid; - reopen: - while(processCount < processPoolSize) { - newPid = fork(); - if(newPid == 0) { - // start serving on the socket - //ubyte[4096] backingBuffer; - for(;;) { - bool closeConnection; + int processCount; + pid_t newPid; + reopen: + while(processCount < processPoolSize) { + newPid = fork(); + if(newPid == 0) { + // start serving on the socket + //ubyte[4096] backingBuffer; + for(;;) { + bool closeConnection; + uint i; + sockaddr addr; + i = addr.sizeof; + version(embedded_httpd_processes_accept_after_fork) { + int s = accept(sock, &addr, &i); + int opt = 1; + import core.sys.posix.netinet.tcp; + // the Cgi class does internal buffering, so disabling this + // helps with latency in many cases... + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } else { + int s; + auto readret = read_fd(pipeReadFd, &s, s.sizeof, &s); + if(readret != s.sizeof) { + import core.stdc.errno; + throw new Exception("pipe read failed " ~ to!string(errno)); + } + + //writeln("process ", getpid(), " got socket ", s); + } + + try { + + if(s == -1) + throw new Exception("accept"); + + scope(failure) close(s); + //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; + auto ir = new BufferedInputRange(s); + //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); + + while(!ir.empty) { + ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; + + Cgi cgi; + try { + cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = s; + // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. + if(processPoolSize <= 1) + closeConnection = true; + //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); + } catch(Throwable t) { + // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P + // anyway let's kill the connection + stderr.writeln(t.toString()); + sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); + closeConnection = true; + break; + } + assert(cgi !is null); + scope(exit) + cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch(ConnectionException ce) { + closeConnection = true; + } catch(Throwable t) { + // a processing error can be recovered from + stderr.writeln(t.toString); + if(!handleException(cgi, t)) + closeConnection = true; + } + + if(closeConnection) { + ir.source.close(); + break; + } else { + if(!ir.empty) + ir.popFront(); // get the next + else if(ir.sourceClosed) { + ir.source.close(); + } + } + } + + ir.source.close(); + } catch(Throwable t) { + debug writeln(t); + // most likely cause is a timeout + } + } + } else { + processCount++; + } + } + + // the parent should wait for its children... + if(newPid) { + import core.sys.posix.sys.wait; + + version(embedded_httpd_processes_accept_after_fork) {} else { + import core.sys.posix.sys.select; + int[] fdQueue; + while(true) { + // writeln("select call"); + int nfds = pipeWriteFd; + if(sock > pipeWriteFd) + nfds = sock; + nfds += 1; + fd_set read_fds; + fd_set write_fds; + FD_ZERO(&read_fds); + FD_ZERO(&write_fds); + FD_SET(sock, &read_fds); + if(fdQueue.length) + FD_SET(pipeWriteFd, &write_fds); + auto ret = select(nfds, &read_fds, &write_fds, null, null); + if(ret == -1) { + import core.stdc.errno; + if(errno == EINTR) + goto try_wait; + else + throw new Exception("wtf select"); + } + + int s = -1; + if(FD_ISSET(sock, &read_fds)) { uint i; sockaddr addr; i = addr.sizeof; - int s = accept(sock, &addr, &i); + s = accept(sock, &addr, &i); + import core.sys.posix.netinet.tcp; + int opt = 1; + setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &opt, opt.sizeof); + } - try { - - if(s == -1) - throw new Exception("accept"); - - scope(failure) close(s); - //ubyte[__traits(classInstanceSize, BufferedInputRange)] bufferedRangeContainer; - auto ir = new BufferedInputRange(s); - //auto ir = emplace!BufferedInputRange(bufferedRangeContainer, s, backingBuffer); - - while(!ir.empty) { - ubyte[__traits(classInstanceSize, CustomCgi)] cgiContainer; - - Cgi cgi; - try { - cgi = new CustomCgi(ir, &closeConnection); - // if we have a single process and the browser tries to leave the connection open while concurrently requesting another, it will block everything an deadlock since there's no other server to accept it. By closing after each request in this situation, it tells the browser to serialize for us. - if(processPoolSize == 1) - closeConnection = true; - //cgi = emplace!CustomCgi(cgiContainer, ir, &closeConnection); - } catch(Throwable t) { - // a construction error is either bad code or bad request; bad request is what it should be since this is bug free :P - // anyway let's kill the connection - stderr.writeln(t.toString()); - sendAll(ir.source, plainHttpError(false, "400 Bad Request", t)); - closeConnection = true; - break; - } - assert(cgi !is null); - scope(exit) - cgi.dispose(); - - try { - fun(cgi); - cgi.close(); - } catch(ConnectionException ce) { - closeConnection = true; - } catch(Throwable t) { - // a processing error can be recovered from - stderr.writeln(t.toString); - if(!handleException(cgi, t)) - closeConnection = true; - } - - if(closeConnection) { - ir.source.close(); - break; - } else { - if(!ir.empty) - ir.popFront(); // get the next - else if(ir.sourceClosed) { - ir.source.close(); - } - } - } - - ir.source.close(); - } catch(Throwable t) { - debug writeln(t); - // most likely cause is a timeout + if(FD_ISSET(pipeWriteFd, &write_fds)) { + if(s == -1 && fdQueue.length) { + s = fdQueue[0]; + fdQueue = fdQueue[1 .. $]; // FIXME reuse buffer } - } - } else { - processCount++; + write_fd(pipeWriteFd, &s, s.sizeof, s); + close(s); // we are done with it, let the other process take ownership + } else + fdQueue ~= s; } } - // the parent should wait for its children... - if(newPid) { - import core.sys.posix.sys.wait; - int status; - // FIXME: maybe we should respawn if one dies unexpectedly - while(-1 != wait(&status)) { + try_wait: + + int status; + while(-1 != wait(&status)) { import std.stdio; writeln("Process died ", status); - processCount--; - goto reopen; - } - close(sock); + processCount--; + goto reopen; } - } else - version(embedded_httpd_threads) { - auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(8085), &doThreadHttpConnection!(CustomCgi, fun)); - manager.listen(); - } else - version(scgi) { - import std.exception; - import al = std.algorithm; - auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(4000), &doThreadScgiConnection!(CustomCgi, fun, maxContentLength)); - manager.listen(); - } else - version(fastcgi) { - // SetHandler fcgid-script - FCGX_Stream* input, output, error; - FCGX_ParamArray env; + close(sock); + } + } else + version(embedded_httpd_threads) { + auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(8085), &doThreadHttpConnection!(CustomCgi, fun)); + manager.listen(); + } else + version(scgi) { + import std.exception; + import al = std.algorithm; + auto manager = new ListeningConnectionManager(listeningHost(), listeningPort(4000), &doThreadScgiConnection!(CustomCgi, fun, maxContentLength)); + manager.listen(); + } else + version(fastcgi) { + // SetHandler fcgid-script + FCGX_Stream* input, output, error; + FCGX_ParamArray env; - const(ubyte)[] getFcgiChunk() { - const(ubyte)[] ret; - while(FCGX_HasSeenEOF(input) != -1) - ret ~= cast(ubyte) FCGX_GetChar(input); - return ret; + const(ubyte)[] getFcgiChunk() { + const(ubyte)[] ret; + while(FCGX_HasSeenEOF(input) != -1) + ret ~= cast(ubyte) FCGX_GetChar(input); + return ret; + } + + void writeFcgi(const(ubyte)[] data) { + FCGX_PutStr(data.ptr, data.length, output); + } + + void doARequest() { + string[string] fcgienv; + + for(auto e = env; e !is null && *e !is null; e++) { + string cur = to!string(*e); + auto idx = cur.indexOf("="); + string name, value; + if(idx == -1) + name = cur; + else { + name = cur[0 .. idx]; + value = cur[idx + 1 .. $]; + } + + fcgienv[name] = value; } - void writeFcgi(const(ubyte)[] data) { - FCGX_PutStr(data.ptr, data.length, output); + void flushFcgi() { + FCGX_FFlush(output); } - void doARequest() { - string[string] fcgienv; - - for(auto e = env; e !is null && *e !is null; e++) { - string cur = to!string(*e); - auto idx = cur.indexOf("="); - string name, value; - if(idx == -1) - name = cur; - else { - name = cur[0 .. idx]; - value = cur[idx + 1 .. $]; - } - - fcgienv[name] = value; - } - - void flushFcgi() { - FCGX_FFlush(output); - } - - Cgi cgi; - try { - cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); - } catch(Throwable t) { - FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); - writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); - return; //continue; - } - assert(cgi !is null); - scope(exit) cgi.dispose(); - try { - fun(cgi); - cgi.close(); - } catch(Throwable t) { - // log it to the error stream - FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); - // handle it for the user, if we can - if(!handleException(cgi, t)) - return; // continue; - } - } - - auto lp = listeningPort(0); - FCGX_Request request; - if(lp) { - // if a listening port was specified on the command line, we want to spawn ourself - // (needed for nginx without spawn-fcgi, e.g. on Windows) - FCGX_Init(); - auto sock = FCGX_OpenSocket(toStringz(listeningHost() ~ ":" ~ to!string(lp)), 12); - if(sock < 0) - throw new Exception("Couldn't listen on the port"); - FCGX_InitRequest(&request, sock, 0); - while(FCGX_Accept_r(&request) >= 0) { - input = request.inStream; - output = request.outStream; - error = request.errStream; - env = request.envp; - doARequest(); - } - } else { - // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) - // using the version with a global variable since we are separate processes anyway - while(FCGX_Accept(&input, &output, &error, &env) >= 0) { - doARequest(); - } - } - } else { - // standard CGI is the default version Cgi cgi; try { - cgi = new CustomCgi(maxContentLength); + cgi = new CustomCgi(maxContentLength, fcgienv, &getFcgiChunk, &writeFcgi, &flushFcgi); } catch(Throwable t) { - stderr.writeln(t.msg); - // the real http server will probably handle this; - // most likely, this is a bug in Cgi. But, oh well. - stdout.write(plainHttpError(true, "400 Bad Request", t)); - return; + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + writeFcgi(cast(const(ubyte)[]) plainHttpError(true, "400 Bad Request", t)); + return; //continue; } assert(cgi !is null); scope(exit) cgi.dispose(); - try { fun(cgi); cgi.close(); - } catch (Throwable t) { - stderr.writeln(t.msg); + } catch(Throwable t) { + // log it to the error stream + FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error); + // handle it for the user, if we can if(!handleException(cgi, t)) - return; + return; // continue; } } + + auto lp = listeningPort(0); + FCGX_Request request; + if(lp) { + // if a listening port was specified on the command line, we want to spawn ourself + // (needed for nginx without spawn-fcgi, e.g. on Windows) + FCGX_Init(); + auto sock = FCGX_OpenSocket(toStringz(listeningHost() ~ ":" ~ to!string(lp)), 12); + if(sock < 0) + throw new Exception("Couldn't listen on the port"); + FCGX_InitRequest(&request, sock, 0); + while(FCGX_Accept_r(&request) >= 0) { + input = request.inStream; + output = request.outStream; + error = request.errStream; + env = request.envp; + doARequest(); + } + } else { + // otherwise, assume the httpd is doing it (the case for Apache, IIS, and Lighttpd) + // using the version with a global variable since we are separate processes anyway + while(FCGX_Accept(&input, &output, &error, &env) >= 0) { + doARequest(); + } + } + } else { + // standard CGI is the default version + Cgi cgi; + try { + cgi = new CustomCgi(maxContentLength); + version(Posix) + cgi._outputFileHandle = 1; // stdout + else version(Windows) + cgi._outputFileHandle = GetStdHandle(STD_OUTPUT_HANDLE); + else static assert(0); + } catch(Throwable t) { + stderr.writeln(t.msg); + // the real http server will probably handle this; + // most likely, this is a bug in Cgi. But, oh well. + stdout.write(plainHttpError(true, "400 Bad Request", t)); + return; + } + assert(cgi !is null); + scope(exit) cgi.dispose(); + + try { + fun(cgi); + cgi.close(); + } catch (Throwable t) { + stderr.writeln(t.msg); + if(!handleException(cgi, t)) + return; + } } } @@ -3161,6 +3332,7 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { Cgi cgi; try { cgi = new CustomCgi(ir, &closeConnection); + cgi._outputFileHandle = connection.handle; } catch(ConnectionException ce) { // broken pipe or something, just abort the connection closeConnection = true; @@ -3276,6 +3448,7 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket Cgi cgi; try { cgi = new CustomCgi(maxContentLength, headers, &getScgiChunk, &writeScgi, &flushScgi); + cgi._outputFileHandle = connection.handle; } catch(Throwable t) { sendAll(connection, plainHttpError(true, "400 Bad Request", t)); connection.close(); @@ -3309,46 +3482,6 @@ string printDate(DateTime date) { } -version(with_cgi_packed) { -// This is temporary until Phobos supports base64 -immutable(ubyte)[] base64UrlDecode(string e) { - string encoded = e.idup; - while (encoded.length % 4) { - encoded ~= "="; // add padding - } - - // convert base64 URL to standard base 64 - encoded = encoded.replace("-", "+"); - encoded = encoded.replace("_", "/"); - - return cast(immutable(ubyte)[]) Base64.decode(encoded); -} - // should be set as arsd_packed_data - string packedDataEncode(in string[string] variables) { - string result; - - bool outputted = false; - foreach(k, v; variables) { - if(outputted) - result ~= "&"; - else - outputted = true; - - result ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); - } - - result = cast(string) Base64.encode(cast(ubyte[]) result); - - // url variant - result.replace("=", ""); - result.replace("+", "-"); - result.replace("/", "_"); - - return result; - } -} - - // Referencing this gigantic typeid seems to remind the compiler // to actually put the symbol in the object file. I guess the immutable // assoc array array isn't actually included in druntime @@ -3438,7 +3571,7 @@ import std.socket; // it is a class primarily for reference semantics // I might change this interface -/// +/// This is NOT ACTUALLY an input range! It is too different. Historical mistake kinda. class BufferedInputRange { version(Posix) this(int source, ubyte[] buffer = null) { @@ -3521,7 +3654,7 @@ class BufferedInputRange { return; } - view = underlyingBuffer[underlyingBuffer.ptr - view.ptr .. view.length + ret]; + view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; } while(view.length < minBytesToSettleFor); } @@ -3619,10 +3752,12 @@ class ListeningConnectionManager { while(!loopBroken && running) { auto sn = listener.accept(); - // disable Nagle's algorithm to avoid a 40ms delay when we send/recv - // on the socket because we do some buffering internally. I think this helps, - // certainly does for small requests, and I think it does for larger ones too - sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + if(tcp) { + // disable Nagle's algorithm to avoid a 40ms delay when we send/recv + // on the socket because we do some buffering internally. I think this helps, + // certainly does for small requests, and I think it does for larger ones too + sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); + } while(queueLength >= queue.length) Thread.sleep(1.msecs); synchronized(this) { @@ -3638,14 +3773,49 @@ class ListeningConnectionManager { } } } + + // FIXME: i typically stop this with ctrl+c which never + // actually gets here. i need to do a sigint handler. + if(cleanup) + cleanup(); } } + bool tcp; + void delegate() cleanup; + this(string host, ushort port, void function(Socket) handler) { this.handler = handler; - listener = new TcpSocket(); - listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); - listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port)); + + if(host.startsWith("unix:")) { + version(Posix) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + string filename = host["unix:".length .. $].idup; + listener.bind(new UnixAddress(filename)); + cleanup = delegate() { + import std.file; + remove(filename); + }; + tcp = false; + } else { + throw new Exception("unix sockets not supported on this system"); + } + } else if(host.startsWith("abstract:")) { + version(linux) { + listener = new Socket(AddressFamily.UNIX, SocketType.STREAM); + string filename = "\0" ~ host["abstract:".length .. $]; + listener.bind(new UnixAddress(filename)); + tcp = false; + } else { + throw new Exception("abstract unix sockets not supported on this system"); + } + } else { + listener = new TcpSocket(); + listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); + listener.bind(host.length ? parseAddress(host, port) : new InternetAddress(port)); + tcp = true; + } + listener.listen(128); } @@ -4036,10 +4206,10 @@ version(cgi_with_websocket) { "sec-websocket-key" in cgi.requestHeaders && "connection" in cgi.requestHeaders && - cgi.requestHeaders["connection"].toLower().indexOf("upgrade") != -1 + cgi.requestHeaders["connection"].asLowerCase().canFind("upgrade") && "upgrade" in cgi.requestHeaders && - cgi.requestHeaders["upgrade"].toLower() == "websocket" + cgi.requestHeaders["upgrade"].asLowerCase().equal("websocket") ; } @@ -4051,10 +4221,11 @@ version(cgi_with_websocket) { cgi.header("Connection: upgrade"); string key = cgi.requestHeaders["sec-websocket-key"]; - key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + key ~= "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; // the defined guid from the websocket spec - import arsd.sha; - auto accept = Base64.encode(SHA1(key)); + import std.digest.sha; + auto hash = sha1Of(key); + auto accept = Base64.encode(hash); cgi.header(("Sec-WebSocket-Accept: " ~ accept).idup); @@ -4266,12 +4437,2720 @@ version(Windows) else static assert(0); } +version(Posix) +private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**); + + +// FIXME: these aren't quite public yet. +//private: + +// template for laziness +void startWebsocketServer()() { + version(linux) { + import core.sys.posix.unistd; + pid_t pid; + const(char)*[16] args; + args[0] = "ARSD_CGI_WEBSOCKET_SERVER"; + args[1] = "--websocket-server"; + posix_spawn(&pid, "/proc/self/exe", + null, + null, + args.ptr, + null // env + ); + } else version(Windows) { + wchar[2048] filename; + auto len = GetModuleFileNameW(null, filename.ptr, cast(DWORD) filename.length); + if(len == 0 || len == filename.length) + throw new Exception("could not get process name to start helper server"); + + STARTUPINFOW startupInfo; + startupInfo.cb = cast(DWORD) startupInfo.sizeof; + PROCESS_INFORMATION processInfo; + + // I *MIGHT* need to run it as a new job or a service... + auto ret = CreateProcessW( + filename.ptr, + "--websocket-server"w, + null, // process attributes + null, // thread attributes + false, // inherit handles + 0, // creation flags + null, // environment + null, // working directory + &startupInfo, + &processInfo + ); + + if(!ret) + throw new Exception("create process failed"); + + // when done with those, if we set them + /* + CloseHandle(hStdInput); + CloseHandle(hStdOutput); + CloseHandle(hStdError); + */ + + } else static assert(0, "Websocket server not implemented on this system yet (email me, i can prolly do it if you need it)"); +} + +// template for laziness /* -Copyright: Adam D. Ruppe, 2008 - 2016 + The websocket server is a single-process, single-thread, event + I/O thing. It is passed websockets from other CGI processes + and is then responsible for handling their messages and responses. + Note that the CGI process is responsible for websocket setup, + including authentication, etc. + + It also gets data sent to it by other processes and is responsible + for distributing that, as necessary. +*/ +void runWebsocketServer()() { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(WebSocket ws, string group) { + assert(0, "not implemented"); +} + +void sendToWebsocketServer(string content, string group) { + assert(0, "not implemented"); +} + + +void runEventServer()() { + runAddonServer("/tmp/arsd_cgi_event_server", new EventSourceServerImplementation()); +} + +version(Posix) { + alias LocalServerConnectionHandle = int; + alias CgiConnectionHandle = int; + alias SocketConnectionHandle = int; + + enum INVALID_CGI_CONNECTION_HANDLE = -1; +} else version(Windows) { + alias LocalServerConnectionHandle = HANDLE; + version(embedded_httpd_threads) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else version(fastcgi) { + alias CgiConnectionHandle = void*; // Doesn't actually work! But I don't want compile to fail pointlessly at this point. + enum INVALID_CGI_CONNECTION_HANDLE = null; + } else version(scgi) { + alias CgiConnectionHandle = SOCKET; + enum INVALID_CGI_CONNECTION_HANDLE = INVALID_SOCKET; + } else { /* version(plain_cgi) */ + alias CgiConnectionHandle = HANDLE; + enum INVALID_CGI_CONNECTION_HANDLE = null; + } + alias SocketConnectionHandle = SOCKET; +} + +LocalServerConnectionHandle openLocalServerConnection(string name) { + version(Posix) { + import core.sys.posix.unistd; + import core.sys.posix.sys.un; + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. name.length + 1] = cast(typeof(addr.sun_path[])) name[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. name.length] = cast(typeof(addr.sun_path[])) name[]; + } + + if(connect(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("connect " ~ to!string(errno)); + + return sock; + } else version(Windows) { + return null; // FIXME + } +} + +void closeLocalServerConnection(LocalServerConnectionHandle handle) { + version(Posix) { + import core.sys.posix.unistd; + close(handle); + } else version(Windows) + CloseHandle(handle); +} + +void runSessionServer()() { + runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation()); +} + +version(Posix) +private void makeNonBlocking(int fd) { + import core.sys.posix.fcntl; + auto flags = fcntl(fd, F_GETFL, 0); + if(flags == -1) + throw new Exception("fcntl get"); + flags |= O_NONBLOCK; + auto s = fcntl(fd, F_SETFL, flags); + if(s == -1) + throw new Exception("fcntl set"); +} + +import core.stdc.errno; + +struct IoOp { + @disable this(); + @disable this(this); + + /* + So we want to be able to eventually handle generic sockets too. + */ + + enum Read = 1; + enum Write = 2; + enum Accept = 3; + enum ReadSocketHandle = 4; + + // Your handler may be called in a different thread than the one that initiated the IO request! + // It is also possible to have multiple io requests being called simultaneously. Use proper thread safety caution. + private void delegate(IoOp*, int) handler; + private void delegate(IoOp*) closeHandler; + private void delegate(IoOp*) completeHandler; + private int internalFd; + private int operation; + private int bufferLengthAllocated; + private int bufferLengthUsed; + private ubyte[1] internalBuffer; // it can be overallocated! + + ubyte[] allocatedBuffer() { + return internalBuffer.ptr[0 .. bufferLengthAllocated]; + } + + ubyte[] usedBuffer() { + return allocatedBuffer[0 .. bufferLengthUsed]; + } + + void reset() { + bufferLengthUsed = 0; + } + + int fd() { + return internalFd; + } +} + +IoOp* allocateIoOp(int fd, int operation, int bufferSize, void delegate(IoOp*, int) handler) { + import core.stdc.stdlib; + + auto ptr = malloc(IoOp.sizeof + bufferSize); + if(ptr is null) + assert(0); // out of memory! + + auto op = cast(IoOp*) ptr; + + op.handler = handler; + op.internalFd = fd; + op.operation = operation; + op.bufferLengthAllocated = bufferSize; + op.bufferLengthUsed = 0; + + return op; +} + +void freeIoOp(ref IoOp* ptr) { + import core.stdc.stdlib; + free(ptr); + ptr = null; +} + +version(Posix) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + import core.sys.posix.unistd; + + auto ret = write(connection, data.ptr, data.length); + if(ret != data.length) { + if(ret == 0 || errno == EPIPE) { + // the file is closed, remove it + eis.fileClosed(connection); + } else + throw new Exception("alas " ~ to!string(ret) ~ " " ~ to!string(errno)); // FIXME + } +} +version(Windows) +void nonBlockingWrite(EventIoServer eis, int connection, const void[] data) { + // FIXME +} + +bool isInvalidHandle(CgiConnectionHandle h) { + return h == INVALID_CGI_CONNECTION_HANDLE; +} + +/+ +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv +https://support.microsoft.com/en-gb/help/181611/socket-overlapped-i-o-versus-blocking-nonblocking-mode +https://stackoverflow.com/questions/18018489/should-i-use-iocps-or-overlapped-wsasend-receive +https://docs.microsoft.com/en-us/windows/desktop/fileio/i-o-completion-ports +https://docs.microsoft.com/en-us/windows/desktop/fileio/createiocompletionport +https://docs.microsoft.com/en-us/windows/desktop/api/mswsock/nf-mswsock-acceptex +https://docs.microsoft.com/en-us/windows/desktop/Sync/waitable-timer-objects +https://docs.microsoft.com/en-us/windows/desktop/api/synchapi/nf-synchapi-setwaitabletimer +https://docs.microsoft.com/en-us/windows/desktop/Sync/using-a-waitable-timer-with-an-asynchronous-procedure-call +https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsagetoverlappedresult + ++/ + +/++ + You can customize your server by subclassing the appropriate server. Then, register your + subclass at compile time with the [registerEventIoServer] template, or implement your own + main function and call it yourself. + + $(TIP If you make your subclass a `final class`, there is a slight performance improvement.) ++/ +interface EventIoServer { + void handleLocalConnectionData(IoOp* op, int receivedFd); + void handleLocalConnectionClose(IoOp* op); + void handleLocalConnectionComplete(IoOp* op); + void wait_timeout(); + void fileClosed(int fd); +} + +// the sink should buffer it +private void serialize(T)(scope void delegate(ubyte[]) sink, T t) { + static if(is(T == struct)) { + foreach(member; __traits(allMembers, T)) + serialize(sink, __traits(getMember, t, member)); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + sink((cast(ubyte*) &t)[0 .. t.sizeof]); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc. + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + sink(cast(ubyte[]) t[]); + } else static if(is(T : A[], A)) { + // generic array is less optimal but still prolly ok + int len = cast(int) t.length; + sink((cast(ubyte*) &len)[0 .. int.sizeof]); + foreach(item; t) + serialize(sink, item); + } else static assert(0, T.stringof); +} + +// all may be stack buffers, so use cautio +private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) { + static if(is(T == struct)) { + T t; + foreach(member; __traits(allMembers, T)) + deserialize!(typeof(__traits(getMember, T, member)))(get, (mbr) { __traits(getMember, t, member) = mbr; }); + dg(t); + } else static if(is(T : int)) { + // no need to think of endianness just because this is only used + // for local, same-machine stuff anyway. thanks private lol + T t; + auto data = get(t.sizeof); + t = (cast(T[]) data)[0]; + dg(t); + } else static if(is(T == string) || is(T : const(ubyte)[])) { + // these are common enough to optimize + int len; + auto data = get(len.sizeof); + len = (cast(int[]) data)[0]; + + /* + typeof(T[0])[2000] stackBuffer; + T buffer; + + if(len < stackBuffer.length) + buffer = stackBuffer[0 .. len]; + else + buffer = new T(len); + + data = get(len * typeof(T[0]).sizeof); + */ + + T t = cast(T) get(len * cast(int) typeof(T.init[0]).sizeof); + + dg(t); + + } else static assert(0, T.stringof); +} + +unittest { + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 1); }); + }, 1); + serialize((ubyte[] b) { + deserialize!int( sz => b[0 .. sz], (t) { assert(t == 56674); }); + }, 56674); + ubyte[1000] buffer; + int bufferPoint; + void add(ubyte[] b) { + buffer[bufferPoint .. bufferPoint + b.length] = b[]; + bufferPoint += b.length; + } + ubyte[] get(int sz) { + auto b = buffer[bufferPoint .. bufferPoint + sz]; + bufferPoint += sz; + return b; + } + serialize(&add, "test here"); + bufferPoint = 0; + deserialize!string(&get, (t) { assert(t == "test here"); }); + bufferPoint = 0; + + struct Foo { + int a; + ubyte c; + string d; + } + serialize(&add, Foo(403, 37, "amazing")); + bufferPoint = 0; + deserialize!Foo(&get, (t) { + assert(t.a == 403); + assert(t.c == 37); + assert(t.d == "amazing"); + }); + bufferPoint = 0; +} + +/* + Here's the way the RPC interface works: + + You define the interface that lists the functions you can call on the remote process. + The interface may also have static methods for convenience. These forward to a singleton + instance of an auto-generated class, which actually sends the args over the pipe. + + An impl class actually implements it. A receiving server deserializes down the pipe and + calls methods on the class. + + I went with the interface to get some nice compiler checking and documentation stuff. + + I could have skipped the interface and just implemented it all from the server class definition + itself, but then the usage may call the method instead of rpcing it; I just like having the user + interface and the implementation separate so you aren't tempted to `new impl` to call the methods. + + + I fiddled with newlines in the mixin string to ensure the assert line numbers matched up to the source code line number. Idk why dmd didn't do this automatically, but it was important to me. + + Realistically though the bodies would just be + connection.call(this.mangleof, args...) sooooo. + + FIXME: overloads aren't supported +*/ + +mixin template ImplementRpcClientInterface(T, string serverPath) { + static import std.traits; + + // derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering. + static foreach(idx, member; __traits(derivedMembers, T)) { + static if(__traits(isVirtualFunction, __traits(getMember, T, member))) + mixin( q{ + std.traits.ReturnType!(__traits(getMember, T, member)) + } ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params) + { + SerializationBuffer buffer; + auto i = cast(ushort) idx; + serialize(&buffer.sink, i); + serialize(&buffer.sink, __traits(getMember, T, member).mangleof); + foreach(param; params) + serialize(&buffer.sink, param); + + auto sendable = buffer.sendable; + + version(Posix) {{ + auto ret = send(connectionHandle, sendable.ptr, sendable.length, 0); + assert(ret == sendable.length); + }} // FIXME Windows impl + + static if(!is(typeof(return) == void)) { + // there is a return value; we need to wait for it too + version(Posix) { + ubyte[3000] revBuffer; + auto ret = recv(connectionHandle, revBuffer.ptr, revBuffer.length, 0); + auto got = revBuffer[0 .. ret]; + + int dataLocation; + ubyte[] grab(int sz) { + auto d = got[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + typeof(return) retu; + deserialize!(typeof(return))(&grab, (a) { retu = a; }); + return retu; + } else { + // FIXME Windows impl + return typeof(return).init; + } + + } + }}); + } + + private static typeof(this) singletonInstance; + private LocalServerConnectionHandle connectionHandle; + + static typeof(this) connection() { + if(singletonInstance is null) { + singletonInstance = new typeof(this)(); + singletonInstance.connect(); + } + return singletonInstance; + } + + void connect() { + connectionHandle = openLocalServerConnection(serverPath); + } + + void disconnect() { + closeLocalServerConnection(connectionHandle); + } +} + +void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(is(Class : Interface)) { + ushort calledIdx; + string calledFunction; + + int dataLocation; + ubyte[] grab(int sz) { + auto d = data[dataLocation .. dataLocation + sz]; + dataLocation += sz; + return d; + } + + again: + + deserialize!ushort(&grab, (a) { calledIdx = a; }); + deserialize!string(&grab, (a) { calledFunction = a; }); + + import std.traits; + + sw: switch(calledIdx) { + static foreach(idx, memberName; __traits(derivedMembers, Interface)) + static if(__traits(isVirtualFunction, __traits(getMember, Interface, memberName))) { + case idx: + assert(calledFunction == __traits(getMember, Interface, memberName).mangleof); + + Parameters!(__traits(getMember, Interface, memberName)) params; + foreach(ref param; params) + deserialize!(typeof(param))(&grab, (a) { param = a; }); + + static if(is(ReturnType!(__traits(getMember, Interface, memberName)) == void)) { + __traits(getMember, this_, memberName)(params); + } else { + auto ret = __traits(getMember, this_, memberName)(params); + SerializationBuffer buffer; + serialize(&buffer.sink, ret); + + auto sendable = buffer.sendable; + + version(Posix) { + auto r = send(fd, sendable.ptr, sendable.length, 0); + assert(r == sendable.length); + } // FIXME Windows impl + } + break sw; + } + default: assert(0); + } + + if(dataLocation != data.length) + goto again; +} + + +private struct SerializationBuffer { + ubyte[2048] bufferBacking; + int bufferLocation; + void sink(scope ubyte[] data) { + bufferBacking[bufferLocation .. bufferLocation + data.length] = data[]; + bufferLocation += data.length; + } + + ubyte[] sendable() { + return bufferBacking[0 .. bufferLocation]; + } +} + +/* + FIXME: + add a version command line arg + version data in the library + management gui as external program + + at server with event_fd for each run + use .mangleof in the at function name + + i think the at server will have to: + pipe args to the child + collect child output for logging + get child return value for logging + + on windows timers work differently. idk how to best combine with the io stuff. + + will have to have dump and restore too, so i can restart without losing stuff. +*/ + +/++ + ++/ +interface BasicDataServer { + /// + void createSession(string sessionId, int lifetime); + /// + void renewSession(string sessionId, int lifetime); + /// + void destroySession(string sessionId); + /// + void renameSession(string oldSessionId, string newSessionId); + + /// + void setSessionData(string sessionId, string dataKey, string dataValue); + /// + string getSessionData(string sessionId, string dataKey); + + /// + static BasicDataServerConnection connection() { + return BasicDataServerConnection.connection(); + } +} + +class BasicDataServerConnection : BasicDataServer { + mixin ImplementRpcClientInterface!(BasicDataServer, "/tmp/arsd_session_server"); +} + +final class BasicDataServerImplementation : BasicDataServer, EventIoServer { + + void createSession(string sessionId, int lifetime) { + sessions[sessionId.idup] = Session(lifetime); + } + void destroySession(string sessionId) { + sessions.remove(sessionId); + } + void renewSession(string sessionId, int lifetime) { + sessions[sessionId].lifetime = lifetime; + } + void renameSession(string oldSessionId, string newSessionId) { + sessions[newSessionId.idup] = sessions[oldSessionId]; + sessions.remove(oldSessionId); + } + void setSessionData(string sessionId, string dataKey, string dataValue) { + sessions[sessionId].values[dataKey.idup] = dataValue.idup; + } + string getSessionData(string sessionId, string dataKey) { + return sessions[sessionId].values[dataKey]; + } + + + protected: + + struct Session { + int lifetime; + + string[string] values; + } + + Session[string] sessions; + + void handleLocalConnectionData(IoOp* op, int receivedFd) { + auto data = op.usedBuffer; + dispatchRpcServer!BasicDataServer(this, data, op.fd); + } + + void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go + void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant + void wait_timeout() {} + void fileClosed(int fd) {} // stateless so irrelevant +} + +/++ + ++/ +struct ScheduledJobHelper { + private string func; + private string[] args; + + /++ + Schedules the job to be run at the given time. + +/ + void at(DateTime when, immutable TimeZone timezone = UTC()) { + + } + + /++ + Schedules the job to run at least after the specified delay. + +/ + void delay(Duration delay) { + + } + + /++ + Runs the job in the background ASAP. + + $(NOTE It may run in a background thread. Don't segfault!) + +/ + void runNowInBackground() { + //delay(0); + } + + /++ + Schedules the job to recur on the given pattern. + +/ + version(none) + void recur(string spec) { + + } +} + +/++ + First step to schedule a job on the scheduled job server. + + You MUST set details on the returned object to actually do anything! ++/ +ScheduledJobHelper schedule(alias fn, T...)(T args) { + return ScheduledJobHelper(); +} + +/// +interface ScheduledJobServer { + /// + int scheduleJob(int whenIs, int when, string executable, string func, string[] args); + /// + void cancelJob(int jobId); +} + +class ScheduledJobServerConnection : ScheduledJobServer { + mixin ImplementRpcClientInterface!(ScheduledJobServer, "/tmp/arsd_scheduled_job_server"); +} + +/// +interface EventSourceServer { + /++ + sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + See_Also: + [sendEvent] + +/ + public static void adoptConnection(Cgi cgi, in char[] eventUrl) { + /* + If lastEventId is missing or empty, you just get new events as they come. + + If it is set from something else, it sends all since then (that are still alive) + down the pipe immediately. + + The reason it can come from the header is that's what the standard defines for + browser reconnects. The reason it can come from a query string is just convenience + in catching up in a user-defined manner. + + The reason the header overrides the query string is if the browser tries to reconnect, + it will send the header AND the query (it reconnects to the same url), so we just + want to do the restart thing. + + Note that if you ask for "0" as the lastEventId, it will get ALL still living events. + */ + string lastEventId = cgi.lastEventId; + if(lastEventId.length == 0 && "lastEventId" in cgi.get) + lastEventId = cgi.get["lastEventId"]; + + cgi.setResponseContentType("text/event-stream"); + cgi.write(":\n", false); // to initialize the chunking and send headers before keeping the fd for later + cgi.flush(); + + cgi.closed = true; + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + version(fastcgi) + throw new Exception("sending fcgi connections not supported"); + else { + auto fd = cgi.getOutputFileHandle(); + if(isInvalidHandle(fd)) + throw new Exception("bad fd from cgi!"); + + EventSourceServerImplementation.SendableEventConnection sec; + sec.populate(cgi.responseChunked, eventUrl, lastEventId); + + version(Posix) { + auto res = write_fd(s, cast(void*) &sec, sec.sizeof, fd); + assert(res == sec.sizeof); + } else version(Windows) { + // FIXME + } + } + } + + /++ + Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later. + + $(WARNING This API is extremely unstable. I might change it or remove it without notice.) + + Params: + url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request. + event = the event type string, which is used in the Javascript addEventListener API on EventSource + data = the event data. Available in JS as `event.data`. + lifetime = the amount of time to keep this event for replaying on the event server. + + See_Also: + [sendEventToEventServer] + +/ + public static void sendEvent(string url, string event, string data, int lifetime) { + auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server"); + scope(exit) + closeLocalServerConnection(s); + + EventSourceServerImplementation.SendableEvent sev; + sev.populate(url, event, data, lifetime); + + version(Posix) { + auto ret = send(s, &sev, sev.sizeof, 0); + assert(ret == sev.sizeof); + } else version(Windows) { + // FIXME + } + } + + /++ + Messages sent to `url` will also be sent to anyone listening on `forwardUrl`. + + See_Also: [disconnect] + +/ + void connect(string url, string forwardUrl); + + /++ + Disconnects `forwardUrl` from `url` + + See_Also: [connect] + +/ + void disconnect(string url, string forwardUrl); +} + +/// +final class EventSourceServerImplementation : EventSourceServer, EventIoServer { + + protected: + + void connect(string url, string forwardUrl) { + pipes[url] ~= forwardUrl; + } + void disconnect(string url, string forwardUrl) { + auto t = url in pipes; + if(t is null) + return; + foreach(idx, n; (*t)) + if(n == forwardUrl) { + (*t)[idx] = (*t)[$-1]; + (*t) = (*t)[0 .. $-1]; + break; + } + } + + void handleLocalConnectionData(IoOp* op, int receivedFd) { + if(receivedFd != -1) { + //writeln("GOT FD ", receivedFd, " -- ", op.usedBuffer); + + //core.sys.posix.unistd.write(receivedFd, "hello".ptr, 5); + + SendableEventConnection* got = cast(SendableEventConnection*) op.usedBuffer.ptr; + + auto url = got.url.idup; + eventConnectionsByUrl[url] ~= EventConnection(receivedFd, got.responseChunked > 0 ? true : false); + + // FIXME: catch up on past messages here + } else { + auto data = op.usedBuffer; + auto event = cast(SendableEvent*) data.ptr; + + if(event.magic == 0xdeadbeef) { + handleInputEvent(event); + + if(event.url in pipes) + foreach(pipe; pipes[event.url]) { + event.url = pipe; + handleInputEvent(event); + } + } else { + dispatchRpcServer!EventSourceServer(this, data, op.fd); + } + } + } + void handleLocalConnectionClose(IoOp* op) {} + void handleLocalConnectionComplete(IoOp* op) {} + + void wait_timeout() { + // just keeping alive + foreach(url, connections; eventConnectionsByUrl) + foreach(connection; connections) + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, "2\r\n:\n"); + else + nonBlockingWrite(this, connection.fd, ":\n"); + } + + void fileClosed(int fd) { + outer: foreach(url, ref connections; eventConnectionsByUrl) { + foreach(idx, conn; connections) { + if(fd == conn.fd) { + connections[idx] = connections[$-1]; + connections = connections[0 .. $ - 1]; + continue outer; + } + } + } + } + + + + private: + + + struct SendableEventConnection { + ubyte responseChunked; + + int urlLength; + char[256] urlBuffer = 0; + + int lastEventIdLength; + char[32] lastEventIdBuffer = 0; + + char[] url() { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + char[] lastEventId() { + return lastEventIdBuffer[0 .. lastEventIdLength]; + } + void populate(bool responseChunked, in char[] url, in char[] lastEventId) + in { + assert(url.length < this.urlBuffer.length); + assert(lastEventId.length < this.lastEventIdBuffer.length); + } + do { + this.responseChunked = responseChunked ? 1 : 0; + this.urlLength = cast(int) url.length; + this.lastEventIdLength = cast(int) lastEventId.length; + + this.urlBuffer[0 .. url.length] = url[]; + this.lastEventIdBuffer[0 .. lastEventId.length] = lastEventId[]; + } + } + + struct SendableEvent { + int magic = 0xdeadbeef; + int urlLength; + char[256] urlBuffer = 0; + int typeLength; + char[32] typeBuffer = 0; + int messageLength; + char[2048] messageBuffer = 0; + int _lifetime; + + char[] message() { + return messageBuffer[0 .. messageLength]; + } + char[] type() { + return typeBuffer[0 .. typeLength]; + } + char[] url() { + return urlBuffer[0 .. urlLength]; + } + void url(in char[] u) { + urlBuffer[0 .. u.length] = u[]; + urlLength = cast(int) u.length; + } + int lifetime() { + return _lifetime; + } + + /// + void populate(string url, string type, string message, int lifetime) + in { + assert(url.length < this.urlBuffer.length); + assert(type.length < this.typeBuffer.length); + assert(message.length < this.messageBuffer.length); + } + do { + this.urlLength = cast(int) url.length; + this.typeLength = cast(int) type.length; + this.messageLength = cast(int) message.length; + this._lifetime = lifetime; + + this.urlBuffer[0 .. url.length] = url[]; + this.typeBuffer[0 .. type.length] = type[]; + this.messageBuffer[0 .. message.length] = message[]; + } + } + + struct EventConnection { + int fd; + bool needsChunking; + } + + private EventConnection[][string] eventConnectionsByUrl; + private string[][string] pipes; + + private void handleInputEvent(scope SendableEvent* event) { + static int eventId; + + static struct StoredEvent { + int id; + string type; + string message; + int lifetimeRemaining; + } + + StoredEvent[][string] byUrl; + + int thisId = ++eventId; + + if(event.lifetime) + byUrl[event.url.idup] ~= StoredEvent(thisId, event.type.idup, event.message.idup, event.lifetime); + + auto connectionsPtr = event.url in eventConnectionsByUrl; + EventConnection[] connections; + if(connectionsPtr is null) + return; + else + connections = *connectionsPtr; + + char[4096] buffer; + char[] formattedMessage; + + void append(const char[] a) { + // the 6's here are to leave room for a HTTP chunk header, if it proves necessary + buffer[6 + formattedMessage.length .. 6 + formattedMessage.length + a.length] = a[]; + formattedMessage = buffer[6 .. 6 + formattedMessage.length + a.length]; + } + + import std.algorithm.iteration; + + if(connections.length) { + append("id: "); + append(to!string(thisId)); + append("\n"); + + append("event: "); + append(event.type); + append("\n"); + + foreach(line; event.message.splitter("\n")) { + append("data: "); + append(line); + append("\n"); + } + + append("\n"); + } + + // chunk it for HTTP! + auto len = toHex(formattedMessage.length); + buffer[4 .. 6] = "\r\n"[]; + buffer[4 - len.length .. 4] = len[]; + + auto chunkedMessage = buffer[4 - len.length .. 6 + formattedMessage.length]; + // done + + // FIXME: send back requests when needed + // FIXME: send a single ":\n" every 15 seconds to keep alive + + foreach(connection; connections) { + if(connection.needsChunking) + nonBlockingWrite(this, connection.fd, chunkedMessage); + else + nonBlockingWrite(this, connection.fd, formattedMessage); + } + } +} + +void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoServer)) { + version(Posix) { + + import core.sys.posix.unistd; + import core.sys.posix.fcntl; + import core.sys.posix.sys.un; + + import core.sys.posix.signal; + signal(SIGPIPE, SIG_IGN); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if(sock == -1) + throw new Exception("socket " ~ to!string(errno)); + + scope(failure) + close(sock); + + // add-on server processes are assumed to be local, and thus will + // use unix domain sockets. Besides, I want to pass sockets to them, + // so it basically must be local (except for the session server, but meh). + sockaddr_un addr; + addr.sun_family = AF_UNIX; + version(linux) { + // on linux, we will use the abstract namespace + addr.sun_path[0] = 0; + addr.sun_path[1 .. localListenerName.length + 1] = cast(typeof(addr.sun_path[])) localListenerName[]; + } else { + // but otherwise, just use a file cuz we must. + addr.sun_path[0 .. localListenerName.length] = cast(typeof(addr.sun_path[])) localListenerName[]; + } + + if(bind(sock, cast(sockaddr*) &addr, addr.sizeof) == -1) + throw new Exception("bind " ~ to!string(errno)); + + if(listen(sock, 128) == -1) + throw new Exception("listen " ~ to!string(errno)); + + version(linux) { + + makeNonBlocking(sock); + + import core.sys.linux.epoll; + auto epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if(epoll_fd == -1) + throw new Exception("epoll_create1 " ~ to!string(errno)); + scope(failure) + close(epoll_fd); + + auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); + scope(exit) + freeIoOp(acceptOp); + + epoll_event ev; + ev.events = EPOLLIN | EPOLLET; + ev.data.ptr = acceptOp; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sock, &ev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + + epoll_event[64] events; + + while(true) { + + // FIXME: it should actually do a timerfd that runs on any thing that hasn't been run recently + + int timeout_milliseconds = 15000; // -1; // infinite + //writeln("waiting for ", name); + auto nfds = epoll_wait(epoll_fd, events.ptr, events.length, timeout_milliseconds); + if(nfds == -1) { + if(errno == EINTR) + continue; + throw new Exception("epoll_wait " ~ to!string(errno)); + } + + if(nfds == 0) { + eis.wait_timeout(); + } + + foreach(idx; 0 .. nfds) { + auto flags = events[idx].events; + auto ioop = cast(IoOp*) events[idx].data.ptr; + + //writeln(flags, " ", ioop.fd); + + if(ioop.fd == sock && (flags & EPOLLIN)) { + // on edge triggering, it is important that we get it all + while(true) { + auto size = cast(uint) addr.sizeof; + auto ns = accept(sock, cast(sockaddr*) &addr, &size); + if(ns == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + break; + } + throw new Exception("accept " ~ to!string(errno)); + } + + makeNonBlocking(ns); + epoll_event nev; + nev.events = EPOLLIN | EPOLLET; + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &eis.handleLocalConnectionData); + niop.closeHandler = &eis.handleLocalConnectionClose; + niop.completeHandler = &eis.handleLocalConnectionComplete; + scope(failure) freeIoOp(niop); + nev.data.ptr = niop; + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ns, &nev) == -1) + throw new Exception("epoll_ctl " ~ to!string(errno)); + } + } else if(ioop.operation == IoOp.ReadSocketHandle) { + while(true) { + int in_fd; + auto got = read_fd(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, &in_fd); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + ioop.handler(ioop, in_fd); + } + } else if(ioop.operation == IoOp.Read) { + while(true) { + auto got = recv(ioop.fd, ioop.allocatedBuffer.ptr, ioop.allocatedBuffer.length, 0); + if(got == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + // all done, got it all + if(ioop.completeHandler) + ioop.completeHandler(ioop); + break; + } + throw new Exception("recv " ~ to!string(errno)); + } + + if(got == 0) { + if(ioop.closeHandler) + ioop.closeHandler(ioop); + close(ioop.fd); + freeIoOp(ioop); + break; + } + + ioop.bufferLengthUsed = cast(int) got; + ioop.handler(ioop, -1); + } + } + + // EPOLLHUP? + } + } + } else { + // this isn't seriously implemented. + static assert(0); + } + } else version(Windows) { + + // set up a named pipe + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724251(v=vs.85).aspx + // https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsaduplicatesocketw + // https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeserverprocessid + + } else static assert(0); +} + + +version(Posix) +// copied from the web and ported from C +// see https://stackoverflow.com/questions/2358684/can-i-share-a-file-descriptor-to-another-process-on-linux-or-are-they-local-to-t +ssize_t write_fd(int fd, void *ptr, size_t nbytes, int sendfd) { + msghdr msg; + iovec[1] iov; + + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + cmptr = CMSG_FIRSTHDR(&msg); + cmptr.cmsg_len = CMSG_LEN(int.sizeof); + cmptr.cmsg_level = SOL_SOCKET; + cmptr.cmsg_type = SCM_RIGHTS; + *(cast(int *) CMSG_DATA(cmptr)) = sendfd; + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + return sendmsg(fd, &msg, 0); +} + +version(Posix) +// copied from the web and ported from C +ssize_t read_fd(int fd, void *ptr, size_t nbytes, int *recvfd) { + msghdr msg; + iovec[1] iov; + ssize_t n; + int newfd; + + union ControlUnion { + cmsghdr cm; + char[CMSG_SPACE(int.sizeof)] control; + } + ControlUnion control_un; + cmsghdr* cmptr; + + msg.msg_control = control_un.control.ptr; + msg.msg_controllen = control_un.control.length; + + msg.msg_name = null; + msg.msg_namelen = 0; + + iov[0].iov_base = ptr; + iov[0].iov_len = nbytes; + msg.msg_iov = iov.ptr; + msg.msg_iovlen = 1; + + if ( (n = recvmsg(fd, &msg, 0)) <= 0) + return n; + + if ( (cmptr = CMSG_FIRSTHDR(&msg)) != null && + cmptr.cmsg_len == CMSG_LEN(int.sizeof)) { + if (cmptr.cmsg_level != SOL_SOCKET) + throw new Exception("control level != SOL_SOCKET"); + if (cmptr.cmsg_type != SCM_RIGHTS) + throw new Exception("control type != SCM_RIGHTS"); + *recvfd = *(cast(int *) CMSG_DATA(cmptr)); + } else + *recvfd = -1; /* descriptor was not passed */ + + return n; +} +/* end read_fd */ + + +/* + Event source stuff + + The api is: + + sendEvent(string url, string type, string data, int timeout = 60*10); + + attachEventListener(string url, int fd, lastId) + + + It just sends to all attached listeners, and stores it until the timeout + for replaying via lastEventId. +*/ + +/* + Session process stuff + + it stores it all. the cgi object has a session object that can grab it + + session may be done in the same process if possible, there is a version + switch to choose if you want to override. +*/ + +struct DispatcherDefinition(alias dispatchHandler) {// if(is(typeof(dispatchHandler("str", Cgi.init) == bool))) { // bool delegate(string urlPrefix, Cgi cgi) dispatchHandler; + alias handler = dispatchHandler; + string urlPrefix; + bool rejectFurther; +} + +private string urlify(string name) { + return name; +} + +private string beautify(string name) { + char[160] buffer; + int bufferIndex = 0; + bool shouldCap = true; + bool shouldSpace; + bool lastWasCap; + foreach(idx, char ch; name) { + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + + if(ch >= 'A' && ch <= 'Z') { + if(lastWasCap) { + // two caps in a row, don't change. Prolly acronym. + } else { + if(idx) + shouldSpace = true; // new word, add space + } + + lastWasCap = true; + } + + if(shouldSpace) { + buffer[bufferIndex++] = ' '; + if(bufferIndex == buffer.length) return name; // out of space, just give up, not that important + } + if(shouldCap) { + if(ch >= 'a' && ch <= 'z') + ch -= 32; + shouldCap = false; + } + buffer[bufferIndex++] = ch; + } + return buffer[0 .. bufferIndex].idup; +} + +/+ + Argument conversions: for the most part, it is to!Thing(string). + + But arrays and structs are a bit different. Arrays come from the cgi array. Thus + they are passed + + arr=foo&arr=bar <-- notice the same name. + + Structs are first declared with an empty thing, then have their members set individually, + with dot notation. The members are not required, just the initial declaration. + + struct Foo { + int a; + string b; + } + void test(Foo foo){} + + foo&foo.a=5&foo.b=str <-- the first foo declares the arg, the others set the members + + Arrays of structs use this declaration. + + void test(Foo[] foo) {} + + foo&foo.a=5&foo.b=bar&foo&foo.a=9 + + You can use a hidden input field in HTML forms to achieve this. The value of the naked name + declaration is ignored. + + Mind that order matters! The declaration MUST come first in the string. + + Arrays of struct members follow this rule recursively. + + struct Foo { + int[] a; + } + + foo&foo.a=1&foo.a=2&foo&foo.a=1 + + + Associative arrays are formatted with brackets, after a declaration, like structs: + + foo&foo[key]=value&foo[other_key]=value + + + Note: for maximum compatibility with outside code, keep your types simple. Some libraries + do not support the strict ordering requirements to work with these struct protocols. + + FIXME: also perhaps accept application/json to better work with outside trash. + + + Return values are also auto-formatted according to user-requested type: + for json, it loops over and converts. + for html, basic types are strings. Arrays are
    . Structs are
    . Arrays of structs are tables! ++/ + +// returns an arsd.dom.Element +static auto elementFor(T)(string displayName, string name) { + import arsd.dom; + import std.traits; + + auto div = Element.make("div"); + div.addClass("form-field"); + + static if(is(T == struct)) { + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto fieldset = div.addChild("fieldset"); + fieldset.addChild("legend", beautify(T.stringof)); // FIXME + fieldset.addChild("input", name); + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + fieldset.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(beautify(memberName), name ~ "." ~ memberName)); + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("input", name); + i.attrs.name = name; + static if(isSomeString!T) + i.attrs.type = "text"; + else + i.attrs.type = "number"; + i.attrs.value = to!string(T.init); + } else static if(is(T == K[], K)) { + auto templ = div.addChild("template"); + templ.appendChild(elementFor!(K)(null, name)); + if(displayName !is null) + div.addChild("span", displayName, "label-text"); + auto btn = div.addChild("button"); + btn.addClass("add-array-button"); + btn.attrs.type = "button"; + btn.innerText = "Add"; + btn.attrs.onclick = q{ + var a = document.importNode(this.parentNode.firstChild.content, true); + this.parentNode.insertBefore(a, this); + }; + } else static if(is(T == V[K], K, V)) { + div.innerText = "assoc array not implemented for automatic form at this time"; + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + + return div; +} + +// actually returns an arsd.dom.Form +auto createAutomaticFormForFunction(alias method, T)(T dg) { + import arsd.dom; + + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, method))); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static if(is(typeof(method) P == __parameters)) + static foreach(idx, _; P) {{ + alias param = P[idx .. idx + 1]; + string displayName = beautify(__traits(identifier, param)); + static foreach(attr; __traits(getAttributes, param)) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + form.appendChild(elementFor!(param)(displayName, __traits(identifier, param))); + }} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; +} + +// actually returns an arsd.dom.Form +auto createAutomaticFormForObject(T)(T obj) { + import arsd.dom; + + auto form = cast(Form) Element.make("form"); + + form.addClass("automatic-form"); + + form.addChild("h3", beautify(__traits(identifier, T))); + + import std.traits; + + //Parameters!method params; + //alias idents = ParameterIdentifierTuple!method; + //alias defaults = ParameterDefaults!method; + + static foreach(idx, memberName; __traits(derivedMembers, T)) {{ + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + string displayName = beautify(memberName); + static foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) + static if(is(typeof(attr) == DisplayName)) + displayName = attr.name; + form.appendChild(elementFor!(typeof(__traits(getMember, T, memberName)))(displayName, memberName)); + + form.setValue(memberName, to!string(__traits(getMember, obj, memberName))); + }}} + + form.addChild("div", Html(``), "submit-button-holder"); + + return form; +} + +/* +string urlFor(alias func)() { + return __traits(identifier, func); +} +*/ + +/++ + UDA: The name displayed to the user in auto-generated HTML. + + Default is `beautify(identifier)`. ++/ +struct DisplayName { + string name; +} + +/++ + UDA: The name used in the URL or web parameter. + + Default is `urlify(identifier)` for functions and `identifier` for parameters and data members. ++/ +struct UrlName { + string name; +} + +class MissingArgumentException : Exception { + string functionName; + string argumentName; + string argumentType; + + this(string functionName, string argumentName, string argumentType, string file = __FILE__, size_t line = __LINE__, Throwable next = null) { + this.functionName = functionName; + this.argumentName = argumentName; + this.argumentType = argumentType; + + super("Missing Argument", file, line, next); + } +} + +auto callFromCgi(alias method, T)(T dg, Cgi cgi) { + + // FIXME: any array of structs should also be settable or gettable from csv as well. + + // FIXME: think more about checkboxes and bools. + + import std.traits; + + Parameters!method params; + alias idents = ParameterIdentifierTuple!method; + alias defaults = ParameterDefaults!method; + + const(string)[] names; + const(string)[] values; + + // first, check for missing arguments and initialize to defaults if necessary + static foreach(idx, param; params) {{ + auto ident = idents[idx]; + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + if(ident !in cgi.post) { + static if(is(defaults[idx] == void)) + throw new MissingArgumentException(__traits(identifier, method), ident, typeof(param).stringof); + else + params[idx] = defaults[idx]; + } + } else { + if(ident !in cgi.get) { + static if(is(defaults[idx] == void)) + throw new MissingArgumentException(__traits(identifier, method), ident, typeof(param).stringof); + else + params[idx] = defaults[idx]; + } + } + }} + + // second, parse the arguments in order to build up arrays, etc. + + static bool setVariable(T)(string name, string paramName, T* what, string value) { + static if(is(T == struct)) { + if(name == paramName) { + *what = T.init; + return true; + } else { + // could be a child + if(name[paramName.length] == '.') { + paramName = name[paramName.length + 1 .. $]; + name = paramName; + int p = 0; + foreach(ch; paramName) { + if(ch == '.' || ch == '[') + break; + p++; + } + + // set the child member + switch(paramName) { + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + // data member! + case memberName: + return setVariable(name, paramName, &(__traits(getMember, *what, memberName)), value); + } + default: + // ok, not a member + } + } + } + } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { + *what = to!T(value); + return true; + } else static if(is(T == K[], K)) { + K tmp; + if(name == paramName) { + // direct - set and append + if(setVariable(name, paramName, &tmp, value)) { + (*what) ~= tmp; + return true; + } else { + return false; + } + } else { + // child, append to last element + // FIXME: what about range violations??? + auto ptr = &(*what)[(*what).length - 1]; + return setVariable(name, paramName, ptr, value); + + } + } else static if(is(T == V[K], K, V)) { + // assoc array, name[key] is valid + if(name == paramName) { + // no action necessary + return true; + } else if(name[paramName.length] == '[') { + int count = 1; + auto idx = paramName.length + 1; + while(idx < name.length && count > 0) { + if(name[idx] == '[') + count++; + else if(name[idx] == ']') { + count--; + if(count == 0) break; + } + idx++; + } + if(idx == name.length) + return false; // malformed + + auto insideBrackets = name[paramName.length + 1 .. idx]; + auto afterName = name[idx + 1 .. $]; + + auto k = to!K(insideBrackets); + V v; + + name = name[0 .. paramName.length]; + writeln(name, afterName, " ", paramName); + + auto ret = setVariable(name ~ afterName, paramName, &v, value); + if(ret) { + (*what)[k] = v; + return true; + } + } + } else { + static assert(0, "unsupported type for cgi call " ~ T.stringof); + } + + return false; + } + + void setArgument(string name, string value) { + int p; + foreach(ch; name) { + if(ch == '.' || ch == '[') + break; + p++; + } + + auto paramName = name[0 .. p]; + + sw: switch(paramName) { + static foreach(idx, param; params) { + case idents[idx]: + setVariable(name, paramName, ¶ms[idx], value); + break sw; + } + default: + // ignore; not relevant argument + } + } + + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + names = cgi.allPostNamesInOrder; + values = cgi.allPostValuesInOrder; + } else { + names = cgi.allGetNamesInOrder; + values = cgi.allGetValuesInOrder; + } + + foreach(idx, name; names) { + setArgument(name, values[idx]); + } + + static if(is(ReturnType!method == void)) { + typeof(null) ret; + dg(params); + } else { + auto ret = dg(params); + } + + // FIXME: format return values + // options are: json, html, csv. + // also may need to wrap in envelope format: none, html, or json. + return ret; +} + +auto formatReturnValueAsHtml(T)(T t) { + import arsd.dom; + import std.traits; + + static if(is(T == typeof(null))) { + return Element.make("span"); + } else static if(isIntegral!T || isSomeString!T || isFloatingPoint!T) { + return Element.make("span", to!string(t), "automatic-data-display"); + } else static if(is(T == V[K], K, V)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display"); + foreach(k, v; t) { + dl.addChild("dt", to!string(k)); + dl.addChild("dd", formatReturnValueAsHtml(v)); + } + return dl; + } else static if(is(T == struct)) { + auto dl = Element.make("dl"); + dl.addClass("automatic-data-display"); + + static foreach(idx, memberName; __traits(allMembers, T)) + static if(__traits(compiles, __traits(getMember, T, memberName).offsetof)) { + dl.addChild("dt", memberName); + dl.addChild("dt", formatReturnValueAsHtml(__traits(getMember, t, memberName))); + } + + return dl; + } else static if(is(T == E[], E)) { + static if(is(E : RestObject!Proxy, Proxy)) { + // treat RestObject similar to struct + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + static foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + static foreach(idx, memberName; __traits(derivedMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + static if(memberName == "id") { + string val = to!string(__traits(getMember, l, memberName)); + tr.addChild("td", Element.make("a", val, E.stringof.toLower ~ "s/" ~ val)); // FIXME + } else { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + } + + return table; + } else static if(is(E == struct)) { + // an array of structs is kinda special in that I like + // having those formatted as tables. + auto table = cast(Table) Element.make("table"); + table.addClass("automatic-data-display"); + string[] names; + static foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + names ~= beautify(memberName); + } + table.appendHeaderRow(names); + + foreach(l; t) { + auto tr = table.appendRow(); + static foreach(idx, memberName; __traits(allMembers, E)) + static if(__traits(compiles, __traits(getMember, E, memberName).offsetof)) { + tr.addChild("td", formatReturnValueAsHtml(__traits(getMember, l, memberName))); + } + } + + return table; + } else { + // otherwise, I will just make a list. + auto ol = Element.make("ol"); + ol.addClass("automatic-data-display"); + foreach(e; t) + ol.addChild("li", formatReturnValueAsHtml(e)); + return ol; + } + } else static assert(0, "bad return value for cgi call " ~ T.stringof); + + assert(0); +} + +/++ + A web presenter is responsible for rendering things to HTML to be usable + in a web browser. + + They are passed as template arguments to the base classes of [WebObject] + + FIXME ++/ +class WebPresenter() { + +} + +/++ + The base class for the [dispatcher] function and object support. ++/ +class WebObject(Helper = void) { + Cgi cgi; + void initialize(Cgi cgi) { + this.cgi = cgi; + } + + string script() { + return ` + `; + } + + string style() { + return ` + :root { + --mild-border: #ccc; + --middle-border: #999; + --accent-color: #e8e8e8; + --sidebar-color: #f2f2f2; + } + ` ~ genericFormStyling() ~ genericSiteStyling(); + } + + string genericFormStyling() { + return ` + table.automatic-data-display { + border-collapse: collapse; + border: solid 1px var(--mild-border); + } + + table.automatic-data-display td { + vertical-align: top; + border: solid 1px var(--mild-border); + padding: 2px 4px; + } + + table.automatic-data-display th { + border: solid 1px var(--mild-border); + border-bottom: solid 1px var(--middle-border); + padding: 2px 4px; + } + + ol.automatic-data-display { + margin: 0px; + list-style-position: inside; + padding: 0px; + } + + .automatic-form { + max-width: 600px; + } + + .form-field { + margin: 0.5em; + padding-left: 0.5em; + } + + .label-text { + display: block; + font-weight: bold; + margin-left: -0.5em; + } + + .add-array-button { + + } + `; + } + + string genericSiteStyling() { + return ` + * { box-sizing: border-box; } + html, body { margin: 0px; } + body { + font-family: sans-serif; + } + #header { + background: var(--accent-color); + height: 64px; + } + #footer { + background: var(--accent-color); + height: 64px; + } + #main-site { + display: flex; + } + #container { + flex: 1 1 auto; + order: 2; + min-height: calc(100vh - 64px - 64px); + padding: 4px; + padding-left: 1em; + } + #sidebar { + flex: 0 0 16em; + order: 1; + background: var(--sidebar-color); + } + `; + } + + import arsd.dom; + Element htmlContainer() { + auto document = new Document(` + + + D Application + + + + +
    +
    + +
    + + + +`, true, true); + + return document.requireElementById("container"); + } +} + +/++ + Serves a class' methods, as a kind of low-state RPC over the web. To be used with [dispatcher]. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + FIXME: explain this better ++/ +auto serveApi(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] == '/'); + + import arsd.dom; + import arsd.jsvar; + + static bool handler(string urlPrefix, Cgi cgi) { + + auto obj = new T(); + obj.initialize(cgi); + + switch(cgi.pathInfo[urlPrefix.length .. $]) { + static foreach(methodName; __traits(derivedMembers, T)){{ + static if(is(typeof(__traits(getMember, T, methodName)) P == __parameters)) + { + case urlify(methodName): + switch(cgi.request("format", "html")) { + case "html": + auto container = obj.htmlContainer(); + try { + auto ret = callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + container.appendChild(formatReturnValueAsHtml(ret)); + } catch(MissingArgumentException mae) { + container.appendChild(Element.make("p", "Argument `" ~ mae.argumentName ~ "` of type `" ~ mae.argumentType ~ "` is missing")); + container.appendChild(createAutomaticFormForFunction!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName))); + } + cgi.write(container.parentDocument.toString(), true); + break; + case "json": + auto ret = callFromCgi!(__traits(getMember, obj, methodName))(&__traits(getMember, obj, methodName), cgi); + var json = ret; + var envelope = var.emptyObject; + envelope.success = true; + envelope.result = json; + envelope.error = null; + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + + break; + default: + } + return true; + } + }} + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(obj.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(obj.style(), true); + return true; + default: + return false; + } + + assert(0); + } + return DispatcherDefinition!handler(urlPrefix, false); +} + + + enum AccessCheck { + allowed, + denied, + nonExistant, + } + + enum Operation { + show, + create, + replace, + remove, + update + } + + enum UpdateResult { + accessDenied, + noSuchResource, + success, + failure, + unnecessary + } + + enum ValidationResult { + valid, + invalid + } + + +/++ + The base of all REST objects, to be used with [serveRestObject] and [serveRestCollectionOf]. ++/ +class RestObject(Helper = void) : WebObject!Helper { + + import arsd.dom; + import arsd.jsvar; + + /// Prepare the object to be shown. + void show() {} + /// ditto + void show(string urlId) { + load(urlId); + show(); + } + + ValidationResult delegate(typeof(this)) validateFromReflection; + Element delegate(typeof(this)) toHtmlFromReflection; + var delegate(typeof(this)) toJsonFromReflection; + + /// Override this to provide access control to this object. + AccessCheck accessCheck(string urlId, Operation operation) { + return AccessCheck.allowed; + } + + ValidationResult validate() { + if(validateFromReflection !is null) + return validateFromReflection(this); + return ValidationResult.valid; + } + + // The functions with more arguments are the low-level ones, + // they forward to the ones with fewer arguments by default. + + // POST on a parent collection - this is called from a collection class after the members are updated + /++ + Given a populated object, this creates a new entry. Returns the url identifier + of the new object. + +/ + string create(scope void delegate() applyChanges) { + return null; + } + + void replace() { + save(); + } + void replace(string urlId, scope void delegate() applyChanges) { + load(urlId); + applyChanges(); + replace(); + } + + void update(string[] fieldList) { + save(); + } + void update(string urlId, scope void delegate() applyChanges, string[] fieldList) { + load(urlId); + applyChanges(); + update(fieldList); + } + + void remove() {} + + void remove(string urlId) { + load(urlId); + remove(); + } + + abstract void load(string urlId); + abstract void save(); + + Element toHtml() { + if(toHtmlFromReflection) + return toHtmlFromReflection(this); + else + assert(0); + } + + var toJson() { + if(toJsonFromReflection) + return toJsonFromReflection(this); + else + assert(0); + } + + /+ + auto structOf(this This) { + + } + +/ +} + +/++ + Responsible for displaying stuff as HTML. You can put this into your own aggregate + and override it. Use forwarding and specialization to customize it. ++/ +mixin template Presenter() { + +} + +/++ + Base class for REST collections. ++/ +class CollectionOf(Obj, Helper = void) : RestObject!(Helper) { + /// You might subclass this and use the cgi object's query params + /// to implement a search filter, for example. + /// + /// FIXME: design a way to auto-generate that form + /// (other than using the WebObject thing above lol + // it'll prolly just be some searchParams UDA or maybe an enum. + // + // pagination too perhaps. + // + // and sorting too + IndexResult index() { return IndexResult.init; } + + string[] sortableFields() { return null; } + string[] searchableFields() { return null; } + + struct IndexResult { + Obj[] results; + + string[] sortableFields; + + string previousPageIdentifier; + string nextPageIdentifier; + string firstPageIdentifier; + string lastPageIdentifier; + + int numberOfPages; + } + + override string create(scope void delegate() applyChanges) { assert(0); } + override void load(string urlId) { assert(0); } + override void save() { assert(0); } + override void show() { + index(); + } + override void show(string urlId) { + show(); + } + + /// Proxy POST requests (create calls) to the child collection + alias PostProxy = Obj; +} + +/++ + Serves a REST object, similar to a Ruby on Rails resource. + + You put data members in your class. cgi.d will automatically make something out of those. + + It will call your constructor with the ID from the URL. This may be null. + It will then populate the data members from the request. + It will then call a method, if present, telling what happened. You don't need to write these! + It finally returns a reply. + + Your methods are passed a list of fields it actually set. + + The URL mapping - despite my general skepticism of the wisdom - matches up with what most REST + APIs I have used seem to follow. (I REALLY want to put trailing slashes on it though. Works better + with relative linking. But meh.) + + GET /items -> index. all values not set. + GET /items/id -> get. only ID will be set, other params ignored. + POST /items -> create. values set as given + PUT /items/id -> replace. values set as given + or POST /items/id with cgi.post["_method"] (thus urlencoded or multipart content-type) set to "PUT" to work around browser/html limitation + a GET with cgi.get["_method"] (in the url) set to "PUT" will render a form. + PATCH /items/id -> update. values set as given, list of changed fields passed + or POST /items/id with cgi.post["_method"] == "PATCH" + DELETE /items/id -> destroy. only ID guaranteed to be set + or POST /items/id with cgi.post["_method"] == "DELETE" + + Following the stupid convention, there will never be a trailing slash here, and if it is there, it will + redirect you away from it. + + API clients should set the `Accept` HTTP header to application/json or the cgi.get["_format"] = "json" var. + + I will also let you change the default, if you must. + + // One add-on is validation. You can issue a HTTP GET to a resource with _method = VALIDATE to check potential changes. + + You can define sub-resources on your object inside the object. These sub-resources are also REST objects + that follow the same thing. They may be individual resources or collections themselves. + + Your class is expected to have at least the following methods: + + FIXME: i kinda wanna add a routes object to the initialize call + + create + Create returns the new address on success, some code on failure. + show + index + update + remove + + You will want to be able to customize the HTTP, HTML, and JSON returns but generally shouldn't have to - the defaults + should usually work. The returned JSON will include a field "href" on all returned objects along with "id". Or omething like that. + + Usage of this function will add a dependency on [arsd.dom] and [arsd.jsvar]. + + NOT IMPLEMENTED + + + Really, a collection is a resource with a bunch of subresources. + + GET /items + index because it is GET on the top resource + + GET /items/foo + item but different than items? + + class Items { + + } + + ... but meh, a collection can be automated. not worth making it + a separate thing, let's look at a real example. Users has many + items and a virtual one, /users/current. + + the individual users have properties and two sub-resources: + session, which is just one, and comments, a collection. + + class User : RestObject!() { // no parent + int id; + string name; + + // the default implementations of the urlId ones is to call load(that_id) then call the arg-less one. + // but you can override them to do it differently. + + // any member which is of type RestObject can be linked automatically via href btw. + + void show() {} + void show(string urlId) {} // automated! GET of this specific thing + void create() {} // POST on a parent collection - this is called from a collection class after the members are updated + void replace(string urlId) {} // this is the PUT; really, it just updates all fields. + void update(string urlId, string[] fieldList) {} // PATCH, it updates some fields. + void remove(string urlId) {} // DELETE + + void load(string urlId) {} // the default implementation of show() populates the id, then + + this() {} + + mixin Subresource!Session; + mixin Subresource!Comment; + } + + class Session : RestObject!() { + // the parent object may not be fully constructed/loaded + this(User parent) {} + + } + + class Comment : CollectionOf!Comment { + this(User parent) {} + } + + class Users : CollectionOf!User { + // but you don't strictly need ANYTHING on a collection; it will just... collect. Implement the subobjects. + void index() {} // GET on this specific thing; just like show really, just different name for the different semantics. + User create() {} // You MAY implement this, but the default is to create a new object, populate it from args, and then call create() on the child + } + ++/ +auto serveRestObject(T)(string urlPrefix) { + assert(urlPrefix[$ - 1] != '/', "Do NOT use a trailing slash on REST objects."); + static bool handler(string urlPrefix, Cgi cgi) { + string url = cgi.pathInfo[urlPrefix.length .. $]; + + if(url.length && url[$ - 1] == '/') { + // remove the final slash... + cgi.setResponseLocation(cgi.scriptName ~ cgi.pathInfo[0 .. $ - 1]); + return true; + } + + return restObjectServeHandler!T(cgi, url); + + } + return DispatcherDefinition!handler(urlPrefix, false); +} + +/// Convenience method for serving a collection. It will be named the same +/// as type T, just with an s at the end. If you need any further, just +/// write the class yourself. +auto serveRestCollectionOf(T)(string urlPrefix) { + mixin(`static class `~T.stringof~`s : CollectionOf!(T) {}`); + return serveRestObject!(mixin(T.stringof ~ "s"))(urlPrefix); +} + +bool restObjectServeHandler(T)(Cgi cgi, string url) { + string urlId = null; + if(url.length && url[0] == '/') { + // asking for a subobject + urlId = url[1 .. $]; + foreach(idx, ch; urlId) { + if(ch == '/') { + urlId = urlId[0 .. idx]; + break; + } + } + } + + // FIXME handle other subresources + + static if(is(T : CollectionOf!(C, P), C, P)) { + if(urlId !is null) { + return restObjectServeHandler!C(cgi, url); // FIXME? urlId); + } + } + + // FIXME: support precondition failed, if-modified-since, expectation failed, etc. + + auto obj = new T(); + obj.toHtmlFromReflection = delegate(t) { + import arsd.dom; + auto div = Element.make("div"); + div.addClass("Dclass_" ~ T.stringof); + div.dataset.url = urlId; + bool first = true; + static foreach(idx, memberName; __traits(derivedMembers, T)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + if(!first) div.addChild("br"); else first = false; + div.appendChild(formatReturnValueAsHtml(__traits(getMember, obj, memberName))); + } + return div; + }; + obj.toJsonFromReflection = delegate(t) { + import arsd.jsvar; + var v = var.emptyObject(); + static foreach(idx, memberName; __traits(derivedMembers, T)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + v[memberName] = __traits(getMember, obj, memberName); + } + return v; + }; + obj.validateFromReflection = delegate(t) { + // FIXME + return ValidationResult.valid; + }; + obj.initialize(cgi); + // FIXME: populate reflection info delegates + + + // FIXME: I am not happy with this. + switch(urlId) { + case "script.js": + cgi.setResponseContentType("text/javascript"); + cgi.gzipResponse = true; + cgi.write(obj.script(), true); + return true; + case "style.css": + cgi.setResponseContentType("text/css"); + cgi.gzipResponse = true; + cgi.write(obj.style(), true); + return true; + default: + // intentionally blank + } + + + + + static void applyChangesTemplate(Obj)(Cgi cgi, Obj obj) { + static foreach(idx, memberName; __traits(derivedMembers, Obj)) + static if(__traits(compiles, __traits(getMember, obj, memberName).offsetof)) { + __traits(getMember, obj, memberName) = cgi.request(memberName, __traits(getMember, obj, memberName)); + } + } + void applyChanges() { + applyChangesTemplate(cgi, obj); + } + + string[] modifiedList; + + void writeObject(bool addFormLinks) { + if(cgi.request("format") == "json") { + cgi.setResponseContentType("application/json"); + cgi.write(obj.toJson().toString, true); + } else { + auto container = obj.htmlContainer(); + if(addFormLinks) { + static if(is(T : CollectionOf!(C, P), C, P)) + container.appendHtml(` +
    + +
    + `); + else + container.appendHtml(` +
    + + +
    + `); + } + container.appendChild(obj.toHtml()); + cgi.write(container.parentDocument.toString, true); + } + } + + // FIXME: I think I need a set type in here.... + // it will be nice to pass sets of members. + + switch(cgi.requestMethod) { + case Cgi.RequestMethod.GET: + // I could prolly use template this parameters in the implementation above for some reflection stuff. + // sure, it doesn't automatically work in subclasses... but I instantiate here anyway... + + // automatic forms here for usable basic auto site from browser. + // even if the format is json, it could actually send out the links and formats, but really there i'ma be meh. + switch(cgi.request("_method", "GET")) { + case "GET": + static if(is(T : CollectionOf!(C, P), C, P)) { + auto results = obj.index(); + if(cgi.request("format", "html") == "html") { + auto container = obj.htmlContainer(); + auto html = formatReturnValueAsHtml(results.results); + container.appendHtml(` +
    + +
    + `); + + container.appendChild(html); + cgi.write(container.parentDocument.toString, true); + } else { + cgi.setResponseContentType("application/json"); + import arsd.jsvar; + var json = var.emptyArray; + foreach(r; results.results) { + var o = var.emptyObject; + static foreach(idx, memberName; __traits(derivedMembers, typeof(r))) + static if(__traits(compiles, __traits(getMember, r, memberName).offsetof)) { + o[memberName] = __traits(getMember, r, memberName); + } + + json ~= o; + } + cgi.write(json.toJson(), true); + } + } else { + obj.show(urlId); + writeObject(true); + } + break; + case "PATCH": + obj.load(urlId); + goto case; + case "PUT": + case "POST": + // an editing form for the object + auto container = obj.htmlContainer(); + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto form = (cgi.request("_method") == "POST") ? createAutomaticFormForObject(new obj.PostProxy()) : createAutomaticFormForObject(obj); + } else { + auto form = createAutomaticFormForObject(obj); + } + form.attrs.method = "POST"; + form.setValue("_method", cgi.request("_method", "GET")); + container.appendChild(form); + cgi.write(container.parentDocument.toString(), true); + break; + case "DELETE": + // FIXME: a delete form for the object (can be phrased "are you sure?") + auto container = obj.htmlContainer(); + container.appendHtml(` +
    + Are you sure you want to delete this item? + + +
    + + `); + cgi.write(container.parentDocument.toString(), true); + break; + default: + cgi.write("bad method\n", true); + } + break; + case Cgi.RequestMethod.POST: + // this is to allow compatibility with HTML forms + switch(cgi.request("_method", "POST")) { + case "PUT": + goto PUT; + case "PATCH": + goto PATCH; + case "DELETE": + goto DELETE; + case "POST": + static if(__traits(compiles, () { auto o = new obj.PostProxy(); })) { + auto p = new obj.PostProxy(); + void specialApplyChanges() { + applyChangesTemplate(cgi, p); + } + string n = p.create(&specialApplyChanges); + } else { + string n = obj.create(&applyChanges); + } + + auto newUrl = cgi.scriptName ~ cgi.pathInfo ~ "/" ~ n; + cgi.setResponseLocation(newUrl); + cgi.setResponseStatus("201 Created"); + cgi.write(`The object has been created.`); + break; + default: + cgi.write("bad method\n", true); + } + // FIXME this should be valid on the collection, but not the child.... + // 303 See Other + break; + case Cgi.RequestMethod.PUT: + PUT: + obj.replace(urlId, &applyChanges); + writeObject(false); + break; + case Cgi.RequestMethod.PATCH: + PATCH: + obj.update(urlId, &applyChanges, modifiedList); + writeObject(false); + break; + case Cgi.RequestMethod.DELETE: + DELETE: + obj.remove(urlId); + cgi.setResponseStatus("204 No Content"); + break; + default: + // FIXME: OPTIONS, HEAD + } + + return true; +} + +/+ +struct SetOfFields(T) { + private void[0][string] storage; + void set(string what) { + //storage[what] = + } + void unset(string what) {} + void setAll() {} + void unsetAll() {} + bool isPresent(string what) { return false; } +} ++/ + +/+ +enum readonly; +enum hideonindex; ++/ + +/++ + Serves a static file. To be used with [dispatcher]. ++/ +auto serveStaticFile(string urlPrefix, string filename = null, string contentType = null) { + if(filename is null) + filename = urlPrefix[1 .. $]; + if(contentType is null) { + + } + static bool handler(string urlPrefix, Cgi cgi) { + //cgi.setResponseContentType(contentType); + //cgi.write(std.file.read(filename), true); + cgi.write(std.file.read(urlPrefix[1 .. $]), true); + return true; + } + return DispatcherDefinition!handler(urlPrefix, true); +} + +auto serveRedirect(string urlPrefix, string redirectTo) { + // FIXME +} + +/+ +/++ + See [serveStaticFile] if you want to serve a file off disk. ++/ +auto serveStaticData(string urlPrefix, const(void)[] data, string contentType) { + +} ++/ + +/++ + A URL dispatcher. + + --- + if(cgi.dispatcher!( + "/api/".serveApi!MyApiClass, + "/objects/lol".serveRestObject!MyRestObject, + "/file.js".serveStaticFile, + )) return; + --- ++/ +bool dispatcher(definitions...)(Cgi cgi) { + // I can prolly make this more efficient later but meh. + foreach(definition; definitions) { + if(definition.rejectFurther) { + if(cgi.pathInfo == definition.urlPrefix) { + auto ret = definition.handler(definition.urlPrefix, cgi); + if(ret) + return true; + } + } else if(cgi.pathInfo.startsWith(definition.urlPrefix)) { + auto ret = definition.handler(definition.urlPrefix, cgi); + if(ret) + return true; + } + } + return false; +} + +/+ +/++ + This is the beginnings of my web.d 2.0 - it dispatches web requests to a class object. + + It relies on jsvar.d and dom.d. + + + You can get javascript out of it to call. The generated functions need to look + like + + function name(a,b,c,d,e) { + return _call("name", {"realName":a,"sds":b}); + } + + And _call returns an object you can call or set up or whatever. ++/ +bool apiDispatcher()(Cgi cgi) { + import arsd.jsvar; + import arsd.dom; +} ++/ +/* +Copyright: Adam D. Ruppe, 2008 - 2019 License: Boost License 1.0. Authors: Adam D. Ruppe - Copyright Adam D. Ruppe 2008 - 2016. + Copyright Adam D. Ruppe 2008 - 2019. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/characterencodings.d b/characterencodings.d index 19eb06a..1a2aac4 100644 --- a/characterencodings.d +++ b/characterencodings.d @@ -18,22 +18,26 @@ ought to just work. Example: + --- auto data = cast(immutable(ubyte)[]) std.file.read("my-windows-file.txt"); string utf8String = convertToUtf8(data, "windows-1252"); // utf8String can now be used + --- The encodings currently implemented for decoding are: - UTF-8 (a no-op; it simply casts the array to string) - UTF-16, - UTF-32, - Windows-1252, - ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16. + $(LIST + * UTF-8 (a no-op; it simply casts the array to string) + * UTF-16, + * UTF-32, + * Windows-1252, + * ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16. + * KOI8-R + ) It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since - those labels are pretty much de-facto the same thing in wild documents. - + those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic). This module currently makes no attempt to look at control characters. */ diff --git a/color.d b/color.d index 35c5aa9..dd863d0 100644 --- a/color.d +++ b/color.d @@ -6,8 +6,8 @@ module arsd.color; // importing phobos explodes the size of this code 10x, so not doing it. private { - real toInternal(T)(scope const(char)[] s) { - real accumulator = 0.0; + double toInternal(T)(scope const(char)[] s) { + double accumulator = 0.0; size_t i = s.length; foreach(idx, c; s) { if(c >= '0' && c <= '9') { @@ -17,21 +17,21 @@ private { i = idx + 1; break; } else { - string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from "; + string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make double from "; wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s; throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute); } } - real accumulator2 = 0.0; - real count = 1; + double accumulator2 = 0.0; + double count = 1; foreach(c; s[i .. $]) { if(c >= '0' && c <= '9') { accumulator2 *= 10; accumulator2 += c - '0'; count *= 10; } else { - string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from "; + string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make double from "; wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s; throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute); } @@ -64,8 +64,8 @@ private { return cast(string) ret; } - string toInternal(T)(real a) { - // a simplifying assumption here is the fact that we only use this in one place: toInternal!string(cast(real) a / 255) + string toInternal(T)(double a) { + // a simplifying assumption here is the fact that we only use this in one place: toInternal!string(cast(double) a / 255) // thus we know this will always be between 0.0 and 1.0, inclusive. if(a <= 0.0) return "0.0"; @@ -78,16 +78,16 @@ private { } nothrow @safe @nogc pure - real absInternal(real a) { return a < 0 ? -a : a; } + double absInternal(double a) { return a < 0 ? -a : a; } nothrow @safe @nogc pure - real minInternal(real a, real b, real c) { + double minInternal(double a, double b, double c) { auto m = a; if(b < m) m = b; if(c < m) m = c; return m; } nothrow @safe @nogc pure - real maxInternal(real a, real b, real c) { + double maxInternal(double a, double b, double c) { auto m = a; if(b > m) m = b; if(c > m) m = c; @@ -217,6 +217,7 @@ struct Color { /// Return black-and-white color Color toBW() () nothrow pure @safe @nogc { + // FIXME: gamma? int intens = clampToByte(cast(int)(0.2126*r+0.7152*g+0.0722*b)); return Color(intens, intens, intens, a); } @@ -226,7 +227,7 @@ struct Color { if(a == 255) return "#" ~ toHexInternal(r) ~ toHexInternal(g) ~ toHexInternal(b); else { - return "rgba("~toInternal!string(r)~", "~toInternal!string(g)~", "~toInternal!string(b)~", "~toInternal!string(cast(real)a / 255.0)~")"; + return "rgba("~toInternal!string(r)~", "~toInternal!string(g)~", "~toInternal!string(b)~", "~toInternal!string(cast(double)a / 255.0)~")"; } } @@ -277,15 +278,15 @@ struct Color { assert(s[$-1] == ')'); s = s[s.startsWithInternal("hsl(") ? 4 : 5 .. $ - 1]; // the closing paren - real[3] hsl; + double[3] hsl; ubyte a = 255; auto parts = s.splitInternal(','); foreach(i, part; parts) { if(i < 3) - hsl[i] = toInternal!real(part.stripInternal); + hsl[i] = toInternal!double(part.stripInternal); else - a = clampToByte(cast(int) (toInternal!real(part.stripInternal) * 255)); + a = clampToByte(cast(int) (toInternal!double(part.stripInternal) * 255)); } c = .fromHsl(hsl); @@ -302,7 +303,7 @@ struct Color { auto parts = s.splitInternal(','); foreach(i, part; parts) { // lol the loop-switch pattern - auto v = toInternal!real(part.stripInternal); + auto v = toInternal!double(part.stripInternal); switch(i) { case 0: // red c.r = clampToByte(cast(int) v); @@ -353,7 +354,7 @@ struct Color { } /// from hsl - static Color fromHsl(real h, real s, real l) { + static Color fromHsl(double h, double s, double l) { return .fromHsl(h, s, l); } @@ -460,22 +461,26 @@ private ubyte fromHexInternal(in char[] s) { /// Converts hsl to rgb Color fromHsl(real[3] hsl) nothrow pure @safe @nogc { + return fromHsl(cast(double) hsl[0], cast(double) hsl[1], cast(double) hsl[2]); +} + +Color fromHsl(double[3] hsl) nothrow pure @safe @nogc { return fromHsl(hsl[0], hsl[1], hsl[2]); } /// Converts hsl to rgb -Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { +Color fromHsl(double h, double s, double l, double a = 255) nothrow pure @safe @nogc { h = h % 360; - real C = (1 - absInternal(2 * l - 1)) * s; + double C = (1 - absInternal(2 * l - 1)) * s; - real hPrime = h / 60; + double hPrime = h / 60; - real X = C * (1 - absInternal(hPrime % 2 - 1)); + double X = C * (1 - absInternal(hPrime % 2 - 1)); - real r, g, b; + double r, g, b; - if(h is real.nan) + if(h is double.nan) r = g = b = 0; else if (hPrime >= 0 && hPrime < 1) { r = C; @@ -503,7 +508,7 @@ Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { b = X; } - real m = l - C / 2; + double m = l - C / 2; r += m; g += m; @@ -516,23 +521,33 @@ Color fromHsl(real h, real s, real l, real a = 255) nothrow pure @safe @nogc { cast(int)(a)); } +/// Assumes the input `u` is already between 0 and 1 fyi. +nothrow pure @safe @nogc +double srgbToLinearRgb(double u) { + if(u < 0.4045) + return u / 12.92; + else + return ((u + 0.055) / 1.055) ^^ 2.4; +} + /// Converts an RGB color into an HSL triplet. useWeightedLightness will try to get a better value for luminosity for the human eye, which is more sensitive to green than red and more to red than blue. If it is false, it just does average of the rgb. -real[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted @nogc { - real r1 = cast(real) c.r / 255; - real g1 = cast(real) c.g / 255; - real b1 = cast(real) c.b / 255; +double[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted @nogc { + double r1 = cast(double) c.r / 255; + double g1 = cast(double) c.g / 255; + double b1 = cast(double) c.b / 255; - real maxColor = maxInternal(r1, g1, b1); - real minColor = minInternal(r1, g1, b1); + double maxColor = maxInternal(r1, g1, b1); + double minColor = minInternal(r1, g1, b1); - real L = (maxColor + minColor) / 2 ; + double L = (maxColor + minColor) / 2 ; if(useWeightedLightness) { // the colors don't affect the eye equally // this is a little more accurate than plain HSL numbers - L = 0.2126*r1 + 0.7152*g1 + 0.0722*b1; + L = 0.2126*srgbToLinearRgb(r1) + 0.7152*srgbToLinearRgb(g1) + 0.0722*srgbToLinearRgb(b1); + // maybe a better number is 299, 587, 114 } - real S = 0; - real H = 0; + double S = 0; + double H = 0; if(maxColor != minColor) { if(L < 0.5) { S = (maxColor - minColor) / (maxColor + minColor); @@ -557,7 +572,7 @@ real[3] toHsl(Color c, bool useWeightedLightness = false) nothrow pure @trusted } /// . -Color lighten(Color c, real percentage) nothrow pure @safe @nogc { +Color lighten(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] *= (1 + percentage); if(hsl[2] > 1) @@ -566,7 +581,7 @@ Color lighten(Color c, real percentage) nothrow pure @safe @nogc { } /// . -Color darken(Color c, real percentage) nothrow pure @safe @nogc { +Color darken(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] *= (1 - percentage); return fromHsl(hsl); @@ -574,7 +589,7 @@ Color darken(Color c, real percentage) nothrow pure @safe @nogc { /// for light colors, call darken. for dark colors, call lighten. /// The goal: get toward center grey. -Color moderate(Color c, real percentage) nothrow pure @safe @nogc { +Color moderate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); if(hsl[2] > 0.5) hsl[2] *= (1 - percentage); @@ -590,7 +605,7 @@ Color moderate(Color c, real percentage) nothrow pure @safe @nogc { } /// the opposite of moderate. Make darks darker and lights lighter -Color extremify(Color c, real percentage) nothrow pure @safe @nogc { +Color extremify(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c, true); if(hsl[2] < 0.5) hsl[2] *= (1 - percentage); @@ -626,7 +641,7 @@ Color makeTextColor(Color c) nothrow pure @safe @nogc { // These provide functional access to hsl manipulation; useful if you need a delegate -Color setLightness(Color c, real lightness) nothrow pure @safe @nogc { +Color setLightness(Color c, double lightness) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[2] = lightness; return fromHsl(hsl); @@ -634,28 +649,28 @@ Color setLightness(Color c, real lightness) nothrow pure @safe @nogc { /// -Color rotateHue(Color c, real degrees) nothrow pure @safe @nogc { +Color rotateHue(Color c, double degrees) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[0] += degrees; return fromHsl(hsl); } /// -Color setHue(Color c, real hue) nothrow pure @safe @nogc { +Color setHue(Color c, double hue) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[0] = hue; return fromHsl(hsl); } /// -Color desaturate(Color c, real percentage) nothrow pure @safe @nogc { +Color desaturate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] *= (1 - percentage); return fromHsl(hsl); } /// -Color saturate(Color c, real percentage) nothrow pure @safe @nogc { +Color saturate(Color c, double percentage) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] *= (1 + percentage); if(hsl[1] > 1) @@ -664,7 +679,7 @@ Color saturate(Color c, real percentage) nothrow pure @safe @nogc { } /// -Color setSaturation(Color c, real saturation) nothrow pure @safe @nogc { +Color setSaturation(Color c, double saturation) nothrow pure @safe @nogc { auto hsl = toHsl(c); hsl[1] = saturation; return fromHsl(hsl); @@ -785,9 +800,9 @@ void main() { import browser.document; foreach(ele; document.querySelectorAll("input")) { ele.addEventListener("change", { - auto h = toInternal!real(document.querySelector("input[name=h]").value); - auto s = toInternal!real(document.querySelector("input[name=s]").value); - auto l = toInternal!real(document.querySelector("input[name=l]").value); + auto h = toInternal!double(document.querySelector("input[name=h]").value); + auto s = toInternal!double(document.querySelector("input[name=s]").value); + auto l = toInternal!double(document.querySelector("input[name=l]").value); Color c = Color.fromHsl(h, s, l); @@ -1081,6 +1096,79 @@ class TrueColorImage : MemoryImage { } } +/+ +/// An RGB array of image data. +class TrueColorImageWithoutAlpha : MemoryImage { + struct Data { + ubyte[] bytes; // the data as rgba bytes. Stored left to right, top to bottom, no padding. + } + + /// . + Data imageData; + + int _width; + int _height; + + override void clearInternal () nothrow @system {// @nogc { + import core.memory : GC; + // it is safe to call [GC.free] with `null` pointer. + GC.free(imageData.bytes.ptr); imageData.bytes = null; + _width = _height = 0; + } + + /// . + override TrueColorImageWithoutAlpha clone() const pure nothrow @trusted { + auto n = new TrueColorImageWithoutAlpha(width, height); + n.imageData.bytes[] = this.imageData.bytes[]; // copy into existing array ctor allocated + return n; + } + + /// . + override int width() const pure nothrow @trusted @nogc { return _width; } + ///. + override int height() const pure nothrow @trusted @nogc { return _height; } + + override Color getPixel(int x, int y) const pure nothrow @trusted @nogc { + if (x >= 0 && y >= 0 && x < _width && y < _height) { + uint pos = (y*_width+x) * 3; + return Color(imageData.bytes[0], imageData.bytes[1], imageData.bytes[2], 255); + } else { + return Color(0, 0, 0, 0); + } + } + + override void setPixel(int x, int y, in Color clr) nothrow @trusted { + if (x >= 0 && y >= 0 && x < _width && y < _height) { + uint pos = y*_width+x; + //if (pos < imageData.bytes.length/4) imageData.colors.ptr[pos] = clr; + // FIXME + } + } + + /// . + this(int w, int h) pure nothrow @safe { + _width = w; + _height = h; + imageData.bytes = new ubyte[w*h*3]; + } + + /// Creates with existing data. The data pointer is stored here. + this(int w, int h, ubyte[] data) pure nothrow @safe { + _width = w; + _height = h; + assert(data.length == w * h * 3); + imageData.bytes = data; + } + + /// + override TrueColorImage getAsTrueColorImage() pure nothrow @safe { + // FIXME + //return this; + } +} ++/ + + alias extern(C) int function(const void*, const void*) @system Comparator; @trusted void nonPhobosSort(T)(T[] obj, Comparator comparator) { import core.stdc.stdlib; diff --git a/csv.d b/csv.d index 72fe093..c770952 100644 --- a/csv.d +++ b/csv.d @@ -6,9 +6,10 @@ import std.array; /// string[][] readCsv(string data) { + data = data.replace("\r\n", "\n"); data = data.replace("\r", ""); - auto idx = data.indexOf("\n"); + //auto idx = data.indexOf("\n"); //data = data[idx + 1 .. $]; // skip headers string[] fields; @@ -39,9 +40,9 @@ string[][] readCsv(string data) { field ~= c; break; case 1: // in quote - if(c == '"') + if(c == '"') { state = 2; - else + } else field ~= c; break; case 2: // is it a closing quote or an escaped one? @@ -55,6 +56,11 @@ string[][] readCsv(string data) { } } + if(field !is null) + current ~= field; + if(current !is null) + records ~= current; + return records; } diff --git a/dom.d b/dom.d index 988b089..1f1cbe0 100644 --- a/dom.d +++ b/dom.d @@ -1,4 +1,8 @@ // FIXME: add classList +// FIXME: add matchesSelector +// FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML +// FIXME: appendChild should not fail if the thing already has a parent; it should just automatically remove it per standard. + /++ This is an html DOM implementation, started with cloning what the browser offers in Javascript, but going well beyond @@ -71,7 +75,7 @@ bool isConvenientAttribute(string name) { /// The main document interface, including a html parser. class Document : FileResource { /// Convenience method for web scraping. Requires [arsd.http2] to be - /// included in the build. + /// included in the build as well as [arsd.characterencodings]. static Document fromUrl()(string url) { import arsd.http2; auto client = new HttpClient(); @@ -79,7 +83,10 @@ class Document : FileResource { auto req = client.navigateTo(Uri(url), HttpVerb.GET); auto res = req.waitForCompletion(); - return new Document(cast(string) res.content); + auto document = new Document(); + document.parseGarbage(cast(string) res.content); + + return document; } ///. @@ -821,6 +828,12 @@ class Document : FileResource { while(pos < data.length && data[pos] != '>') pos++; + + if(pos >= data.length) { + // the tag never closed + assert(data.length != 0); + pos = data.length - 1; // rewinding so it hits the end at the bottom.. + } } auto whereThisTagStarted = pos; // for better error messages @@ -1403,7 +1416,7 @@ class Element { body { auto e = cast(SomeElementType) getElementById(id); if(e is null) - throw new ElementNotFoundException(SomeElementType.stringof, "id=" ~ id, file, line); + throw new ElementNotFoundException(SomeElementType.stringof, "id=" ~ id, this, file, line); return e; } @@ -1418,7 +1431,7 @@ class Element { body { auto e = cast(SomeElementType) querySelector(selector); if(e is null) - throw new ElementNotFoundException(SomeElementType.stringof, selector, file, line); + throw new ElementNotFoundException(SomeElementType.stringof, selector, this, file, line); return e; } @@ -2127,7 +2140,7 @@ class Element { static if(!is(T == Element)) { auto t = cast(T) par; if(t is null) - throw new ElementNotFoundException("", tagName ~ " parent not found"); + throw new ElementNotFoundException("", tagName ~ " parent not found", this); } else auto t = par; @@ -3494,8 +3507,14 @@ struct ElementCollection { return !elements.length; } - /// Collects strings from the collection, concatenating them together - /// Kinda like running reduce and ~= on it. + /++ + Collects strings from the collection, concatenating them together + Kinda like running reduce and ~= on it. + + --- + document["p"].collect!"innerText"; + --- + +/ string collect(string method)(string separator = "") { string text; foreach(e; elements) { @@ -3514,6 +3533,17 @@ struct ElementCollection { return this; } + /++ + Calls [Element.wrapIn] on each member of the collection, but clones the argument `what` for each one. + +/ + ElementCollection wrapIn(Element what) { + foreach(e; elements) { + e.wrapIn(what.cloneNode(false)); + } + + return this; + } + /// Concatenates two ElementCollection together. ElementCollection opBinary(string op : "~")(ElementCollection rhs) { return ElementCollection(this.elements ~ rhs.elements); @@ -3776,7 +3806,7 @@ T require(T = Element, string file = __FILE__, int line = __LINE__)(Element e) i body { auto ret = cast(T) e; if(ret is null) - throw new ElementNotFoundException(T.stringof, "passed value", file, line); + throw new ElementNotFoundException(T.stringof, "passed value", e, file, line); return ret; } @@ -5042,7 +5072,7 @@ class Table : Element { return position; } - foreach(int i, rowElement; rows) { + foreach(i, rowElement; rows) { auto row = cast(TableRow) rowElement; assert(row !is null); assert(i < ret.length); @@ -5059,7 +5089,7 @@ class Table : Element { foreach(int j; 0 .. cell.colspan) { foreach(int k; 0 .. cell.rowspan) // if the first row, always append. - insertCell(k + i, k == 0 ? -1 : position, cell); + insertCell(k + cast(int) i, k == 0 ? -1 : position, cell); position++; } } @@ -5139,9 +5169,12 @@ class MarkupException : Exception { class ElementNotFoundException : Exception { /// type == kind of element you were looking for and search == a selector describing the search. - this(string type, string search, string file = __FILE__, size_t line = __LINE__) { + this(string type, string search, Element searchContext, string file = __FILE__, size_t line = __LINE__) { + this.searchContext = searchContext; super("Element of type '"~type~"' matching {"~search~"} not found.", file, line); } + + Element searchContext; } /// The html struct is used to differentiate between regular text nodes and html in certain functions diff --git a/dub.json b/dub.json index e91aa02..364dda4 100644 --- a/dub.json +++ b/dub.json @@ -95,6 +95,8 @@ "description": "MySQL client library. Wraps the official C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs-posix": ["mysqlclient"], + "libs-windows": ["libmysql"], "sourceFiles": ["mysql.d"] }, { @@ -102,6 +104,7 @@ "description": "Postgresql client library. Wraps the libpq C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs": ["pq"], "sourceFiles": ["postgres.d"] }, @@ -110,6 +113,8 @@ "description": "sqlite wrapper. Wraps the official C library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs": ["sqlite3"], + "libs-posix": ["dl"], "sourceFiles": ["sqlite.d"] }, @@ -118,6 +123,7 @@ "description": "Microsoft SQL Server client library. Wraps the official ODBC library with my database.d interface.", "targetType": "sourceLibrary", "dependencies": {"arsd-official:database_base":"*"}, + "libs-windows": ["odbc32"], "sourceFiles": ["mssql.d"] }, diff --git a/gamehelpers.d b/gamehelpers.d index dd86c02..c2c8d42 100644 --- a/gamehelpers.d +++ b/gamehelpers.d @@ -31,9 +31,10 @@ } int x, y; - override void update(Duration deltaTime) { + override bool update(Duration deltaTime) { x += 1; y += 1; + return true; } override SimpleWindow getWindow() { @@ -100,7 +101,8 @@ class GameHelperBase { abstract void drawFrame(); /// Implement this to update. The deltaTime tells how much real time has passed since the last update. - abstract void update(Duration deltaTime); + /// Returns true if anything changed, which will queue up a redraw + abstract bool update(Duration deltaTime); //abstract void fillAudioBuffer(short[] buffer); /// Returns the main game window. This function will only be @@ -155,6 +157,14 @@ void runGame(T : GameHelperBase)(T game, int maxUpdateRate = 20, int maxRedrawRa delegate (KeyEvent ke) { game.keyboardState[ke.hardwareCode] = ke.pressed; + /* + switch(ke.key) { + case Key.UpArrow: + game.joysticks[0] + break; + default: + } + */ // FIXME } ); diff --git a/htmltotext.d b/htmltotext.d index 272dadc..781e3b5 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -7,10 +7,47 @@ import std.string; import std.uni : isWhite; +/// class HtmlConverter { int width; + /++ + Will enable color output using VT codes. Determines color through dom.d's css support, which means you need to apply a stylesheet first. + + --- + import arsd.dom; + + auto document = new Document(source_code_for_html); + auto stylesheet = new Stylesheet(source_code_for_css); + stylesheet.apply(document); + --- + +/ + bool enableVtOutput; + + + string color; + string backgroundColor; + + /// void htmlToText(Element element, bool preformatted, int width) { + string color, backgroundColor; + if(enableVtOutput) { + color = element.computedStyle.getValue("color"); + backgroundColor = element.computedStyle.getValue("background-color"); + } + + string originalColor = this.color, originalBackgroundColor = this.backgroundColor; + + this.color = color.length ? color : this.color; + this.backgroundColor = backgroundColor.length ? backgroundColor : this.backgroundColor; + + scope(exit) { + // the idea is as we pop working back up the tree, it restores what it was here + this.color = originalColor; + this.backgroundColor = originalBackgroundColor; + } + + this.width = width; if(auto tn = cast(TextNode) element) { foreach(dchar ch; tn.nodeValue) { @@ -28,7 +65,7 @@ class HtmlConverter { // The table stuff is removed right now because while it looks // ok for test tables, it isn't working well for the emails I have // - it handles data ok but not really nested layouts. - case "trfixme": + case "trlol": auto children = element.childElements; auto tdWidth = (width - cast(int)(children.length)*3) / cast(int)(children.length); @@ -91,6 +128,16 @@ class HtmlConverter { s ~= "\n"; } break; + case "tr": + startBlock(2); + sinkChildren(); + endBlock(); + break; + case "td": + startBlock(0); + sinkChildren(); + endBlock(); + break; case "a": sinkChildren(); if(element.href != element.innerText) { @@ -105,17 +152,27 @@ class HtmlConverter { } break; case "span": - /* - auto csc = element.computedStyle.getValue("color"); - if(csc.length) { - auto c = Color.fromString(csc); - s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b); - } - sinkChildren(); + if(enableVtOutput) { + auto csc = color; // element.computedStyle.getValue("color"); + if(csc.length) { + auto c = Color.fromString(csc); + s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b); + } - if(csc.length) - s ~= "\033[39m"; - */ + bool bold = element.computedStyle.getValue("font-weight") == "bold"; + + if(bold) + s ~= "\033[1m"; + + sinkChildren(); + + if(bold) + s ~= "\033[0m"; + if(csc.length) + s ~= "\033[39m"; + } else { + sinkChildren(); + } break; case "p": startBlock(); @@ -126,9 +183,15 @@ class HtmlConverter { case "em", "i": if(element.innerText.length == 0) break; - sink('*', false); - sinkChildren(); - sink('*', false); + if(enableVtOutput) { + s ~= "\033[1m"; + sinkChildren(); + s ~= "\033[0m"; + } else { + sink('*', false); + sinkChildren(); + sink('*', false); + } break; case "u": if(element.innerText.length == 0) @@ -139,20 +202,28 @@ class HtmlConverter { break; case "ul": ulDepth++; + startBlock(2); sinkChildren(); + endBlock(); ulDepth--; break; case "ol": olDepth++; + startBlock(2); sinkChildren(); + endBlock(); olDepth--; break; case "li": startBlock(); //sink('\t', true); - sink(' ', true); - sink(' ', true); + /* + foreach(cnt; 0 .. olDepth + ulDepth) { + sink(' ', true); + sink(' ', true); + } + */ if(olDepth) sink('*', false); if(ulDepth) @@ -164,15 +235,33 @@ class HtmlConverter { endBlock(); break; - case "h1", "h2": + case "dl": + case "dt": + case "dd": + startBlock(element.tagName == "dd" ? 2 : 0); + sinkChildren(); + endBlock(); + break; + + case "h1": + startBlock(); + sink('#', true); + sink('#', true); + sink(' ', true); + sinkChildren(); + sink(' ', true); + sink('#', true); + sink('#', true); + endBlock(); + break; + case "h2", "h3": startBlock(); sinkChildren(); sink('\n', true); foreach(dchar ch; element.innerText) - sink(element.tagName == "h1" ? '=' : '-', false); + sink(element.tagName == "h2" ? '=' : '-', false); endBlock(); break; - case "hr": startBlock(); foreach(i; 0 .. width / 4) @@ -185,7 +274,6 @@ class HtmlConverter { case "br": sink('\n', true); break; - case "tr": case "div": startBlock(); @@ -207,7 +295,7 @@ class HtmlConverter { endBlock(); break; case "pre": - startBlock(); + startBlock(4); foreach(child; element.childNodes) htmlToText(child, true, width); endBlock(); @@ -221,6 +309,7 @@ class HtmlConverter { int olDepth; int ulDepth; + /// string convert(string html, bool wantWordWrap = true, int wrapAmount = 74) { Document document = new Document; @@ -237,10 +326,16 @@ class HtmlConverter { //auto stylesheet = new StyleSheet(readText("/var/www/dpldocs.info/experimental-docs/style.css")); //stylesheet.apply(document); + return convert(start, wantWordWrap, wrapAmount); + } + + /// + string convert(Element start, bool wantWordWrap = true, int wrapAmount = 74) { htmlToText(start, false, wrapAmount); return s; } + /// void reset() { s = null; justOutputWhitespace = true; @@ -248,6 +343,7 @@ class HtmlConverter { justOutputMargin = true; } + /// string s; bool justOutputWhitespace = true; bool justOutputBlock = true; @@ -255,6 +351,12 @@ class HtmlConverter { int lineLength; void sink(dchar item, bool preformatted, int lineWidthOverride = int.min) { + + if(needsIndent && item != '\n') { + lineLength += doIndent(); + needsIndent = false; + } + int width = lineWidthOverride == int.min ? this.width : lineWidthOverride; if(!preformatted && isWhite(item)) { if(!justOutputWhitespace) { @@ -282,8 +384,9 @@ class HtmlConverter { auto os = s; s = os[0 .. idx]; s ~= '\n'; - s ~= os[idx + 1 .. $]; lineLength = cast(int)(os[idx+1..$].length); + lineLength += doIndent(); + s ~= os[idx + 1 .. $]; broken = true; break; } @@ -295,15 +398,17 @@ class HtmlConverter { if(!broken) { s ~= '\n'; lineLength = 0; + needsIndent = true; justOutputWhitespace = true; } } - if(item == '\n') + if(item == '\n') { lineLength = 0; - else + needsIndent = true; + } else lineLength ++; @@ -312,163 +417,53 @@ class HtmlConverter { justOutputMargin = false; } } - void startBlock() { + + int doIndent() { + int cnt = 0; + foreach(i; indentStack) + foreach(lol; 0 .. i) { + s ~= ' '; + cnt++; + } + return cnt; + } + + int[] indentStack; + bool needsIndent = false; + + void startBlock(int indent = 0) { + + indentStack ~= indent; + if(!justOutputBlock) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputBlock = true; } if(!justOutputMargin) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputMargin = true; } } void endBlock() { + if(indentStack.length) + indentStack = indentStack[0 .. $ - 1]; + if(!justOutputMargin) { s ~= "\n"; lineLength = 0; + needsIndent = true; justOutputMargin = true; } } } +/// string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { auto converter = new HtmlConverter(); return converter.convert(html, true, wrapAmount); } -string repeat(string s, ulong num) { - string ret; - foreach(i; 0 .. num) - ret ~= s; - return ret; -} - -import std.stdio; -version(none) -void penis() { - - again: - string result = ""; - foreach(ele; start.tree) { - if(ele is start) continue; - if(ele.nodeType != 1) continue; - - switch(ele.tagName) { - goto again; - case "h1": - ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("=", ele.innerText.length) ~ "\r"; - ele.stripOut(); - goto again; - case "h2": - ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("-", ele.innerText.length) ~ "\r"; - ele.stripOut(); - goto again; - case "h3": - ele.innerText = "\r" ~ ele.innerText.toUpper ~ "\r"; - ele.stripOut(); - goto again; - case "td": - case "p": - /* - if(ele.innerHTML.length > 1) - ele.innerHTML = "\r" ~ wrap(ele.innerHTML) ~ "\r"; - ele.stripOut(); - goto again; - */ - break; - case "a": - string href = ele.getAttribute("href"); - if(href && !ele.hasClass("no-brackets")) { - if(ele.hasClass("href-text")) - ele.innerText = href; - else { - if(ele.innerText != href) - ele.innerText = ele.innerText ~ " <" ~ href ~ "> "; - } - } - ele.stripOut(); - goto again; - case "ol": - case "ul": - ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r"; - break; - case "li": - if(!ele.innerHTML.startsWith("* ")) - ele.innerHTML = "* " ~ ele.innerHTML ~ "\r"; - // ele.stripOut(); - break; - case "sup": - ele.innerText = "^" ~ ele.innerText; - ele.stripOut(); - break; - /* - case "img": - string alt = ele.getAttribute("alt"); - if(alt) - result ~= ele.alt; - break; - */ - default: - ele.stripOut(); - goto again; - } - } - - again2: - //start.innerHTML = start.innerHTML().replace("\u0001", "\n"); - - foreach(ele; start.tree) { - if(ele.tagName == "td") { - if(ele.directText().strip().length) { - ele.prependText("\r"); - ele.appendText("\r"); - } - ele.stripOut(); - goto again2; - } else if(ele.tagName == "p") { - if(strip(ele.innerText()).length > 1) { - string res = ""; - string all = ele.innerText().replace("\n \n", "\n\n"); - foreach(part; all.split("\n\n")) - res ~= "\r" ~ strip( wantWordWrap ? wrap(part, /*74*/ wrapAmount) : part ) ~ "\r"; - ele.innerText = res; - } else - ele.innerText = strip(ele.innerText); - ele.stripOut(); - goto again2; - } else if(ele.tagName == "li") { - auto part = ele.innerText; - part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part ); - part = " " ~ part.replace("\n", "\n\v") ~ "\r"; - ele.innerText = part; - ele.stripOut(); - goto again2; - } - } - - result = start.innerText(); - result = squeeze(result, " "); - - result = result.replace("\r ", "\r"); - result = result.replace(" \r", "\r"); - - //result = result.replace("\u00a0", " "); - - - result = squeeze(result, "\r"); - result = result.replace("\r", "\n\n"); - - result = result.replace("\v", " "); - - result = result.replace("舗", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere - result = result.replace(""", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere - //result = htmlEntitiesDecode(result); // for special chars mainly - - result = result.replace("\u0001 ", "\n"); - result = result.replace("\u0001", "\n"); - - //a = std.regex.replace(a, std.regex.regex("(\n\t)+", "g"), "\n"); //\t"); - return result.strip; -} diff --git a/http.d b/http.d index 624c2dc..28cc3f1 100644 --- a/http.d +++ b/http.d @@ -3,7 +3,7 @@ I no longer work on this, use http2.d instead. +/ -module arsd.http; +deprecated module arsd.http; import std.socket; diff --git a/http2.d b/http2.d index 614781b..1f97919 100644 --- a/http2.d +++ b/http2.d @@ -75,6 +75,20 @@ HttpRequest get(string url) { return request; } +HttpRequest post(string url, string[string] req) { + auto client = new HttpClient(); + ubyte[] bdata; + foreach(k, v; req) { + if(bdata.length) + bdata ~= cast(ubyte[]) "&"; + bdata ~= cast(ubyte[]) encodeComponent(k); + bdata ~= cast(ubyte[]) "="; + bdata ~= cast(ubyte[]) encodeComponent(v); + } + auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); + return request; +} + /// gets the text off a url. basic operation only. string getText(string url) { auto request = get(url); @@ -1090,7 +1104,6 @@ class HttpRequest { ubyte[] sendBuffer; HttpResponse responseData; - HttpRequestParameters parameters; private HttpClient parentClient; size_t bodyBytesSent; diff --git a/joystick.d b/joystick.d index 27c4a09..7a76069 100644 --- a/joystick.d +++ b/joystick.d @@ -167,6 +167,8 @@ version(linux) { // I'd just use my xbox controller. ); + /// For Linux only, reads the latest joystick events into the change buffer, if available. + /// It is non-blocking void readJoystickEvents(int fd) { js_event event; @@ -312,6 +314,7 @@ int enableJoystickInput( // return 0; } +/// void closeJoysticks() { version(linux) { foreach(ref fd; joystickFds) { @@ -330,33 +333,36 @@ void closeJoysticks() { } else static assert(0); } +/// struct JoystickUpdate { + /// int player; JoystickState old; JoystickState current; - // changes from last update + /// changes from last update bool buttonWasJustPressed(Button button) { return buttonIsPressed(button) && !oldButtonIsPressed(button); } + /// ditto bool buttonWasJustReleased(Button button) { return !buttonIsPressed(button) && oldButtonIsPressed(button); } - // this is normalized down to a 16 step change - // and ignores a dead zone near the middle + /// this is normalized down to a 16 step change + /// and ignores a dead zone near the middle short axisChange(Axis axis) { return cast(short) (axisPosition(axis) - oldAxisPosition(axis)); } - // current state + /// current state bool buttonIsPressed(Button button) { return buttonIsPressedHelper(button, ¤t); } - // Note: UP is negative! + /// Note: UP is negative! short axisPosition(Axis axis, short digitalFallbackValue = short.max) { return axisPositionHelper(axis, ¤t, digitalFallbackValue); } @@ -521,6 +527,7 @@ struct JoystickUpdate { } } +/// JoystickUpdate getJoystickUpdate(int player) { static JoystickState[4] previous; diff --git a/jpeg.d b/jpeg.d index f1c51c7..1bd106e 100644 --- a/jpeg.d +++ b/jpeg.d @@ -2971,9 +2971,10 @@ public bool detect_jpeg_image_from_file (const(char)[] filename, out int width, bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + if (filename.length < 512) { + char[513] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); @@ -3147,9 +3148,10 @@ public ubyte[] decompress_jpeg_image_from_file(bool useMalloc=false) (const(char bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + if (filename.length < 512) { + char[513] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); @@ -3338,9 +3340,10 @@ public MemoryImage readJpeg (const(char)[] filename) { bool m_eof_flag, m_error_flag; if (filename.length == 0) throw new Exception("cannot open unnamed file"); - if (filename.length < 2048) { - import core.stdc.stdlib : alloca; - auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1]; + if (filename.length < 512) { + char[513] buffer; + //import core.stdc.stdlib : alloca; + auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1]; tfn[0..filename.length] = filename[]; tfn[filename.length] = 0; m_pFile = fopen(tfn.ptr, "rb"); diff --git a/jsvar.d b/jsvar.d index 5752aeb..696a490 100644 --- a/jsvar.d +++ b/jsvar.d @@ -709,7 +709,7 @@ struct var { case Type.String: case Type.Function: assert(0); // FIXME - break; + //break; case Type.Integral: return var(-this.get!long); case Type.Floating: diff --git a/minigui.d b/minigui.d index 9699c96..a489c36 100644 --- a/minigui.d +++ b/minigui.d @@ -1,5 +1,16 @@ // http://msdn.microsoft.com/en-us/library/windows/desktop/bb775498%28v=vs.85%29.aspx +// So a window needs to have a selection, and that can be represented by a type. This is manipulated by various +// functions like cut, copy, paste. Widgets can have a selection and that would assert teh selection ownership for +// the window. + +// so what about context menus? + +// https://docs.microsoft.com/en-us/windows/desktop/Controls/about-custom-draw + +// FIXME: add a command search thingy built in and implement tip. +// FIXME: omg omg what if menu functions have arguments and it can pop up a gui or command line script them?! + // On Windows: // FIXME: various labels look broken in high contrast mode // FIXME: changing themes while the program is upen doesn't trigger a redraw @@ -3910,8 +3921,10 @@ class MainWindow : Window { .toolbar toolbar; bool separator; .accelerator accelerator; + .hotkey hotkey; .icon icon; string label; + string tip; foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) { static if(is(typeof(attr) == .menu)) menu = attr; @@ -3921,10 +3934,14 @@ class MainWindow : Window { separator = true; else static if(is(typeof(attr) == .accelerator)) accelerator = attr; + else static if(is(typeof(attr) == .hotkey)) + hotkey = attr; else static if(is(typeof(attr) == .icon)) icon = attr; else static if(is(typeof(attr) == .label)) label = attr.label; + else static if(is(typeof(attr) == .tip)) + tip = attr.tip; } if(menu !is .menu.init || toolbar !is .toolbar.init) { @@ -5504,6 +5521,15 @@ abstract class EditableTextWidget : EditableTextWidgetParent { super(parent); } + bool wordWrapEnabled_ = false; + void wordWrapEnabled(bool enabled) { + version(win32_widgets) { + SendMessageW(hwnd, EM_FMTLINES, enabled ? 1 : 0, 0); + } else version(custom_widgets) { + wordWrapEnabled_ = enabled; // FIXME + } else static assert(false); + } + override int minWidth() { return 16; } override int minHeight() { return Window.lineHeight + 0; } // the +0 is to leave room for the padding override int widthStretchiness() { return 7; } @@ -5563,6 +5589,7 @@ abstract class EditableTextWidget : EditableTextWidgetParent { void addText(string txt) { version(custom_widgets) { + textLayout.addText(txt); { @@ -5579,7 +5606,7 @@ abstract class EditableTextWidget : EditableTextWidgetParent { SendMessageW( hwnd, EM_GETSEL, cast(WPARAM)(&StartPos), cast(LPARAM)(&EndPos) ); // move the caret to the end of the text - int outLength = GetWindowTextLengthW( hwnd ); + int outLength = GetWindowTextLengthW(hwnd); SendMessageW( hwnd, EM_SETSEL, outLength, outLength ); // insert the text at the new caret position @@ -6721,6 +6748,12 @@ struct icon { ushort id; } /// /// Group: generating_from_code struct label { string label; } +/// +/// Group: generating_from_code +struct hotkey { dchar ch; } +/// +/// Group: generating_from_code +struct tip { string tip; } /++ diff --git a/minigui_addons/color_dialog.d b/minigui_addons/color_dialog.d index b29ed0e..18cce3f 100644 --- a/minigui_addons/color_dialog.d +++ b/minigui_addons/color_dialog.d @@ -11,7 +11,7 @@ static if(UsingWin32Widgets) /++ +/ -void showColorDialog(Window owner, Color current, void delegate(Color choice) onOK, void delegate() onCancel = null) { +auto showColorDialog(Window owner, Color current, void delegate(Color choice) onOK, void delegate() onCancel = null) { static if(UsingWin32Widgets) { import core.sys.windows.windows; static COLORREF[16] customColors; @@ -30,6 +30,7 @@ void showColorDialog(Window owner, Color current, void delegate(Color choice) on } else static if(UsingCustomWidgets) { auto cpd = new ColorPickerDialog(current, onOK, owner); cpd.show(); + return cpd; } else static assert(0); } @@ -53,7 +54,7 @@ class ColorPickerDialog : Dialog { void delegate(Color) onOK; this(Color current, void delegate(Color) onOK, Window owner) { - super(360, 350, "Color picker"); + super(360, 460, "Color picker"); this.onOK = onOK; @@ -79,7 +80,7 @@ class ColorPickerDialog : Dialog { canUseImage = true; if(hslImage is null && canUseImage) { - auto img = new TrueColorImage(180, 128); + auto img = new TrueColorImage(360, 255); double h = 0.0, s = 1.0, l = 0.5; foreach(y; 0 .. img.height) { foreach(x; 0 .. img.width) { @@ -99,60 +100,112 @@ class ColorPickerDialog : Dialog { this() { super(t); } override int minHeight() { return hslImage ? hslImage.height : 4; } override int maxHeight() { return hslImage ? hslImage.height : 4; } + override int marginBottom() { return 4; } override void paint(ScreenPainter painter) { if(hslImage) hslImage.drawAt(painter, Point(0, 0)); } }; + auto hr = new HorizontalLayout(t); + auto vlRgb = new class VerticalLayout { this() { - super(t); + super(hr); } override int maxWidth() { return 150; }; }; + auto vlHsl = new class VerticalLayout { + this() { + super(hr); + } + override int maxWidth() { return 150; }; + }; + + h = new LabeledLineEdit("Hue:", vlHsl); + s = new LabeledLineEdit("Saturation:", vlHsl); + l = new LabeledLineEdit("Lightness:", vlHsl); + + css = new LabeledLineEdit("CSS:", vlHsl); + r = new LabeledLineEdit("Red:", vlRgb); g = new LabeledLineEdit("Green:", vlRgb); b = new LabeledLineEdit("Blue:", vlRgb); a = new LabeledLineEdit("Alpha:", vlRgb); import std.conv; + import std.format; - r.content = to!string(current.r); - g.content = to!string(current.g); - b.content = to!string(current.b); - a.content = to!string(current.a); + void updateCurrent() { + r.content = to!string(current.r); + g.content = to!string(current.g); + b.content = to!string(current.b); + a.content = to!string(current.a); + + auto hsl = current.toHsl; + + h.content = format("%0.2f", hsl[0]); + s.content = format("%0.2f", hsl[1]); + l.content = format("%0.2f", hsl[2]); + + css.content = current.toCssString(); + } + + updateCurrent(); r.addEventListener("focus", &r.selectAll); g.addEventListener("focus", &g.selectAll); b.addEventListener("focus", &b.selectAll); a.addEventListener("focus", &a.selectAll); + h.addEventListener("focus", &h.selectAll); + s.addEventListener("focus", &s.selectAll); + l.addEventListener("focus", &l.selectAll); - if(hslImage !is null) - wid.addEventListener("mousedown", (Event event) { - auto h = cast(double) event.clientX / hslImage.width * 360.0; - auto s = 1.0 - (cast(double) event.clientY / hslImage.height * 1.0); - auto l = 0.5; + css.addEventListener("focus", &css.selectAll); - auto color = Color.fromHsl(h, s, l); - - r.content = to!string(color.r); - g.content = to!string(color.g); - b.content = to!string(color.b); - a.content = to!string(color.a); - - }); - - Color currentColor() { + void convertFromHsl() { try { - return Color(to!int(r.content), to!int(g.content), to!int(b.content), to!int(a.content)); + auto c = Color.fromHsl(h.content.to!double, s.content.to!double, l.content.to!double); + c.a = a.content.to!ubyte; + current = c; + updateCurrent(); } catch(Exception e) { - return Color.transparent; } } + h.addEventListener("change", &convertFromHsl); + s.addEventListener("change", &convertFromHsl); + l.addEventListener("change", &convertFromHsl); + + css.addEventListener("change", () { + current = Color.fromString(css.content); + updateCurrent(); + }); + + void helper(Event event) { + auto h = cast(double) event.clientX / hslImage.width * 360.0; + auto s = 1.0 - (cast(double) event.clientY / hslImage.height * 1.0); + auto l = this.l.content.to!double; + + current = Color.fromHsl(h, s, l); + current.a = a.content.to!ubyte; + + updateCurrent(); + + auto e2 = new Event("change", this); + e2.dispatch(); + } + + if(hslImage !is null) + wid.addEventListener("mousedown", &helper); + if(hslImage !is null) + wid.addEventListener("mousemove", (Event event) { + if(event.state & ModifierState.leftButtonDown) + helper(event); + }); + this.addEventListener("keydown", (Event event) { if(event.key == Key.Enter || event.key == Key.PadEnter) OK(); @@ -205,6 +258,22 @@ class ColorPickerDialog : Dialog { LabeledLineEdit b; LabeledLineEdit a; + LabeledLineEdit h; + LabeledLineEdit s; + LabeledLineEdit l; + + LabeledLineEdit css; + + Color currentColor() { + import std.conv; + try { + return Color(to!int(r.content), to!int(g.content), to!int(b.content), to!int(a.content)); + } catch(Exception e) { + return Color.transparent; + } + } + + override void OK() { import std.conv; try { diff --git a/mssql.d b/mssql.d index 8957bd8..1cbc9b3 100644 --- a/mssql.d +++ b/mssql.d @@ -155,16 +155,16 @@ class MsSqlResult : ResultSet { string a; more: - SQLCHAR[255] buf; - if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 255, &ptr) != SQL_SUCCESS) + SQLCHAR[1024] buf; + if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 1024, &ptr) != SQL_SUCCESS) throw new DatabaseException("get data: " ~ getSQLError(SQL_HANDLE_STMT, statement)); assert(ptr != SQL_NO_TOTAL); if(ptr == SQL_NULL_DATA) a = null; else { - a ~= cast(string) buf[0 .. ptr > 255 ? 255 : ptr].idup; - ptr -= ptr > 255 ? 255 : ptr; + a ~= cast(string) buf[0 .. ptr > 1024 ? 1024 : ptr].idup; + ptr -= ptr > 1024 ? 1024 : ptr; if(ptr) goto more; } @@ -181,11 +181,11 @@ class MsSqlResult : ResultSet { void makeFieldMapping() { for(int i = 0; i < numFields; i++) { SQLSMALLINT len; - SQLCHAR[255] buf; + SQLCHAR[1024] buf; auto ret = SQLDescribeCol(statement, cast(ushort)(i+1), cast(ubyte*)buf.ptr, - 255, + 1024, &len, null, null, null, null); if (ret != SQL_SUCCESS) diff --git a/nanovega.d b/nanovega.d index fc7e4d1..d3f22e6 100644 --- a/nanovega.d +++ b/nanovega.d @@ -1,4 +1,7 @@ // +// Would be nice: way to take output of the canvas to an image file (raster and/or svg) +// +// // Copyright (c) 2013 Mikko Mononen memon@inside.org // // This software is provided 'as-is', without any express or implied diff --git a/png.d b/png.d index e92e040..4c417b6 100644 --- a/png.d +++ b/png.d @@ -22,6 +22,31 @@ void writePng(string filename, MemoryImage mi) { std.file.write(filename, writePng(png)); } +/// +enum PngType { + greyscale = 0, /// The data must be `depth` bits per pixel + truecolor = 2, /// The data will be RGB triples, so `depth * 3` bits per pixel. Depth must be 8 or 16. + indexed = 3, /// The data must be `depth` bits per pixel, with a palette attached. Use [writePng] with [IndexedImage] for this mode. Depth must be <= 8. + greyscale_with_alpha = 4, /// The data must be (grey, alpha) byte pairs for each pixel. Thus `depth * 2` bits per pixel. Depth must be 8 or 16. + truecolor_with_alpha = 6 /// The data must be RGBA quads for each pixel. Thus, `depth * 4` bits per pixel. Depth must be 8 or 16. +} + +/// Saves an image from an existing array. Note that depth other than 8 may not be implemented yet. Also note depth of 16 must be stored big endian +void writePng(string filename, const ubyte[] data, int width, int height, PngType type, ubyte depth = 8) { + PngHeader h; + h.width = width; + h.height = height; + h.type = cast(ubyte) type; + h.depth = depth; + + auto png = blankPNG(h); + addImageDatastreamToPng(data, png); + + import std.file; + std.file.write(filename, writePng(png)); +} + + /* //Here's a simple test program that shows how to write a quick image viewer with simpledisplay: @@ -629,9 +654,28 @@ void addImageDatastreamToPng(const(ubyte)[] data, PNG* png) { PngHeader h = getHeader(png); - auto bytesPerLine = h.width * 4; - if(h.type == 3) - bytesPerLine = h.width * h.depth / 8; + int bytesPerLine; + switch(h.type) { + case 0: + // FIXME: < 8 depth not supported here but should be + bytesPerLine = h.width * 1 * h.depth / 8; + break; + case 2: + bytesPerLine = h.width * 3 * h.depth / 8; + break; + case 3: + bytesPerLine = h.width * 1 * h.depth / 8; + break; + case 4: + // FIXME: < 8 depth not supported here but should be + bytesPerLine = h.width * 2 * h.depth / 8; + break; + case 6: + bytesPerLine = h.width * 4 * h.depth / 8; + break; + default: assert(0); + + } Chunk dat; dat.type = ['I', 'D', 'A', 'T']; int pos = 0; diff --git a/script.d b/script.d index 1cd764d..ae106c9 100644 --- a/script.d +++ b/script.d @@ -90,6 +90,8 @@ obj.__prop("name", value); // bypasses operator overloading, useful for use inside the opIndexAssign especially Note: if opIndex is not overloaded, getting a non-existent member will actually add it to the member. This might be a bug but is needed right now in the D impl for nice chaining. Or is it? FIXME + + FIXME: it doesn't do opIndex with multiple args. * if/else * array slicing, but note that slices are rvalues currently * variables must start with A-Z, a-z, _, or $, then must be [A-Za-z0-9_]*. @@ -2304,7 +2306,9 @@ Expression parseExpression(MyTokenStreamHere)(ref MyTokenStreamHere tokens, bool auto ident = tokens.requireNextToken(ScriptToken.Type.identifier); tokens.requireNextToken(ScriptToken.Type.symbol, "("); - auto args = parseVariableDeclaration(tokens, ")"); + VariableDeclaration args; + if(!tokens.peekNextToken(ScriptToken.Type.symbol, ")")) + args = parseVariableDeclaration(tokens, ")"); tokens.requireNextToken(ScriptToken.Type.symbol, ")"); auto bod = parseExpression(tokens); @@ -2538,7 +2542,7 @@ Expression parseStatement(MyTokenStreamHere)(ref MyTokenStreamHere tokens, strin return parseFunctionCall(tokens, new AssertKeyword(token)); - break; + //break; // declarations case "var": return parseVariableDeclaration(tokens, ";"); diff --git a/simpledisplay.d b/simpledisplay.d index 7ab6158..3753be7 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -1158,6 +1158,64 @@ string sdpyWindowClass () { return null; } +/++ + Returns the DPI of the default monitor. [0] is width, [1] is height (they are usually the same though). You may wish to round the numbers off. ++/ +float[2] getDpi() { + float[2] dpi; + version(Windows) { + HDC screen = GetDC(null); + dpi[0] = GetDeviceCaps(screen, LOGPIXELSX); + dpi[1] = GetDeviceCaps(screen, LOGPIXELSY); + } else version(X11) { + auto display = XDisplayConnection.get; + auto screen = DefaultScreen(display); + + void fallback() { + // 25.4 millimeters in an inch... + dpi[0] = cast(float) DisplayWidth(display, screen) / DisplayWidthMM(display, screen) * 25.4; + dpi[1] = cast(float) DisplayHeight(display, screen) / DisplayHeightMM(display, screen) * 25.4; + } + + char* resourceString = XResourceManagerString(display); + XrmInitialize(); + + auto db = XrmGetStringDatabase(resourceString); + + if (resourceString) { + XrmValue value; + char* type; + if (XrmGetResource(db, "Xft.dpi", "String", &type, &value) == true) { + if (value.addr) { + import core.stdc.stdlib; + dpi[0] = atof(cast(char*) value.addr); + dpi[1] = dpi[0]; + } else { + fallback(); + } + } else { + fallback(); + } + } else { + fallback(); + } + } + + return dpi; +} + +version(X11) { + extern(C) char* XResourceManagerString(Display*); + extern(C) void XrmInitialize(); + extern(C) XrmDatabase XrmGetStringDatabase(char* data); + extern(C) bool XrmGetResource(XrmDatabase, const char*, const char*, char**, XrmValue*); + alias XrmDatabase = void*; + struct XrmValue { + uint size; + void* addr; + } +} + TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int width, int height) { throw new Exception("not implemented"); version(none) { @@ -1165,6 +1223,8 @@ TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int wid auto display = XDisplayConnection.get; auto image = XGetImage(display, handle, 0, 0, width, height, (cast(c_ulong) ~0) /*AllPlanes*/, ZPixmap); + // https://github.com/adamdruppe/arsd/issues/98 + // FIXME: copy that shit XDestroyImage(image); @@ -1990,11 +2050,15 @@ class SimpleWindow : CapableOfHandlingNativeEvent, CapableOfBeingDrawnUpon { +/ @property void cursor(MouseCursor cursor) { - auto ch = cursor.cursorHandle; + version(OSXCocoa) + featureNotImplemented(); + else if(this.impl.curHidden <= 0) { static if(UsingSimpledisplayX11) { + auto ch = cursor.cursorHandle; XDefineCursor(XDisplayConnection.get(), this.impl.window, ch); } else version(Windows) { + auto ch = cursor.cursorHandle; impl.currentCursor = ch; SetCursor(ch); // redraw without waiting for mouse movement to update } else featureNotImplemented(); @@ -2602,10 +2666,13 @@ static struct GenericCursor { struct EventLoop { @disable this(); + /// Gets a reference to an existing event loop static EventLoop get() { return EventLoop(0, null); } + /// Construct an application-global event loop for yourself + /// See_Also: [SimpleWindow.setEventHandlers] this(long pulseTimeout, void delegate() handlePulse) { if(impl is null) impl = new EventLoopImpl(pulseTimeout, handlePulse); @@ -2633,12 +2700,14 @@ struct EventLoop { impl.refcount++; } + /// Runs the event loop until the whileCondition, if present, returns false int run(bool delegate() whileCondition = null) { assert(impl !is null); impl.notExited = true; return impl.run(whileCondition); } + /// Exits the event loop void exit() { assert(impl !is null); impl.notExited = false; @@ -11895,6 +11964,8 @@ struct Visual int DefaultDepth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).root_depth; } int DisplayWidth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).width; } int DisplayHeight(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).height; } + int DisplayWidthMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mwidth; } + int DisplayHeightMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mheight; } auto DefaultColormap(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).cmap; } int ConnectionNumber(Display* dpy) { return dpy.fd; } @@ -12189,7 +12260,7 @@ private: alias const(void)* CGContextRef; alias const(void)* CGColorSpaceRef; alias const(void)* CGImageRef; - alias uint CGBitmapInfo; + alias ulong CGBitmapInfo; struct objc_super { id self; @@ -12197,18 +12268,18 @@ private: } struct CFRange { - int location, length; + long location, length; } struct NSPoint { - float x, y; + double x, y; static fromTuple(T)(T tupl) { return NSPoint(tupl.tupleof); } } struct NSSize { - float width, height; + double width, height; } struct NSRect { NSPoint origin; @@ -12219,7 +12290,7 @@ private: alias NSRect CGRect; struct CGAffineTransform { - float a, b, c, d, tx, ty; + double a, b, c, d, tx, ty; } enum NSApplicationActivationPolicyRegular = 0; @@ -12235,7 +12306,7 @@ private: NSTexturedBackgroundWindowMask = 1 << 8 } - enum : uint { + enum : ulong { kCGImageAlphaNone, kCGImageAlphaPremultipliedLast, kCGImageAlphaPremultipliedFirst, @@ -12244,7 +12315,7 @@ private: kCGImageAlphaNoneSkipLast, kCGImageAlphaNoneSkipFirst } - enum : uint { + enum : ulong { kCGBitmapAlphaInfoMask = 0x1F, kCGBitmapFloatComponents = (1 << 8), kCGBitmapByteOrderMask = 0x7000, @@ -12293,12 +12364,12 @@ private: CFStringRef CFStringCreateWithBytes(CFAllocatorRef allocator, const(char)* bytes, long numBytes, - int encoding, + long encoding, BOOL isExternalRepresentation); - int CFStringGetBytes(CFStringRef theString, CFRange range, int encoding, + long CFStringGetBytes(CFStringRef theString, CFRange range, long encoding, char lossByte, bool isExternalRepresentation, char* buffer, long maxBufLen, long* usedBufLen); - int CFStringGetLength(CFStringRef theString); + long CFStringGetLength(CFStringRef theString); CGContextRef CGBitmapContextCreate(void* data, size_t width, size_t height, @@ -12316,13 +12387,13 @@ private: void CGColorSpaceRelease(CGColorSpaceRef cs); void CGContextSetRGBStrokeColor(CGContextRef c, - float red, float green, float blue, - float alpha); + double red, double green, double blue, + double alpha); void CGContextSetRGBFillColor(CGContextRef c, - float red, float green, float blue, - float alpha); + double red, double green, double blue, + double alpha); void CGContextDrawImage(CGContextRef c, CGRect rect, CGImageRef image); - void CGContextShowTextAtPoint(CGContextRef c, float x, float y, + void CGContextShowTextAtPoint(CGContextRef c, double x, double y, const(char)* str, size_t length); void CGContextStrokeLineSegments(CGContextRef c, const(CGPoint)* points, size_t count); @@ -12330,15 +12401,15 @@ private: void CGContextBeginPath(CGContextRef c); void CGContextDrawPath(CGContextRef c, CGPathDrawingMode mode); void CGContextAddEllipseInRect(CGContextRef c, CGRect rect); - void CGContextAddArc(CGContextRef c, float x, float y, float radius, - float startAngle, float endAngle, int clockwise); + void CGContextAddArc(CGContextRef c, double x, double y, double radius, + double startAngle, double endAngle, long clockwise); void CGContextAddRect(CGContextRef c, CGRect rect); void CGContextAddLines(CGContextRef c, const(CGPoint)* points, size_t count); void CGContextSaveGState(CGContextRef c); void CGContextRestoreGState(CGContextRef c); - void CGContextSelectFont(CGContextRef c, const(char)* name, float size, - uint textEncoding); + void CGContextSelectFont(CGContextRef c, const(char)* name, double size, + ulong textEncoding); CGAffineTransform CGContextGetTextMatrix(CGContextRef c); void CGContextSetTextMatrix(CGContextRef c, CGAffineTransform t); @@ -12348,7 +12419,7 @@ private: private: // A convenient method to create a CFString (=NSString) from a D string. CFStringRef createCFString(string str) { - return CFStringCreateWithBytes(null, str.ptr, cast(int) str.length, + return CFStringCreateWithBytes(null, str.ptr, cast(long) str.length, kCFStringEncodingUTF8, false); } @@ -12435,7 +12506,7 @@ version(OSXCocoa) { // if rawData had a length.... //assert(rawData.length == where.length); - for(int idx = 0; idx < where.length; idx += 4) { + for(long idx = 0; idx < where.length; idx += 4) { auto alpha = rawData[idx + 3]; if(alpha == 255) { where[idx + 0] = rawData[idx + 0]; // r @@ -12458,7 +12529,7 @@ version(OSXCocoa) { // if rawData had a length.... //assert(rawData.length == where.length); - for(int idx = 0; idx < where.length; idx += 4) { + for(long idx = 0; idx < where.length; idx += 4) { auto alpha = rawData[idx + 3]; if(alpha == 255) { rawData[idx + 0] = where[idx + 0]; // r @@ -12508,17 +12579,19 @@ version(OSXCocoa) { mixin template NativeScreenPainterImplementation() { CGContextRef context; ubyte[4] _outlineComponents; + id view; void create(NativeWindowHandle window) { context = window.drawingContext; + view = window.view; } void dispose() { + setNeedsDisplay(view, true); } // NotYetImplementedException Size textSize(in char[] txt) { return Size(32, 16); throw new NotYetImplementedException(); } - void pen(Pen p) {} void rasterOp(RasterOp op) {} Pen _activePen; Color _fillColor; @@ -12529,8 +12602,10 @@ version(OSXCocoa) { // end - @property void outlineColor(Color color) { - float alphaComponent = color.a/255.0f; + void pen(Pen pen) { + _activePen = pen; + auto color = pen.color; // FIXME + double alphaComponent = color.a/255.0f; CGContextSetRGBStrokeColor(context, color.r/255.0f, color.g/255.0f, color.b/255.0f, alphaComponent); @@ -12581,7 +12656,7 @@ version(OSXCocoa) { _outlineComponents[1]*invAlpha, _outlineComponents[2]*invAlpha, _outlineComponents[3]/255.0f); - CGContextShowTextAtPoint(context, x, y, text.ptr, text.length); + CGContextShowTextAtPoint(context, x, y + 12 /* this is cuz this picks baseline but i want bounding box */, text.ptr, text.length); // auto cfstr = cast(id)createCFString(text); // objc_msgSend(cfstr, sel_registerName("drawAtPoint:withAttributes:"), // NSPoint(x, y), null); @@ -12748,9 +12823,13 @@ version(OSXCocoa) { // will like it. Let's leave it to the native handler. // perform the default action. - auto superData = objc_super(self, superclass(self)); - alias extern(C) void function(objc_super*, SEL, id) T; - (cast(T)&objc_msgSendSuper)(&superData, _cmd, event); + + // so the default action is to make a bomp sound and i dont want that + // sooooooooo yeah not gonna do that. + + //auto superData = objc_super(self, superclass(self)); + //alias extern(C) void function(objc_super*, SEL, id) T; + //(cast(T)&objc_msgSendSuper)(&superData, _cmd, event); } } diff --git a/terminal.d b/terminal.d index 49ab246..bec8ef3 100644 --- a/terminal.d +++ b/terminal.d @@ -1,4 +1,3 @@ -// FIXME: have a simple function that integrates with sdpy event loop. it can be a template // for optional dependency /++ Module for interacting with the user's terminal, including color output, cursor manipulation, and full-featured real-time mouse and keyboard input. Also includes high-level convenience methods, like [Terminal.getline], which gives the user a line editor with history, completion, etc. See the [#examples]. @@ -68,7 +67,7 @@ module arsd.terminal; This example will demonstrate the high-level getline interface. - The user will be able to type a line and navigate around it with cursor keys and even the mouse on some systems, as well as perform editing as they expect (e.g. the backspace and delete keys work normally) until they press enter. Then, the final line will be returned to your program, which the example will simply print back to the user. + The user will be able to type a line and navigate around it with cursor keys and even the mouse on some systems, as well as perform editing as they expect (e.g. the backspace and delete keys work normally) until they press enter. Then, the final line will be returned to your program, which the example will simply print back to the user. +/ unittest { import arsd.terminal; @@ -78,6 +77,8 @@ unittest { string line = terminal.getline(); terminal.writeln("You wrote: ", line); } + + main; // exclude from docs } /++ @@ -88,6 +89,7 @@ unittest { +/ unittest { import arsd.terminal; + void main() { auto terminal = Terminal(ConsoleOutputType.linear); terminal.color(Color.green, Color.black); @@ -95,6 +97,8 @@ unittest { terminal.color(Color.DEFAULT, Color.DEFAULT); terminal.writeln("And back to normal."); } + + main; // exclude from docs } /++ @@ -105,6 +109,7 @@ unittest { +/ unittest { import arsd.terminal; + void main() { auto terminal = Terminal(ConsoleOutputType.linear); auto input = RealTimeConsoleInput(&terminal, ConsoleInputFlags.raw); @@ -113,6 +118,8 @@ unittest { auto ch = input.getch(); terminal.writeln("You pressed ", ch); } + + main; // exclude from docs } /* @@ -427,6 +434,25 @@ struct Terminal { @disable this(this); private ConsoleOutputType type; + /++ + Terminal is only valid to use on an actual console device or terminal + handle. You should not attempt to construct a Terminal instance if this + returns false; + +/ + static bool stdoutIsTerminal() { + version(Posix) { + import core.sys.posix.unistd; + return cast(bool) isatty(1); + } else version(Windows) { + auto hConsole = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO originalSbi; + if(GetConsoleScreenBufferInfo(hConsole, &originalSbi) == 0) + return false; + else + return true; + } else static assert(0); + } + version(Posix) { private int fdOut; private int fdIn; @@ -797,11 +823,19 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as defaultForegroundColor = cast(Color) (originalSbi.wAttributes & 0x0f); defaultBackgroundColor = cast(Color) ((originalSbi.wAttributes >> 4) & 0x0f); + + oldCp = GetConsoleOutputCP(); + SetConsoleOutputCP(65001); // UTF-8 + + oldCpIn = GetConsoleCP(); + SetConsoleCP(65001); // UTF-8 } version(Windows) { private Color defaultBackgroundColor = Color.black; private Color defaultForegroundColor = Color.white; + UINT oldCp; + UINT oldCpIn; } // only use this if you are sure you know what you want, since the terminal is a shared resource you generally really want to reset it to normal when you leave... @@ -836,6 +870,10 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as if(lineGetter !is null) lineGetter.dispose(); + + SetConsoleOutputCP(oldCp); + SetConsoleCP(oldCpIn); + auto stdo = GetStdHandle(STD_OUTPUT_HANDLE); SetConsoleActiveScreenBuffer(stdo); if(hConsole !is stdo) @@ -1587,6 +1625,16 @@ struct RealTimeConsoleInput { void delegate(InputEvent) userEventHandler; + /++ + If you are using [arsd.simpledisplay] and want terminal interop too, you can call + this function to add it to the sdpy event loop and get the callback called on new + input. + + Note that you will probably need to call `terminal.flush()` when you are doing doing + output, as the sdpy event loop doesn't know to do that (yet). I will probably change + that in a future version, but it doesn't hurt to call it twice anyway, so I recommend + calling flush yourself in any code you write using this. + +/ void integrateWithSimpleDisplayEventLoop()(void delegate(InputEvent) userEventHandler) { this.userEventHandler = userEventHandler; import arsd.simpledisplay; @@ -3577,7 +3625,7 @@ struct ScrollbackBuffer { } void clear() { - lines = null; + lines.clear(); clickRegions = null; scrollbackPosition = 0; } @@ -3679,9 +3727,87 @@ struct ScrollbackBuffer { } } - // FIXME: limit scrollback lines.length + static struct CircularBuffer(T) { + T[] backing; - Line[] lines; + enum maxScrollback = 8192; // as a power of 2, i hope the compiler optimizes the % below to a simple bit mask... + + int start; + int length_; + + void clear() { + backing = null; + start = 0; + length_ = 0; + } + + size_t length() { + return length_; + } + + void opOpAssign(string op : "~")(T line) { + if(length_ < maxScrollback) { + backing.assumeSafeAppend(); + backing ~= line; + length_++; + } else { + backing[start] = line; + start++; + if(start == maxScrollback) + start = 0; + } + } + + T opIndex(int idx) { + return backing[(start + idx) % maxScrollback]; + } + T opIndex(Dollar idx) { + return backing[(start + (length + idx.offsetFromEnd)) % maxScrollback]; + } + + CircularBufferRange opSlice(int startOfIteration, Dollar end) { + return CircularBufferRange(&this, startOfIteration, cast(int) length - startOfIteration + end.offsetFromEnd); + } + CircularBufferRange opSlice(int startOfIteration, int end) { + return CircularBufferRange(&this, startOfIteration, end - startOfIteration); + } + CircularBufferRange opSlice() { + return CircularBufferRange(&this, 0, cast(int) length); + } + + static struct CircularBufferRange { + CircularBuffer* item; + int position; + int remaining; + this(CircularBuffer* item, int startOfIteration, int count) { + this.item = item; + position = startOfIteration; + remaining = count; + } + + T front() { return (*item)[position]; } + bool empty() { return remaining <= 0; } + void popFront() { + position++; + remaining--; + } + + T back() { return (*item)[remaining - 1 - position]; } + void popBack() { + remaining--; + } + } + + static struct Dollar { + int offsetFromEnd; + Dollar opBinary(string op : "-")(int rhs) { + return Dollar(offsetFromEnd - rhs); + } + } + Dollar opDollar() { return Dollar(0); } + } + + CircularBuffer!Line lines; string name; int x, y, width, height; @@ -3811,7 +3937,7 @@ struct ScrollbackBuffer { // second pass: actually draw it int linePos = remaining; - foreach(idx, line; lines[start .. start + howMany]) { + foreach(line; lines[start .. start + howMany]) { int written = 0; if(linePos < 0) {