forks
/
arsd
mirror of https://github.com/adamdruppe/arsd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:adamdruppe/arsd

This commit is contained in:
Adam D. Ruppe 2018-11-10 21:30:05 -05:00
parent 2a99aaf7ec fc83e264aa
commit 9cb2c9f539
8 changed files with 260 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
cgi.d
View File

@ -1,12 +1,33 @@
// FIXME: if an exception is thrown, we shouldn't necessarily cache...
// FIXME: there's some annoying duplication of code in the various versioned mains
// FIXME: new ConnectionThread is done a lot, no pooling implemented
// Note: spawn-fcgi can help with fastcgi on nginx
// FIXME: to do: add openssl optionally
// make sure embedded_httpd doesn't send two answers if one writes() then dies
// future direction: websocket as a separate process that you can sendfile to for an async passoff of those long-lived connections
/*
Session manager process: it spawns a new process, passing a
command line argument, to just be a little key/value store
of some serializable struct. On Windows, it CreateProcess.
On Linux, it can just fork or maybe fork/exec. The session
key is in a cookie.
Server-side event process: spawns an async manager. You can
push stuff out to channel ids and the clients listen to it.
websocket process: spawns an async handler. They can talk to
each other or get info from a cgi request.
Tempting to put web.d 2.0 in here. It would:
* map urls and form generation to functions
* have data presentation magic
* do the skeleton stuff like 1.0
* auto-cache generated stuff in files (at least if pure?)
*/
/++
Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications.
@ -1518,8 +1539,10 @@ class Cgi {
if(header.indexOf("HTTP/1.0") != -1) {
http10 = true;
autoBuffer = true;
if(closeConnection)
if(closeConnection) {
// on http 1.0, close is assumed (unlike http/1.1 where we assume keep alive)
*closeConnection = true;
}
}
} else {
// other header
@ -1540,8 +1563,16 @@ class Cgi {
else if (name == "connection") {
if(value == "close" && closeConnection)
*closeConnection = true;
if(value.toLower().indexOf("keep-alive") != -1)
if(value.toLower().indexOf("keep-alive") != -1) {
keepAliveRequested = true;
// on http 1.0, the connection is closed by default,
// but not if they request keep-alive. then we don't close
// anymore - undoing the set above
if(http10 && closeConnection) {
*closeConnection = false;
}
}
}
else if (name == "transfer-encoding") {
if(value == "chunked")
@ -2737,6 +2768,9 @@ mixin template CustomCgiMainImpl(CustomCgi, alias fun, long maxContentLength = d
throw new Exception("bind");
}
// FIXME: if this queue is full, it will just ignore it
// and wait for the client to retransmit it. This is an
// obnoxious timeout condition there.
if(sock.listen(128) == -1) {
close(sock);
throw new Exception("listen");
@ -2968,6 +3002,9 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) {
sendAll(connection, plainHttpError(false, "500 Internal Server Error", null));
connection.close();
}
connection.setOption(SocketOptionLevel.SOCKET, SocketOption.RCVTIMEO, dur!"seconds"(10));
bool closeConnection;
auto ir = new BufferedInputRange(connection);
@ -3100,6 +3137,7 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket
try {
fun(cgi);
cgi.close();
connection.close();
} catch(Throwable t) {
// no std err
if(!handleException(cgi, t)) {
@ -3316,6 +3354,11 @@ class BufferedInputRange {
try_again:
auto ret = source.receive(freeSpace);
if(ret == Socket.ERROR) {
if(wouldHaveBlocked()) {
// gonna treat a timeout here as a close
sourceClosed = true;
return;
}
version(Posix) {
import core.stdc.errno;
if(errno == EINTR || errno == EAGAIN) {
@ -3374,32 +3417,8 @@ class BufferedInputRange {
bool sourceClosed;
}
class ConnectionThread2 : Thread {
import std.concurrency;
this(void function(Socket) handler) {
this.handler = handler;
super(&run);
}
void run() {
tid = thisTid();
available = true;
while(true)
receive(
(/*Socket*/ size_t s) {
available = false;
try {
handler(cast(Socket) cast(void*) s);
} catch(Throwable t) {}
available = true;
}
);
}
bool available;
Tid tid;
void function(Socket) handler;
}
import core.sync.semaphore;
import core.atomic;
/**
To use this thing:
@ -3415,38 +3434,61 @@ class ConnectionThread2 : Thread {
FIXME: should I offer an event based async thing like netman did too? Yeah, probably.
*/
class ListeningConnectionManager {
Semaphore semaphore;
Socket[256] queue;
shared(ubyte) nextIndexFront;
ubyte nextIndexBack;
shared(int) queueLength;
void listen() {
version(cgi_multiple_connections_per_thread) {
import std.concurrency;
import std.random;
ConnectionThread2[16] pool;
foreach(ref p; pool) {
p = new ConnectionThread2(handler);
p.start();
}
running = true;
shared(int) loopBroken;
while(true) {
auto connection = listener.accept();
version(cgi_no_threads) {
// NEVER USE THIS
// it exists only for debugging and other special occasions
bool handled = false;
retry:
foreach(p; pool)
if(p.available) {
handled = true;
send(p.tid, cast(size_t) cast(void*) connection);
break;
}
// none available right now, make it wait a bit then try again
if(!handled) {
Thread.sleep(dur!"msecs"(25));
goto retry;
// the thread mode is faster and less likely to stall the whole
// thing when a request is slow
while(!loopBroken && running) {
auto sn = listener.accept();
try {
handler(sn);
} catch(Exception e) {
// if a connection goes wrong, we want to just say no, but try to carry on unless it is an Error of some sort (in which case, we'll die. You might want an external helper program to revive the server when it dies)
sn.close();
}
}
} else {
foreach(connection; this)
handler(connection);
semaphore = new Semaphore();
ConnectionThread[16] threads;
foreach(ref thread; threads) {
thread = new ConnectionThread(this, handler);
thread.start();
}
while(!loopBroken && running) {
auto sn = listener.accept();
// disable Nagle's algorithm to avoid a 40ms delay when we send/recv
// on the socket because we do some buffering internally. I think this helps,
// certainly does for small requests, and I think it does for larger ones too
sn.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1);
while(queueLength >= queue.length)
Thread.sleep(1.msecs);
synchronized(this) {
queue[nextIndexBack] = sn;
nextIndexBack++;
atomicOp!"+="(queueLength, 1);
}
semaphore.notify();
foreach(thread; threads) {
if(!thread.isRunning) {
thread.join();
}
}
}
}
}
@ -3465,50 +3507,6 @@ class ListeningConnectionManager {
void quit() {
running = false;
}
int opApply(scope CMT dg) {
running = true;
shared(int) loopBroken;
while(!loopBroken && running) {
auto sn = listener.accept();
try {
version(cgi_no_threads) {
// NEVER USE THIS
// it exists only for debugging and other special occasions
// the thread mode is faster and less likely to stall the whole
// thing when a request is slow
dg(sn);
} else {
/*
version(cgi_multiple_connections_per_thread) {
bool foundOne = false;
tryAgain:
foreach(t; pool)
if(t.s is null) {
t.s = sn;
foundOne = true;
break;
}
Thread.sleep(dur!"msecs"(1));
if(!foundOne)
goto tryAgain;
} else {
*/
auto thread = new ConnectionThread(sn, &loopBroken, dg);
thread.start();
//}
}
// loopBroken = dg(sn);
} catch(Exception e) {
// if a connection goes wrong, we want to just say no, but try to carry on unless it is an Error of some sort (in which case, we'll die. You might want an external helper program to revive the server when it dies)
sn.close();
}
}
return loopBroken;
}
}
// helper function to send a lot to a socket. Since this blocks for the buffer (possibly several times), you should probably call it in a separate thread or something.
@ -3532,46 +3530,35 @@ class ConnectionException : Exception {
}
}
alias int delegate(Socket) CMT;
alias void function(Socket) CMT;
import core.thread;
class ConnectionThread : Thread {
this(Socket s, shared(int)* breakSignifier, CMT dg) {
this.s = s;
this.breakSignifier = breakSignifier;
this(ListeningConnectionManager lcm, CMT dg) {
this.lcm = lcm;
this.dg = dg;
super(&runAll);
}
void runAll() {
if(s !is null)
run();
/*
version(cgi_multiple_connections_per_thread) {
while(1) {
while(s is null)
sleep(dur!"msecs"(1));
run();
}
}
*/
super(&run);
}
void run() {
scope(exit) {
// I don't want to double close it, and it does this on close() according to source
// might be fragile, but meh
if(s.handle() != socket_t.init)
s.close();
s = null; // so we know this thread is clear
}
if(auto result = dg(s)) {
*breakSignifier = result;
while(true) {
lcm.semaphore.wait();
Socket socket;
synchronized(lcm) {
auto idx = lcm.nextIndexFront;
socket = lcm.queue[idx];
lcm.queue[idx] = null;
atomicOp!"+="(lcm.nextIndexFront, 1);
atomicOp!"-="(lcm.queueLength, 1);
}
try
dg(socket);
catch(Exception e)
socket.close();
}
}
Socket s;
shared(int)* breakSignifier;
ListeningConnectionManager lcm;
CMT dg;
}

28
color.d
View File

@ -6,7 +6,7 @@ module arsd.color;
// importing phobos explodes the size of this code 10x, so not doing it.
private {
real toInternal(T)(string s) {
real toInternal(T)(scope const(char)[] s) {
real accumulator = 0.0;
size_t i = s.length;
foreach(idx, c; s) {
@ -16,8 +16,11 @@ private {
} else if(c == '.') {
i = idx + 1;
break;
} else
throw new Exception("bad char to make real from " ~ s);
} else {
string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from ";
wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s;
throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute);
}
}
real accumulator2 = 0.0;
@ -27,8 +30,11 @@ private {
accumulator2 *= 10;
accumulator2 += c - '0';
count *= 10;
} else
throw new Exception("bad char to make real from " ~ s);
} else {
string wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute = "bad char to make real from ";
wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute ~= s;
throw new Exception(wtfIsWrongWithThisStupidLanguageWithItsBrokenSafeAttribute);
}
}
return accumulator + accumulator2 / count;
@ -80,11 +86,11 @@ private {
return m;
}
nothrow @safe @nogc pure
bool startsWithInternal(string a, string b) {
bool startsWithInternal(in char[] a, in char[] b) {
return (a.length >= b.length && a[0 .. b.length] == b);
}
string[] splitInternal(string a, char c) {
string[] ret;
inout(char)[][] splitInternal(inout(char)[] a, char c) {
inout(char)[][] ret;
size_t previous = 0;
foreach(i, char ch; a) {
if(ch == c) {
@ -97,7 +103,7 @@ private {
return ret;
}
nothrow @safe @nogc pure
string stripInternal(string s) {
inout(char)[] stripInternal(inout(char)[] s) {
foreach(i, char c; s)
if(c != ' ' && c != '\t' && c != '\n') {
s = s[i .. $];
@ -242,7 +248,7 @@ struct Color {
}
/// Reads a CSS style string to get the color. Understands #rrggbb, rgba(), hsl(), and rrggbbaa
static Color fromString(string s) {
static Color fromString(scope const(char)[] s) {
s = s.stripInternal();
Color c;
@ -422,7 +428,7 @@ private string toHexInternal(ubyte b) {
}
nothrow @safe @nogc pure
private ubyte fromHexInternal(string s) {
private ubyte fromHexInternal(in char[] s) {
int result = 0;
int exp = 1;

4
dom.d
View File

@ -4819,7 +4819,7 @@ class Table : Element {
tagName = "table";
}
///.
/// Creates an element with the given type and content.
Element th(T)(T t) {
Element e;
if(parentDocument !is null)
@ -4833,7 +4833,7 @@ class Table : Element {
return e;
}
///.
/// ditto
Element td(T)(T t) {
Element e;
if(parentDocument !is null)

38
gamehelpers.d
View File

@ -226,6 +226,23 @@ final class OpenGlTexture {
/// Make a texture from an image.
this(TrueColorImage from) {
bindFrom(from);
}
/// Generates from text. Requires ttf.d
/// pass a pointer to the TtfFont as the first arg (it is template cuz of lazy importing, not because it actually works with different types)
this(T, FONT)(FONT* font, int size, in T[] text) if(is(T == char)) {
bindFrom(font, size, text);
}
/// Creates an empty texture class for you to use with [bindFrom] later
/// Using it when not bound is undefined behavior.
this() {}
/// After you delete it with dispose, you may rebind it to something else with this.
void bindFrom(TrueColorImage from) {
assert(from.width > 0 && from.height > 0);
import core.stdc.stdlib;
@ -295,9 +312,8 @@ final class OpenGlTexture {
free(cast(void*) data);
}
/// Generates from text. Requires ttf.d
/// pass a pointer to the TtfFont as the first arg (it is template cuz of lazy importing, not because it actually works with different types)
this(T, FONT)(FONT* font, int size, in T[] text) if(is(T == char)) {
/// ditto
void bindFrom(T, FONT)(FONT* font, int size, in T[] text) if(is(T == char)) {
assert(font !is null);
int width, height;
auto data = font.renderString(text, size, width, height);
@ -313,14 +329,26 @@ final class OpenGlTexture {
}
assert(data.length == 0);
this(image);
bindFrom(image);
}
/// Deletes the texture. Using it after calling this is undefined behavior
void dispose() {
glDeleteTextures(1, &_tex);
_tex = 0;
}
~this() {
glDeleteTextures(1, &_tex);
if(_tex > 0)
dispose();
}
}
///
void clearOpenGlScreen(SimpleWindow window) {
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_ACCUM_BUFFER_BIT);
}
// Some math helpers

6
mysql.d
View File

@ -197,10 +197,10 @@ class MySqlResult : ResultSet {
+/
class MySql : Database {
this(string host, string user, string pass, string db, uint port = 0) {
mysql = enforceEx!(DatabaseException)(
mysql = enforce!(DatabaseException)(
mysql_init(null),
"Couldn't init mysql");
enforceEx!(DatabaseException)(
enforce!(DatabaseException)(
mysql_real_connect(mysql, toCstring(host), toCstring(user), toCstring(pass), toCstring(db), port, null, 0),
error());
@ -371,7 +371,7 @@ class MySql : Database {
override ResultSet queryImpl(string sql, Variant[] args...) {
sql = escapedVariants(this, sql, args);
enforceEx!(DatabaseException)(
enforce!(DatabaseException)(
!mysql_query(mysql, toCstring(sql)),
error() ~ " :::: " ~ sql);

45
simpledisplay.d
View File

@ -9674,6 +9674,9 @@ version(X11) {
scope(exit) XLockDisplay(display);
win.setSelectionHandler(e);
}
break;
case EventType.PropertyNotify:
break;
case EventType.SelectionNotify:
if(auto win = e.xselection.requestor in SimpleWindow.nativeMapping)
@ -9694,11 +9697,11 @@ version(X11) {
e.xselection.property,
0,
100000 /* length */,
false,
//false, /* don't erase it */
true, /* do erase it lol */
0 /*AnyPropertyType*/,
&target, &format, &length, &bytesafter, &value);
// FIXME: it might be sent in pieces...
// FIXME: I don't have to copy it now since it is in char[] instead of string
{
@ -9728,15 +9731,51 @@ version(X11) {
}
} else if(target == GetAtom!"UTF8_STRING"(display) || target == XA_STRING) {
win.getSelectionHandler((cast(char[]) value[0 .. length]).idup);
} else if(target == GetAtom!"INCR"(display)) {
// incremental
//sdpyGettingPaste = true; // FIXME: should prolly be separate for the different selections
// FIXME: handle other events while it goes so app doesn't lock up with big pastes
// can also be optimized if it chunks to the api somehow
char[] s;
do {
XEvent subevent;
do {
XMaskEvent(display, EventMask.PropertyChangeMask, &subevent);
} while(subevent.type != EventType.PropertyNotify || subevent.xproperty.atom != e.xselection.property || subevent.xproperty.state != PropertyNotification.PropertyNewValue);
void* subvalue;
XGetWindowProperty(
e.xselection.display,
e.xselection.requestor,
e.xselection.property,
0,
100000 /* length */,
true, /* erase it to signal we got it and want more */
0 /*AnyPropertyType*/,
&target, &format, &length, &bytesafter, &subvalue);
s ~= (cast(char*) subvalue)[0 .. length];
XFree(subvalue);
} while(length > 0);
win.getSelectionHandler(s);
} else {
// unsupported type
}
}
XFree(value);
/*
XDeleteProperty(
e.xselection.display,
e.xselection.requestor,
e.xselection.property);
*/
}
}
break;
@ -10853,6 +10892,8 @@ int XNextEvent(
XEvent* /* event_return */
);
int XMaskEvent(Display*, arch_long, XEvent*);
Bool XFilterEvent(XEvent *event, Window window);
int XRefreshKeyboardMapping(XMappingEvent *event_map);

1
ttf.d
View File

@ -92,6 +92,7 @@ struct TtfFont {
stbtt_GetCodepointHMetrics(&font, ch, &advance, &lsb);
int cw, cheight;
auto c = renderCharacter(ch, size, cw, cheight, x_shift, 0.0);
scope(exit) stbtt_FreeBitmap(c.ptr, null);
int x0, y0, x1, y1;
stbtt_GetCodepointBitmapBoxSubpixel(&font, ch, scale,scale,x_shift,0, &x0,&y0,&x1,&y1);

56
web.d
View File

@ -9,6 +9,11 @@ enum RequirePost;
enum RequireHttps;
enum NoAutomaticForm;
///
struct GenericContainerType {
string type; ///
}
/// Attribute for the default formatting (html, table, json, etc)
struct DefaultFormat {
string format;
@ -577,7 +582,7 @@ class ApiProvider : WebDotDBaseType {
/// Returns a list of links to all functions in this class or sub-classes
/// You can expose it publicly with alias: "alias _sitemap sitemap;" for example.
Element _sitemap() {
auto container = _getGenericContainer();
auto container = Element.make("div", "", "sitemap");
void writeFunctions(Element list, in ReflectionInfo* reflection, string base) {
string[string] handled;
@ -617,7 +622,7 @@ class ApiProvider : WebDotDBaseType {
starting = cgi.logicalScriptName ~ cgi.pathInfo; // FIXME
writeFunctions(list, reflection, starting ~ "/");
return list.parentNode.removeChild(list);
return container;
}
/// If the user goes to your program without specifying a path, this function is called.
@ -626,13 +631,18 @@ class ApiProvider : WebDotDBaseType {
throw new Exception("no default");
}
/// forwards to [_getGenericContainer]("default")
Element _getGenericContainer() {
return _getGenericContainer("default");
}
/// When the html document envelope is used, this function is used to get a html element
/// where the return value is appended.
/// It's the main function to override to provide custom HTML templates.
///
/// The default document provides a default stylesheet, our default javascript, and some timezone cookie handling (which you must handle on the server. Eventually I'll open source my date-time helpers that do this, but the basic idea is it sends an hour offset, and you can add that to any UTC time you have to get a local time).
Element _getGenericContainer()
Element _getGenericContainer(string containerName)
out(ret) {
assert(ret !is null);
}
@ -766,7 +776,7 @@ struct ReflectionInfo {
// these might go away.
string defaultOutputFormat = "html";
string defaultOutputFormat = "default";
int versionOfOutputFormat = 2; // change this in your constructor if you still need the (deprecated) old behavior
// bool apiMode = false; // no longer used - if format is json, apiMode behavior is assumed. if format is html, it is not.
// FIXME: what if you want the data formatted server side, but still in a json envelope?
@ -815,6 +825,8 @@ struct FunctionInfo {
bool requireHttps;
string genericContainerType = "default";
Document delegate(in string[string] args) createForm; /// This is used if you want a custom form - normally, on insufficient parameters, an automatic form is created. But if there's a functionName_Form method, it is used instead. FIXME: this used to work but not sure if it still does
}
@ -1000,6 +1012,8 @@ immutable(ReflectionInfo*) prepareReflectionImpl(alias PM, alias Parent)(Parent
f.returnType = ReturnType!(__traits(getMember, Class, member)).stringof;
f.returnTypeIsDocument = is(ReturnType!(__traits(getMember, Class, member)) : Document);
f.returnTypeIsElement = is(ReturnType!(__traits(getMember, Class, member)) : Element);
static if(hasValueAnnotation!(__traits(getMember, Class, member), GenericContainerType))
f.genericContainerType = getAnnotation!(__traits(getMember, Class, member), GenericContainerType).type;
f.parentObject = reflection;
@ -1573,9 +1587,9 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
Element e;
auto hack = cast(ApiProvider) realObject;
if(hack !is null)
e = hack._getGenericContainer();
e = hack._getGenericContainer(fun is null ? "default" : fun.genericContainerType);
else
e = instantiation._getGenericContainer();
e = instantiation._getGenericContainer(fun is null ? "default" : fun.genericContainerType);
document = e.parentDocument;
@ -1881,6 +1895,7 @@ Form createAutomaticForm(Document document, string action, in Parameter[] parame
auto fmt = Element.make("select");
fmt.name = "format";
fmt.addChild("option", "Automatic").setAttribute("value", "default");
fmt.addChild("option", "html").setAttribute("value", "html");
fmt.addChild("option", "table").setAttribute("value", "table");
fmt.addChild("option", "json").setAttribute("value", "json");
@ -2741,6 +2756,15 @@ WrapperFunction generateWrapper(alias ObjectType, string funName, alias f, R)(Re
// FIXME: it's awkward to call manually due to the JSONValue ref thing. Returning a string would be mega nice.
string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue = null, string formatJsonToStringAs = null) if(is(R : ApiProvider)) {
if(format == "default") {
static if(is(typeof(ret) : K[N][V], size_t N, K, V)) {
format = "table";
} else {
format = "html";
}
}
string retstr;
if(api !is null) {
static if(__traits(compiles, api._customFormat(ret, format))) {
@ -2803,8 +2827,18 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue
goto badType;
gotATable(table);
break;
}
else
} else static if(is(typeof(ret) : K[N][V], size_t N, K, V)) {
auto table = cast(Table) Element.make("table");
table.addClass("data-display");
foreach(k, v; ret) {
auto row = table.addChild("tr");
foreach(cell; v)
table.addChild("td", to!string(cell));
}
gotATable(table);
break;
} else
goto badType;
default:
badType:
@ -3899,7 +3933,7 @@ bool checkPassword(string saltedPasswordHash, string userSuppliedPassword) {
/// implements the "table" format option. Works on structs and associative arrays (string[string][])
Table structToTable(T)(Document document, T arr, string[] fieldsToSkip = null) if(isArray!(T) && !isAssociativeArray!(T)) {
auto t = cast(Table) document.createElement("table");
t.border = "1";
t.attrs.border = "1";
static if(is(T == string[string][])) {
string[string] allKeys;
@ -3929,7 +3963,7 @@ Table structToTable(T)(Document document, T arr, string[] fieldsToSkip = null) i
odd = !odd;
}
} else static if(is(typeof(T[0]) == struct)) {
} else static if(is(typeof(arr[0]) == struct)) {
{
auto thead = t.addChild("thead");
auto tr = thead.addChild("tr");
@ -3951,7 +3985,7 @@ Table structToTable(T)(Document document, T arr, string[] fieldsToSkip = null) i
odd = !odd;
}
} else static assert(0);
} else static assert(0, T.stringof);
return t;
}