catching up on lots of changes.

cgi.d

@@ -421,6 +421,58 @@ class Cgi {
 		PostParserState pps;
 	}
 
+	/// This represents a file the user uploaded via a POST request.
+	static struct UploadedFile {
+		/// If you want to create one of these structs for yourself from some data,
+		/// use this function.
+		static UploadedFile fromData(immutable(void)[] data, string name = null) {
+			Cgi.UploadedFile f;
+			f.filename = name;
+			f.content = cast(immutable(ubyte)[]) data;
+			f.contentInMemory = true;
+			return f;
+		}
+
+		string name; 		/// The name of the form element.
+		string filename; 	/// The filename the user set.
+		string contentType; 	/// The MIME type the user's browser reported. (Not reliable.)
+
+		/**
+			For small files, cgi.d will buffer the uploaded file in memory, and make it
+			directly accessible to you through the content member. I find this very convenient
+			and somewhat efficient, since it can avoid hitting the disk entirely. (I
+			often want to inspect and modify the file anyway!)
+
+			I find the file is very large, it is undesirable to eat that much memory just
+			for a file buffer. In those cases, if you pass a large enough value for maxContentLength
+			to the constructor so they are accepted, cgi.d will write the content to a temporary
+			file that you can re-read later.
+
+			You can override this behavior by subclassing Cgi and overriding the protected
+			handlePostChunk method. Note that the object is not initialized when you
+			write that method - the http headers are available, but the cgi.post method
+			is not. You may parse the file as it streams in using this method.
+
+
+			Anyway, if the file is small enough to be in memory, contentInMemory will be
+			set to true, and the content is available in the content member.
+
+			If not, contentInMemory will be set to false, and the content saved in a file,
+			whose name will be available in the contentFilename member.
+
+
+			Tip: if you know you are always dealing with small files, and want the convenience
+			of ignoring this member, construct Cgi with a small maxContentLength. Then, if
+			a large file comes in, it simply throws an exception (and HTTP error response)
+			instead of trying to handle it.
+
+			The default value of maxContentLength in the constructor is for small files.
+		*/
+		bool contentInMemory = true; // the default ought to always be true
+		immutable(ubyte)[] content; /// The actual content of the file, if contentInMemory == true
+		string contentFilename; /// the file where we dumped the content, if contentInMemory == false. Note that if you want to keep it, you MUST move the file, since otherwise it is considered garbage when cgi is disposed.
+	}
+
 	// given a content type and length, decide what we're going to do with the data..
 	protected void prepareForIncomingDataChunks(string contentType, ulong contentLength) {
 		pps.expectedLength = contentLength;
@@ -840,10 +892,11 @@ class Cgi {
 		// streaming parser
 		import al = std.algorithm;
 
-		auto idx = al.indexOf(inputData.front(), "\r\n\r\n");
+			// FIXME: tis cast is technically wrong, but Phobos deprecated al.indexOf... for some reason.
+		auto idx = indexOf(cast(string) inputData.front(), "\r\n\r\n");
 		while(idx == -1) {
 			inputData.popFront(0);
-			idx = al.indexOf(inputData.front(), "\r\n\r\n");
+			idx = indexOf(cast(string) inputData.front(), "\r\n\r\n");
 		}
 
 		assert(idx != -1);
@@ -1047,58 +1100,6 @@ class Cgi {
 
 		return null;
 	}
-
-	/// This represents a file the user uploaded via a POST request.
-	static struct UploadedFile {
-		/// If you want to create one of these structs for yourself from some data,
-		/// use this function.
-		static UploadedFile fromData(immutable(void)[] data) {
-			Cgi.UploadedFile f;
-			f.content = cast(immutable(ubyte)[]) data;
-			f.contentInMemory = true;
-			return f;
-		}
-
-		string name; 		/// The name of the form element.
-		string filename; 	/// The filename the user set.
-		string contentType; 	/// The MIME type the user's browser reported. (Not reliable.)
-
-		/**
-			For small files, cgi.d will buffer the uploaded file in memory, and make it
-			directly accessible to you through the content member. I find this very convenient
-			and somewhat efficient, since it can avoid hitting the disk entirely. (I
-			often want to inspect and modify the file anyway!)
-
-			I find the file is very large, it is undesirable to eat that much memory just
-			for a file buffer. In those cases, if you pass a large enough value for maxContentLength
-			to the constructor so they are accepted, cgi.d will write the content to a temporary
-			file that you can re-read later.
-
-			You can override this behavior by subclassing Cgi and overriding the protected
-			handlePostChunk method. Note that the object is not initialized when you
-			write that method - the http headers are available, but the cgi.post method
-			is not. You may parse the file as it streams in using this method.
-
-
-			Anyway, if the file is small enough to be in memory, contentInMemory will be
-			set to true, and the content is available in the content member.
-
-			If not, contentInMemory will be set to false, and the content saved in a file,
-			whose name will be available in the contentFilename member.
-
-
-			Tip: if you know you are always dealing with small files, and want the convenience
-			of ignoring this member, construct Cgi with a small maxContentLength. Then, if
-			a large file comes in, it simply throws an exception (and HTTP error response)
-			instead of trying to handle it.
-
-			The default value of maxContentLength in the constructor is for small files.
-		*/
-		bool contentInMemory = true; // the default ought to always be true
-		immutable(ubyte)[] content; /// The actual content of the file, if contentInMemory == true
-		string contentFilename; /// the file where we dumped the content, if contentInMemory == false. Note that if you want to keep it, you MUST move the file, since otherwise it is considered garbage when cgi is disposed.
-	}
-
 	/// Very simple method to require a basic auth username and password.
 	/// If the http request doesn't include the required credentials, it throws a
 	/// HTTP 401 error, and an exception.
@@ -1988,7 +1989,7 @@ mixin template CustomCgiMain(CustomCgi, alias fun, T...) if(is(CustomCgi : Cgi))
 				more_data:
 				auto chunk = range.front();
 				// waiting for colon for header length
-				auto idx = al.indexOf(chunk, ':');
+				auto idx = indexOf(cast(string) chunk, ':');
 				if(idx == -1) {
 					range.popFront();
 					goto more_data;
@@ -2063,6 +2064,8 @@ mixin template CustomCgiMain(CustomCgi, alias fun, T...) if(is(CustomCgi : Cgi))
 			}
 		} else
 		version(fastcgi) {
+			//         SetHandler fcgid-script
+
 			FCGX_Stream* input, output, error;
 			FCGX_ParamArray env;
 
@@ -2421,7 +2424,7 @@ void sendAll(Socket s, const(void)[] data) {
 	do {
 		amount = s.send(data);
 		if(amount == Socket.ERROR)
-			throw new Exception("wtf in send: " ~ lastSocketError());
+			throw new Exception("wtf in send: " ~ lastSocketError);
 		assert(amount > 0);
 		data = data[amount .. $];
 	} while(data.length);

characterencodings.d

@@ -117,7 +117,7 @@ string tryToDetermineEncoding(in ubyte[] rawdata) {
 		validate!string(cast(string) rawdata);
 		// the odds of non stuff validating as utf-8 are pretty low
 		return "UTF-8";
-	} catch(UtfException t) {
+	} catch(UTFException t) {
 		// it's definitely not UTF-8!
 		// we'll look at the first few characters. If there's a
 		// BOM, it's probably UTF-16 or UTF-32

color.d

@@ -98,7 +98,7 @@ Color fromHsl(real h, real s, real l) {
 		255);
 }
 
-real[3] toHsl(Color c) {
+real[3] toHsl(Color c, bool useWeightedLightness = false) {
 	real r1 = cast(real) c.r / 255;
 	real g1 = cast(real) c.g / 255;
 	real b1 = cast(real) c.b / 255;
@@ -107,6 +107,11 @@ real[3] toHsl(Color c) {
 	real minColor = min(r1, g1, b1);
 
 	real L = (maxColor + minColor) / 2 ;
+	if(useWeightedLightness) {
+		// the colors don't affect the eye equally
+		// this is a little more accurate than plain HSL numbers
+		L = 0.2126*r1 + 0.7152*g1 + 0.0722*b1;
+	}
 	real S = 0;
 	real H = 0;
 	if(maxColor != minColor) {
@@ -153,8 +158,12 @@ Color moderate(Color c, real percentage) {
 	auto hsl = toHsl(c);
 	if(hsl[2] > 0.5)
 		hsl[2] *= (1 - percentage);
-	else
+	else {
-		hsl[2] *= (1 + percentage);
+		if(hsl[2] <= 0.01) // if we are given black, moderating it means getting *something* out
+			hsl[2] = percentage;
+		else
+			hsl[2] *= (1 + percentage);
+	}
 	if(hsl[2] > 1)
 		hsl[2] = 1;
 	return fromHsl(hsl);
@@ -162,7 +171,7 @@ Color moderate(Color c, real percentage) {
 
 /// the opposite of moderate. Make darks darker and lights lighter
 Color extremify(Color c, real percentage) {
-	auto hsl = toHsl(c);
+	auto hsl = toHsl(c, true);
 	if(hsl[2] < 0.5)
 		hsl[2] *= (1 - percentage);
 	else
@@ -186,6 +195,15 @@ Color oppositeLightness(Color c) {
 	return fromHsl(hsl);
 }
 
+/// Try to determine a text color - either white or black - based on the input
+Color makeTextColor(Color c) {
+	auto hsl = toHsl(c, true); // give green a bonus for contrast
+	if(hsl[2] > 0.5)
+		return Color(0, 0, 0);
+	else
+		return Color(255, 255, 255);
+}
+
 Color setLightness(Color c, real lightness) {
 	auto hsl = toHsl(c);
 	hsl[2] = lightness;
@@ -296,3 +314,45 @@ ubyte makeAlpha(ubyte colorYouHave, ubyte backgroundColor/*, ubyte foreground =
 		return 255;
 	return cast(ubyte) alphaf;
 }
+
+
+int fromHex(string s) {
+	import std.range;
+	int result = 0;
+
+	int exp = 1;
+	foreach(c; retro(s)) {
+		if(c >= 'A' && c <= 'F')
+			result += exp * (c - 'A' + 10);
+		else if(c >= 'a' && c <= 'f')
+			result += exp * (c - 'a' + 10);
+		else if(c >= '0' && c <= '9')
+			result += exp * (c - '0');
+		else
+			throw new Exception("invalid hex character: " ~ cast(char) c);
+
+		exp *= 16;
+	}
+
+	return result;
+}
+
+Color colorFromString(string s) {
+	if(s.length == 0)
+		return Color(0,0,0,255);
+	if(s[0] == '#')
+		s = s[1..$];
+	assert(s.length == 6 || s.length == 8);
+
+	Color c;
+
+	c.r = cast(ubyte) fromHex(s[0..2]);
+	c.g = cast(ubyte) fromHex(s[2..4]);
+	c.b = cast(ubyte) fromHex(s[4..6]);
+	if(s.length == 8)
+		c.a = cast(ubyte) fromHex(s[6..8]);
+	else
+		c.a = 255;
+
+	return c;
+}

curl.d

@@ -106,7 +106,7 @@ string curlAuth(string url, string data = null, string username = null, string p
 
 	int res;
 
-	//curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+	// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
 
 	res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url));
 	if(res != 0) throw new CurlException(res);
@@ -122,6 +122,7 @@ string curlAuth(string url, string data = null, string username = null, string p
 	curl_slist* headers = null;
 	//if(data !is null)
 	//	contentType = "";
+	if(contentType.length)
 	headers = curl_slist_append(headers, toStringz("Content-Type: " ~ contentType));
 
 	foreach(h; customHeaders) {
@@ -157,8 +158,10 @@ string curlAuth(string url, string data = null, string username = null, string p
 	//res = curl_easy_setopt(curl, 81, 0); // FIXME verify host
 	//if(res != 0) throw new CurlException(res);
 
-	res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+	version(no_curl_follow) {} else {
-	if(res != 0) throw new CurlException(res);
+		res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+		if(res != 0) throw new CurlException(res);
+	}
 
 	if(methodOverride !is null) {
 		switch(methodOverride) {

database.d

@@ -142,6 +142,90 @@ class DatabaseException : Exception {
 
 
 
+abstract class SqlBuilder { }
+
+/// WARNING: this is as susceptible to SQL injections as you would be writing it out by hand
+class SelectBuilder : SqlBuilder {
+	string[] fields;
+	string table;
+	string[] joins;
+	string[] wheres;
+	string[] orderBys;
+	string[] groupBys;
+
+	int limit;
+	int limitStart;
+
+	string toString() {
+		string sql = "SELECT ";
+
+		// the fields first
+		{
+			bool outputted = false;
+			foreach(field; fields) {
+				if(outputted)
+					sql ~= ", ";
+				else
+					outputted = true;
+
+				sql ~= field; // "`" ~ field ~ "`";
+			}
+		}
+
+		sql ~= " FROM " ~ table;
+
+		if(joins.length) {
+			foreach(join; joins)
+				sql ~= " " ~ join;
+		}
+
+		if(wheres.length) {
+			bool outputted = false;
+			sql ~= " WHERE ";
+			foreach(w; wheres) {
+				if(outputted)
+					sql ~= " AND ";
+				else
+					outputted = true;
+				sql ~= "(" ~ w ~ ")";
+			}
+		}
+
+		if(groupBys.length) {
+			bool outputted = false;
+			sql ~= " GROUP BY ";
+			foreach(o; groupBys) {
+				if(outputted)
+					sql ~= ", ";
+				else
+					outputted = true;
+				sql ~= o;
+			}
+		}
+		
+		if(orderBys.length) {
+			bool outputted = false;
+			sql ~= " ORDER BY ";
+			foreach(o; orderBys) {
+				if(outputted)
+					sql ~= ", ";
+				else
+					outputted = true;
+				sql ~= o;
+			}
+		}
+
+		if(limit) {
+			sql ~= " LIMIT ";
+			if(limitStart)
+				sql ~= to!string(limitStart) ~ ", ";
+			sql ~= to!string(limit);
+		}
+
+		return sql;
+	}
+}
+
 
 // ///////////////////////////////////////////////////////
 
@@ -464,12 +548,12 @@ class DataObject {
 	Tuple!(string, string)[string] mappings;
 
 	// vararg hack so property assignment works right, even with null
-	string opDispatch(string field)(...)
+	string opDispatch(string field, string file = __FILE__, size_t line = __LINE__)(...)
 		if((field.length < 8 || field[0..8] != "id_from_") && field != "popFront")
 	{
 		if(_arguments.length == 0) {
 			if(field !in fields)
-				throw new Exception("no such field " ~ field);
+				throw new Exception("no such field " ~ field, file, line);
 
 			return fields[field];
 		} else if(_arguments.length == 1) {
@@ -513,6 +597,10 @@ class DataObject {
 		fields[field] = value;
 	}
 
+	public void setWithoutChange(string field, string value) {
+		fields[field] = value;
+	}
+
 	int opApply(int delegate(ref string) dg) {
 		foreach(a; fields)
 			mixin(yield("a"));

domconvenience.d

@@ -0,0 +1,6 @@
+/**
+
+*/
+module arsd.domconvenience;
+
+// the contents of this file are back in dom.d for now. I might split them out later.

html.d

@@ -198,7 +198,96 @@ string recommendedBasicCssForUserContent = `
 	}
 `;
 
+Html linkify(string text) {
+	auto div = Element.make("div");
 
+	while(text.length) {
+		auto idx = text.indexOf("http");
+		if(idx == -1) {
+			idx = text.length;
+		}
+
+		div.appendText(text[0 .. idx]);
+		text = text[idx .. $];
+
+		if(text.length) {
+			// where does it end? whitespace I guess
+			auto idxSpace = text.indexOf(" ");
+			if(idxSpace == -1) idxSpace = text.length;
+			auto idxLine = text.indexOf("\n");
+			if(idxLine == -1) idxLine = text.length;
+
+
+			auto idxEnd = idxSpace < idxLine ? idxSpace : idxLine;
+
+			auto link = text[0 .. idxEnd];
+			text = text[idxEnd .. $];
+
+			div.addChild("a", link, link);
+		}
+	}
+
+	return Html(div.innerHTML);
+}
+
+/// Returns true of the string appears to be html/xml - if it matches the pattern
+/// for tags or entities.
+bool appearsToBeHtml(string src) {
+	return false;
+}
+
+/+
+void qsaFilter(string logicalScriptName) {
+	string logicalScriptName = siteBase[0 .. $-1];
+
+	foreach(a; document.querySelectorAll("a[qsa]")) {
+		string href = logicalScriptName ~ _cgi.pathInfo ~ "?";
+
+		int matches, possibilities;
+
+		string[][string] vars;
+		foreach(k, v; _cgi.getArray)
+			vars[k] = cast(string[]) v;
+		foreach(k, v; decodeVariablesSingle(a.qsa)) {
+			if(k in _cgi.get && _cgi.get[k] == v)
+				matches++;
+			possibilities++;
+
+			if(k !in vars || vars[k].length <= 1)
+				vars[k] = [v];
+			else
+				assert(0, "qsa doesn't work here");
+		}
+
+		string[] clear = a.getAttribute("qsa-clear").split("&");
+		clear ~= "ajaxLoading";
+		if(a.parentNode !is null)
+			clear ~= a.parentNode.getAttribute("qsa-clear").split("&");
+
+		bool outputted = false;
+		varskip: foreach(k, varr; vars) {
+			foreach(item; clear)
+				if(k == item)
+					continue varskip;
+			foreach(v; varr) {
+				if(outputted)
+					href ~= "&";
+				else
+					outputted = true;
+
+				href ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
+			}
+		}
+
+		a.href = href;
+
+		a.removeAttribute("qsa");
+
+		if(matches == possibilities)
+			a.addClass("current");
+	}
+}
++/
 string favicon(Document document) {
 	auto item = document.querySelector("link[rel~=icon]");
 	if(item !is null)
@@ -206,6 +295,21 @@ string favicon(Document document) {
 	return "/favicon.ico"; // it pisses me off that the fucking browsers do this.... but they do, so I will too.
 }
 
+Element checkbox(string name, string value, string label, bool checked = false) {
+	auto lbl = Element.make("label");
+	auto input = lbl.addChild("input");
+	input.type = "checkbox";
+	input.name = name;
+	input.value = value;
+	if(checked)
+		input.checked = "checked";
+
+	lbl.appendText(" ");
+	lbl.addChild("span", label);
+
+	return lbl;
+}
+
 
 /++ Convenience function to create a small <form> to POST, but the creation function is more like a link
     than a DOM form.
@@ -321,7 +425,7 @@ void translateValidation(Document document) {
 			if(i.tagName != "input" && i.tagName != "select")
 				continue;
 			if(i.getAttribute("id") is null)
-				i.id = i.name;
+				i.id = "form-input-" ~ i.name;
 			auto validate = i.getAttribute("validate");
 			if(validate is null)
 				continue;
@@ -588,7 +692,7 @@ void translateInputTitles(Document document) {
 void translateInputTitles(Element rootElement) {
 	foreach(form; rootElement.getElementsByTagName("form")) {
 		string os;
-		foreach(e; form.getElementsBySelector("input[type=text][title]")) {
+		foreach(e; form.getElementsBySelector("input[type=text][title], textarea[title]")) {
 			if(e.hasClass("has-placeholder"))
 				continue;
 			e.addClass("has-placeholder");
@@ -611,9 +715,16 @@ void translateInputTitles(Element rootElement) {
 					temporaryItem.value = '';
 			`;
 
-			if(e.value == "") {
+			if(e.tagName == "input") {
-				e.value = e.title;
+				if(e.value == "") {
-				e.addClass("default");
+					e.value = e.title;
+					e.addClass("default");
+				}
+			} else {
+				if(e.innerText.length == 0) {
+					e.innerText = e.title;
+					e.addClass("default");
+				}
 			}
 		}
 
@@ -1434,6 +1545,11 @@ class MacroExpander {
 	dstring delegate(dstring[])[dstring] functions;
 	dstring[dstring] variables;
 
+	/// This sets a variable inside the macro system
+	void setValue(string key, string value) {
+		variables[to!dstring(key)] = to!dstring(value);
+	}
+
 	struct Macro {
 		dstring name;
 		dstring[] args;
@@ -1464,12 +1580,18 @@ class MacroExpander {
 			return ret;
 		};
 
+		functions["uriEncode"] = delegate dstring(dstring[] args) {
+			return to!dstring(std.uri.encodeComponent(to!string(args[0])));
+		};
+
 		functions["test"] = delegate dstring(dstring[] args) {
 			assert(0, to!string(args.length) ~ " args: " ~ to!string(args));
 			return null;
 		};
 	}
 
+	// the following are used inside the user text
+
 	dstring define(dstring[] args) {
 		enforce(args.length > 1, "requires at least a macro name and definition");
 
@@ -1518,12 +1640,14 @@ class MacroExpander {
 		return returned;
 	}
 
+	/// Performs the expansion
 	string expand(string srcutf8) {
 		auto src = expand(to!dstring(srcutf8));
 		return to!string(src);
 	}
 
 	private int depth = 0;
+	/// ditto
 	dstring expand(dstring src) {
 		return expandImpl(src, null);
 	}
@@ -1764,6 +1888,7 @@ class CssMacroExpander : MacroExpander {
 		functions["darken"] = &(colorFunctionWrapper!darken);
 		functions["moderate"] = &(colorFunctionWrapper!moderate);
 		functions["extremify"] = &(colorFunctionWrapper!extremify);
+		functions["makeTextColor"] = &(oneArgColorFunctionWrapper!makeTextColor);
 
 		functions["oppositeLightness"] = &(oneArgColorFunctionWrapper!oppositeLightness);
 
@@ -1785,11 +1910,12 @@ class CssMacroExpander : MacroExpander {
 		return ret;
 	}
 
+	/// Runs the macro expansion but then a CSS densesting
 	string expandAndDenest(string cssSrc) {
 		return cssToString(denestCss(lexCss(this.expand(cssSrc))));
 	}
 
-
+	// internal things
 	dstring colorFunctionWrapper(alias func)(dstring[] args) {
 		auto color = readCssColor(to!string(args[0]));
 		auto percentage = readCssNumber(args[1]);

mysql.d

@@ -325,6 +325,13 @@ class MySql : Database {
 	}
 
 
+	ResultByDataObject!R queryDataObjectWithCustomKeys(R = DataObject, T...)(string[string] keyMapping, string sql, T t) {
+		sql = fixupSqlForDataObjectUse(sql, keyMapping);
+
+		auto magic = query(sql, t);
+		return ResultByDataObject!R(cast(MySqlResult) magic, this);
+	}
+
 
 
 

png.d

@@ -547,6 +547,17 @@ RGBQUAD[] fetchPaletteWin32(PNG* p) {
 Color[] fetchPalette(PNG* p) {
 	Color[] colors;
 
+	auto header = getHeader(p);
+	if(header.type == 0) { // greyscale
+		colors.length = 256;
+		foreach(i; 0..256)
+			colors[i] = Color(cast(ubyte) i, cast(ubyte) i, cast(ubyte) i);
+		return colors;
+	}
+
+	// assuming this is indexed
+	assert(header.type == 3);
+
 	auto palette = p.getChunk("PLTE");
 
 	Chunk* alpha = p.getChunkNullable("tRNS");
@@ -572,7 +583,12 @@ void replacePalette(PNG* p, Color[] colors) {
 	auto palette = p.getChunk("PLTE");
 	auto alpha = p.getChunk("tRNS");
 
-	assert(colors.length == alpha.size);
+	//import std.string;
+	//assert(0, format("%s %s", colors.length, alpha.size));
+	//assert(colors.length == alpha.size);
+	alpha.size = colors.length;
+	alpha.payload.length = colors.length; // we make sure there's room for our simple method below
+	p.length = 0; // so write will recalculate
 
 	for(int i = 0; i < colors.length; i++) {
 		palette.payload[i*3+0] = colors[i].r;

rtud.d

@@ -129,6 +129,11 @@ void writeToFd(int fd, string s) {
 		goto again;
 }
 
+__gshared bool deathRequested = false;
+extern(C)
+void requestDeath(int sig) {
+	deathRequested = true;
+}
 
 import arsd.cgi;
 /// The throttledConnection param is useful for helping to get
@@ -144,6 +149,14 @@ import arsd.cgi;
 int handleListenerGateway(Cgi cgi, string channelPrefix, bool throttledConnection = false) {
 	cgi.setCache(false);
 
+	import core.sys.posix.signal;
+	sigaction_t act;
+	// I want all zero everywhere else; the read() must not automatically restart for this to work.
+	act.sa_handler = &requestDeath;
+
+	if(linux.sigaction(linux.SIGTERM, &act, null) != 0)
+		throw new Exception("sig err");
+
 	auto f = openNetworkFd("localhost", 7070);
 	scope(exit) linux.close(f);
 
@@ -185,7 +198,7 @@ int handleListenerGateway(Cgi cgi, string channelPrefix, bool throttledConnectio
 
 	string[4096] buffer;
 
-	for(;;) {
+	for(; !deathRequested ;) {
 		auto num = linux.read(f, buffer.ptr, buffer.length);
 		if(num < 0)
 			throw new Exception("read error");
@@ -202,7 +215,7 @@ int handleListenerGateway(Cgi cgi, string channelPrefix, bool throttledConnectio
 	}
 
 	// this is to support older browsers
-	if(!isSse) {
+	if(!isSse && !deathRequested) {
 		// we have to parse it out and reformat for plain cgi...
 		auto lol = parseMessages(wegot);
 		//cgi.setResponseContentType("text/json");

sha.d

@@ -134,7 +134,7 @@ struct FileByByte {
 			fclose(fp);
 	}
 
-	@property void popFront() {
+	void popFront() {
 		f = cast(ubyte) fgetc(fp);
 	}
 

web.d

@@ -304,6 +304,22 @@ class ApiProvider : WebDotDBaseType {
 		}
 	}
 
+	protected bool isCsrfTokenCorrect() {
+		auto tokenInfo = _getCsrfInfo();
+		if(tokenInfo is null)
+			return false; // this means we aren't doing checks (probably because there is no session), but it is a failure nonetheless
+
+		auto token = tokenInfo["key"] ~ "=" ~ tokenInfo["token"];
+		if("x-arsd-csrf-pair" in cgi.requestHeaders)
+			return cgi.requestHeaders["x-arsd-csrf-pair"] == token;
+		if(tokenInfo["key"] in cgi.post)
+			return cgi.post[tokenInfo["key"]] == tokenInfo["token"];
+		if(tokenInfo["key"] in cgi.get)
+			return cgi.get[tokenInfo["key"]] == tokenInfo["token"];
+
+		return false;
+	}
+
 	/// Shorthand for ensurePost and checkCsrfToken. You should use this on non-indempotent 
 	/// functions. Override it if doing some custom checking.
 	void ensureGoodPost() {
@@ -341,6 +357,9 @@ class ApiProvider : WebDotDBaseType {
 
 	/// we have to add these things to the document...
 	override void _postProcess(Document document) {
+		foreach(pp; documentPostProcessors)
+			pp(document);
+
 		addCsrfTokens(document);
 		super._postProcess(document);
 	}
@@ -365,6 +384,9 @@ class ApiProvider : WebDotDBaseType {
 
 	// and added to ajax forms..
 	override void _postProcessElement(Element element) {
+		foreach(pp; elementPostProcessors)
+			pp(element);
+
 		addCsrfTokens(element);
 		super._postProcessElement(element);
 	}
@@ -380,6 +402,38 @@ class ApiProvider : WebDotDBaseType {
 	/// It should be used instead of the constructor for most work.
 	void _initialize() {}
 
+	/// On each call, you can register another post processor for the generated html. If your delegate takes a Document, it will only run on document envelopes (full pages generated). If you take an Element, it will apply on almost any generated html.
+	///
+	/// Note: if you override _postProcess or _postProcessElement, be sure to call the superclass version for these registered functions to run.
+	void _registerPostProcessor(void delegate(Document) pp) {
+		documentPostProcessors ~= pp;
+	}
+
+	/// ditto
+	void _registerPostProcessor(void delegate(Element) pp) {
+		elementPostProcessors ~= pp;
+	}
+
+	/// ditto
+	void _registerPostProcessor(void function(Document) pp) {
+		documentPostProcessors ~= delegate void(Document d) { pp(d); };
+	}
+
+	/// ditto
+	void _registerPostProcessor(void function(Element) pp) {
+		elementPostProcessors ~= delegate void(Element d) { pp(d); };
+	}
+
+	// these only work for one particular call
+	private void delegate(Document d)[] documentPostProcessors;
+	private void delegate(Element d)[] elementPostProcessors;
+	private void _initializePerCallInternal() {
+		documentPostProcessors = null;
+		elementPostProcessors = null;
+
+		_initializePerCall();
+	}
+
 	/// This one is called at least once per call. (_initialize is only called once per process)
 	void _initializePerCall() {}
 
@@ -464,20 +518,44 @@ class ApiProvider : WebDotDBaseType {
 	/// where the return value is appended.
 
 	/// It's the main function to override to provide custom HTML templates.
+	///
+	/// The default document provides a default stylesheet, our default javascript, and some timezone cookie handling (which you must handle on the server. Eventually I'll open source my date-time helpers that do this, but the basic idea is it sends an hour offset, and you can add that to any UTC time you have to get a local time).
 	Element _getGenericContainer()
 	out(ret) {
 		assert(ret !is null);
 	}
 	body {
-		auto document = new Document("<!DOCTYPE html><html><head><title></title><style>.format-row { display: none; }</style><link rel=\"stylesheet\" href=\"styles.css\" /></head><body><div id=\"body\"></div><script src=\"functions.js\"></script></body></html>", true, true);
+		auto document = new TemplatedDocument(
+"<!DOCTYPE html>
+<html>
+<head>
+	<title></title>
+	<link rel=\"stylesheet\" href=\"styles.css\" />
+	<script> var delayedExecutionQueue = []; </script> <!-- FIXME do some better separation -->
+	<script>
+		if(document.cookie.indexOf(\"timezone=\") == -1) {
+			var d = new Date();
+			var tz = -d.getTimezoneOffset() / 60;
+			document.cookie = \"timezone=\" + tz + \"; path=/\";
+		}
+	</script>
+	<style>.format-row { display: none; }</style>
+</head>
+<body>
+	<div id=\"body\"></div>
+	<script src=\"functions.js\"></script>
+</body>
+</html>");
 		if(this.reflection !is null)
 			document.title = this.reflection.name;
-		auto container = document.getElementById("body");
+		auto container = document.requireElementById("body");
 		return container;
 	}
 
+	// FIXME: set a generic container for a particular call
+
 	/// If the given url path didn't match a function, it is passed to this function
-	/// for further handling. By default, it throws a NoSuchPageException.
+	/// for further handling. By default, it throws a NoSuchFunctionException.
 
 	/// Overriding it might be useful if you want to serve generic filenames or an opDispatch kind of thing.
 	/// (opDispatch itself won't work because it's name argument needs to be known at compile time!)
@@ -485,7 +563,7 @@ class ApiProvider : WebDotDBaseType {
 	/// Note that you can return Documents here as they implement
 	/// the FileResource interface too.
 	FileResource _catchAll(string path) {
-		throw new NoSuchPageException(_errorMessageForCatchAll);
+		throw new NoSuchFunctionException(_errorMessageForCatchAll);
 	}
 
 	private string _errorMessageForCatchAll;
@@ -1055,12 +1133,20 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 	WebDotDBaseType realObject = instantiation;
 	if(instantiator.length == 0)
 	if(fun !is null && fun.parentObject !is null && fun.parentObject.instantiation !is null)
-		realObject = fun.parentObject.instantiation;
+		realObject = cast() fun.parentObject.instantiation; // casting away transitive immutable...
 
 	// FIXME
 	if(cgi.pathInfo.indexOf("builtin.") != -1 && instantiation.builtInFunctions !is null)
 		base = instantiation.builtInFunctions;
 
+	if(base !is realObject) {
+		auto hack1 = cast(ApiProvider) base;
+		auto hack2 = cast(ApiProvider) realObject;
+
+		if(hack1 !is null && hack2 !is null && hack2.session is null)
+			hack2.session = hack1.session;
+	}
+
 	try {
 		if(fun is null) {
 			auto d = instantiation._catchallEntry(
@@ -1232,6 +1318,35 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 				auto json = toJsonValue(result);
 				cgi.write(toJSON(&json), true);
 			break;
+			case "script":
+			case "jsonp":
+				bool securityPass = false;
+				version(web_d_unrestricted_jsonp) {
+					// unrestricted is opt-in because i worry about fetching user info from across sites
+					securityPass = true;
+				} else {
+					// we check this on both get and post to ensure they can't fetch user private data cross domain.
+					auto hack1 = cast(ApiProvider) base;
+					if(hack1)
+						securityPass = hack1.isCsrfTokenCorrect();
+				}
+
+				if(securityPass) {
+					if(envelopeFormat == "script")
+						cgi.setResponseContentType("text/html");
+					else
+						cgi.setResponseContentType("application/javascript");
+
+					auto json = cgi.request("jsonp", "throw new Error") ~ "(" ~ toJson(result) ~ ");";
+
+					if(envelopeFormat == "script")
+						json = "<script type=\"text/javascript\">" ~ json ~ "</script>";
+					cgi.write(json, true);
+				} else {
+					// if the security check fails, you just don't get anything at all data wise...
+					cgi.setResponseStatus("403 Forbidden");
+				}
+			break;
 			case "none":
 				cgi.setResponseContentType("text/plain");
 
@@ -1259,7 +1374,15 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 							// probably not super efficient...
 							document = new TemplatedDocument(returned);
 						} else {
-							auto e = instantiation._getGenericContainer();
+							// auto e = instantiation._getGenericContainer();
+							Element e;
+							auto hack = cast(ApiProvider) realObject;
+							if(hack !is null)
+								e = hack._getGenericContainer();
+							else
+								e = instantiation._getGenericContainer();
+
+
 							document = e.parentDocument;
 							// FIXME: a wee bit slow, esp if func return element
 							e.innerHTML = returned;
@@ -1964,6 +2087,11 @@ class NoSuchPageException : Exception {
 	}
 }
 
+class NoSuchFunctionException : NoSuchPageException {
+	this(string msg, string file = __FILE__, int line = __LINE__) {
+		super(msg, file, line);
+	}
+}
 
 type fromUrlParam(type)(string ofInterest) {
 	type ret;
@@ -1971,6 +2099,7 @@ type fromUrlParam(type)(string ofInterest) {
 	static if(isArray!(type) && !isSomeString!(type)) {
 		// how do we get an array out of a simple string?
 		// FIXME
+		static assert(0);
 	} else static if(__traits(compiles, ret = type.fromWebString(ofInterest))) { // for custom object handling...
 		ret = type.fromWebString(ofInterest);
 	} else static if(is(type : Element)) {
@@ -2000,7 +2129,11 @@ type fromUrlParam(type)(string[] ofInterest) {
 	type ret;
 
 	// Arrays in a query string are sent as the name repeating...
-	static if(isArray!(type) && !isSomeString!(type)) {
+	static if(isArray!(type) && !isSomeString!type) {
+		foreach(a; ofInterest) {
+			ret ~= fromUrlParam!(ElementType!(type))(a);
+		}
+	} else static if(isArray!(type) && isSomeString!(ElementType!type)) {
 		foreach(a; ofInterest) {
 			ret ~= fromUrlParam!(ElementType!(type))(a);
 		}
@@ -2012,7 +2145,7 @@ type fromUrlParam(type)(string[] ofInterest) {
 
 auto getMemberDelegate(alias ObjectType, string member)(ObjectType object) if(is(ObjectType : WebDotDBaseType)) {
 	if(object is null)
-		throw new NoSuchPageException("no such object " ~ ObjectType.stringof);
+		throw new NoSuchFunctionException("no such object " ~ ObjectType.stringof);
 	return &__traits(getMember, object, member);
 }
 
@@ -2028,7 +2161,7 @@ WrapperFunction generateWrapper(alias ObjectType, string funName, alias f, R)(Re
 
 		auto instantiation = getMemberDelegate!(ObjectType, funName)(cast(ObjectType) object);
 
-		api._initializePerCall();
+		api._initializePerCallInternal();
 
 		ParameterTypeTuple!(f) args;
 
@@ -2100,7 +2233,7 @@ WrapperFunction generateWrapper(alias ObjectType, string funName, alias f, R)(Re
 
 							// find it in reflection
 							ofInterest ~= reflection.functions[callingName].
-								dispatcher(cgi, null, decodeVariables(callingArguments), "string").str;
+								dispatcher(cgi, null, decodeVariables(callingArguments), "string", null).str;
 						}
 					}
 
@@ -2543,7 +2676,7 @@ class Session {
 	}
 
 	/// get/set for strings
-	string opDispatch(string name)(string v = null) if(name != "popFront") {
+	@property string opDispatch(string name)(string v = null) if(name != "popFront") {
 		if(v !is null)
 			set(name, v);
 		if(hasKey(name))
@@ -2596,6 +2729,9 @@ class Session {
 				case JSON_TYPE.STRING:
 					ret = v.str;
 				break;
+				case JSON_TYPE.UINTEGER:
+					ret = to!string(v.integer);
+				break;
 				case JSON_TYPE.INTEGER:
 					ret = to!string(v.integer);
 				break;
@@ -3073,7 +3209,7 @@ string makeJavascriptApi(const ReflectionInfo* mod, string base, bool isNested =
 
 	string script;
 
-	if(isNested)
+	if(0 && isNested)
 		script = `'`~mod.name~`': {
 	"_apiBase":'`~base~`',`;
 	else
@@ -3154,16 +3290,6 @@ string makeJavascriptApi(const ReflectionInfo* mod, string base, bool isNested =
 		script ~= "\n\t}; }";
 	}
 
-	// FIXME: it should output the classes too
-	foreach(obj; mod.objects) {
-		if(outp)
-			script ~= ",\n\t";
-		else
-			outp = true;
-
-		script ~= makeJavascriptApi(obj, base ~ obj.name ~ "/", true);
-	}
-
 	foreach(key, func; mod.functions) {
 		if(func.originalName in alreadyDone)
 			continue; // there's url friendly and code friendly, only need one
@@ -3234,6 +3360,18 @@ string makeJavascriptApi(const ReflectionInfo* mod, string base, bool isNested =
 		}
 	`;
 
+	// FIXME: it should output the classes too
+	// FIXME: hax hax hax
+	foreach(n, obj; mod.objects) {
+		script ~= ";";
+		//if(outp)
+		//	script ~= ",\n\t";
+		//else
+		//	outp = true;
+
+		script ~= makeJavascriptApi(obj, base ~ n ~ "/", true);
+	}
+
 	return script;
 }
 
@@ -3372,21 +3510,29 @@ enum string javascriptBaseImpl = q{
 
 		var a = "";
 
+		var csrfKey = document.body.getAttribute("data-csrf-key");
+		var csrfToken = document.body.getAttribute("data-csrf-token");
+		var csrfPair = "";
+		if(csrfKey && csrfKey.length > 0 && csrfToken && csrfToken.length > 0) {
+			csrfPair = encodeURIComponent(csrfKey) + "=" + encodeURIComponent(csrfToken);
+			// we send this so it can be easily verified for things like restricted jsonp
+			xmlHttp.setRequestHeader("X-Arsd-Csrf-Pair", csrfPair);
+		}
+
 		if(method == "POST") {
 			xmlHttp.setRequestHeader("Content-type","application/x-www-form-urlencoded");
 			a = argString;
 			// adding the CSRF stuff, if necessary
-			var csrfKey = document.body.getAttribute("data-csrf-key");
+			if(csrfPair.length) {
-			var csrfToken = document.body.getAttribute("data-csrf-token");
-			if(csrfKey && csrfKey.length > 0 && csrfToken && csrfToken.length > 0) {
 				if(a.length > 0)
 					a += "&";
-				a += encodeURIComponent(csrfKey) + "=" + encodeURIComponent(csrfToken);
+				a += csrfPair;
 			}
 		} else {
 			xmlHttp.setRequestHeader("Content-type", "text/plain");
 		}
 
+		xmlHttp.setRequestHeader("X-Requested-With", "XMLHttpRequest");
 		xmlHttp.send(a);
 
 		if(!async && callback) {
@@ -3487,7 +3633,7 @@ enum string javascriptBaseImpl = q{
 
 			// lower level implementation
 			"_get":function(callback, onError, async) {
-				var resObj = this;
+				var resObj = this; // the request/response object. var me is the ApiObject.
 				if(args == null)
 					args = {};
 				if(!args.format)
@@ -3512,6 +3658,8 @@ enum string javascriptBaseImpl = q{
 							obj = eval("(" + t + ")");
 					//}
 
+					var returnValue;
+
 					if(obj.success) {
 						if(typeof callback == "function")
 							callback(obj.result);
@@ -3527,7 +3675,7 @@ enum string javascriptBaseImpl = q{
 							// FIXME: meh just do something here.
 						}
 
-						return obj.result;
+						returnValue = obj.result;
 					} else {
 						// how should we handle the error? I guess throwing is better than nothing
 						// but should there be an error callback too?
@@ -3554,15 +3702,25 @@ enum string javascriptBaseImpl = q{
 						}
 
 						if(onError) // local override first...
-							return onError(error);
+							returnValue = onError(error);
 						else if(resObj.onError) // then this object
-							return resObj.onError(error);
+							returnValue = resObj.onError(error);
 						else if(me._onError) // then the global object
-							return me._onError(error);
+							returnValue = me._onError(error);
-
+						else
-						throw error; // if all else fails...
+							throw error; // if all else fails...
 					}
 
+					if(typeof resObj.onComplete == "function") {
+						resObj.onComplete();
+					}
+
+					if(typeof me._onComplete == "function") {
+						me._onComplete(resObj);
+					}
+
+					return returnValue;
+
 					// assert(0); // not reached
 				}, (name.indexOf("get") == 0) ? "GET" : "POST", async); // FIXME: hack: naming convention used to figure out method to use
 			},