catch up

cgi.d

@@ -1793,6 +1793,14 @@ class Cgi {
 	immutable(string[][string]) postArray; /// ditto for post
 	immutable(string[][string]) cookiesArray; /// ditto for cookies
 
+	// convenience function for appending to a uri without extra ?
+	// matches the name and effect of javascript's location.search property
+	string search() const {
+		if(queryString.length)
+			return "?" ~ queryString;
+		return "";
+	}
+
 	// FIXME: what about multiple files with the same name?
   private:
 	//RequestMethod _requestMethod;
@@ -1997,6 +2005,14 @@ struct Uri {
 
 		// FIXME: add something for port too
 	}
+
+	// these are like javascript's location.search and location.hash
+	string search() const {
+		return query.length ? ("?" ~ query) : "";
+	}
+	string hash() const {
+		return fragment.length ? ("#" ~ fragment) : "";
+	}
 }
 
 

characterencodings.d

@@ -1,3 +1,5 @@
+// helper program is in ~me/encodings.d to make more tables from wikipedia
+
 /**
 	This is meant to help get data from the wild into utf8 strings
 	so you can work with them easily inside D.
@@ -69,6 +71,8 @@ string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) {
 		// and now the various 8 bit encodings we support.
 		case "windows1252":
 			return decodeImpl(data, ISO_8859_1, Windows_1252);
+		case "windows1251":
+			return decodeImpl(data, Windows_1251, Windows_1251_Lower);
 		case "koi8r":
 			return decodeImpl(data, KOI8_R, KOI8_R_Lower);
 		case "latin1":
@@ -430,3 +434,24 @@ immutable dchar[] KOI8_R = [
 	'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о',
 	'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в',
 	'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ'];
+
+immutable dchar[] Windows_1251_Lower = [
+	'Ђ', 'Ѓ', '‚', 'ѓ', '„', '…', '†', '‡',
+	'€', '‰', 'Љ', '‹', 'Њ', 'Ќ', 'Ћ', 'Џ',
+	'ђ', '‘', '’', '“', '”', '•', '–', '—',
+	' ', '™', 'љ', '›', 'њ', 'ќ', 'ћ', 'џ'];
+
+immutable dchar[] Windows_1251 = [
+	' ', 'Ў', 'ў', 'Ј', '¤', 'Ґ', '¦', '§',
+	'Ё', '©', 'Є', '«', '¬', '­', '®', 'Ї',
+	'°', '±', 'І', 'і', 'ґ', 'µ', '¶', '·',
+	'ё', '№', 'є', '»', 'ј', 'Ѕ', 'ѕ', 'ї',
+	'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З',
+	'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П',
+	'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч',
+	'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я',
+	'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з',
+	'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п',
+	'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч',
+	'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я'];
+

curl.d

@@ -107,7 +107,8 @@ string curlAuth(string url, string data = null, string username = null, string p
 
 	int res;
 
-	// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+	debug(arsd_curl_verbose)
+		curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
 
 	res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url));
 	if(res != 0) throw new CurlException(res);

database.d

@@ -28,16 +28,19 @@ interface Database {
 			if(arg == typeid(string) || arg == typeid(immutable(string)) || arg == typeid(const(string)))
 				a = va_arg!string(_argptr);
 			else if (arg == typeid(int) || arg == typeid(immutable(int)) || arg == typeid(const(int))) {
-				int e = va_arg!int(_argptr);
+				auto e = va_arg!int(_argptr);
 				a = to!string(e);
 			} else if (arg == typeid(uint) || arg == typeid(immutable(uint)) || arg == typeid(const(uint))) {
-				int e = va_arg!uint(_argptr);
+				auto e = va_arg!uint(_argptr);
 				a = to!string(e);
 			} else if (arg == typeid(immutable(char))) {
-				char e = va_arg!char(_argptr);
+				auto e = va_arg!char(_argptr);
 				a = to!string(e);
 			} else if (arg == typeid(long) || arg == typeid(const(long)) || arg == typeid(immutable(long))) {
-				long e = va_arg!long(_argptr);
+				auto e = va_arg!long(_argptr);
+				a = to!string(e);
+			} else if (arg == typeid(ulong) || arg == typeid(const(ulong)) || arg == typeid(immutable(ulong))) {
+				auto e = va_arg!ulong(_argptr);
 				a = to!string(e);
 			} else if (arg == typeid(null)) {
 				a = null;
@@ -51,10 +54,10 @@ interface Database {
 }
 import std.stdio;
 
-Ret queryOneColumn(Ret, T...)(Database db, string sql, T t) {
+Ret queryOneColumn(Ret, string file = __FILE__, size_t line = __LINE__, T...)(Database db, string sql, T t) {
 	auto res = db.query(sql, t);
 	if(res.empty)
-		throw new Exception("no row in result");
+		throw new Exception("no row in result", file, line);
 	auto row = res.front;
 	return to!Ret(row[0]);
 }

dom.d

@@ -885,6 +885,8 @@ class Element {
 
 		version(dom_node_indexes)
 			this.dataset.nodeIndex = to!string(&(this.attributes));
+
+		assert(_tagName.indexOf(" ") == -1);
 	}
 
 	/// Convenience constructor when you don't care about the parentDocument. Note this might break things on the document.
@@ -3245,9 +3247,17 @@ class Document : FileResource {
 			}
 		}
 
-		if(dataEncoding != "UTF-8")
+		if(dataEncoding != "UTF-8") {
-			data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
+			if(strict)
-		else
+				data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
+			else {
+				try {
+					data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
+				} catch(Exception e) {
+					data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, "Windows 1252");
+				}
+			}
+		} else
 			data = rawdata;
 		assert(data !is null);
 

email.d

@@ -402,7 +402,7 @@ string[string] breakUpHeaderParts(string headerContent) {
 
 	string currentName = "root";
 	string currentContent;
-	bool inQuote;
+	bool inQuote = false;
 	bool gettingName = false;
 	bool ignoringSpaces = false;
 	foreach(char c; headerContent) {
@@ -423,7 +423,7 @@ string[string] breakUpHeaderParts(string headerContent) {
 
 		if(c == '"') {
 			inQuote = !inQuote;
-			break;
+			continue;
 		}
 
 		if(!inQuote && c == ';') {
@@ -519,6 +519,8 @@ class IncomingEmailMessage {
 
 		string charset = "latin-1";
 
+		string contentTransferEncoding;
+
 		string headerName;
 		string headerContent;
 		void commitHeader() {
@@ -556,12 +558,14 @@ class IncomingEmailMessage {
 				}
 			} else if(headerName == "from") {
 				this.from = headerContent;
+			} else if(headerName == "to") {
+				this.to = headerContent;
 			} else if(headerName == "subject") {
 				this.subject = headerContent;
+			} else if(headerName == "content-transfer-encoding") {
+				contentTransferEncoding = headerContent;
 			}
 
-			// FIXME: do I have to worry about content-transfer-encoding here? I think procmail takes care of it but I'm not entirely sure
-
 			headers[headerName] = headerContent;
 			headerName = null;
 			headerContent = null;
@@ -667,8 +671,25 @@ class IncomingEmailMessage {
 					// if inline it is prolly an image to be concated in the other body
 					// if attachment, it is an attachment
 				break;
+				case "multipart/signed":
+					// FIXME: it would be cool to actually check the signature
 				default:
 					// FIXME: correctly handle more
+					if(part.stuff.length) {
+						part = part.stuff[0];
+						goto deeperInTheMimeTree;
+					}
+			}
+		} else {
+			switch(contentTransferEncoding) {
+				case "quoted-printable":
+					if(textMessageBody.length)
+						textMessageBody = cast(string) decodeQuotedPrintable(textMessageBody);
+					if(htmlMessageBody.length)
+						htmlMessageBody = cast(string) decodeQuotedPrintable(htmlMessageBody);
+				break;
+				default:
+					// nothing needed
 			}
 		}
 
@@ -687,6 +708,7 @@ class IncomingEmailMessage {
 	string textMessageBody;
 
 	string from;
+	string to;
 
 	bool textAutoConverted;
 
@@ -850,3 +872,17 @@ immutable(ubyte)[] decodeQuotedPrintable(string text) {
 
 	return ret;
 }
+
+/+
+void main() {
+	import std.file;
+	import std.stdio;
+
+	auto data = cast(immutable(ubyte)[]) std.file.read("/home/me/test_email_data");
+	foreach(message; processMboxData(data)) {
+		writeln(message.subject);
+		writeln(message.textMessageBody);
+		writeln("**************** END MESSSAGE **************");
+	}
+}
++/

htmltotext.d

@@ -86,9 +86,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 			break;
 			case "a":
 				string href = ele.getAttribute("href");
-				if(href) {
+				if(href && !ele.hasClass("no-brackets")) {
-					if(ele.innerText != href)
+					if(ele.hasClass("href-text"))
-						ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
+						ele.innerText = href;
+					else {
+						if(ele.innerText != href)
+							ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
+					}
 				}
 				ele.stripOut();
 				goto again;
@@ -98,8 +102,9 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 				ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r";
 			break;
 			case "li":
-				ele.innerHTML = "\t* " ~ ele.innerHTML ~ "\r";
+				if(!ele.innerHTML.startsWith("* "))
-				ele.stripOut();
+					ele.innerHTML = "* " ~ ele.innerHTML ~ "\r";
+				// ele.stripOut();
 			break;
 			case "sup":
 				ele.innerText = "^" ~ ele.innerText;
@@ -140,6 +145,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 				ele.innerText = strip(ele.innerText);
 			ele.stripOut();
 			goto again2;
+		} else if(ele.tagName == "li") {
+			auto part = ele.innerText;
+			part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part );
+			part = "  " ~ part.replace("\n", "\n\v") ~ "\r";
+			ele.innerText = part;
+			ele.stripOut();
+			goto again2;
 		}
 	}
 
@@ -155,6 +167,8 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 	result = squeeze(result, "\r");
 	result = result.replace("\r", "\n\n");
 
+	result = result.replace("\v", "  ");
+
 	result = result.replace("&#33303;", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
 	result = result.replace("&quot;", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
 	//result = htmlEntitiesDecode(result);  // for special chars mainly

mysql.d

@@ -185,10 +185,14 @@ class MySql : Database {
 		return fromCstring(mysql_error(mysql));
 	}
 
-	~this() {
+	void close() {
 		mysql_close(mysql);
 	}
 
+	~this() {
+		close();
+	}
+
 	int lastInsertId() {
 		return cast(int) mysql_insert_id(mysql);
 	}

oauth.d

@@ -30,7 +30,9 @@ class FacebookApiException : Exception {
 
 import arsd.curl;
 import arsd.sha;
-import std.digest.md;
+
+import std.md5;
+// import std.digest.md;
 
 import std.file;
 
@@ -39,7 +41,6 @@ import std.file;
 Variant[string] postToFacebookWall(string[] info, string id, string message, string picture = null, string link = null, long when = 0, string linkDescription = null) {
 	string url = "https://graph.facebook.com/" ~ id ~ "/feed";
 
-
 	string data = "access_token=" ~ std.uri.encodeComponent(info[1]);
 	data ~= "&message=" ~ std.uri.encodeComponent(message);
 
@@ -224,8 +225,8 @@ OAuthParams twitter(string apiKey, string apiSecret) {
 	params.apiKey = apiKey;
 	params.apiSecret = apiSecret;
 
-	//params.baseUrl = "https://api.twitter.com";
+	params.baseUrl = "https://api.twitter.com";
-	params.baseUrl = "http://twitter.com";
+	//params.baseUrl = "http://twitter.com";
 	params.requestTokenPath = "/oauth/request_token";
 	params.authorizePath = "/oauth/authorize";
 	params.accessTokenPath = "/oauth/access_token";

web.d

@@ -250,6 +250,10 @@ class WebDotDBaseType {
 
 	/// By default, it forwards the document root to _postProcess(Element).
 	void _postProcess(Document document) {
+		auto td = cast(TemplatedDocument) document;
+		if(td !is null)
+			td.vars["compile.timestamp"] = compiliationStamp;
+
 		if(document !is null && document.root !is null)
 			_postProcessElement(document.root);
 	}
@@ -617,7 +621,7 @@ class ApiProvider : WebDotDBaseType {
 <html>
 <head>
 	<title></title>
-	<link rel=\"stylesheet\" href=\"styles.css\" />
+	<link rel=\"stylesheet\" href=\"styles.css?"~compiliationStamp~"\" />
 	<script> var delayedExecutionQueue = []; </script> <!-- FIXME do some better separation -->
 	<script>
 		if(document.cookie.indexOf(\"timezone=\") == -1) {
@@ -633,7 +637,7 @@ class ApiProvider : WebDotDBaseType {
 </head>
 <body>
 	<div id=\"body\"></div>
-	<script src=\"functions.js\"></script>
+	<script src=\"functions.js?"~compiliationStamp~"\"></script>
 	" ~ deqFoot ~ "
 </body>
 </html>");
@@ -1198,6 +1202,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 	if(funName == "functions.js") {
 		cgi.gzipResponse = true;
 		cgi.setResponseContentType("text/javascript");
+		cgi.setCache(true);
 		cgi.write(makeJavascriptApi(reflection, replace(cast(string) cgi.requestUri, "functions.js", "")), true);
 		cgi.close();
 		return;
@@ -1205,6 +1210,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 	if(funName == "styles.css") {
 		cgi.gzipResponse = true;
 		cgi.setResponseContentType("text/css");
+		cgi.setCache(true);
 		cgi.write(instantiation.stylesheet(), true);
 		cgi.close();
 		return;
@@ -1376,7 +1382,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
 						f.setValue(k, v[$-1]);
 
 					foreach(idx, failure; fve.failed) {
-						auto ele = f.querySelector("[name=\""~failure~"\"]");
+						auto ele = f.requireSelector("[name=\""~failure~"\"]");
 						ele.addClass("validation-failed");
 						ele.dataset.validationMessage = fve.messagesForUser[idx];
 						ele.parentNode.addChild("span", fve.messagesForUser[idx]).addClass("validation-message");
@@ -1673,7 +1679,6 @@ mixin template FancyMain(T, Args...) {
 	mixin CustomCgiFancyMain!(Cgi, T, Args);
 }
 
-
 /// Like FancyMain, but you can pass a custom subclass of Cgi
 mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi)) {
 	void fancyMainFunction(Cgi cgi) { //string[] args) {
@@ -1700,7 +1705,13 @@ mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi))
 		version(no_automatic_session) {}
 		else {
 			auto session = new Session(cgi);
-			scope(exit) session.commit();
+			scope(exit) {
+				// I only commit automatically on non-bots to avoid writing too many files
+				// looking for bot should catch most them without false positives...
+				// empty user agent is prolly a tester too so i'll let that slide
+				if(cgi.userAgent.length && cgi.userAgent.toLower.indexOf("bot") == -1)
+					session.commit();
+			}
 			instantiation.session = session;
 		}
 
@@ -2300,15 +2311,42 @@ class ParamCheckHelper {
 		messagesForUser ~= messageForUser;
 	}
 
-	string checkParam(in string[string] among, string name, bool delegate(string) ok, string messageForUser = null) {
+	string checkParam(in string[string] among, in string name, bool delegate(string) ok, string messageForUser = null) {
-		string value = null;
+		return checkTypedParam!string(among, name, ok, messageForUser);
-		auto ptr = "name" in among;
+	}
-		if(ptr !is null) {
+
-			value = *ptr;
+	T checkTypedParam(T)(in string[string] among, in string name, bool delegate(T) ok, string messageForUser = null) {
+		T value;
+
+		bool isOk = false;
+		string genericErrorMessage = "Please complete this field";
+
+
+		try {
+			//auto ptr = "name" in among;
+			//if(ptr !is null) {
+			//	value = *ptr;
+			//} else {
+				// D's in operator is sometimes buggy, so let's confirm this with a linear search ugh)
+				// FIXME: fix D's AA
+				foreach(k, v; among)
+					if(k == name) {
+						value = fromUrlParam!T(v);
+						break;
+					}
+			//}
+
+			if(ok !is null)
+				isOk = ok(value);
+			else
+				isOk = true; // no checker means if we were able to convert above, we're ok
+		} catch(Exception e) {
+			genericErrorMessage = e.msg;
+			isOk = false;
 		}
 
-		if(!ok(value)) {
+		if(!isOk) {
-			failure(name, messageForUser is null ? "Please complete this field" : messageForUser);
+			failure(name, messageForUser is null ? genericErrorMessage : messageForUser);
 		}
 
 		return value;
@@ -2346,6 +2384,14 @@ auto check(alias field)(ParamCheckHelper helper, bool delegate(typeof(field)) ok
 	return field;
 }
 
+bool isConvertableTo(T)(string v) {
+	try {
+		auto t = fromUrlParam!(T)(v);
+		return true;
+	} catch(Exception e) {
+		return false;
+	}
+}
 
 class FormValidationException : Exception {
 	this(
@@ -2674,9 +2720,8 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue
 		case "table":
 		case "csv":
 			auto document = new Document("<root></root>");
-			static if(__traits(compiles, structToTable(document, ret)))
+
-			{
+			void gotATable(Table table) {
-				auto table = structToTable(document, ret);
 				if(format == "csv") {
 					retstr = tableToCsv(table);
 				} else if(format == "table")
@@ -2686,6 +2731,18 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue
 
 				if(returnValue !is null)
 					returnValue.str = retstr;
+			}
+
+			static if(__traits(compiles, structToTable(document, ret)))
+			{
+				auto table = structToTable(document, ret);
+				gotATable(table);
+				break;
+			} else static if(is(typeof(ret) : Element)) {
+				auto table = cast(Table) ret;
+				if(table is null)
+					goto badType;
+				gotATable(table);
 				break;
 			}
 			else
@@ -2706,7 +2763,7 @@ string tableToCsv(Table table) {
 	string csv;
 	foreach(tr; table.querySelectorAll("tr")) {
 		if(csv.length)
-			csv ~= "\n";
+			csv ~= "\r\n";
 
 		bool outputted = false;
 		foreach(item; tr.querySelectorAll("td, th")) {
@@ -3402,6 +3459,18 @@ struct TemplateFilters {
 }
 
 
+string applyTemplateToText(
+	string text,
+	in string[string] vars,
+	in string delegate(string, string[], in Element, string)[string] pipeFunctions = TemplateFilters.defaultThings())
+{
+	// kinda hacky, but meh
+	auto element = Element.make("body");
+	element.innerText = text;
+	applyTemplateToElement(element, vars, pipeFunctions);
+	return element.innerText;
+}
+
 void applyTemplateToElement(
 	Element e,
 	in string[string] vars,
@@ -4630,7 +4699,16 @@ template getAnnotation(alias f, Attr) if(hasValueAnnotation!(f, Attr)) {
 	enum getAnnotation = helper;
 }
 
+// use this as a query string param to all forever-cached resources
+string makeCompileTimestamp(string ts) {
+	string ret;
+	foreach(t; ts)
+		if((t >= '0' && t <= '9'))
+			ret ~= t;
+	return ret;
+}
 
+enum compiliationStamp = makeCompileTimestamp(__TIMESTAMP__);
 
 /*
 Copyright: Adam D. Ruppe, 2010 - 2012