From 65513f5ccf259bcde069186a6123f286708e6dde Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 11 Apr 2013 07:59:49 -0400 Subject: [PATCH] catch up --- cgi.d | 16 +++++++ characterencodings.d | 25 ++++++++++ curl.d | 3 +- database.d | 15 +++--- dom.d | 16 +++++-- email.d | 44 +++++++++++++++-- htmltotext.d | 24 ++++++++-- mysql.d | 6 ++- oauth.d | 9 ++-- web.d | 110 ++++++++++++++++++++++++++++++++++++------- 10 files changed, 228 insertions(+), 40 deletions(-) diff --git a/cgi.d b/cgi.d index 639c939..f6b07c8 100644 --- a/cgi.d +++ b/cgi.d @@ -1793,6 +1793,14 @@ class Cgi { immutable(string[][string]) postArray; /// ditto for post immutable(string[][string]) cookiesArray; /// ditto for cookies + // convenience function for appending to a uri without extra ? + // matches the name and effect of javascript's location.search property + string search() const { + if(queryString.length) + return "?" ~ queryString; + return ""; + } + // FIXME: what about multiple files with the same name? private: //RequestMethod _requestMethod; @@ -1997,6 +2005,14 @@ struct Uri { // FIXME: add something for port too } + + // these are like javascript's location.search and location.hash + string search() const { + return query.length ? ("?" ~ query) : ""; + } + string hash() const { + return fragment.length ? ("#" ~ fragment) : ""; + } } diff --git a/characterencodings.d b/characterencodings.d index e3b7caa..a75e76d 100644 --- a/characterencodings.d +++ b/characterencodings.d @@ -1,3 +1,5 @@ +// helper program is in ~me/encodings.d to make more tables from wikipedia + /** This is meant to help get data from the wild into utf8 strings so you can work with them easily inside D. @@ -69,6 +71,8 @@ string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) { // and now the various 8 bit encodings we support. case "windows1252": return decodeImpl(data, ISO_8859_1, Windows_1252); + case "windows1251": + return decodeImpl(data, Windows_1251, Windows_1251_Lower); case "koi8r": return decodeImpl(data, KOI8_R, KOI8_R_Lower); case "latin1": @@ -430,3 +434,24 @@ immutable dchar[] KOI8_R = [ 'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в', 'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ']; + +immutable dchar[] Windows_1251_Lower = [ + 'Ђ', 'Ѓ', '‚', 'ѓ', '„', '…', '†', '‡', + '€', '‰', 'Љ', '‹', 'Њ', 'Ќ', 'Ћ', 'Џ', + 'ђ', '‘', '’', '“', '”', '•', '–', '—', + ' ', '™', 'љ', '›', 'њ', 'ќ', 'ћ', 'џ']; + +immutable dchar[] Windows_1251 = [ + ' ', 'Ў', 'ў', 'Ј', '¤', 'Ґ', '¦', '§', + 'Ё', '©', 'Є', '«', '¬', '­', '®', 'Ї', + '°', '±', 'І', 'і', 'ґ', 'µ', '¶', '·', + 'ё', '№', 'є', '»', 'ј', 'Ѕ', 'ѕ', 'ї', + 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З', + 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', + 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', + 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', + 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', + 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', + 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', + 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я']; + diff --git a/curl.d b/curl.d index 45ceee7..d79f945 100644 --- a/curl.d +++ b/curl.d @@ -107,7 +107,8 @@ string curlAuth(string url, string data = null, string username = null, string p int res; - // curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + debug(arsd_curl_verbose) + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url)); if(res != 0) throw new CurlException(res); diff --git a/database.d b/database.d index 28c033c..718939c 100644 --- a/database.d +++ b/database.d @@ -28,16 +28,19 @@ interface Database { if(arg == typeid(string) || arg == typeid(immutable(string)) || arg == typeid(const(string))) a = va_arg!string(_argptr); else if (arg == typeid(int) || arg == typeid(immutable(int)) || arg == typeid(const(int))) { - int e = va_arg!int(_argptr); + auto e = va_arg!int(_argptr); a = to!string(e); } else if (arg == typeid(uint) || arg == typeid(immutable(uint)) || arg == typeid(const(uint))) { - int e = va_arg!uint(_argptr); + auto e = va_arg!uint(_argptr); a = to!string(e); } else if (arg == typeid(immutable(char))) { - char e = va_arg!char(_argptr); + auto e = va_arg!char(_argptr); a = to!string(e); } else if (arg == typeid(long) || arg == typeid(const(long)) || arg == typeid(immutable(long))) { - long e = va_arg!long(_argptr); + auto e = va_arg!long(_argptr); + a = to!string(e); + } else if (arg == typeid(ulong) || arg == typeid(const(ulong)) || arg == typeid(immutable(ulong))) { + auto e = va_arg!ulong(_argptr); a = to!string(e); } else if (arg == typeid(null)) { a = null; @@ -51,10 +54,10 @@ interface Database { } import std.stdio; -Ret queryOneColumn(Ret, T...)(Database db, string sql, T t) { +Ret queryOneColumn(Ret, string file = __FILE__, size_t line = __LINE__, T...)(Database db, string sql, T t) { auto res = db.query(sql, t); if(res.empty) - throw new Exception("no row in result"); + throw new Exception("no row in result", file, line); auto row = res.front; return to!Ret(row[0]); } diff --git a/dom.d b/dom.d index c623928..b0a78d1 100644 --- a/dom.d +++ b/dom.d @@ -885,6 +885,8 @@ class Element { version(dom_node_indexes) this.dataset.nodeIndex = to!string(&(this.attributes)); + + assert(_tagName.indexOf(" ") == -1); } /// Convenience constructor when you don't care about the parentDocument. Note this might break things on the document. @@ -3245,9 +3247,17 @@ class Document : FileResource { } } - if(dataEncoding != "UTF-8") - data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding); - else + if(dataEncoding != "UTF-8") { + if(strict) + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding); + else { + try { + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding); + } catch(Exception e) { + data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, "Windows 1252"); + } + } + } else data = rawdata; assert(data !is null); diff --git a/email.d b/email.d index ddc3c60..0ddf06a 100644 --- a/email.d +++ b/email.d @@ -402,7 +402,7 @@ string[string] breakUpHeaderParts(string headerContent) { string currentName = "root"; string currentContent; - bool inQuote; + bool inQuote = false; bool gettingName = false; bool ignoringSpaces = false; foreach(char c; headerContent) { @@ -423,7 +423,7 @@ string[string] breakUpHeaderParts(string headerContent) { if(c == '"') { inQuote = !inQuote; - break; + continue; } if(!inQuote && c == ';') { @@ -519,6 +519,8 @@ class IncomingEmailMessage { string charset = "latin-1"; + string contentTransferEncoding; + string headerName; string headerContent; void commitHeader() { @@ -556,12 +558,14 @@ class IncomingEmailMessage { } } else if(headerName == "from") { this.from = headerContent; + } else if(headerName == "to") { + this.to = headerContent; } else if(headerName == "subject") { this.subject = headerContent; + } else if(headerName == "content-transfer-encoding") { + contentTransferEncoding = headerContent; } - // FIXME: do I have to worry about content-transfer-encoding here? I think procmail takes care of it but I'm not entirely sure - headers[headerName] = headerContent; headerName = null; headerContent = null; @@ -667,8 +671,25 @@ class IncomingEmailMessage { // if inline it is prolly an image to be concated in the other body // if attachment, it is an attachment break; + case "multipart/signed": + // FIXME: it would be cool to actually check the signature default: // FIXME: correctly handle more + if(part.stuff.length) { + part = part.stuff[0]; + goto deeperInTheMimeTree; + } + } + } else { + switch(contentTransferEncoding) { + case "quoted-printable": + if(textMessageBody.length) + textMessageBody = cast(string) decodeQuotedPrintable(textMessageBody); + if(htmlMessageBody.length) + htmlMessageBody = cast(string) decodeQuotedPrintable(htmlMessageBody); + break; + default: + // nothing needed } } @@ -687,6 +708,7 @@ class IncomingEmailMessage { string textMessageBody; string from; + string to; bool textAutoConverted; @@ -850,3 +872,17 @@ immutable(ubyte)[] decodeQuotedPrintable(string text) { return ret; } + +/+ +void main() { + import std.file; + import std.stdio; + + auto data = cast(immutable(ubyte)[]) std.file.read("/home/me/test_email_data"); + foreach(message; processMboxData(data)) { + writeln(message.subject); + writeln(message.textMessageBody); + writeln("**************** END MESSSAGE **************"); + } +} ++/ diff --git a/htmltotext.d b/htmltotext.d index ec67193..f0ba596 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -86,9 +86,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { break; case "a": string href = ele.getAttribute("href"); - if(href) { - if(ele.innerText != href) - ele.innerText = ele.innerText ~ " <" ~ href ~ "> "; + if(href && !ele.hasClass("no-brackets")) { + if(ele.hasClass("href-text")) + ele.innerText = href; + else { + if(ele.innerText != href) + ele.innerText = ele.innerText ~ " <" ~ href ~ "> "; + } } ele.stripOut(); goto again; @@ -98,8 +102,9 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r"; break; case "li": - ele.innerHTML = "\t* " ~ ele.innerHTML ~ "\r"; - ele.stripOut(); + if(!ele.innerHTML.startsWith("* ")) + ele.innerHTML = "* " ~ ele.innerHTML ~ "\r"; + // ele.stripOut(); break; case "sup": ele.innerText = "^" ~ ele.innerText; @@ -140,6 +145,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { ele.innerText = strip(ele.innerText); ele.stripOut(); goto again2; + } else if(ele.tagName == "li") { + auto part = ele.innerText; + part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part ); + part = " " ~ part.replace("\n", "\n\v") ~ "\r"; + ele.innerText = part; + ele.stripOut(); + goto again2; } } @@ -155,6 +167,8 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { result = squeeze(result, "\r"); result = result.replace("\r", "\n\n"); + result = result.replace("\v", " "); + result = result.replace("舗", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere result = result.replace(""", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere //result = htmlEntitiesDecode(result); // for special chars mainly diff --git a/mysql.d b/mysql.d index 3b32954..7a1459f 100644 --- a/mysql.d +++ b/mysql.d @@ -185,10 +185,14 @@ class MySql : Database { return fromCstring(mysql_error(mysql)); } - ~this() { + void close() { mysql_close(mysql); } + ~this() { + close(); + } + int lastInsertId() { return cast(int) mysql_insert_id(mysql); } diff --git a/oauth.d b/oauth.d index c121b07..f6f6167 100644 --- a/oauth.d +++ b/oauth.d @@ -30,7 +30,9 @@ class FacebookApiException : Exception { import arsd.curl; import arsd.sha; -import std.digest.md; + +import std.md5; +// import std.digest.md; import std.file; @@ -39,7 +41,6 @@ import std.file; Variant[string] postToFacebookWall(string[] info, string id, string message, string picture = null, string link = null, long when = 0, string linkDescription = null) { string url = "https://graph.facebook.com/" ~ id ~ "/feed"; - string data = "access_token=" ~ std.uri.encodeComponent(info[1]); data ~= "&message=" ~ std.uri.encodeComponent(message); @@ -224,8 +225,8 @@ OAuthParams twitter(string apiKey, string apiSecret) { params.apiKey = apiKey; params.apiSecret = apiSecret; - //params.baseUrl = "https://api.twitter.com"; - params.baseUrl = "http://twitter.com"; + params.baseUrl = "https://api.twitter.com"; + //params.baseUrl = "http://twitter.com"; params.requestTokenPath = "/oauth/request_token"; params.authorizePath = "/oauth/authorize"; params.accessTokenPath = "/oauth/access_token"; diff --git a/web.d b/web.d index 5f10723..13ab480 100644 --- a/web.d +++ b/web.d @@ -250,6 +250,10 @@ class WebDotDBaseType { /// By default, it forwards the document root to _postProcess(Element). void _postProcess(Document document) { + auto td = cast(TemplatedDocument) document; + if(td !is null) + td.vars["compile.timestamp"] = compiliationStamp; + if(document !is null && document.root !is null) _postProcessElement(document.root); } @@ -617,7 +621,7 @@ class ApiProvider : WebDotDBaseType { - + + " ~ deqFoot ~ " "); @@ -1198,6 +1202,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint if(funName == "functions.js") { cgi.gzipResponse = true; cgi.setResponseContentType("text/javascript"); + cgi.setCache(true); cgi.write(makeJavascriptApi(reflection, replace(cast(string) cgi.requestUri, "functions.js", "")), true); cgi.close(); return; @@ -1205,6 +1210,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint if(funName == "styles.css") { cgi.gzipResponse = true; cgi.setResponseContentType("text/css"); + cgi.setCache(true); cgi.write(instantiation.stylesheet(), true); cgi.close(); return; @@ -1376,7 +1382,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint f.setValue(k, v[$-1]); foreach(idx, failure; fve.failed) { - auto ele = f.querySelector("[name=\""~failure~"\"]"); + auto ele = f.requireSelector("[name=\""~failure~"\"]"); ele.addClass("validation-failed"); ele.dataset.validationMessage = fve.messagesForUser[idx]; ele.parentNode.addChild("span", fve.messagesForUser[idx]).addClass("validation-message"); @@ -1673,7 +1679,6 @@ mixin template FancyMain(T, Args...) { mixin CustomCgiFancyMain!(Cgi, T, Args); } - /// Like FancyMain, but you can pass a custom subclass of Cgi mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi)) { void fancyMainFunction(Cgi cgi) { //string[] args) { @@ -1700,7 +1705,13 @@ mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi)) version(no_automatic_session) {} else { auto session = new Session(cgi); - scope(exit) session.commit(); + scope(exit) { + // I only commit automatically on non-bots to avoid writing too many files + // looking for bot should catch most them without false positives... + // empty user agent is prolly a tester too so i'll let that slide + if(cgi.userAgent.length && cgi.userAgent.toLower.indexOf("bot") == -1) + session.commit(); + } instantiation.session = session; } @@ -2300,15 +2311,42 @@ class ParamCheckHelper { messagesForUser ~= messageForUser; } - string checkParam(in string[string] among, string name, bool delegate(string) ok, string messageForUser = null) { - string value = null; - auto ptr = "name" in among; - if(ptr !is null) { - value = *ptr; + string checkParam(in string[string] among, in string name, bool delegate(string) ok, string messageForUser = null) { + return checkTypedParam!string(among, name, ok, messageForUser); + } + + T checkTypedParam(T)(in string[string] among, in string name, bool delegate(T) ok, string messageForUser = null) { + T value; + + bool isOk = false; + string genericErrorMessage = "Please complete this field"; + + + try { + //auto ptr = "name" in among; + //if(ptr !is null) { + // value = *ptr; + //} else { + // D's in operator is sometimes buggy, so let's confirm this with a linear search ugh) + // FIXME: fix D's AA + foreach(k, v; among) + if(k == name) { + value = fromUrlParam!T(v); + break; + } + //} + + if(ok !is null) + isOk = ok(value); + else + isOk = true; // no checker means if we were able to convert above, we're ok + } catch(Exception e) { + genericErrorMessage = e.msg; + isOk = false; } - if(!ok(value)) { - failure(name, messageForUser is null ? "Please complete this field" : messageForUser); + if(!isOk) { + failure(name, messageForUser is null ? genericErrorMessage : messageForUser); } return value; @@ -2346,6 +2384,14 @@ auto check(alias field)(ParamCheckHelper helper, bool delegate(typeof(field)) ok return field; } +bool isConvertableTo(T)(string v) { + try { + auto t = fromUrlParam!(T)(v); + return true; + } catch(Exception e) { + return false; + } +} class FormValidationException : Exception { this( @@ -2674,9 +2720,8 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue case "table": case "csv": auto document = new Document(""); - static if(__traits(compiles, structToTable(document, ret))) - { - auto table = structToTable(document, ret); + + void gotATable(Table table) { if(format == "csv") { retstr = tableToCsv(table); } else if(format == "table") @@ -2686,6 +2731,18 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue if(returnValue !is null) returnValue.str = retstr; + } + + static if(__traits(compiles, structToTable(document, ret))) + { + auto table = structToTable(document, ret); + gotATable(table); + break; + } else static if(is(typeof(ret) : Element)) { + auto table = cast(Table) ret; + if(table is null) + goto badType; + gotATable(table); break; } else @@ -2706,7 +2763,7 @@ string tableToCsv(Table table) { string csv; foreach(tr; table.querySelectorAll("tr")) { if(csv.length) - csv ~= "\n"; + csv ~= "\r\n"; bool outputted = false; foreach(item; tr.querySelectorAll("td, th")) { @@ -3402,6 +3459,18 @@ struct TemplateFilters { } +string applyTemplateToText( + string text, + in string[string] vars, + in string delegate(string, string[], in Element, string)[string] pipeFunctions = TemplateFilters.defaultThings()) +{ + // kinda hacky, but meh + auto element = Element.make("body"); + element.innerText = text; + applyTemplateToElement(element, vars, pipeFunctions); + return element.innerText; +} + void applyTemplateToElement( Element e, in string[string] vars, @@ -4630,7 +4699,16 @@ template getAnnotation(alias f, Attr) if(hasValueAnnotation!(f, Attr)) { enum getAnnotation = helper; } +// use this as a query string param to all forever-cached resources +string makeCompileTimestamp(string ts) { + string ret; + foreach(t; ts) + if((t >= '0' && t <= '9')) + ret ~= t; + return ret; +} +enum compiliationStamp = makeCompileTimestamp(__TIMESTAMP__); /* Copyright: Adam D. Ruppe, 2010 - 2012