forks
/
arsd
mirror of https://github.com/adamdruppe/arsd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

catch up

This commit is contained in:
Adam D. Ruppe 2013-04-11 07:59:49 -04:00
parent 290f6c794c
commit 65513f5ccf
10 changed files with 228 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cgi.d
View File

@ -1793,6 +1793,14 @@ class Cgi {
immutable(string[][string]) postArray; /// ditto for post
immutable(string[][string]) cookiesArray; /// ditto for cookies
// convenience function for appending to a uri without extra ?
// matches the name and effect of javascript's location.search property
string search() const {
if(queryString.length)
return "?" ~ queryString;
return "";
}
// FIXME: what about multiple files with the same name?
private:
//RequestMethod _requestMethod;
@ -1997,6 +2005,14 @@ struct Uri {
// FIXME: add something for port too
}
// these are like javascript's location.search and location.hash
string search() const {
return query.length ? ("?" ~ query) : "";
}
string hash() const {
return fragment.length ? ("#" ~ fragment) : "";
}
}

25
characterencodings.d
View File

@ -1,3 +1,5 @@
// helper program is in ~me/encodings.d to make more tables from wikipedia
/**
This is meant to help get data from the wild into utf8 strings
so you can work with them easily inside D.
@ -69,6 +71,8 @@ string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) {
// and now the various 8 bit encodings we support.
case "windows1252":
return decodeImpl(data, ISO_8859_1, Windows_1252);
case "windows1251":
return decodeImpl(data, Windows_1251, Windows_1251_Lower);
case "koi8r":
return decodeImpl(data, KOI8_R, KOI8_R_Lower);
case "latin1":
@ -430,3 +434,24 @@ immutable dchar[] KOI8_R = [
'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о',
'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в',
'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ'];
immutable dchar[] Windows_1251_Lower = [
'Ђ', 'Ѓ', '‚', 'ѓ', '„', '…', '†', '‡',
'€', '‰', 'Љ', '‹', 'Њ', 'Ќ', 'Ћ', 'Џ',
'ђ', '‘', '’', '“', '”', '•', '–', '—',
' ', '™', 'љ', '›', 'њ', 'ќ', 'ћ', 'џ'];
immutable dchar[] Windows_1251 = [
' ', 'Ў', 'ў', 'Ј', '¤', 'Ґ', '¦', '§',
'Ё', '©', 'Є', '«', '¬', '­', '®', 'Ї',
'°', '±', 'І', 'і', 'ґ', 'µ', '¶', '·',
'ё', '№', 'є', '»', 'ј', 'Ѕ', 'ѕ', 'ї',
'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З',
'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П',
'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч',
'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я',
'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з',
'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п',
'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч',
'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я'];

3
curl.d
View File

@ -107,7 +107,8 @@ string curlAuth(string url, string data = null, string username = null, string p
int res;
// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
debug(arsd_curl_verbose)
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url));
if(res != 0) throw new CurlException(res);

15
database.d
View File

@ -28,16 +28,19 @@ interface Database {
if(arg == typeid(string) || arg == typeid(immutable(string)) || arg == typeid(const(string)))
a = va_arg!string(_argptr);
else if (arg == typeid(int) || arg == typeid(immutable(int)) || arg == typeid(const(int))) {
int e = va_arg!int(_argptr);
auto e = va_arg!int(_argptr);
a = to!string(e);
} else if (arg == typeid(uint) || arg == typeid(immutable(uint)) || arg == typeid(const(uint))) {
int e = va_arg!uint(_argptr);
auto e = va_arg!uint(_argptr);
a = to!string(e);
} else if (arg == typeid(immutable(char))) {
char e = va_arg!char(_argptr);
auto e = va_arg!char(_argptr);
a = to!string(e);
} else if (arg == typeid(long) || arg == typeid(const(long)) || arg == typeid(immutable(long))) {
long e = va_arg!long(_argptr);
auto e = va_arg!long(_argptr);
a = to!string(e);
} else if (arg == typeid(ulong) || arg == typeid(const(ulong)) || arg == typeid(immutable(ulong))) {
auto e = va_arg!ulong(_argptr);
a = to!string(e);
} else if (arg == typeid(null)) {
a = null;
@ -51,10 +54,10 @@ interface Database {
}
import std.stdio;
Ret queryOneColumn(Ret, T...)(Database db, string sql, T t) {
Ret queryOneColumn(Ret, string file = __FILE__, size_t line = __LINE__, T...)(Database db, string sql, T t) {
auto res = db.query(sql, t);
if(res.empty)
throw new Exception("no row in result");
throw new Exception("no row in result", file, line);
auto row = res.front;
return to!Ret(row[0]);
}

16
dom.d
View File

@ -885,6 +885,8 @@ class Element {
version(dom_node_indexes)
this.dataset.nodeIndex = to!string(&(this.attributes));
assert(_tagName.indexOf(" ") == -1);
}
/// Convenience constructor when you don't care about the parentDocument. Note this might break things on the document.
@ -3245,9 +3247,17 @@ class Document : FileResource {
}
}
if(dataEncoding != "UTF-8")
data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
else
if(dataEncoding != "UTF-8") {
if(strict)
data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
else {
try {
data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, dataEncoding);
} catch(Exception e) {
data = convertToUtf8(cast(immutable(ubyte)[]) rawdata, "Windows 1252");
}
}
} else
data = rawdata;
assert(data !is null);

44
email.d
View File

@ -402,7 +402,7 @@ string[string] breakUpHeaderParts(string headerContent) {
string currentName = "root";
string currentContent;
bool inQuote;
bool inQuote = false;
bool gettingName = false;
bool ignoringSpaces = false;
foreach(char c; headerContent) {
@ -423,7 +423,7 @@ string[string] breakUpHeaderParts(string headerContent) {
if(c == '"') {
inQuote = !inQuote;
break;
continue;
}
if(!inQuote && c == ';') {
@ -519,6 +519,8 @@ class IncomingEmailMessage {
string charset = "latin-1";
string contentTransferEncoding;
string headerName;
string headerContent;
void commitHeader() {
@ -556,12 +558,14 @@ class IncomingEmailMessage {
}
} else if(headerName == "from") {
this.from = headerContent;
} else if(headerName == "to") {
this.to = headerContent;
} else if(headerName == "subject") {
this.subject = headerContent;
} else if(headerName == "content-transfer-encoding") {
contentTransferEncoding = headerContent;
}
// FIXME: do I have to worry about content-transfer-encoding here? I think procmail takes care of it but I'm not entirely sure
headers[headerName] = headerContent;
headerName = null;
headerContent = null;
@ -667,8 +671,25 @@ class IncomingEmailMessage {
// if inline it is prolly an image to be concated in the other body
// if attachment, it is an attachment
break;
case "multipart/signed":
// FIXME: it would be cool to actually check the signature
default:
// FIXME: correctly handle more
if(part.stuff.length) {
part = part.stuff[0];
goto deeperInTheMimeTree;
}
}
} else {
switch(contentTransferEncoding) {
case "quoted-printable":
if(textMessageBody.length)
textMessageBody = cast(string) decodeQuotedPrintable(textMessageBody);
if(htmlMessageBody.length)
htmlMessageBody = cast(string) decodeQuotedPrintable(htmlMessageBody);
break;
default:
// nothing needed
}
}
@ -687,6 +708,7 @@ class IncomingEmailMessage {
string textMessageBody;
string from;
string to;
bool textAutoConverted;
@ -850,3 +872,17 @@ immutable(ubyte)[] decodeQuotedPrintable(string text) {
return ret;
}
/+
void main() {
import std.file;
import std.stdio;
auto data = cast(immutable(ubyte)[]) std.file.read("/home/me/test_email_data");
foreach(message; processMboxData(data)) {
writeln(message.subject);
writeln(message.textMessageBody);
writeln("**************** END MESSSAGE **************");
}
}
+/

24
htmltotext.d
View File

@ -86,9 +86,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
break;
case "a":
string href = ele.getAttribute("href");
if(href) {
if(ele.innerText != href)
ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
if(href && !ele.hasClass("no-brackets")) {
if(ele.hasClass("href-text"))
ele.innerText = href;
else {
if(ele.innerText != href)
ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
}
}
ele.stripOut();
goto again;
@ -98,8 +102,9 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r";
break;
case "li":
ele.innerHTML = "\t* " ~ ele.innerHTML ~ "\r";
ele.stripOut();
if(!ele.innerHTML.startsWith("* "))
ele.innerHTML = "* " ~ ele.innerHTML ~ "\r";
// ele.stripOut();
break;
case "sup":
ele.innerText = "^" ~ ele.innerText;
@ -140,6 +145,13 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
ele.innerText = strip(ele.innerText);
ele.stripOut();
goto again2;
} else if(ele.tagName == "li") {
auto part = ele.innerText;
part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part );
part = " " ~ part.replace("\n", "\n\v") ~ "\r";
ele.innerText = part;
ele.stripOut();
goto again2;
}
}
@ -155,6 +167,8 @@ string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
result = squeeze(result, "\r");
result = result.replace("\r", "\n\n");
result = result.replace("\v", " ");
result = result.replace("&#33303;", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
result = result.replace("&quot;", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
//result = htmlEntitiesDecode(result); // for special chars mainly

6
mysql.d
View File

@ -185,10 +185,14 @@ class MySql : Database {
return fromCstring(mysql_error(mysql));
}
~this() {
void close() {
mysql_close(mysql);
}
~this() {
close();
}
int lastInsertId() {
return cast(int) mysql_insert_id(mysql);
}

9
oauth.d
View File

@ -30,7 +30,9 @@ class FacebookApiException : Exception {
import arsd.curl;
import arsd.sha;
import std.digest.md;
import std.md5;
// import std.digest.md;
import std.file;
@ -39,7 +41,6 @@ import std.file;
Variant[string] postToFacebookWall(string[] info, string id, string message, string picture = null, string link = null, long when = 0, string linkDescription = null) {
string url = "https://graph.facebook.com/" ~ id ~ "/feed";
string data = "access_token=" ~ std.uri.encodeComponent(info[1]);
data ~= "&message=" ~ std.uri.encodeComponent(message);
@ -224,8 +225,8 @@ OAuthParams twitter(string apiKey, string apiSecret) {
params.apiKey = apiKey;
params.apiSecret = apiSecret;
//params.baseUrl = "https://api.twitter.com";
params.baseUrl = "http://twitter.com";
params.baseUrl = "https://api.twitter.com";
//params.baseUrl = "http://twitter.com";
params.requestTokenPath = "/oauth/request_token";
params.authorizePath = "/oauth/authorize";
params.accessTokenPath = "/oauth/access_token";

110
web.d
View File

@ -250,6 +250,10 @@ class WebDotDBaseType {
/// By default, it forwards the document root to _postProcess(Element).
void _postProcess(Document document) {
auto td = cast(TemplatedDocument) document;
if(td !is null)
td.vars["compile.timestamp"] = compiliationStamp;
if(document !is null && document.root !is null)
_postProcessElement(document.root);
}
@ -617,7 +621,7 @@ class ApiProvider : WebDotDBaseType {
<html>
<head>
<title></title>
<link rel=\"stylesheet\" href=\"styles.css\" />
<link rel=\"stylesheet\" href=\"styles.css?"~compiliationStamp~"\" />
<script> var delayedExecutionQueue = []; </script> <!-- FIXME do some better separation -->
<script>
if(document.cookie.indexOf(\"timezone=\") == -1) {
@ -633,7 +637,7 @@ class ApiProvider : WebDotDBaseType {
</head>
<body>
<div id=\"body\"></div>
<script src=\"functions.js\"></script>
<script src=\"functions.js?"~compiliationStamp~"\"></script>
" ~ deqFoot ~ "
</body>
</html>");
@ -1198,6 +1202,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
if(funName == "functions.js") {
cgi.gzipResponse = true;
cgi.setResponseContentType("text/javascript");
cgi.setCache(true);
cgi.write(makeJavascriptApi(reflection, replace(cast(string) cgi.requestUri, "functions.js", "")), true);
cgi.close();
return;
@ -1205,6 +1210,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
if(funName == "styles.css") {
cgi.gzipResponse = true;
cgi.setResponseContentType("text/css");
cgi.setCache(true);
cgi.write(instantiation.stylesheet(), true);
cgi.close();
return;
@ -1376,7 +1382,7 @@ void run(Provider)(Cgi cgi, Provider instantiation, size_t pathInfoStartingPoint
f.setValue(k, v[$-1]);
foreach(idx, failure; fve.failed) {
auto ele = f.querySelector("[name=\""~failure~"\"]");
auto ele = f.requireSelector("[name=\""~failure~"\"]");
ele.addClass("validation-failed");
ele.dataset.validationMessage = fve.messagesForUser[idx];
ele.parentNode.addChild("span", fve.messagesForUser[idx]).addClass("validation-message");
@ -1673,7 +1679,6 @@ mixin template FancyMain(T, Args...) {
mixin CustomCgiFancyMain!(Cgi, T, Args);
}
/// Like FancyMain, but you can pass a custom subclass of Cgi
mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi)) {
void fancyMainFunction(Cgi cgi) { //string[] args) {
@ -1700,7 +1705,13 @@ mixin template CustomCgiFancyMain(CustomCgi, T, Args...) if(is(CustomCgi : Cgi))
version(no_automatic_session) {}
else {
auto session = new Session(cgi);
scope(exit) session.commit();
scope(exit) {
// I only commit automatically on non-bots to avoid writing too many files
// looking for bot should catch most them without false positives...
// empty user agent is prolly a tester too so i'll let that slide
if(cgi.userAgent.length && cgi.userAgent.toLower.indexOf("bot") == -1)
session.commit();
}
instantiation.session = session;
}
@ -2300,15 +2311,42 @@ class ParamCheckHelper {
messagesForUser ~= messageForUser;
}
string checkParam(in string[string] among, string name, bool delegate(string) ok, string messageForUser = null) {
string value = null;
auto ptr = "name" in among;
if(ptr !is null) {
value = *ptr;
string checkParam(in string[string] among, in string name, bool delegate(string) ok, string messageForUser = null) {
return checkTypedParam!string(among, name, ok, messageForUser);
}
T checkTypedParam(T)(in string[string] among, in string name, bool delegate(T) ok, string messageForUser = null) {
T value;
bool isOk = false;
string genericErrorMessage = "Please complete this field";
try {
//auto ptr = "name" in among;
//if(ptr !is null) {
// value = *ptr;
//} else {
// D's in operator is sometimes buggy, so let's confirm this with a linear search ugh)
// FIXME: fix D's AA
foreach(k, v; among)
if(k == name) {
value = fromUrlParam!T(v);
break;
}
//}
if(ok !is null)
isOk = ok(value);
else
isOk = true; // no checker means if we were able to convert above, we're ok
} catch(Exception e) {
genericErrorMessage = e.msg;
isOk = false;
}
if(!ok(value)) {
failure(name, messageForUser is null ? "Please complete this field" : messageForUser);
if(!isOk) {
failure(name, messageForUser is null ? genericErrorMessage : messageForUser);
}
return value;
@ -2346,6 +2384,14 @@ auto check(alias field)(ParamCheckHelper helper, bool delegate(typeof(field)) ok
return field;
}
bool isConvertableTo(T)(string v) {
try {
auto t = fromUrlParam!(T)(v);
return true;
} catch(Exception e) {
return false;
}
}
class FormValidationException : Exception {
this(
@ -2674,9 +2720,8 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue
case "table":
case "csv":
auto document = new Document("<root></root>");
static if(__traits(compiles, structToTable(document, ret)))
{
auto table = structToTable(document, ret);
void gotATable(Table table) {
if(format == "csv") {
retstr = tableToCsv(table);
} else if(format == "table")
@ -2686,6 +2731,18 @@ string formatAs(T, R)(T ret, string format, R api = null, JSONValue* returnValue
if(returnValue !is null)
returnValue.str = retstr;
}
static if(__traits(compiles, structToTable(document, ret)))
{
auto table = structToTable(document, ret);
gotATable(table);
break;
} else static if(is(typeof(ret) : Element)) {
auto table = cast(Table) ret;
if(table is null)
goto badType;
gotATable(table);
break;
}
else
@ -2706,7 +2763,7 @@ string tableToCsv(Table table) {
string csv;
foreach(tr; table.querySelectorAll("tr")) {
if(csv.length)
csv ~= "\n";
csv ~= "\r\n";
bool outputted = false;
foreach(item; tr.querySelectorAll("td, th")) {
@ -3402,6 +3459,18 @@ struct TemplateFilters {
}
string applyTemplateToText(
string text,
in string[string] vars,
in string delegate(string, string[], in Element, string)[string] pipeFunctions = TemplateFilters.defaultThings())
{
// kinda hacky, but meh
auto element = Element.make("body");
element.innerText = text;
applyTemplateToElement(element, vars, pipeFunctions);
return element.innerText;
}
void applyTemplateToElement(
Element e,
in string[string] vars,
@ -4630,7 +4699,16 @@ template getAnnotation(alias f, Attr) if(hasValueAnnotation!(f, Attr)) {
enum getAnnotation = helper;
}
// use this as a query string param to all forever-cached resources
string makeCompileTimestamp(string ts) {
string ret;
foreach(t; ts)
if((t >= '0' && t <= '9'))
ret ~= t;
return ret;
}
enum compiliationStamp = makeCompileTimestamp(__TIMESTAMP__);
/*
Copyright: Adam D. Ruppe, 2010 - 2012