From 391628e3d0df8787d147d237a33705ee3079d364 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 15 Jul 2011 08:48:59 -0400 Subject: [PATCH] adding the existing stuff --- README | 55 ++ cgi.d | 1 + csv.d | 58 ++ curl.d | 188 ++++ database.d | 1010 ++++++++++++++++++++ dom.d | 1 + http.d | 221 +++++ mysql.d | 733 +++++++++++++++ png.d | 600 ++++++++++++ postgres.d | 218 +++++ sha.d | 366 ++++++++ simpleaudio.d | 1 + simpledisplay.d | 1 + sqlite.d | 740 +++++++++++++++ web.d | 2348 +++++++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 6541 insertions(+) create mode 100644 README create mode 120000 cgi.d create mode 100644 csv.d create mode 100644 curl.d create mode 100644 database.d create mode 120000 dom.d create mode 100644 http.d create mode 100644 mysql.d create mode 100644 png.d create mode 100644 postgres.d create mode 100644 sha.d create mode 120000 simpleaudio.d create mode 120000 simpledisplay.d create mode 100644 sqlite.d create mode 100644 web.d diff --git a/README b/README new file mode 100644 index 0000000..3fb3706 --- /dev/null +++ b/README @@ -0,0 +1,55 @@ +This is a collection of modules I find generally useful. + +Modules are usually independent; you don't need this whole directory +but it doesn't hurt to grab it all either. + +Currently included are: + +Web related +================ + +cgi.d - base module for making webapps in D +dom.d - an xml/html DOM based on what Javascript provides in browsers +web.d - a fancier way to write web apps. Uses reflection to make functions + accessible via url with minimal boilerplate in your code + + +Database related +================ + +database.d - main interface to databases. Includes DataObject +mysql.d - a mysql engine for database.d (most mature of the three) +postgres.d - a postgres engne for database.d +sqlite.d - a sqlite engine for database.d + +Desktop app stuff +================ + +simpledisplay.d - gives quick and easy access to a window for drawing +simpleaudio.d - gives minimal audio output + +Other +================ + +sha.d - implementations of the SHA1 and SHA256 algorithms +png.d - provides some png read/write support +curl.d - a small wrapper around the curl library +csv.d - gives read support to csv files +http.d - a lighterweight alternative to curl.d + + + +Things I might add once I clean up the files (this can be expedited upon +request, to an extent): + +httpd.d - an embedded web server +oauth.d - client/server stuff for oauth1 +html.d - a bunch of dom translation functions. Think unobstructive javascript + on the server side +browser.d - a very small html widget +netman.d - handles net connections (required by httpd.d) +imagedraft.d - (temporary name) has algorithms for images +bmp.d - gives .bmp read/write +dws.d - a draft of my D windowing system (also includes some Qt code) +wav.d - reading and writing WAV files +midi.d - reading and writing MIDI files diff --git a/cgi.d b/cgi.d new file mode 120000 index 0000000..283c969 --- /dev/null +++ b/cgi.d @@ -0,0 +1 @@ +../../djs/proxy/cgi.d \ No newline at end of file diff --git a/csv.d b/csv.d new file mode 100644 index 0000000..9466a66 --- /dev/null +++ b/csv.d @@ -0,0 +1,58 @@ +module arsd.csv; + +import std.string; +import std.array; + +string[][] readCsv(string data) { + data = data.replace("\r", ""); + + auto idx = data.indexOf("\n"); + //data = data[idx + 1 .. $]; // skip headers + + string[] fields; + string[][] records; + + string[] current; + + int state = 0; + string field; + foreach(c; data) { + tryit: switch(state) { + default: assert(0); + case 0: // normal + if(c == '"') + state = 1; + else if(c == ',') { + // commit field + current ~= field; + field = null; + } else if(c == '\n') { + // commit record + current ~= field; + + records ~= current; + current = null; + field = null; + } else + field ~= c; + break; + case 1: // in quote + if(c == '"') + state = 2; + else + field ~= c; + break; + case 2: // is it a closing quote or an escaped one? + if(c == '"') { + field ~= c; + state = 1; + } else { + state = 0; + goto tryit; + } + } + } + + + return records; +} diff --git a/curl.d b/curl.d new file mode 100644 index 0000000..0aca0d8 --- /dev/null +++ b/curl.d @@ -0,0 +1,188 @@ +module arsd.curl; + +pragma(lib, "curl"); + +import std.string; +extern(C) { + typedef void CURL; + typedef void curl_slist; + + alias int CURLcode; + alias int CURLoption; + + enum int CURLOPT_URL = 10002; + enum int CURLOPT_WRITEFUNCTION = 20011; + enum int CURLOPT_WRITEDATA = 10001; + enum int CURLOPT_POSTFIELDS = 10015; + enum int CURLOPT_POSTFIELDSIZE = 60; + enum int CURLOPT_POST = 47; + enum int CURLOPT_HTTPHEADER = 10023; + enum int CURLOPT_USERPWD = 0x00002715; + + enum int CURLOPT_VERBOSE = 41; + +// enum int CURLOPT_COOKIE = 22; + enum int CURLOPT_COOKIEFILE = 10031; + enum int CURLOPT_COOKIEJAR = 10082; + + enum int CURLOPT_SSL_VERIFYPEER = 64; + + enum int CURLOPT_FOLLOWLOCATION = 52; + + CURL* curl_easy_init(); + void curl_easy_cleanup(CURL* handle); + CURLcode curl_easy_perform(CURL* curl); + + void curl_global_init(int flags); + + enum int CURL_GLOBAL_ALL = 0b1111; + + CURLcode curl_easy_setopt(CURL* handle, CURLoption option, ...); + curl_slist* curl_slist_append(curl_slist*, const char*); + void curl_slist_free_all(curl_slist*); + + // size is size of item, count is how many items + size_t write_data(void* buffer, size_t size, size_t count, void* user) { + string* str = cast(string*) user; + char* data = cast(char*) buffer; + + assert(size == 1); + + *str ~= data[0..count]; + + return count; + } + + char* curl_easy_strerror(CURLcode errornum ); +} +/* +struct CurlOptions { + string username; + string password; +} +*/ + +import std.md5; +import std.file; +/// this automatically caches to a local file for the given time. it ignores the expires header in favor of your time to keep. +version(linux) +string cachedCurl(string url, int maxCacheHours) { + string res; + + auto cacheFile = "/tmp/arsd-curl-cache-" ~ getDigestString(url); + + if(!std.file.exists(cacheFile) || std.file.lastModified(cacheFile) > 1000 * 60 * 60 * maxCacheHours) { + res = curl(url); + std.file.write(cacheFile, res); + } else { + res = readText(cacheFile); + } + + return res; +} + + +string curl(string url, string data = null, string contentType = "application/x-www-form-urlencoded") { + return curlAuth(url, data, null, null, contentType); +} + +string curlCookie(string cookieFile, string url, string data = null, string contentType = "application/x-www-form-urlencoded") { + return curlAuth(url, data, null, null, contentType, null, null, cookieFile); +} + +string curlAuth(string url, string data = null, string username = null, string password = null, string contentType = "application/x-www-form-urlencoded", string methodOverride = null, string[] customHeaders = null, string cookieJar = null) { + CURL* curl = curl_easy_init(); + if(curl is null) + throw new Exception("curl init"); + scope(exit) + curl_easy_cleanup(curl); + + string ret; + + int res; + + //curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + + res = curl_easy_setopt(curl, CURLOPT_URL, std.string.toStringz(url)); + if(res != 0) throw new CurlException(res); + if(username !is null) { + res = curl_easy_setopt(curl, CURLOPT_USERPWD, std.string.toStringz(username ~ ":" ~ password)); + if(res != 0) throw new CurlException(res); + } + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ret); + if(res != 0) throw new CurlException(res); + + curl_slist* headers = null; + //if(data !is null) + // contentType = ""; + headers = curl_slist_append(headers, toStringz("Content-Type: " ~ contentType)); + + foreach(h; customHeaders) { + headers = curl_slist_append(headers, toStringz(h)); + } + scope(exit) + curl_slist_free_all(headers); + + if(data) { + res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.ptr); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.length); + if(res != 0) throw new CurlException(res); + } + + res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + if(res != 0) throw new CurlException(res); + + if(cookieJar !is null) { + res = curl_easy_setopt(curl, CURLOPT_COOKIEJAR, toStringz(cookieJar)); + if(res != 0) throw new CurlException(res); + res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz(cookieJar)); + if(res != 0) throw new CurlException(res); + } + + res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + if(res != 0) throw new CurlException(res); + //res = curl_easy_setopt(curl, 81, 0); // FIXME verify host + //if(res != 0) throw new CurlException(res); + + res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + if(res != 0) throw new CurlException(res); + + if(methodOverride !is null) { + switch(methodOverride) { + default: assert(0); + case "POST": + res = curl_easy_setopt(curl, CURLOPT_POST, 1); + break; + case "GET": + //curl_easy_setopt(curl, CURLOPT_POST, 0); + break; + } + } + + auto failure = curl_easy_perform(curl); + if(failure != 0) + throw new CurlException(failure, "\nURL" ~ url); + + return ret; +} + +class CurlException : Exception { + this(CURLcode code, string msg = null, string file = __FILE__, int line = __LINE__) { + string message = file ~ ":" ~ to!string(line) ~ " (" ~ to!string(code) ~ ") "; + + auto strerror = curl_easy_strerror(code); + + while(*strerror) { + message ~= *strerror; + strerror++; + } + + super(message ~ msg); + } +} + + +import std.conv; diff --git a/database.d b/database.d new file mode 100644 index 0000000..b8c8268 --- /dev/null +++ b/database.d @@ -0,0 +1,1010 @@ +module arsd.database; + +public import std.variant; +import std.string; + +import core.vararg; + +interface Database { + /// Actually implements the query for the database. The query() method + /// below might be easier to use. + ResultSet queryImpl(string sql, Variant[] args...); + + /// Escapes data for inclusion into an sql string literal + string escape(string sqlData); + + /// query to start a transaction, only here because sqlite is apparently different in syntax... + void startTransaction(); + + // FIXME: this would be better as a template, but can't because it is an interface + + /// Just executes a query. It supports placeholders for parameters + /// by using ? in the sql string. NOTE: it only accepts string, int, long, and null types. + /// Others will fail runtime asserts. + final ResultSet query(string sql, ...) { + Variant[] args; + foreach(arg; _arguments) { + string a; + if(arg == typeid(string)) { + a = va_arg!(string)(_argptr); + } else if(arg == typeid(immutable(string))) { + a = va_arg!(immutable(string))(_argptr); + } else if(arg == typeid(const(immutable(char)[]))) { + a = va_arg!(const(immutable(char)[]))(_argptr); + } else if (arg == typeid(int)) { + auto e = va_arg!(int)(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(int))) { + auto e = va_arg!(immutable(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(const(int))) { + auto e = va_arg!(const(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(char))) { + auto e = va_arg!(immutable(char))(_argptr); + a = to!string(e); + } else if (arg == typeid(long)) { + auto e = va_arg!(long)(_argptr); + a = to!string(e); + } else if (arg == typeid(const(long))) { + auto e = va_arg!(const(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(long))) { + auto e = va_arg!(immutable(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(void*)) { + auto e = va_arg!(void*)(_argptr); + assert(e is null, "can only pass null pointer"); + a = null; + } else assert(0, "invalid type " ~ arg.toString ); + + args ~= Variant(a); + } + + return queryImpl(sql, args); + } +} +import std.stdio; + +struct Row { + package string[] row; + package ResultSet resultSet; + + string opIndex(size_t idx) { + return row[idx]; + } + + string opIndex(string idx) { + return row[resultSet.getFieldIndex(idx)]; + } + + string toString() { + return to!string(row); + } + + string[string] toAA() { + string[string] a; + + string[] fn = resultSet.fieldNames(); + + foreach(i, r; row) + a[fn[i]] = r; + + return a; + } + + int opApply(int delegate(ref string, ref string) dg) { + foreach(a, b; toAA) + mixin(yield("a, b")); + + return 0; + } + + + + string[] toStringArray() { + return row; + } +} +import std.conv; + +interface ResultSet { + // name for associative array to result index + int getFieldIndex(string field); + string[] fieldNames(); + + // this is a range that can offer other ranges to access it + bool empty(); + Row front(); + void popFront(); + int length(); + + /* deprecated */ final ResultSet byAssoc() { return this; } +} + +class DatabaseException : Exception { + this(string msg) { + super(msg); + } +} + + + + +// /////////////////////////////////////////////////////// + + +/// Note: ?n params are zero based! +string escapedVariants(Database db, in string sql, Variant[] t) { + + string toSql(Variant a) { + auto v = a.peek!(void*); + if(v && (*v is null)) + return "NULL"; + else { + string str = to!string(a); + return '\'' ~ db.escape(str) ~ '\''; + } + + assert(0); + } + + + + // if nothing to escape or nothing to escape with, don't bother + if(t.length > 0 && sql.indexOf("?") != -1) { + string fixedup; + int currentIndex; + int currentStart = 0; + foreach(i, dchar c; sql) { + if(c == '?') { + fixedup ~= sql[currentStart .. i]; + + int idx = -1; + currentStart = i + 1; + if((i + 1) < sql.length) { + auto n = sql[i + 1]; + if(n >= '0' && n <= '9') { + currentStart = i + 2; + idx = n - '0'; + } + } + if(idx == -1) { + idx = currentIndex; + currentIndex++; + } + + if(idx < 0 || idx >= t.length) + throw new Exception("SQL Parameter index is out of bounds: " ~ to!string(idx) ~ " at `"~sql[0 .. i]~"`"); + + fixedup ~= toSql(t[idx]); + } + } + + fixedup ~= sql[currentStart .. $]; + + return fixedup; + /* + string fixedup; + int pos = 0; + + + void escAndAdd(string str, int q) { + fixedup ~= sql[pos..q] ~ '\'' ~ db.escape(str) ~ '\''; + + } + + foreach(a; t) { + int q = sql[pos..$].indexOf("?"); + if(q == -1) + break; + q += pos; + + auto v = a.peek!(void*); + if(v && (*v is null)) + fixedup ~= sql[pos..q] ~ "NULL"; + else { + string str = to!string(a); + escAndAdd(str, q); + } + + pos = q+1; + } + + fixedup ~= sql[pos..$]; + + sql = fixedup; + */ + } + + return sql; +} + + + + + + +enum UpdateOrInsertMode { + CheckForMe, + AlwaysUpdate, + AlwaysInsert +} + +int updateOrInsert(Database db, string table, string[string] values, string where, UpdateOrInsertMode mode = UpdateOrInsertMode.CheckForMe, string key = "id") { + bool insert = false; + + final switch(mode) { + case UpdateOrInsertMode.CheckForMe: + auto res = db.query("SELECT "~key~" FROM `"~db.escape(table)~"` WHERE " ~ where); + insert = res.empty; + + break; + case UpdateOrInsertMode.AlwaysInsert: + insert = true; + break; + case UpdateOrInsertMode.AlwaysUpdate: + insert = false; + break; + } + + + if(insert) { + string insertSql = "INSERT INTO `" ~ db.escape(table) ~ "` "; + + bool outputted = false; + string vs, cs; + foreach(column, value; values) { + if(column is null) + continue; + if(outputted) { + vs ~= ", "; + cs ~= ", "; + } else + outputted = true; + + //cs ~= "`" ~ db.escape(column) ~ "`"; + cs ~= "`" ~ column ~ "`"; // FIXME: possible insecure + vs ~= "'" ~ db.escape(value) ~ "'"; + } + + if(!outputted) + return 0; + + + insertSql ~= "(" ~ cs ~ ")"; + insertSql ~= " VALUES "; + insertSql ~= "(" ~ vs ~ ")"; + + db.query(insertSql); + + return 0; // db.lastInsertId; + } else { + string updateSql = "UPDATE `"~db.escape(table)~"` SET "; + + bool outputted = false; + foreach(column, value; values) { + if(column is null) + continue; + if(outputted) + updateSql ~= ", "; + else + outputted = true; + + updateSql ~= "`" ~ db.escape(column) ~ "` = '" ~ db.escape(value) ~ "'"; + } + + if(!outputted) + return 0; + + updateSql ~= " WHERE " ~ where; + + db.query(updateSql); + return 0; + } +} + + + + + +string fixupSqlForDataObjectUse(string sql) { + + string[] tableNames; + + string piece = sql; + int idx; + while((idx = piece.indexOf("JOIN")) != -1) { + auto start = idx + 5; + auto i = start; + while(piece[i] != ' ' && piece[i] != '\n' && piece[i] != '\t' && piece[i] != ',') + i++; + auto end = i; + + tableNames ~= strip(piece[start..end]); + + piece = piece[end..$]; + } + + idx = sql.indexOf("FROM"); + if(idx != -1) { + auto start = idx + 5; + auto i = start; + start = i; + while(i < sql.length && !(sql[i] > 'A' && sql[i] <= 'Z')) // if not uppercase, except for A (for AS) to avoid SQL keywords (hack) + i++; + + auto from = sql[start..i]; + auto pieces = from.split(","); + foreach(p; pieces) { + p = p.strip; + start = 0; + i = 0; + while(i < p.length && p[i] != ' ' && p[i] != '\n' && p[i] != '\t' && p[i] != ',') + i++; + + tableNames ~= strip(p[start..i]); + } + + string sqlToAdd; + foreach(tbl; tableNames) { + if(tbl.length) { + sqlToAdd ~= ", " ~ tbl ~ ".id" ~ " AS " ~ "id_from_" ~ tbl; + } + } + + sqlToAdd ~= " "; + + sql = sql[0..idx] ~ sqlToAdd ~ sql[idx..$]; + } + + return sql; +} + + + + + +/* + This is like a result set + + + DataObject res = [...]; + + res.name = "Something"; + + res.commit; // runs the actual update or insert + + + res = new DataObject(fields, tables + + + + + + + + when doing a select, we need to figure out all the tables and modify the query to include the ids we need + + + search for FROM and JOIN + the next token is the table name + + right before the FROM, add the ids of each table + + + given: + SELECT name, phone FROM customers LEFT JOIN phones ON customer.id = phones.cust_id + + we want: + SELECT name, phone, customers.id AS id_from_customers, phones.id AS id_from_phones FROM customers LEFT JOIN phones ON customer.id[...]; + +*/ + +string yield(string what) { return `if(auto result = dg(`~what~`)) return result;`; } + +import std.typecons; +import std.json; // for json value making +class DataObject { + // lets you just free-form set fields, assuming they all come from the given table + // note it doesn't try to handle joins for new rows. you've gotta do that yourself + this(Database db, string table) { + assert(db !is null); + this.db = db; + this.table = table; + + mode = UpdateOrInsertMode.CheckForMe; + } + + JSONValue makeJsonValue() { + JSONValue val; + val.type = JSON_TYPE.OBJECT; + foreach(k, v; fields) { + JSONValue s; + s.type = JSON_TYPE.STRING; + s.str = v; + val.object[k] = s; + } + return val; + } + + this(Database db, string[string] res, Tuple!(string, string)[string] mappings) { + this.db = db; + this.mappings = mappings; + this.fields = res; + + mode = UpdateOrInsertMode.AlwaysUpdate; + } + + string table; + // table, column [alias] + Tuple!(string, string)[string] mappings; + + // vararg hack so property assignment works right, even with null + string opDispatch(string field)(...) + if((field.length < 8 || field[0..8] != "id_from_") && field != "popFront") + { + if(_arguments.length == 0) { + if(field !in fields) + throw new Exception("no such field " ~ field); + + return fields[field]; + } else if(_arguments.length == 1) { + auto arg = _arguments[0]; + + string a; + if(arg == typeid(string)) { + a = va_arg!(string)(_argptr); + } else if(arg == typeid(immutable(string))) { + a = va_arg!(immutable(string))(_argptr); + } else if(arg == typeid(const(immutable(char)[]))) { + a = va_arg!(const(immutable(char)[]))(_argptr); + } else if (arg == typeid(int)) { + auto e = va_arg!(int)(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(int))) { + auto e = va_arg!(immutable(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(const(int))) { + auto e = va_arg!(const(int))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(char))) { + auto e = va_arg!(immutable(char))(_argptr); + a = to!string(e); + } else if (arg == typeid(long)) { + auto e = va_arg!(long)(_argptr); + a = to!string(e); + } else if (arg == typeid(const(long))) { + auto e = va_arg!(const(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(immutable(long))) { + auto e = va_arg!(immutable(long))(_argptr); + a = to!string(e); + } else if (arg == typeid(void*)) { + auto e = va_arg!(void*)(_argptr); + assert(e is null, "can only pass null pointer"); + a = null; + } else assert(0, "invalid type " ~ arg.toString ); + + + auto setTo = a; + setImpl(field, setTo); + + return setTo; + + } else assert(0, "too many arguments"); + + assert(0); // should never be reached + } + + private void setImpl(string field, string value) { + if(field in fields) { + if(fields[field] != value) + changed[field] = true; + } else { + changed[field] = true; + } + + fields[field] = value; + } + + int opApply(int delegate(ref string) dg) { + foreach(a; fields) + mixin(yield("a")); + + return 0; + } + + int opApply(int delegate(ref string, ref string) dg) { + foreach(a, b; fields) + mixin(yield("a, b")); + + return 0; + } + + + string opIndex(string field) { + if(field !in fields) + throw new DatabaseException("No such field in data object: " ~ field); + return fields[field]; + } + + string opIndexAssign(string value, string field) { + setImpl(field, value); + return value; + } + + string* opBinary(string op)(string key) if(op == "in") { + return key in fields; + } + + string[string] fields; + bool[string] changed; + + void commitChanges() { + commitChanges(cast(string) null, null); + } + + void commitChanges(string key, string keyField) { + commitChanges(key is null ? null : [key], keyField is null ? null : [keyField]); + } + + void commitChanges(string[] keys, string[] keyFields = null) { + string[string][string] toUpdate; + int updateCount = 0; + foreach(field, c; changed) { + if(c) { + string tbl, col; + if(mappings is null) { + tbl = this.table; + col = field; + } else { + if(field !in mappings) + assert(0, "no such mapping for " ~ field); + auto m = mappings[field]; + tbl = m[0]; + col = m[1]; + } + + toUpdate[tbl][col] = fields[field]; + updateCount++; + } + } + + if(updateCount) { + db.startTransaction(); + scope(success) db.query("COMMIT"); + scope(failure) db.query("ROLLBACK"); + + foreach(tbl, values; toUpdate) { + string where, keyFieldToPass; + + if(keys is null) { + keys = [null]; + } + + foreach(i, key; keys) { + string keyField; + + if(key is null) { + key = "id_from_" ~ tbl; + if(key !in fields) + key = "id"; + } + + if(i >= keyFields.length || keyFields[i] is null) { + if(key == "id_from_" ~ tbl) + keyField = "id"; + else + keyField = key; + } else { + keyField = keyFields[i]; + } + + + if(where.length) + where ~= " AND "; + + where ~= keyField ~ " = '"~db.escape(key in fields ? fields[key] : null)~"'" ; + if(keyFieldToPass.length) + keyFieldToPass ~= ", "; + + keyFieldToPass ~= keyField; + } + + + + updateOrInsert(db, tbl, values, where, mode, keyFieldToPass); + } + + changed = null; + } + } + + void commitDelete() { + if(mode == UpdateOrInsertMode.AlwaysInsert) + throw new Exception("Cannot delete an item not in the database"); + + assert(table.length); // FIXME, should work with fancy items too + + // FIXME: escaping and primary key questions + db.query("DELETE FROM " ~ table ~ " WHERE id = '" ~ db.escape(fields["id"]) ~ "'"); + } + + string getAlias(string table, string column) { + string ali; + if(mappings is null) { + if(this.table is null) { + mappings[column] = tuple(table, column); + return column; + } else { + assert(table == this.table); + ali = column; + } + } else { + foreach(a, what; mappings) + if(what[0] == table && what[1] == column + && a.indexOf("id_from_") == -1) { + ali = a; + break; + } + } + + return ali; + } + + void set(string table, string column, string value) { + string ali = getAlias(table, column); + //assert(ali in fields); + setImpl(ali, value); + } + + string select(string table, string column) { + string ali = getAlias(table, column); + //assert(ali in fields); + if(ali in fields) + return fields[ali]; + return null; + } + + DataObject addNew() { + auto n = new DataObject(db, null); + + n.db = this.db; + n.table = this.table; + n.mappings = this.mappings; + + foreach(k, v; this.fields) + if(k.indexOf("id_from_") == -1) + n.fields[k] = v; + else + n.fields[k] = null; // don't copy ids + + n.mode = UpdateOrInsertMode.AlwaysInsert; + + return n; + } + + Database db; + UpdateOrInsertMode mode; +} + +/** + You can subclass DataObject if you want to + get some compile time checks or better types. + + You'll want to disable opDispatch, then forward your + properties to the super opDispatch. +*/ + +/*mixin*/ string DataObjectField(T, string table, string column, string aliasAs = null)() { + string aliasAs_; + if(aliasAs is null) + aliasAs_ = column; + else + aliasAs_ = aliasAs; + return ` + @property void `~aliasAs_~`(`~T.stringof~` setTo) { + super.set("`~table~`", "`~column~`", to!string(setTo)); + } + + @property `~T.stringof~` `~aliasAs_~` () { + return to!(`~T.stringof~`)(super.select("`~table~`", "`~column~`")); + } + `; +} + +mixin template StrictDataObject() { + // disable opdispatch + string opDispatch(string name)(...) if (0) {} +} + + +string createDataObjectFieldsFromAlias(string table, fieldsToUse)() { + string ret; + + fieldsToUse f; + foreach(member; __traits(allMembers, fieldsToUse)) { + ret ~= DataObjectField!(typeof(__traits(getMember, f, member)), table, member); + } + + return ret; +} + + +/** + This creates an editable data object out of a simple struct. + + struct MyFields { + int id; + string name; + } + + alias SimpleDataObject!("my_table", MyFields) User; + + + User a = new User(db); + + a.id = 30; + a.name = "hello"; + a.commitChanges(); // tries an update or insert on the my_table table + + + Unlike the base DataObject class, this template provides compile time + checking for types and names, based on the struct you pass in: + + a.id = "aa"; // compile error + + a.notAField; // compile error +*/ +class SimpleDataObject(string tableToUse, fieldsToUse) : DataObject { + mixin StrictDataObject!(); + + mixin(createDataObjectFieldsFromAlias!(tableToUse, fieldsToUse)()); + + this(Database db) { + super(db, tableToUse); + } +} + +/** + Given some SQL, it finds the CREATE TABLE + instruction for the given tableName. + (this is so it can find one entry from + a file with several SQL commands. But it + may break on a complex file, so try to only + feed it simple sql files.) + + From that, it pulls out the members to create a + simple struct based on it. + + It's not terribly smart, so it will probably + break on complex tables. + + Data types handled: + INTEGER, SMALLINT, MEDIUMINT -> D's int + TINYINT -> D's bool + BIGINT -> D's long + TEXT, VARCHAR -> D's string + FLOAT, DOUBLE -> D's double + + It also reads DEFAULT values to pass to D, except for NULL. + It ignores any length restrictions. + + Bugs: + Skips all constraints + Doesn't handle nullable fields, except with strings + It only handles SQL keywords if they are all caps + + This, when combined with SimpleDataObject!(), + can automatically create usable D classes from + SQL input. +*/ +struct StructFromCreateTable(string sql, string tableName) { + mixin(getCreateTable(sql, tableName)); +} + +string getCreateTable(string sql, string tableName) { + skip: + while(readWord(sql) != "CREATE") {} + + assert(readWord(sql) == "TABLE"); + + if(readWord(sql) != tableName) + goto skip; + + assert(readWord(sql) == "("); + + int state; + int parens; + + struct Field { + string name; + string type; + string defaultValue; + } + Field[] fields; + + string word = readWord(sql); + Field current; + while(word != ")" || parens) { + if(word == ")") { + parens --; + word = readWord(sql); + continue; + } + if(word == "(") { + parens ++; + word = readWord(sql); + continue; + } + switch(state) { + default: assert(0); + case 0: + if(word[0] >= 'A' && word[0] <= 'Z') { + state = 4; + break; // we want to skip this since it starts with a keyword (we hope) + } + current.name = word; + state = 1; + break; + case 1: + current.type ~= word; + state = 2; + break; + case 2: + if(word == "DEFAULT") + state = 3; + else if (word == ",") { + fields ~= current; + current = Field(); + state = 0; // next + } + break; + case 3: + current.defaultValue = word; + state = 2; // back to skipping + break; + case 4: + if(word == ",") + state = 0; + } + + word = readWord(sql); + } + + if(current.name !is null) + fields ~= current; + + + string structCode; + foreach(field; fields) { + structCode ~= "\t"; + + switch(field.type) { + case "INTEGER": + case "SMALLINT": + case "MEDIUMINT": + structCode ~= "int"; + break; + case "BOOLEAN": + case "TINYINT": + structCode ~= "bool"; + break; + case "BIGINT": + structCode ~= "long"; + break; + case "CHAR": + case "char": + case "VARCHAR": + case "varchar": + case "TEXT": + case "text": + structCode ~= "string"; + break; + case "FLOAT": + case "DOUBLE": + structCode ~= "double"; + break; + default: + assert(0, "unknown type " ~ field.type ~ " for " ~ field.name); + } + + structCode ~= " "; + structCode ~= field.name; + + if(field.defaultValue !is null) { + structCode ~= " = " ~ field.defaultValue; + } + + structCode ~= ";\n"; + } + + return structCode; +} + +string readWord(ref string src) { + reset: + while(src[0] == ' ' || src[0] == '\t' || src[0] == '\n') + src = src[1..$]; + if(src.length >= 2 && src[0] == '-' && src[1] == '-') { // a comment, skip it + while(src[0] != '\n') + src = src[1..$]; + goto reset; + } + + int start, pos; + if(src[0] == '`') { + src = src[1..$]; + while(src[pos] != '`') + pos++; + goto gotit; + } + + + while( + (src[pos] >= 'A' && src[pos] <= 'Z') + || + (src[pos] >= 'a' && src[pos] <= 'z') + || + (src[pos] >= '0' && src[pos] <= '9') + || + src[pos] == '_' + ) + pos++; + gotit: + if(pos == 0) + pos = 1; + + string tmp = src[0..pos]; + + if(src[pos] == '`') + pos++; // skip the ending quote; + + src = src[pos..$]; + + return tmp; +} + +/// Combines StructFromCreateTable and SimpleDataObject into a one-stop template. +/// alias DataObjectFromSqlCreateTable(import("file.sql"), "my_table") MyTable; +template DataObjectFromSqlCreateTable(string sql, string tableName) { + alias SimpleDataObject!(tableName, StructFromCreateTable!(sql, tableName)) DataObjectFromSqlCreateTable; +} + +/+ +class MyDataObject : DataObject { + this() { + super(new Database("localhost", "root", "pass", "social"), null); + } + + mixin StrictDataObject!(); + + mixin(DataObjectField!(int, "users", "id")); +} + +void main() { + auto a = new MyDataObject; + + a.fields["id"] = "10"; + + a.id = 34; + + a.commitChanges; +} ++/ + +/* +alias DataObjectFromSqlCreateTable!(import("db.sql"), "users") Test; + +void main() { + auto a = new Test(null); + + a.cool = "way"; + a.value = 100; +} +*/ + +void typeinfoBugWorkaround() { + assert(0, to!string(typeid(immutable(char[])[immutable(char)[]]))); +} diff --git a/dom.d b/dom.d new file mode 120000 index 0000000..505b2fa --- /dev/null +++ b/dom.d @@ -0,0 +1 @@ +/home/me/program/djs/dom.d \ No newline at end of file diff --git a/http.d b/http.d new file mode 100644 index 0000000..aa88f54 --- /dev/null +++ b/http.d @@ -0,0 +1,221 @@ +module arsd.http; + +import std.stdio; + + +/** + Gets a textual document, ignoring headers. Throws on non-text or error. +*/ +string get(string url) { + auto hr = httpRequest("GET", url); + if(hr.code != 200) + throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); + if(hr.contentType.indexOf("text/") == -1) + throw new Exception(hr.contentType ~ " is bad content for conversion to string"); + return cast(string) hr.content; + +} + +static import std.uri; + +string post(string url, string[string] args) { + string content; + + foreach(name, arg; args) { + if(content.length) + content ~= "&"; + content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg); + } + + auto hr = httpRequest("POST", url, cast(ubyte[]) content, ["Content-Type: application/x-www-form-urlencoded"]); + if(hr.code != 200) + throw new Exception(format("HTTP answered %d instead of 200", hr.code)); + if(hr.contentType.indexOf("text/") == -1) + throw new Exception(hr.contentType ~ " is bad content for conversion to string"); + + return cast(string) hr.content; +} + +struct HttpResponse { + int code; + string contentType; + string[] headers; + ubyte[] content; +} + +import std.string; +static import std.algorithm; +import std.conv; + +struct UriParts { + string original; + string method; + string host; + ushort port; + string path; + + this(string uri) { + original = uri; + if(uri[0..7] != "http://") + throw new Exception("You must use an absolute, unencrypted URL."); + + int posSlash = uri[7..$].indexOf("/"); + if(posSlash != -1) + posSlash += 7; + + if(posSlash == -1) + posSlash = uri.length; + + int posColon = uri[7..$].indexOf(":"); + if(posColon != -1) + posColon += 7; + + port = 80; + + if(posColon != -1 && posColon < posSlash) { + host = uri[7..posColon]; + port = to!ushort(uri[posColon+1..posSlash]); + } else + host = uri[7..posSlash]; + + path = uri[posSlash..$]; + if(path == "") + path = "/"; + } +} + +HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string headers[] = null) { + auto u = UriParts(uri); + auto f = openNetwork(u.host, u.port); + + return doHttpRequestOnFile(f, method, uri, content, headers); +} + +/** + Executes a generic http request, returning the full result. The correct formatting + of the parameters are the caller's responsibility. Content-Length is added automatically, + but YOU must give Content-Type! +*/ +HttpResponse doHttpRequestOnFile(File f, string method, string uri, const(ubyte)[] content = null, string headers[] = null) + in { + assert(method == "POST" || method == "GET"); + } +body { + auto u = UriParts(uri); + + f.writefln("%s %s HTTP/1.1", method, u.path); + f.writefln("Host: %s", u.host); + f.writefln("Connection: close"); + if(content !is null) + f.writefln("Content-Length: %d", content.length); + if(headers !is null) + foreach(header; headers) + f.writefln("%s", header); + f.writefln(""); + if(content !is null) + f.rawWrite(content); + + + HttpResponse hr; + cont: + string l = f.readln(); + if(l[0..9] != "HTTP/1.1 ") + throw new Exception("Not talking to a http server"); + + hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK + + if(hr.code == 100) { // continue + do { + l = readln(); + } while(l.length > 1); + + goto cont; + } + + bool chunked = false; + + foreach(line; f.byLine) { + if(line.length <= 1) + break; + hr.headers ~= line.idup; + if(line.startsWith("Content-Type: ")) + hr.contentType = line[14..$-1].idup; + if(line.startsWith("Transfer-Encoding: chunked")) + chunked = true; + } + + ubyte[] response; + foreach(ubyte[] chunk; f.byChunk(4096)) { + response ~= chunk; + } + + + if(chunked) { + // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character + // read binary data of that length. it is our content + // repeat until a zero sized chunk + // then read footers as headers. + + int state = 0; + int size; + int start = 0; + for(int a = 0; a < response.length; a++) { + switch(state) { + case 0: // reading hex + char c = response[a]; + if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) { + // just keep reading + } else { + int power = 1; + size = 0; + for(int b = a-1; b >= start; b--) { + char cc = response[b]; + if(cc >= 'a' && cc <= 'z') + cc -= 0x20; + int val = 0; + if(cc >= '0' && cc <= '9') + val = cc - '0'; + else + val = cc - 'A'; + + size += power * val; + power *= 16; + } + state++; + continue; + } + break; + case 1: // reading until end of line + char c = response[a]; + if(c == '\n') { + if(size == 0) + state = 3; + else + state = 2; + } + break; + case 2: // reading data + hr.content ~= response[a..a+size]; + a += size; + a+= 2; // skipping a 13 10 + start = a; + state = 0; + break; + case 3: // reading footers + goto done; // FIXME + break; + } + } + } else + hr.content = response; + done: + + return hr; +} + + +/* +void main(string args[]) { + write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); +} +*/ diff --git a/mysql.d b/mysql.d new file mode 100644 index 0000000..ae50ffc --- /dev/null +++ b/mysql.d @@ -0,0 +1,733 @@ +module arsd.mysql; +pragma(lib, "mysqlclient"); + +public import arsd.database; + +import std.stdio; +import std.exception; +import std.string; +import std.conv; +import std.typecons; + +class MySqlResult : ResultSet { + private int[string] mapping; + private MYSQL_RES* result; + + private int itemsTotal; + private int itemsUsed; + + string sql; + + this(MYSQL_RES* r, string sql) { + result = r; + itemsTotal = length(); + itemsUsed = 0; + + this.sql = sql; + + // prime it + if(itemsTotal) + fetchNext(); + } + + ~this() { + if(result !is null) + mysql_free_result(result); + } + + + MYSQL_FIELD[] fields() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + MYSQL_FIELD[] ret; + for(int i = 0; i < numFields; i++) { + ret ~= fields[i]; + } + + return ret; + } + + + override int length() { + if(result is null) + return 0; + return cast(int) mysql_num_rows(result); + } + + override bool empty() { + return itemsUsed == itemsTotal; + } + + override Row front() { + return row; + } + + override void popFront() { + itemsUsed++; + if(itemsUsed < itemsTotal) { + fetchNext(); + } + } + + override int getFieldIndex(string field) { + if(mapping is null) + makeFieldMapping(); + debug { + if(field !in mapping) + throw new Exception(field ~ " not in result"); + } + return mapping[field]; + } + + private void makeFieldMapping() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + for(int i = 0; i < numFields; i++) { + mapping[fromCstring(fields[i].name)] = i; + } + } + + private void fetchNext() { + assert(result); + auto r = mysql_fetch_row(result); + uint numFields = mysql_num_fields(result); + uint* lengths = mysql_fetch_lengths(result); + string[] row; + // potential FIXME: not really binary safe + + columnIsNull.length = numFields; + for(int a = 0; a < numFields; a++) { + if(*(r+a) is null) { + row ~= null; + columnIsNull[a] = true; + } else { + row ~= fromCstring(*(r+a), *(lengths + a)); + columnIsNull[a] = false; + } + } + + this.row.row = row; + this.row.resultSet = this; + } + + + override string[] fieldNames() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + string[] names; + for(int i = 0; i < numFields; i++) { + names ~= fromCstring(fields[i].name); + } + + return names; + } + + + + bool[] columnIsNull; + Row row; +} + + + + +class MySql : Database { + this(string host, string user, string pass, string db) { + mysql = enforceEx!(DatabaseException)( + mysql_init(null), + "Couldn't init mysql"); + enforceEx!(DatabaseException)( + mysql_real_connect(mysql, toCstring(host), toCstring(user), toCstring(pass), toCstring(db), 0, null, 0), + error()); + + dbname = db; + + // we want UTF8 for everything + + query("SET NAMES 'utf8'"); + //query("SET CHARACTER SET utf8"); + } + + string dbname; + + override void startTransaction() { + query("START TRANSACTION"); + } + + string error() { + return fromCstring(mysql_error(mysql)); + } + + ~this() { + mysql_close(mysql); + } + + int lastInsertId() { + return cast(int) mysql_insert_id(mysql); + } + + + + int insert(string table, MySqlResult result, string[string] columnsToModify, string[] columnsToSkip) { + assert(!result.empty); + string sql = "INSERT INTO `" ~ table ~ "` "; + + string cols = "("; + string vals = "("; + bool outputted = false; + + string[string] columns; + auto cnames = result.fieldNames; + foreach(i, col; result.front.toStringArray) { + bool skipMe = false; + foreach(skip; columnsToSkip) { + if(cnames[i] == skip) { + skipMe = true; + break; + } + } + if(skipMe) + continue; + + if(outputted) { + cols ~= ","; + vals ~= ","; + } else + outputted = true; + + cols ~= cnames[i]; + + if(result.columnIsNull[i] && cnames[i] !in columnsToModify) + vals ~= "NULL"; + else { + string v = col; + if(cnames[i] in columnsToModify) + v = columnsToModify[cnames[i]]; + + vals ~= "'" ~ escape(v) ~ "'"; + + } + } + + cols ~= ")"; + vals ~= ")"; + + sql ~= cols ~ " VALUES " ~ vals; + + query(sql); + + result.popFront; + + return lastInsertId; + } + + string escape(string str) { + ubyte[] buffer = new ubyte[str.length * 2 + 1]; + buffer.length = mysql_real_escape_string(mysql, buffer.ptr, cast(cstring) str.ptr, str.length); + + return cast(string) buffer; + } + + string escaped(T...)(string sql, T t) { + static if(t.length > 0) { + string fixedup; + int pos = 0; + + + void escAndAdd(string str, int q) { + ubyte[] buffer = new ubyte[str.length * 2 + 1]; + buffer.length = mysql_real_escape_string(mysql, buffer.ptr, cast(cstring) str.ptr, str.length); + + fixedup ~= sql[pos..q] ~ '\'' ~ cast(string) buffer ~ '\''; + + } + + foreach(a; t) { + int q = sql[pos..$].indexOf("?"); + if(q == -1) + break; + q += pos; + + static if(__traits(compiles, t is null)) { + if(t is null) + fixedup ~= sql[pos..q] ~ "NULL"; + else + escAndAdd(to!string(*a), q); + } else { + string str = to!string(a); + escAndAdd(str, q); + } + + pos = q+1; + } + + fixedup ~= sql[pos..$]; + + sql = fixedup; + + //writefln("\n\nExecuting sql: %s", sql); + } + + return sql; + } + + + + + + + + + + + + + + + + + + + + ResultByDataObject queryDataObject(T...)(string sql, T t) { + // modify sql for the best data object grabbing + sql = fixupSqlForDataObjectUse(sql); + + auto magic = query(sql, t); + return ResultByDataObject(cast(MySqlResult) magic, this); + } + + + + + + + + int affectedRows() { + return cast(int) mysql_affected_rows(mysql); + } + + override ResultSet queryImpl(string sql, Variant[] args...) { + sql = escapedVariants(this, sql, args); + + enforceEx!(DatabaseException)( + !mysql_query(mysql, toCstring(sql)), + error() ~ " :::: " ~ sql); + + return new MySqlResult(mysql_store_result(mysql), sql); + } +/+ + Result queryOld(T...)(string sql, T t) { + sql = escaped(sql, t); + + if(sql.length == 0) + throw new DatabaseException("empty query"); + /* + static int queryCount = 0; + queryCount++; + if(sql.indexOf("INSERT") != -1) + stderr.writefln("%d: %s", queryCount, sql.replace("\n", " ").replace("\t", "")); + */ + + version(dryRun) { + pragma(msg, "This is a dry run compile, no queries will be run"); + writeln(sql); + return Result(null, null); + } + + enforceEx!(DatabaseException)( + !mysql_query(mysql, toCstring(sql)), + error() ~ " :::: " ~ sql); + + return Result(mysql_store_result(mysql), sql); + } ++/ +/+ + struct ResultByAssoc { + this(Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + string[string] front() { + auto r = result.front; + string[string] ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + return ret; + } + + @disable this(this) { } + + string[] fields; + Result* result; + } + + + struct ResultByStruct(T) { + this(Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + T front() { + auto r = result.front; + string[string] ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + T s; + // FIXME: should use tupleOf + foreach(member; s.tupleof) { + if(member.stringof in ret) + member = to!(typeof(member))(ret[member]); + } + + return s; + } + + @disable this(this) { } + + string[] fields; + Result* result; + } ++/ + +/+ + + + struct Result { + private Result* heaped() { + auto r = new Result(result, sql, false); + + r.tupleof = this.tupleof; + + this.itemsTotal = 0; + this.result = null; + + return r; + } + + this(MYSQL_RES* r, string sql, bool prime = true) { + result = r; + itemsTotal = length; + itemsUsed = 0; + this.sql = sql; + // prime it here + if(prime && itemsTotal) + fetchNext(); + } + + string sql; + + ~this() { + if(result !is null) + mysql_free_result(result); + } + + /+ + string[string][] fetchAssoc() { + + } + +/ + + ResultByAssoc byAssoc() { + return ResultByAssoc(&this); + } + + ResultByStruct!(T) byStruct(T)() { + return ResultByStruct!(T)(&this); + } + + string[] fieldNames() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + string[] names; + for(int i = 0; i < numFields; i++) { + names ~= fromCstring(fields[i].name); + } + + return names; + } + + MYSQL_FIELD[] fields() { + int numFields = mysql_num_fields(result); + auto fields = mysql_fetch_fields(result); + + MYSQL_FIELD[] ret; + for(int i = 0; i < numFields; i++) { + ret ~= fields[i]; + } + + return ret; + } + + ulong length() { + if(result is null) + return 0; + return mysql_num_rows(result); + } + + bool empty() { + return itemsUsed == itemsTotal; + } + + Row front() { + return row; + } + + void popFront() { + itemsUsed++; + if(itemsUsed < itemsTotal) { + fetchNext(); + } + } + + void fetchNext() { + auto r = mysql_fetch_row(result); + uint numFields = mysql_num_fields(result); + uint* lengths = mysql_fetch_lengths(result); + row.length = 0; + // potential FIXME: not really binary safe + + columnIsNull.length = numFields; + for(int a = 0; a < numFields; a++) { + if(*(r+a) is null) { + row ~= null; + columnIsNull[a] = true; + } else { + row ~= fromCstring(*(r+a), *(lengths + a)); + columnIsNull[a] = false; + } + } + } + + @disable this(this) {} + private MYSQL_RES* result; + + ulong itemsTotal; + ulong itemsUsed; + + alias string[] Row; + + Row row; + bool[] columnIsNull; // FIXME: should be part of the row + } ++/ + private: + MYSQL* mysql; +} + +struct ResultByDataObject { + this(MySqlResult r, MySql mysql) { + result = r; + auto fields = r.fields(); + this.mysql = mysql; + + foreach(i, f; fields) { + string tbl = fromCstring(f.org_table is null ? f.table : f.org_table); + mappings[fromCstring(f.name)] = tuple( + tbl, + fromCstring(f.org_name is null ? f.name : f.org_name)); + } + + + } + + Tuple!(string, string)[string] mappings; + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + DataObject front() { + return new DataObject(mysql, result.front.toAA, mappings); + } + // would it be good to add a new() method? would be valid even if empty + // it'd just fill in the ID's at random and allow you to do the rest + + @disable this(this) { } + + MySqlResult result; + MySql mysql; +} + +extern(C) { + typedef void MYSQL; + typedef void MYSQL_RES; + typedef const(ubyte)* cstring; + + struct MYSQL_FIELD { + cstring name; /* Name of column */ + cstring org_name; /* Original column name, if an alias */ + cstring table; /* Table of column if column was a field */ + cstring org_table; /* Org table name, if table was an alias */ + cstring db; /* Database for table */ + cstring catalog; /* Catalog for table */ + cstring def; /* Default value (set by mysql_list_fields) */ + uint length; /* Width of column (create length) */ + uint max_length; /* Max width for selected set */ + uint name_length; + uint org_name_length; + uint table_length; + uint org_table_length; + uint db_length; + uint catalog_length; + uint def_length; + uint flags; /* Div flags */ + uint decimals; /* Number of decimals in field */ + uint charsetnr; /* Character set */ + uint type; /* Type of field. See mysql_com.h for types */ + // type is actually an enum btw + } + + typedef cstring* MYSQL_ROW; + + cstring mysql_get_client_info(); + MYSQL* mysql_init(MYSQL*); + uint mysql_errno(MYSQL*); + cstring mysql_error(MYSQL*); + + MYSQL* mysql_real_connect(MYSQL*, cstring, cstring, cstring, cstring, uint, cstring, ulong); + + int mysql_query(MYSQL*, cstring); + + void mysql_close(MYSQL*); + + ulong mysql_num_rows(MYSQL_RES*); + uint mysql_num_fields(MYSQL_RES*); + bool mysql_eof(MYSQL_RES*); + + ulong mysql_affected_rows(MYSQL*); + ulong mysql_insert_id(MYSQL*); + + MYSQL_RES* mysql_store_result(MYSQL*); + MYSQL_RES* mysql_use_result(MYSQL*); + + MYSQL_ROW mysql_fetch_row(MYSQL_RES *); + uint* mysql_fetch_lengths(MYSQL_RES*); + MYSQL_FIELD* mysql_fetch_field(MYSQL_RES*); + MYSQL_FIELD* mysql_fetch_fields(MYSQL_RES*); + + uint mysql_real_escape_string(MYSQL*, ubyte* to, cstring from, uint length); + + void mysql_free_result(MYSQL_RES*); + +} + +import std.string; +cstring toCstring(string c) { + return cast(cstring) toStringz(c); +} + +import std.array; +string fromCstring(cstring c, int len = -1) { + string ret; + if(c is null) + return null; + if(len == -1) { + while(*c) { + ret ~= cast(char) *c; + c++; + } + } else + for(int a = 0; a < len; a++) + ret ~= cast(char) *(a+c); + + return ret; +} + +/* +void main() { + auto mysql = new MySql("localhost", "uname", "password", "test"); + scope(exit) delete mysql; + + mysql.query("INSERT INTO users (id, password) VALUES (?, ?)", 10, "lol"); + + foreach(row; mysql.query("SELECT * FROM users")) { + writefln("%s %s %s %s", row["id"], row[0], row[1], row["username"]); + } +} +*/ + +/* +struct ResultByStruct(T) { + this(MySql.Result* r) { + result = r; + fields = r.fieldNames(); + } + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + T front() { + auto r = result.front; + T ret; + foreach(i, a; r) { + ret[fields[i]] = a; + } + + return ret; + } + + @disable this(this) { } + + string[] fields; + MySql.Result* result; +} +*/ + + +/+ + mysql.linq.tablename.field[key] // select field from tablename where id = key + + mysql.link["name"].table.field[key] // select field from table where name = key + + + auto q = mysql.prepQuery("select id from table where something"); + q.sort("name"); + q.limit(start, count); + q.page(3, pagelength = ?); + + q.execute(params here); // returns the same Result range as query ++/ + +/* +void main() { + auto db = new MySql("localhost", "uname", "password", "test"); + foreach(item; db.queryDataObject("SELECT users.*, username + FROM users, password_manager_accounts + WHERE password_manager_accounts.user_id = users.id LIMIT 5")) { + writefln("item: %s, %s", item.id, item.username); + item.first = "new"; + item.last = "new2"; + item.username = "kill"; + //item.commitChanges(); + } +} +*/ + + +/* +Copyright: Adam D. Ruppe, 2009 - 2011 +License: Boost License 1.0. +Authors: Adam D. Ruppe + + Copyright Adam D. Ruppe 2009 - 2011. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + diff --git a/png.d b/png.d new file mode 100644 index 0000000..de2b091 --- /dev/null +++ b/png.d @@ -0,0 +1,600 @@ +module arsd.png; + +// By Adam D. Ruppe, 2009-2010, released into the public domain +import std.stdio; +import std.conv; +import std.file; + +import std.zlib; + +public import arsd.image; + +/** + The return value should be casted to indexed or truecolor depending on what you need. + + To get an image from a png file, do this: + + auto i = cast(TrueColorImage) imageFromPng(readPng(cast(ubyte)[]) std.file.read("file.png"))); +*/ +Image imageFromPng(PNG* png) { + PNGHeader h = getHeader(png); + + return new IndexedImage(h.width, h.height); +} + +/* +struct PNGHeader { + uint width; + uint height; + ubyte depth = 8; + ubyte type = 6; // 0 - greyscale, 2 - truecolor, 3 - indexed color, 4 - grey with alpha, 6 - true with alpha + ubyte compressionMethod = 0; // should be zero + ubyte filterMethod = 0; // should be zero + ubyte interlaceMethod = 0; // bool +} +*/ + + +PNG* pngFromImage(IndexedImage i) { + PNGHeader h; + h.width = i.width; + h.height = i.height; + h.type = 3; + if(i.numColors() <= 2) + h.depth = 1; + else if(i.numColors() <= 4) + h.depth = 2; + else if(i.numColors() <= 16) + h.depth = 4; + else if(i.numColors() <= 256) + h.depth = 8; + else throw new Exception("can't save this as an indexed png"); + + auto png = blankPNG(h); + + // do palette and alpha + // FIXME: if there is only one transparent color, set it as the special chunk for that + + // FIXME: we'd get a smaller file size if the transparent pixels were arranged first + Chunk palette; + palette.type = ['P', 'L', 'T', 'E']; + palette.size = i.palette.length * 3; + palette.payload.length = palette.size; + + Chunk alpha; + if(i.hasAlpha) { + alpha.type = ['t', 'R', 'N', 'S']; + alpha.size = i.palette.length; + alpha.payload.length = alpha.size; + } + + for(int a = 0; a < i.palette.length; a++) { + palette.payload[a*3+0] = i.palette[a].r; + palette.payload[a*3+1] = i.palette[a].g; + palette.payload[a*3+2] = i.palette[a].b; + if(i.hasAlpha) + alpha.payload[a] = i.palette[a].a; + } + + palette.checksum = crc("PLTE", palette.payload); + png.chunks ~= palette; + if(i.hasAlpha) { + alpha.checksum = crc("tRNS", alpha.payload); + png.chunks ~= alpha; + } + + // do the datastream + if(h.depth == 8) { + addImageDatastreamToPng(i.data, png); + } else { + // gotta convert it + ubyte[] datastream = new ubyte[i.width * i.height * 8 / h.depth]; // FIXME? + int shift = 0; + + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + int dsp = 0; + int dpos = 0; + bool justAdvanced; + for(int y = 0; y < i.height; y++) { + for(int x = 0; x < i.width; x++) { + datastream[dsp] |= i.data[dpos++] << shift; + + switch(h.depth) { + case 1: shift-= 1; break; + case 2: shift-= 2; break; + case 4: shift-= 4; break; + case 8: shift-= 8; break; + } + + justAdvanced = shift < 0; + if(shift < 0) { + dsp++; + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + } + } + if(!justAdvanced) + dsp++; + switch(h.depth) { + case 1: shift = 7; break; + case 2: shift = 6; break; + case 4: shift = 4; break; + case 8: shift = 0; break; + } + + } + + addImageDatastreamToPng(datastream, png); + } + + return png; +} + +PNG* pngFromImage(TrueColorImage i) { + PNGHeader h; + h.width = i.width; + h.height = i.height; + // FIXME: optimize it if it is greyscale or doesn't use alpha alpha + + auto png = blankPNG(h); + addImageDatastreamToPng(i.data, png); + + return png; +} + +/* +void main(string[] args) { + auto a = readPng(cast(ubyte[]) read(args[1])); + auto f = getDatastream(a); + + foreach(i; f) { + writef("%d ", i); + } + + writefln("\n\n%d", f.length); +} +*/ + +struct Chunk { + uint size; + ubyte[4] type; + ubyte[] payload; + uint checksum; +} + +struct PNG { + uint length; + ubyte[8] header; + Chunk[] chunks; + + Chunk* getChunk(string what) { + foreach(ref c; chunks) { + if(cast(string) c.type == what) + return &c; + } + throw new Exception("no such chunk " ~ what); + } + + Chunk* getChunkNullable(string what) { + foreach(ref c; chunks) { + if(cast(string) c.type == what) + return &c; + } + return null; + } +} + +ubyte[] writePng(PNG* p) { + ubyte[] a; + if(p.length) + a.length = p.length; + else { + a.length = 8; + foreach(c; p.chunks) + a.length += c.size + 12; + } + uint pos; + + a[0..8] = p.header[0..8]; + pos = 8; + foreach(c; p.chunks) { + a[pos++] = (c.size & 0xff000000) >> 24; + a[pos++] = (c.size & 0x00ff0000) >> 16; + a[pos++] = (c.size & 0x0000ff00) >> 8; + a[pos++] = (c.size & 0x000000ff) >> 0; + + a[pos..pos+4] = c.type[0..4]; + pos += 4; + a[pos..pos+c.size] = c.payload[0..c.size]; + pos += c.size; + + a[pos++] = (c.checksum & 0xff000000) >> 24; + a[pos++] = (c.checksum & 0x00ff0000) >> 16; + a[pos++] = (c.checksum & 0x0000ff00) >> 8; + a[pos++] = (c.checksum & 0x000000ff) >> 0; + } + + return a; +} + +PNG* readPng(ubyte[] data) { + auto p = new PNG; + + p.length = data.length; + p.header[0..8] = data[0..8]; + + uint pos = 8; + + while(pos < data.length) { + Chunk n; + n.size |= data[pos++] << 24; + n.size |= data[pos++] << 16; + n.size |= data[pos++] << 8; + n.size |= data[pos++] << 0; + n.type[0..4] = data[pos..pos+4]; + pos += 4; + n.payload.length = n.size; + n.payload[0..n.size] = data[pos..pos+n.size]; + pos += n.size; + + n.checksum |= data[pos++] << 24; + n.checksum |= data[pos++] << 16; + n.checksum |= data[pos++] << 8; + n.checksum |= data[pos++] << 0; + + p.chunks ~= n; + } + + return p; +} + +PNG* blankPNG(PNGHeader h) { + auto p = new PNG; + p.header = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + + Chunk c; + + c.size = 13; + c.type = ['I', 'H', 'D', 'R']; + + c.payload.length = 13; + int pos = 0; + + c.payload[pos++] = h.width >> 24; + c.payload[pos++] = (h.width >> 16) & 0xff; + c.payload[pos++] = (h.width >> 8) & 0xff; + c.payload[pos++] = h.width & 0xff; + + c.payload[pos++] = h.height >> 24; + c.payload[pos++] = (h.height >> 16) & 0xff; + c.payload[pos++] = (h.height >> 8) & 0xff; + c.payload[pos++] = h.height & 0xff; + + c.payload[pos++] = h.depth; + c.payload[pos++] = h.type; + c.payload[pos++] = h.compressionMethod; + c.payload[pos++] = h.filterMethod; + c.payload[pos++] = h.interlaceMethod; + + + c.checksum = crc("IHDR", c.payload); + + p.chunks ~= c; + + return p; +} + +// should NOT have any idata already. +// FIXME: doesn't handle palettes +void addImageDatastreamToPng(const(ubyte)[] data, PNG* png) { + // we need to go through the lines and add the filter byte + // then compress it into an IDAT chunk + // then add the IEND chunk + + PNGHeader h = getHeader(png); + + auto bytesPerLine = h.width * 4; + if(h.type == 3) + bytesPerLine = h.width * 8 / h.depth; + Chunk dat; + dat.type = ['I', 'D', 'A', 'T']; + int pos = 0; + + const(ubyte)[] output; + while(pos+bytesPerLine <= data.length) { + output ~= 0; + output ~= data[pos..pos+bytesPerLine]; + pos += bytesPerLine; + } + + auto com = cast(ubyte[]) compress(output); + dat.size = com.length; + dat.payload = com; + dat.checksum = crc("IDAT", dat.payload); + + png.chunks ~= dat; + + Chunk c; + + c.size = 0; + c.type = ['I', 'E', 'N', 'D']; + c.checksum = crc("IEND", c.payload); + + png.chunks ~= c; + +} + +struct PNGHeader { + uint width; + uint height; + ubyte depth = 8; + ubyte type = 6; // 0 - greyscale, 2 - truecolor, 3 - indexed color, 4 - grey with alpha, 6 - true with alpha + ubyte compressionMethod = 0; // should be zero + ubyte filterMethod = 0; // should be zero + ubyte interlaceMethod = 0; // bool +} + +// bKGD - palette entry for background or the RGB (16 bits each) for that. or 16 bits of grey + +ubyte[] getDatastream(PNG* p) { + ubyte[] compressed; + + foreach(c; p.chunks) { + if(cast(string) c.type != "IDAT") + continue; + compressed ~= c.payload; + } + + return cast(ubyte[]) uncompress(compressed); +} + +// FIXME: Assuming 8 bits per pixel +ubyte[] getUnfilteredDatastream(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8); // FIXME + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[data.length - h.height]; + + int bytesPerLine = ufdata.length / h.height; + + int pos = 0, pos2 = 0; + for(int a = 0; a < h.height; a++) { + assert(data[pos2] == 0); + ufdata[pos..pos+bytesPerLine] = data[pos2+1..pos2+bytesPerLine+1]; + pos+= bytesPerLine; + pos2+= bytesPerLine + 1; + } + + return ufdata; +} + +ubyte[] getFlippedUnfilteredDatastream(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8 || h.depth == 4); // FIXME + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[data.length - h.height]; + + int bytesPerLine = ufdata.length / h.height; + + + int pos = ufdata.length - bytesPerLine, pos2 = 0; + for(int a = 0; a < h.height; a++) { + assert(data[pos2] == 0); + ufdata[pos..pos+bytesPerLine] = data[pos2+1..pos2+bytesPerLine+1]; + pos-= bytesPerLine; + pos2+= bytesPerLine + 1; + } + + return ufdata; +} + +ubyte getHighNybble(ubyte a) { + return cast(ubyte)(a >> 4); // FIXME +} + +ubyte getLowNybble(ubyte a) { + return a & 0x0f; +} + +// Takes the transparency info and returns +ubyte[] getANDMask(PNG* p) { + PNGHeader h = getHeader(p); + assert(h.filterMethod == 0); + + assert(h.type == 3); // FIXME + assert(h.depth == 8 || h.depth == 4); // FIXME + + assert(h.width % 8 == 0); // might actually be %2 + + ubyte[] data = getDatastream(p); + ubyte[] ufdata = new ubyte[h.height*((((h.width+7)/8)+3)&~3)]; // gotta pad to DWORDs... + + Color[] colors = fetchPalette(p); + + int pos = 0, pos2 = (h.width/((h.depth == 8) ? 1 : 2)+1)*(h.height-1); + bool bits = false; + for(int a = 0; a < h.height; a++) { + assert(data[pos2++] == 0); + for(int b = 0; b < h.width; b++) { + if(h.depth == 4) { + ufdata[pos/8] |= ((colors[bits? getLowNybble(data[pos2]) : getHighNybble(data[pos2])].a <= 30) << (7-(pos%8))); + } else + ufdata[pos/8] |= ((colors[data[pos2]].a == 0) << (7-(pos%8))); + pos++; + if(h.depth == 4) { + if(bits) { + pos2++; + } + bits = !bits; + } else + pos2++; + } + + int pad = 0; + for(; pad < ((pos/8) % 4); pad++) { + ufdata[pos/8] = 0; + pos+=8; + } + if(h.depth == 4) + pos2 -= h.width + 2; + else + pos2-= 2*(h.width) +2; + } + + return ufdata; +} + +// Done with assumption + +PNGHeader getHeader(PNG* p) { + PNGHeader h; + ubyte[] data = p.getChunk("IHDR").payload; + + int pos = 0; + + h.width |= data[pos++] << 24; + h.width |= data[pos++] << 16; + h.width |= data[pos++] << 8; + h.width |= data[pos++] << 0; + + h.height |= data[pos++] << 24; + h.height |= data[pos++] << 16; + h.height |= data[pos++] << 8; + h.height |= data[pos++] << 0; + + h.depth = data[pos++]; + h.type = data[pos++]; + h.compressionMethod = data[pos++]; + h.filterMethod = data[pos++]; + h.interlaceMethod = data[pos++]; + + return h; +} + +struct Color { + ubyte r; + ubyte g; + ubyte b; + ubyte a; +} + +/+ +class Image { + Color[][] trueColorData; + ubyte[] indexData; + + Color[] palette; + + uint width; + uint height; + + this(uint w, uint h) {} +} + +Image fromPNG(PNG* p) { + +} + +PNG* toPNG(Image i) { + +} ++/ struct RGBQUAD { + ubyte rgbBlue; + ubyte rgbGreen; + ubyte rgbRed; + ubyte rgbReserved; + } + +RGBQUAD[] fetchPaletteWin32(PNG* p) { + RGBQUAD[] colors; + + auto palette = p.getChunk("PLTE"); + + colors.length = (palette.size) / 3; + + for(int i = 0; i < colors.length; i++) { + colors[i].rgbRed = palette.payload[i*3+0]; + colors[i].rgbGreen = palette.payload[i*3+1]; + colors[i].rgbBlue = palette.payload[i*3+2]; + colors[i].rgbReserved = 0; + } + + return colors; + +} + +Color[] fetchPalette(PNG* p) { + Color[] colors; + + auto palette = p.getChunk("PLTE"); + + Chunk* alpha = p.getChunkNullable("tRNS"); + + colors.length = palette.size / 3; + + for(int i = 0; i < colors.length; i++) { + colors[i].r = palette.payload[i*3+0]; + colors[i].g = palette.payload[i*3+1]; + colors[i].b = palette.payload[i*3+2]; + if(alpha !is null && i < alpha.size) + colors[i].a = alpha.payload[i]; + else + colors[i].a = 255; + + //writefln("%2d: %3d %3d %3d %3d", i, colors[i].r, colors[i].g, colors[i].b, colors[i].a); + } + + return colors; +} + +void replacePalette(PNG* p, Color[] colors) { + auto palette = p.getChunk("PLTE"); + auto alpha = p.getChunk("tRNS"); + + assert(colors.length == alpha.size); + + for(int i = 0; i < colors.length; i++) { + palette.payload[i*3+0] = colors[i].r; + palette.payload[i*3+1] = colors[i].g; + palette.payload[i*3+2] = colors[i].b; + alpha.payload[i] = colors[i].a; + } + + palette.checksum = crc("PLTE", palette.payload); + alpha.checksum = crc("tRNS", alpha.payload); +} + +uint update_crc(in uint crc, in ubyte[] buf){ + static const uint[256] crc_table = [0, 1996959894, 3993919788, 2567524794, 124634137, 1886057615, 3915621685, 2657392035, 249268274, 2044508324, 3772115230, 2547177864, 162941995, 2125561021, 3887607047, 2428444049, 498536548, 1789927666, 4089016648, 2227061214, 450548861, 1843258603, 4107580753, 2211677639, 325883990, 1684777152, 4251122042, 2321926636, 335633487, 1661365465, 4195302755, 2366115317, 997073096, 1281953886, 3579855332, 2724688242, 1006888145, 1258607687, 3524101629, 2768942443, 901097722, 1119000684, 3686517206, 2898065728, 853044451, 1172266101, 3705015759, 2882616665, 651767980, 1373503546, 3369554304, 3218104598, 565507253, 1454621731, 3485111705, 3099436303, 671266974, 1594198024, 3322730930, 2970347812, 795835527, 1483230225, 3244367275, 3060149565, 1994146192, 31158534, 2563907772, 4023717930, 1907459465, 112637215, 2680153253, 3904427059, 2013776290, 251722036, 2517215374, 3775830040, 2137656763, 141376813, 2439277719, 3865271297, 1802195444, 476864866, 2238001368, 4066508878, 1812370925, 453092731, 2181625025, 4111451223, 1706088902, 314042704, 2344532202, 4240017532, 1658658271, 366619977, 2362670323, 4224994405, 1303535960, 984961486, 2747007092, 3569037538, 1256170817, 1037604311, 2765210733, 3554079995, 1131014506, 879679996, 2909243462, 3663771856, 1141124467, 855842277, 2852801631, 3708648649, 1342533948, 654459306, 3188396048, 3373015174, 1466479909, 544179635, 3110523913, 3462522015, 1591671054, 702138776, 2966460450, 3352799412, 1504918807, 783551873, 3082640443, 3233442989, 3988292384, 2596254646, 62317068, 1957810842, 3939845945, 2647816111, 81470997, 1943803523, 3814918930, 2489596804, 225274430, 2053790376, 3826175755, 2466906013, 167816743, 2097651377, 4027552580, 2265490386, 503444072, 1762050814, 4150417245, 2154129355, 426522225, 1852507879, 4275313526, 2312317920, 282753626, 1742555852, 4189708143, 2394877945, 397917763, 1622183637, 3604390888, 2714866558, 953729732, 1340076626, 3518719985, 2797360999, 1068828381, 1219638859, 3624741850, 2936675148, 906185462, 1090812512, 3747672003, 2825379669, 829329135, 1181335161, 3412177804, 3160834842, 628085408, 1382605366, 3423369109, 3138078467, 570562233, 1426400815, 3317316542, 2998733608, 733239954, 1555261956, 3268935591, 3050360625, 752459403, 1541320221, 2607071920, 3965973030, 1969922972, 40735498, 2617837225, 3943577151, 1913087877, 83908371, 2512341634, 3803740692, 2075208622, 213261112, 2463272603, 3855990285, 2094854071, 198958881, 2262029012, 4057260610, 1759359992, 534414190, 2176718541, 4139329115, 1873836001, 414664567, 2282248934, 4279200368, 1711684554, 285281116, 2405801727, 4167216745, 1634467795, 376229701, 2685067896, 3608007406, 1308918612, 956543938, 2808555105, 3495958263, 1231636301, 1047427035, 2932959818, 3654703836, 1088359270, 936918000, 2847714899, 3736837829, 1202900863, 817233897, 3183342108, 3401237130, 1404277552, 615818150, 3134207493, 3453421203, 1423857449, 601450431, 3009837614, 3294710456, 1567103746, 711928724, 3020668471, 3272380065, 1510334235, 755167117]; + + uint c = crc; + + foreach(b; buf) + c = crc_table[(c ^ b) & 0xff] ^ (c >> 8); + + return c; +} + +// lol is just the chunk name +uint crc(in string lol, in ubyte[] buf){ + uint c = update_crc(0xffffffffL, cast(ubyte[]) lol); + return update_crc(c, buf) ^ 0xffffffffL; +} + diff --git a/postgres.d b/postgres.d new file mode 100644 index 0000000..23f9820 --- /dev/null +++ b/postgres.d @@ -0,0 +1,218 @@ +module arsd.postgres; +pragma(lib, "pq"); + +public import arsd.database; + +import std.string; +import std.exception; + +// remember to CREATE DATABASE name WITH ENCODING 'utf8' + +class PostgreSql : Database { + // dbname = name is probably the most common connection string + this(string connectionString) { + conn = PQconnectdb(toStringz(connectionString)); + if(conn is null) + throw new DatabaseException("Unable to allocate PG connection object"); + if(PQstatus(conn) != CONNECTION_OK) + throw new DatabaseException(error()); + query("SET NAMES 'utf8'"); // D does everything with utf8 + } + + ~this() { + PQfinish(conn); + } + + override void startTransaction() { + query("START TRANSACTION"); + } + + ResultSet queryImpl(string sql, Variant[] args...) { + sql = escapedVariants(this, sql, args); + + auto res = PQexec(conn, toStringz(sql)); + int ress = PQresultStatus(res); + if(ress != PGRES_TUPLES_OK + && ress != PGRES_COMMAND_OK) + throw new DatabaseException(error()); + + return new PostgresResult(res); + } + + string escape(string sqlData) { + char* buffer = (new char[sqlData.length * 2 + 1]).ptr; + int size = PQescapeString (buffer, sqlData.ptr, sqlData.length); + + string ret = assumeUnique(buffer[0..size]); + + return ret; + } + + + string error() { + return copyCString(PQerrorMessage(conn)); + } + + private: + PGconn* conn; +} + +class PostgresResult : ResultSet { + // name for associative array to result index + int getFieldIndex(string field) { + if(mapping is null) + makeFieldMapping(); + return mapping[field]; + } + + + string[] fieldNames() { + if(mapping is null) + makeFieldMapping(); + return columnNames; + } + + // this is a range that can offer other ranges to access it + bool empty() { + return position == numRows; + } + + Row front() { + return row; + } + + void popFront() { + position++; + if(position < numRows) + fetchNext; + } + + int length() { + return numRows; + } + + this(PGresult* res) { + this.res = res; + numFields = PQnfields(res); + numRows = PQntuples(res); + + if(numRows) + fetchNext(); + } + + ~this() { + PQclear(res); + } + + private: + PGresult* res; + int[string] mapping; + string[] columnNames; + int numFields; + + int position; + + int numRows; + + Row row; + + void fetchNext() { + Row r; + r.resultSet = this; + string[] row; + + for(int i = 0; i < numFields; i++) { + string a; + + if(PQgetisnull(res, position, i)) + a = null; + else { + a = copyCString(PQgetvalue(res, position, i), PQgetlength(res, position, i)); + + } + row ~= a; + } + + r.row = row; + this.row = r; + } + + void makeFieldMapping() { + for(int i = 0; i < numFields; i++) { + string a = copyCString(PQfname(res, i)); + + columnNames ~= a; + mapping[a] = i; + } + + } +} + +string copyCString(const char* c, int actualLength = -1) { + const(char)* a = c; + if(a is null) + return null; + + string ret; + if(actualLength == -1) + while(*a) { + ret ~= *a; + a++; + } + else { + ret = a[0..actualLength].idup; + } + + return ret; +} + +extern(C) { + struct PGconn; + struct PGresult; + + void PQfinish(PGconn*); + PGconn* PQconnectdb(const char*); + + int PQstatus(PGconn*); // FIXME check return value + + const (char*) PQerrorMessage(PGconn*); + + PGresult* PQexec(PGconn*, const char*); + void PQclear(PGresult*); + + int PQresultStatus(PGresult*); // FIXME check return value + + int PQnfields(PGresult*); // number of fields in a result + const(char*) PQfname(PGresult*, int); // name of field + + int PQntuples(PGresult*); // number of rows in result + const(char*) PQgetvalue(PGresult*, int row, int column); + + size_t PQescapeString (char *to, const char *from, size_t length); + + enum int CONNECTION_OK = 0; + enum int PGRES_COMMAND_OK = 1; + enum int PGRES_TUPLES_OK = 2; + + int PQgetlength(const PGresult *res, + int row_number, + int column_number); + int PQgetisnull(const PGresult *res, + int row_number, + int column_number); + + +} + +/* +import std.stdio; +void main() { + auto db = new PostgreSql("dbname = test"); + + db.query("INSERT INTO users (id, name) values (?, ?)", 30, "hello mang"); + + foreach(line; db.query("SELECT * FROM users")) { + writeln(line[0], line["name"]); + } +} +*/ diff --git a/sha.d b/sha.d new file mode 100644 index 0000000..1aaedbc --- /dev/null +++ b/sha.d @@ -0,0 +1,366 @@ +module arsd.sha; + +/* + By Adam D. Ruppe, 26 Nov 2009 + I release this file into the public domain +*/ +import std.stdio; + +immutable(ubyte)[/*20*/] SHA1(T)(T data) if(isInputRange!(T)) /*const(ubyte)[] data)*/ { + uint[5] h = [0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0]; + + SHARange!(T) range; + static if(is(data == SHARange)) + range = data; + else { + range.r = data; + } + /* + ubyte[] message = data.dup; + message ~= 0b1000_0000; + while(((message.length+8) * 8) % 512) + message ~= 0; + + ulong originalLength = cast(ulong) data.length * 8; + + for(int a = 7; a >= 0; a--) + message ~= (originalLength >> (a*8)) & 0xff; // to big-endian + + assert(((message.length * 8) % 512) == 0); + + uint pos = 0; + while(pos < message.length) { + */ + while(!range.empty) { + uint[80] words; + + for(int a = 0; a < 16; a++) { + for(int b = 3; b >= 0; b--) { + words[a] |= cast(uint)(range.front()) << (b*8); + range.popFront; + // words[a] |= cast(uint)(message[pos]) << (b*8); + // pos++; + } + } + + for(int a = 16; a < 80; a++) { + uint t = words[a-3]; + t ^= words[a-8]; + t ^= words[a-14]; + t ^= words[a-16]; + asm { rol t, 1; } + words[a] = t; + } + + uint a = h[0]; + uint b = h[1]; + uint c = h[2]; + uint d = h[3]; + uint e = h[4]; + + for(int i = 0; i < 80; i++) { + uint f, k; + if(i >= 0 && i < 20) { + f = (b & c) | ((~b) & d); + k = 0x5A827999; + } else + if(i >= 20 && i < 40) { + f = b ^ c ^ d; + k = 0x6ED9EBA1; + } else + if(i >= 40 && i < 60) { + f = (b & c) | (b & d) | (c & d); + k = 0x8F1BBCDC; + } else + if(i >= 60 && i < 80) { + f = b ^ c ^ d; + k = 0xCA62C1D6; + } else assert(0); + + uint temp; + asm { + mov EAX, a; + rol EAX, 5; + add EAX, f; + add EAX, e; + add EAX, k; + mov temp, EAX; + } + temp += words[i]; + e = d; + d = c; + asm { + mov EAX, b; + rol EAX, 30; + mov c, EAX; + } + b = a; + a = temp; + } + + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + } + + + ubyte[] hash; + for(int j = 0; j < 5; j++) + for(int i = 3; i >= 0; i--) { + hash ~= cast(ubyte)(h[j] >> (i*8))&0xff; + } + + return hash.idup; +} + +import std.range; + +// This does the preprocessing of input data, fetching one byte at a time of the data until it is empty, then the padding and length at the end +template SHARange(T) if(isInputRange!(T)) { + struct SHARange { + T r; + + bool empty() { + return state == 5; + } + + void popFront() { + static int lol = 0; + if(state == 0) { + r.popFront; + /* + static if(__traits(compiles, r.front.length)) + length += r.front.length; + else + length += r.front().sizeof; + */ + length++; // FIXME + + if(r.empty) { + state = 1; + position = 2; + current = 0x80; + } + } else { + if(state == 1) { + current = 0x0; + state = 2; + position++; + } else if( state == 2) { + if(!(((position + length + 8) * 8) % 512)) { + state = 3; + position = 7; + length *= 8; + } else + position++; + } else if (state == 3) { + current = (length >> (position*8)) & 0xff; + if(position == 0) + state = 4; + else + position--; + } else if (state == 4) { + current = 0xff; + state = 5; + } + } + } + + ubyte front() { + if(state == 0) { + return cast(ubyte) r.front(); + } + return current; + } + + ubyte current; + uint position; + ulong length; + int state = 0; // reading range, reading appended bit, reading padding, reading length, done + } +} + +immutable(ubyte)[] SHA256(T)(T data) if ( isInputRange!(T)) { + uint[8] h = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19]; + immutable(uint[64]) k = [0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2]; + + SHARange!(T) range; + static if(is(data == SHARange)) + range = data; + else { + range.r = data; + } +/* + ubyte[] message = cast(ubyte[]) data.dup; + message ~= 0b1000_0000; + while(((message.length+8) * 8) % 512) + message ~= 0; + + ulong originalLength = cast(ulong) data.length * 8; + + for(int a = 7; a >= 0; a--) + message ~= (originalLength >> (a*8)) & 0xff; // to big-endian + + assert(((message.length * 8) % 512) == 0); +*/ +// uint pos = 0; + while(!range.empty) { +// while(pos < message.length) { + uint[64] words; + + for(int a = 0; a < 16; a++) { + for(int b = 3; b >= 0; b--) { + words[a] |= cast(uint)(range.front()) << (b*8); + //words[a] |= cast(uint)(message[pos]) << (b*8); + range.popFront; +// pos++; + } + } + + for(int a = 16; a < 64; a++) { + uint t1 = words[a-15]; + asm { + mov EAX, t1; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 7; + ror EBX, 18; + shr ECX, 3; + xor EAX, EBX; + xor EAX, ECX; + mov t1, EAX; + } + uint t2 = words[a-2]; + asm { + mov EAX, t2; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 17; + ror EBX, 19; + shr ECX, 10; + xor EAX, EBX; + xor EAX, ECX; + mov t2, EAX; + } + + words[a] = words[a-16] + t1 + words[a-7] + t2; + } + + uint A = h[0]; + uint B = h[1]; + uint C = h[2]; + uint D = h[3]; + uint E = h[4]; + uint F = h[5]; + uint G = h[6]; + uint H = h[7]; + + for(int i = 0; i < 64; i++) { + uint s0; + asm { + mov EAX, A; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 2; + ror EBX, 13; + ror ECX, 22; + xor EAX, EBX; + xor EAX, ECX; + mov s0, EAX; + } + uint maj = (A & B) ^ (A & C) ^ (B & C); + uint t2 = s0 + maj; + uint s1; + asm { + mov EAX, E; + mov EBX, EAX; + mov ECX, EAX; + ror EAX, 6; + ror EBX, 11; + ror ECX, 25; + xor EAX, EBX; + xor EAX, ECX; + mov s1, EAX; + } + uint ch = (E & F) ^ ((~E) & G); + uint t1 = H + s1 + ch + k[i] + words[i]; + + H = G; + G = F; + F = E; + E = D + t1; + D = C; + C = B; + B = A; + A = t1 + t2; + } + + h[0] += A; + h[1] += B; + h[2] += C; + h[3] += D; + h[4] += E; + h[5] += F; + h[6] += G; + h[7] += H; + } + + ubyte[] hash; + for(int j = 0; j < 8; j++) + for(int i = 3; i >= 0; i--) { + hash ~= cast(ubyte)(h[j] >> (i*8))&0xff; + } + + return hash.idup; +} + +import std.exception; + +string hashToString(const(ubyte)[] hash) { + char[] s; + + s.length = hash.length * 2; + + char toHex(int a) { + if(a < 10) + return cast(char) (a + '0'); + else + return cast(char) (a + 'a' - 10); + } + + for(int a = 0; a < hash.length; a++) { + s[a*2] = toHex(hash[a] >> 4); + s[a*2+1] = toHex(hash[a] & 0x0f); + } + + return assumeUnique(s); +} +/* +string tee(string t) { + writefln("%s", t); + return t; +} +*/ +unittest { + assert(hashToString(SHA1("abc")) == "a9993e364706816aba3e25717850c26c9cd0d89d"); + assert(hashToString(SHA1("sdfj983yr2ih")) == "335f1f5a4af4aa2c8e93b88d69dda2c22baeb94d"); + assert(hashToString(SHA1("$%&^54ylkufg09fd7f09sa7udsiouhcx987yw98etf7yew98yfds987f632uw90ruds09fudsf09dsuhfoidschyds98fydovipsdaidsd9fsa GA UIA duisguifgsuifgusaufisgfuisafguisagasuidgsaufsauifhuisahfuisafaoisahasiosafhffdasasdisayhfdoisayf8saiuhgduifyds8fiydsufisafoisayf8sayfd98wqyr98wqy98sayd98sayd098sayd09sayd98sayd98saicxyhckxnvjbpovc pousa09cusa 09csau csa9 dusa90d usa9d0sau dsa90 as09posufpodsufodspufdspofuds 9tu sapfusaa daosjdoisajdsapoihdsaiodyhsaioyfg d98ytewq89rysa 98yc98sdxych sa89ydsa89dy sa98ydas98c ysx9v8y cxv89ysd f8ysa89f ysa89fd sg8yhds9g8 rfjcxhvslkhdaiosy09wq7r987t98e7ys98aIYOIYOIY)(*YE (*WY *A(YSA* HDUIHDUIAYT&*ATDAUID AUI DUIAT DUIAG saoidusaoid ysqoid yhsaduiayh UIZYzuI YUIYEDSA UIDYUIADYISA YTDGS UITGUID")) == "e38a1220eaf8103d6176df2e0dd0a933e2f52001"); + + assert(hashToString(SHA256("abc")) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert(hashToString(SHA256("$%&^54ylkufg09fd7f09sa7udsiouhcx987yw98etf7yew98yfds987f632uw90ruds09fudsf09dsuhfoidschyds98fydovipsdaidsd9fsa GA UIA duisguifgsuifgusaufisgfuisafguisagasuidgsaufsauifhuisahfuisafaoisahasiosafhffdasasdisayhfdoisayf8saiuhgduifyds8fiydsufisafoisayf8sayfd98wqyr98wqy98sayd98sayd098sayd09sayd98sayd98saicxyhckxnvjbpovc pousa09cusa 09csau csa9 dusa90d usa9d0sau dsa90 as09posufpodsufodspufdspofuds 9tu sapfusaa daosjdoisajdsapoihdsaiodyhsaioyfg d98ytewq89rysa 98yc98sdxych sa89ydsa89dy sa98ydas98c ysx9v8y cxv89ysd f8ysa89f ysa89fd sg8yhds9g8 rfjcxhvslkhdaiosy09wq7r987t98e7ys98aIYOIYOIY)(*YE (*WY *A(YSA* HDUIHDUIAYT&*ATDAUID AUI DUIAT DUIAG saoidusaoid ysqoid yhsaduiayh UIZYzuI YUIYEDSA UIDYUIADYISA YTDGS UITGUID")) == "64ff79c67ad5ddf9ba5b2d83e07a6937ef9a5b4eb39c54fe1e913e21aad0e95c"); +} +/* +void main() { + auto hash = SHA256(InputByChar(stdin)); + writefln("%s", hashToString(hash)); +} +*/ diff --git a/simpleaudio.d b/simpleaudio.d new file mode 120000 index 0000000..128220c --- /dev/null +++ b/simpleaudio.d @@ -0,0 +1 @@ +../../dimage/simpleaudio.d \ No newline at end of file diff --git a/simpledisplay.d b/simpledisplay.d new file mode 120000 index 0000000..e93495b --- /dev/null +++ b/simpledisplay.d @@ -0,0 +1 @@ +../../dimage/simpledisplay.d \ No newline at end of file diff --git a/sqlite.d b/sqlite.d new file mode 100644 index 0000000..4cc1ec2 --- /dev/null +++ b/sqlite.d @@ -0,0 +1,740 @@ +/* + Compile with version=sqlite_extended_metadata_available + if your sqlite is compiled with the + SQLITE_ENABLE_COLUMN_METADATA C-preprocessor symbol. + + If you enable that, you get the ability to use the + queryDataObject() function with sqlite. (You can still + use DataObjects, but you'll have to set up the mappings + manually without the extended metadata.) +*/ + +module arsd.sqlite; +pragma(lib, "sqlite3"); +version(linux) +pragma(lib, "dl"); // apparently sqlite3 depends on this +public import arsd.database; + +import std.exception; + +import std.string; + +import std.c.stdlib; +import core.exception; +import core.memory; +import std.file; +import std.conv; +/* + NOTE: + + This only works correctly on INSERTs if the user can grow the + database file! This means he must have permission to write to + both the file and the directory it is in. + +*/ + + +/** + The Database interface provides a consistent and safe way to access sql RDBMSs. + + Why are all the classes scope? To ensure the database connection is closed when you are done with it. + The destructor cleans everything up. + + (maybe including rolling back a transaction if one is going and it errors.... maybe, or that could bne + scope(exit)) +*/ + +Sqlite openDBAndCreateIfNotPresent(string filename, string sql, void delegate(Sqlite db) initalize = null){ + if(exists(filename)) + return new Sqlite(filename); + else { + auto db = new Sqlite(filename); + db.exec(sql); + if(initalize !is null) + initalize(db); + return db; + } +} + +/* +import std.stdio; +void main() { + Database db = new Sqlite("test.sqlite.db"); + + db.query("CREATE TABLE users (id integer, name text)"); + + db.query("INSERT INTO users values (?, ?)", 1, "hello"); + + foreach(line; db.query("SELECT * FROM users")) { + writefln("%s %s", line[0], line["name"]); + } +} +*/ + +class Sqlite : Database { + public: + this(string filename, int flags = SQLITE_OPEN_READWRITE) { + /+ + int error = sqlite3_open_v2(toStringz(filename), &db, flags, null); + if(error == SQLITE_CANTOPEN) + throw new DatabaseException("omg cant open"); + if(error != SQLITE_OK) + throw new DatabaseException("db open " ~ error()); + +/ + int error = sqlite3_open(toStringz(filename), &db); + if(error != SQLITE_OK) + throw new DatabaseException(this.error()); + } + + ~this(){ + if(sqlite3_close(db) != SQLITE_OK) + throw new DatabaseException(error()); + } + + // my extension for easier editing + version(sqlite_extended_metadata_available) { + ResultByDataObject queryDataObject(T...)(string sql, T t) { + // modify sql for the best data object grabbing + sql = fixupSqlForDataObjectUse(sql); + + auto s = Statement(this, sql); + foreach(i, arg; t) { + s.bind(i + 1, arg); + } + + auto magic = s.execute(true); // fetch extended metadata + + return ResultByDataObject(cast(SqliteResult) magic, magic.extendedMetadata, this); + } + } + + override void startTransaction() { + query("BEGIN TRANSACTION"); + } + + override ResultSet queryImpl(string sql, Variant[] args...) { + auto s = Statement(this, sql); + foreach(i, arg; args) { + s.bind(i + 1, arg); + } + + return s.execute(); + } + + override string escape(string sql) { + if(sql is null) + return null; + char* got = sqlite3_mprintf("%q", toStringz(sql)); // FIXME: might have to be %Q, need to check this, but I think the other impls do the same as %q + auto orig = got; + string esc; + while(*got) { + esc ~= (*got); + got++; + } + + sqlite3_free(orig); + + return esc; + } + + string error(){ + char* mesg = sqlite3_errmsg(db); + char[] m; + int a = std.c.string.strlen(mesg); + m.length = a; + for(int v = 0; v < a; v++) + m[v] = mesg[v]; + + return assumeUnique(m); + } + + int affectedRows(){ + return sqlite3_changes(db); + } + + int lastInsertId(){ + return cast(int) sqlite3_last_insert_rowid(db); + } + + + int exec(string sql, void delegate (char[][char[]]) onEach = null) { + char* mesg; + if(sqlite3_exec(db, toStringz(sql), &callback, &onEach, &mesg) != SQLITE_OK) { + char[] m; + int a = std.c.string.strlen(mesg); + m.length = a; + for(int v = 0; v < a; v++) + m[v] = mesg[v]; + + sqlite3_free(mesg); + throw new DatabaseException("exec " ~ m.idup); + } + + return 0; + } +/* + Statement prepare(string sql){ + sqlite3_stmt * s; + if(sqlite3_prepare_v2(db, toStringz(sql), cast(int) sql.length, &s, null) != SQLITE_OK) + throw new DatabaseException("prepare " ~ error()); + + Statement a = new Statement(s); + + return a; + } +*/ + private: + sqlite3* db; +} + + + + + + +class SqliteResult : ResultSet { + int getFieldIndex(string field) { + foreach(i, n; columnNames) + if(n == field) + return i; + throw new Exception("no such field " ~ field); + } + + string[] fieldNames() { + return columnNames; + } + + // this is a range that can offer other ranges to access it + bool empty() { + return position == rows.length; + } + + Row front() { + Row r; + + r.resultSet = this; + if(rows.length <= position) + throw new Exception("Result is empty"); + foreach(c; rows[position]) { + r.row ~= c.coerce!(string); + } + + return r; + } + + void popFront() { + position++; + } + + int length() { + return rows.length; + } + + this(Variant[][] rows, char[][] columnNames) { + this.rows = rows; + foreach(c; columnNames) + this.columnNames ~= c.idup; + } + + private: + string[] columnNames; + Variant[][] rows; + int position = 0; +} + + + + + + +struct Statement { + private this(Sqlite db, sqlite3_stmt * S) { + this.db = db; + s = S; + finalized = false; + } + + Sqlite db; + + this(Sqlite db, string sql) { + this.db = db; + if(sqlite3_prepare_v2(db.db, toStringz(sql), cast(int) sql.length, &s, null) != SQLITE_OK) + throw new DatabaseException(db.error()); + } + + version(sqlite_extended_metadata_available) + Tuple!(string, string)[string] extendedMetadata; + + ResultSet execute(bool fetchExtendedMetadata = false) { + bool first = true; + int count; + int numRows = 0; + int r = 0; + // FIXME: doesn't handle busy database + while( SQLITE_ROW == sqlite3_step(s) ){ + numRows++; + if(numRows >= rows.length) + rows.length = rows.length + 8; + + if(first){ + count = sqlite3_column_count(s); + + columnNames.length = count; + for(int a = 0; a < count; a++){ + char* str = sqlite3_column_name(s, a); + int l = std.c.string.strlen(str); + columnNames[a].length = l; + for(int b = 0; b < l; b++) + columnNames[a][b] = str[b]; + + version(sqlite_extended_metadata_available) { + if(fetchExtendedMetadata) { + string origtbl; + string origcol; + + const(char)* rofl; + + rofl = sqlite3_column_table_name(s, a); + if(rofl is null) + throw new Exception("null table name pointer"); + while(*rofl) { + origtbl ~= *rofl; + rofl++; + } + rofl = sqlite3_column_origin_name(s, a); + if(rofl is null) + throw new Exception("null colum name pointer"); + while(*rofl) { + origcol ~= *rofl; + rofl++; + } + extendedMetadata[columnNames[a].idup] = tuple(origtbl, origcol); + } + } + } + + first = false; + } + + + rows[r].length = count; + + for(int a = 0; a < count; a++){ + Variant v; + switch(sqlite3_column_type(s, a)){ + case SQLITE_INTEGER: + v = sqlite3_column_int(s, a); + break; + case SQLITE_FLOAT: + v = sqlite3_column_double(s, a); + break; + case SQLITE3_TEXT: + char* str = sqlite3_column_text(s, a); + char[] st; + + int l = std.c.string.strlen(str); + st.length = l; + for(int aa = 0; aa < l; aa++) + st[aa] = str[aa]; + + v = assumeUnique(st); + break; + case SQLITE_BLOB: + byte* str = cast(byte*) sqlite3_column_blob(s, a); + byte[] st; + + int l = sqlite3_column_bytes(s, a); + st.length = l; + for(int aa = 0; aa < l; aa++) + st[aa] = str[aa]; + + v = assumeUnique(st); + + break; + case SQLITE_NULL: + v = null; + break; + } + + rows[r][a] = v; + } + + r++; + } + + rows.length = numRows; + length = numRows; + position = 0; + executed = true; + reset(); + + return new SqliteResult(rows.dup, columnNames); + } + +/* +template extract(A, T, R...){ + void extract(A args, out T t, out R r){ + if(r.length + 1 != args.length) + throw new DatabaseException("wrong places"); + args[0].to(t); + static if(r.length) + extract(args[1..$], r); + } +} +*/ +/* + bool next(T, R...)(out T t, out R r){ + if(position == length) + return false; + + extract(rows[position], t, r); + + position++; + return true; + } +*/ + bool step(out Variant[] row){ + assert(executed); + if(position == length) + return false; + + row = rows[position]; + position++; + + return true; + } + + bool step(out Variant[char[]] row){ + assert(executed); + if(position == length) + return false; + + for(int a = 0; a < length; a++) + row[columnNames[a].idup] = rows[position][a]; + + position++; + + return true; + } + + void reset(){ + if(sqlite3_reset(s) != SQLITE_OK) + throw new DatabaseException("reset " ~ db.error()); + } + + void resetBindings(){ + sqlite3_clear_bindings(s); + } + + void resetAll(){ + reset; + resetBindings; + executed = false; + } + + int bindNameLookUp(const char[] name){ + int a = sqlite3_bind_parameter_index(s, toStringz(name)); + if(a == 0) + throw new DatabaseException("bind name lookup failed " ~ db.error()); + return a; + } + + bool next(T, R...)(out T t, out R r){ + assert(executed); + if(position == length) + return false; + + extract(rows[position], t, r); + + position++; + return true; + } + + template bindAll(T, R...){ + void bindAll(T what, R more){ + bindAllHelper(1, what, more); + } + } + + template exec(T, R...){ + void exec(T what, R more){ + bindAllHelper(1, what, more); + execute(); + } + } + + void bindAllHelper(A, T, R...)(A where, T what, R more){ + bind(where, what); + static if(more.length) + bindAllHelper(where + 1, more); + } + + //void bind(T)(string name, T value) { + //bind(bindNameLookUp(name), value); + //} + + // This should be a template, but grrrr. + void bind (const char[] name, const char[] value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, int value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, float value){ bind(bindNameLookUp(name), value); } + void bind (const char[] name, const byte[] value){ bind(bindNameLookUp(name), value); } + + void bind(int col, const char[] value){ + if(value is null) { + if(sqlite3_bind_null(s, col) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } else { + if(sqlite3_bind_text(s, col, value.ptr, value.length, cast(void*)-1) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + } + + void bind(int col, float value){ + if(sqlite3_bind_double(s, col, value) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + + void bind(int col, int value){ + if(sqlite3_bind_int(s, col, value) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + + void bind(int col, const byte[] value){ + if(value is null) { + if(sqlite3_bind_null(s, col) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } else { + if(sqlite3_bind_blob(s, col, cast(void*)value.ptr, value.length, cast(void*)-1) != SQLITE_OK) + throw new DatabaseException("bind " ~ db.error()); + } + } + + void bind(int col, Variant v) { + if(v.peek!int) + bind(col, v.get!int); + if(v.peek!string) + bind(col, v.get!string); + if(v.peek!float) + bind(col, v.get!float); + if(v.peek!(byte[])) + bind(col, v.get!(byte[])); + if(v.peek!(void*) && v.get!(void*) is null) + bind(col, cast(string) null); + } + + ~this(){ + if(!finalized) + finalize(); + } + + void finalize(){ + if(finalized) + return; + if(sqlite3_finalize(s) != SQLITE_OK) + throw new DatabaseException("finalize " ~ db.error()); + finalized = true; + } + private: + Variant[][] rows; + char[][] columnNames; + int length; + int position; + bool finalized; + + sqlite3_stmt * s; + + bool executed; + + + + + + + + new(size_t sz) + { + void* p; + + p = std.c.stdlib.malloc(sz); + if (!p) + throw new OutOfMemoryError(__FILE__, __LINE__); + GC.addRange(p, sz); + return p; + } + + delete(void* p) + { + if (p) + { GC.removeRange(p); + std.c.stdlib.free(p); + } + } + + + + +} + + + +version(sqlite_extended_metadata_available) { +import std.typecons; +struct ResultByDataObject { + this(SqliteResult r, Tuple!(string, string)[string] mappings, Sqlite db) { + result = r; + this.db = db; + this.mappings = mappings; + } + + Tuple!(string, string)[string] mappings; + + ulong length() { return result.length; } + bool empty() { return result.empty; } + void popFront() { result.popFront(); } + DataObject front() { + return new DataObject(db, result.front.toAA, mappings); + } + // would it be good to add a new() method? would be valid even if empty + // it'd just fill in the ID's at random and allow you to do the rest + + @disable this(this) { } + + SqliteResult result; + Sqlite db; +} +} + + + + + + + + + + +extern(C) int callback(void* cb, int howmany, char** text, char** columns){ + if(cb is null) + return 0; + + void delegate(char[][char[]]) onEach = *cast(void delegate(char[][char[]])*)cb; + + + char[][char[]] row; + + for(int a = 0; a < howmany; a++){ + int b = std.c.string.strlen(columns[a]); + char[] buf; + buf.length = b; + for(int c = 0; c < b; c++) + buf[c] = columns[a][c]; + + int d = std.c.string.strlen(text[a]); + char[] t; + t.length = d; + for(int c = 0; c < d; c++) + t[c] = text[a][c]; + + row[buf.idup] = t; + } + + onEach(row); + + return 0; +} + + + + + + + + + + +extern(C){ + typedef void sqlite3; + typedef void sqlite3_stmt; + int sqlite3_changes(sqlite3*); + int sqlite3_close(sqlite3 *); + int sqlite3_exec( + sqlite3*, /* An open database */ + const(char) *sql, /* SQL to be evaluted */ + int function(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ + ); + + int sqlite3_open( + const(char) *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ + ); + +/+ +int sqlite3_open_v2( + char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + char *zVfs /* Name of VFS module to use */ +); ++/ + int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const(char) *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +int sqlite3_finalize(sqlite3_stmt *pStmt); +int sqlite3_step(sqlite3_stmt*); +long sqlite3_last_insert_rowid(sqlite3*); + +const int SQLITE_OK = 0; +const int SQLITE_ROW = 100; +const int SQLITE_DONE = 101; + +const int SQLITE_INTEGER = 1; // int +const int SQLITE_FLOAT = 2; // float +const int SQLITE3_TEXT = 3; // char[] +const int SQLITE_BLOB = 4; // byte[] +const int SQLITE_NULL = 5; // void* = null + +char *sqlite3_mprintf(const char*,...); + + +int sqlite3_reset(sqlite3_stmt *pStmt); +int sqlite3_clear_bindings(sqlite3_stmt*); +int sqlite3_bind_parameter_index(sqlite3_stmt*, const(char) *zName); + +int sqlite3_bind_blob(sqlite3_stmt*, int, void*, int n, void*); +//int sqlite3_bind_blob(sqlite3_stmt*, int, void*, int n, void(*)(void*)); +int sqlite3_bind_double(sqlite3_stmt*, int, double); +int sqlite3_bind_int(sqlite3_stmt*, int, int); +int sqlite3_bind_null(sqlite3_stmt*, int); +int sqlite3_bind_text(sqlite3_stmt*, int, const(char)*, int n, void*); +//int sqlite3_bind_text(sqlite3_stmt*, int, char*, int n, void(*)(void*)); + +void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +double sqlite3_column_double(sqlite3_stmt*, int iCol); +int sqlite3_column_int(sqlite3_stmt*, int iCol); +char *sqlite3_column_text(sqlite3_stmt*, int iCol); +int sqlite3_column_type(sqlite3_stmt*, int iCol); +char *sqlite3_column_name(sqlite3_stmt*, int N); + +int sqlite3_column_count(sqlite3_stmt *pStmt); +void sqlite3_free(void*); + char *sqlite3_errmsg(sqlite3*); + + const int SQLITE_OPEN_READONLY = 0x1; + const int SQLITE_OPEN_READWRITE = 0x2; + const int SQLITE_OPEN_CREATE = 0x4; + const int SQLITE_CANTOPEN = 14; + + +// will need these to enable support for DataObjects here +const (char *)sqlite3_column_database_name(sqlite3_stmt*,int); +const (char *)sqlite3_column_table_name(sqlite3_stmt*,int); +const (char *)sqlite3_column_origin_name(sqlite3_stmt*,int); +} + diff --git a/web.d b/web.d new file mode 100644 index 0000000..b942690 --- /dev/null +++ b/web.d @@ -0,0 +1,2348 @@ +module arsd.web; + +/* + Running from the command line: + + ./myapp function positional args.... + ./myapp --format=json function + + _GET + _POST + _PUT + _DELETE + + ./myapp --make-nested-call + + + + + + + + Procedural vs Object Oriented + + right now it is procedural: + root/function + root/module/function + + what about an object approach: + root/object + root/class/object + + static ApiProvider.getObject + + + Formatting data: + + CoolApi.myFunc().getFormat('Element', [...same as get...]); + + You should also be able to ask for json, but with a particular format available as toString + + format("json", "html") -- gets json, but each object has it's own toString. Actually, the object adds + a member called formattedSecondarily that is the other thing. + Note: the array itself cannot be changed in format, only it's members. + Note: the literal string of the formatted object is often returned. This may more than double the bandwidth of the call + + Note: BUG: it only works with built in formats right now when doing secondary + + + // formats are: text, html, json, table, and xml + // except json, they are all represented as strings in json values + + string toString -> formatting as text + Element makeHtmlElement -> making it html (same as fragment) + JSONValue makeJsonValue -> formatting to json + Table makeHtmlTable -> making a table + (not implemented) toXml -> making it into an xml document + + + Arrays can be handled too: + + static (converts to) string makeHtmlArray(typeof(this)[] arr); + + + Envelope format: + + document (default), json, none +*/ + +public import arsd.dom; +public import arsd.cgi; // you have to import this in the actual usage file or else it won't link; surely a compiler bug +import arsd.sha; + +public import std.string; +public import std.array; +public import std.stdio : writefln; +public import std.conv; +import std.random; + +public import std.range; + +public import std.traits; +import std.json; + +struct Envelope { + bool success; + string type; + string errorMessage; + string userData; + JSONValue result; // use result.str if the format was anything other than json + debug string dFullString; +} + +string linkTo(alias func, T...)(T args) { + auto reflection = __traits(parent, func).reflection; + assert(reflection !is null); + + auto name = func.stringof; + int idx = name.indexOf("("); + if(idx != -1) + name = name[0 .. idx]; + + auto funinfo = reflection.functions[name]; + + return funinfo.originalName; +} + +/// Everything should derive from this instead of the old struct namespace used before +/// Your class must provide a default constructor. +class ApiProvider { + Cgi cgi; + static immutable(ReflectionInfo)* reflection; + string _baseUrl; // filled based on where this is called from on this request + + /// Override this if you have initialization work that must be done *after* cgi and reflection is ready. + /// It should be used instead of the constructor for most work. + void _initialize() {} + + /// This one is called at least once per call. (_initialize is only called once per process) + void _initializePerCall() {} + + /// Override this if you want to do something special to the document + void _postProcess(Document document) {} + + /// This tentatively redirects the user - depends on the envelope fomat + void redirect(string location) { + if(cgi.request("envelopeFormat", "document") == "document") + cgi.setResponseLocation(location, false); + } + + Element _sitemap() { + auto container = _getGenericContainer(); + + auto list = container.addChild("ul"); + + string[string] handled; + foreach(func; reflection.functions) { + if(func.originalName in handled) + continue; + handled[func.originalName] = func.originalName; + list.addChild("li", new Link(_baseUrl ~ "/" ~ func.name, beautify(func.originalName))); + } + + return list.parentNode.removeChild(list); + } + + Document _defaultPage() { + throw new Exception("no default"); + return null; + } + + Element _getGenericContainer() + out(ret) { + assert(ret !is null); + } + body { + auto document = new Document(""); + auto container = document.getElementById("body"); + return container; + } + + /// When in website mode, you can use this to beautify the error message + Document delegate(Throwable) _errorFunction; +} + +class ApiObject { + /* abstract this(ApiProvider parent, string identifier) */ +} + + + +struct ReflectionInfo { + FunctionInfo[string] functions; + EnumInfo[string] enums; + StructInfo[string] structs; + const(ReflectionInfo)*[string] objects; + + bool needsInstantiation; + + // the overall namespace + string name; // this is also used as the object name in the JS api + + string defaultOutputFormat = "html"; + int versionOfOutputFormat = 2; // change this in your constructor if you still need the (deprecated) old behavior + // bool apiMode = false; // no longer used - if format is json, apiMode behavior is assumed. if format is html, it is not. + // FIXME: what if you want the data formatted server side, but still in a json envelope? + // should add format-payload: +} + +struct EnumInfo { + string name; + int[] values; + string[] names; +} + +struct StructInfo { + string name; + // a struct is sort of like a function constructor... + StructMemberInfo[] members; +} + +struct StructMemberInfo { + string name; + string staticType; + string defaultValue; +} + +struct FunctionInfo { + WrapperFunction dispatcher; + + JSONValue delegate(Cgi cgi, in string[][string] sargs) documentDispatcher; + // should I also offer dispatchers for other formats like Variant[]? + string name; + string originalName; + + //string uriPath; + + Parameter[] parameters; + + string returnType; + bool returnTypeIsDocument; + + Document function(in string[string] args) createForm; +} + +struct Parameter { + string name; + string value; + + string type; + string staticType; + string validator; + + // for radio and select boxes + string[] options; + string[] optionValues; +} + +string makeJavascriptApi(const ReflectionInfo* mod, string base) { + assert(mod !is null); + + string script = `var `~mod.name~` = { + "_apiBase":'`~base~`',`; + + script ~= javascriptBase; + + script ~= "\n\t"; + + bool[string] alreadyDone; + + bool outp = false; + + foreach(s; mod.enums) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= "'"~s.name~"': {\n"; + + bool outp2 = false; + foreach(i, n; s.names) { + if(outp2) + script ~= ",\n"; + else + outp2 = true; + + // auto v = s.values[i]; + auto v = "'" ~ n ~ "'"; // we actually want to use the name here because to!enum() uses member name. + + script ~= "\t\t'"~n~"':" ~ to!string(v); + } + + script ~= "\n\t}"; + } + + foreach(s; mod.structs) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= "'"~s.name~"': function("; + + bool outp2 = false; + foreach(n; s.members) { + if(outp2) + script ~= ", "; + else + outp2 = true; + + script ~= n.name; + + } + script ~= ") { return {\n"; + + outp2 = false; + + script ~= "\t\t'_arsdTypeOf':'"~s.name~"'"; + if(s.members.length) + script ~= ","; + script ~= " // metadata, ought to be read only\n"; + + // outp2 is still false because I put the comma above + foreach(n; s.members) { + if(outp2) + script ~= ",\n"; + else + outp2 = true; + + auto v = n.defaultValue; + + script ~= "\t\t'"~n.name~"': (typeof "~n.name~" == 'undefined') ? "~n.name~" : '" ~ to!string(v) ~ "'"; + } + + script ~= "\n\t}; }"; + } + + // FIXME: it should output the classes too +/* + foreach(obj; mod.objects) { + if(outp) + script ~= ",\n\t"; + else + outp = true; + + script ~= makeJavascriptApi(obj, base); + } +*/ + + foreach(func; mod.functions) { + if(func.originalName in alreadyDone) + continue; // there's url friendly and code friendly, only need one + + alreadyDone[func.originalName] = true; + + if(outp) + script ~= ",\n\t"; + else + outp = true; + + + string args; + string obj; + bool outputted = false; + foreach(i, arg; func.parameters) { + if(outputted) { + args ~= ","; + obj ~= ","; + } else + outputted = true; + + args ~= arg.name; + + // FIXME: we could probably do better checks here too like on type + obj ~= `'`~arg.name~`':(typeof `~arg.name ~ ` == "undefined" ? this._raiseError('InsufficientParametersException', '`~func.originalName~`: argument `~to!string(i) ~ " (" ~ arg.staticType~` `~arg.name~`) is not present') : `~arg.name~`)`; + } + + /* + if(outputted) + args ~= ","; + args ~= "callback"; + */ + + script ~= `'` ~ func.originalName ~ `'`; + script ~= ":"; + script ~= `function(`~args~`) {`; + if(obj.length) + script ~= ` + var argumentsObject = { + `~obj~` + }; + return this._serverCall('`~func.name~`', argumentsObject, '`~func.returnType~`');`; + else + script ~= ` + return this._serverCall('`~func.name~`', null, '`~func.returnType~`');`; + + script ~= ` + }`; + } + + script ~= "\n}"; + + // some global stuff to put in + script ~= ` + if(typeof arsdGlobalStuffLoadedForWebDotD == "undefined") { + arsdGlobalStuffLoadedForWebDotD = true; + var oldObjectDotPrototypeDotToString = Object.prototype.toString; + Object.prototype.toString = function() { + if(this.formattedSecondarily) + return this.formattedSecondarily; + + return oldObjectDotPrototypeDotToString.call(this); + } + } + `; + + return script; +} + +template isEnum(alias T) if(is(T)) { + static if (is(T == enum)) + enum bool isEnum = true; + else + enum bool isEnum = false; +} + +// WTF, shouldn't is(T == xxx) already do this? +template isEnum(T) if(!is(T)) { + enum bool isEnum = false; +} + +template isStruct(alias T) if(is(T)) { + static if (is(T == struct)) + enum bool isStruct = true; + else + enum bool isStruct = false; +} + +// WTF +template isStruct(T) if(!is(T)) { + enum bool isStruct = false; +} + + +template isApiObject(alias T) if(is(T)) { + static if (is(T : ApiObject)) + enum bool isApiObject = true; + else + enum bool isApiObject = false; +} + +// WTF +template isApiObject(T) if(!is(T)) { + enum bool isApiObject = false; +} + +template isApiProvider(alias T) if(is(T)) { + static if (is(T : ApiProvider)) + enum bool isApiProvider = true; + else + enum bool isApiProvider = false; +} + +// WTF +template isApiProvider(T) if(!is(T)) { + enum bool isApiProvider = false; +} + + +template Passthrough(T) { + T Passthrough; +} + +template PassthroughType(T) { + alias T PassthroughType; +} + +auto generateGetter(PM, Parent, string member, alias hackToEnsureMultipleFunctionsWithTheSameSignatureGetTheirOwnInstantiations)(string io, Parent instantiation) { + static if(is(PM : ApiObject)) { + auto i = new PM(instantiation, io); + return &__traits(getMember, i, member); + } else { + return &__traits(getMember, instantiation, member); + } +} + + + +immutable(ReflectionInfo*) prepareReflection(alias PM)(Cgi cgi, PM instantiation, ApiObject delegate(string) instantiateObject = null) if(is(PM : ApiProvider) || is(PM: ApiObject) ) { + return prepareReflectionImpl!(PM, PM)(cgi, instantiation, instantiateObject); +} + +immutable(ReflectionInfo*) prepareReflectionImpl(alias PM, alias Parent)(Cgi cgi, Parent instantiation, ApiObject delegate(string) instantiateObject = null) if((is(PM : ApiProvider) || is(PM: ApiObject)) && is(Parent : ApiProvider) ) { + + assert(instantiation !is null); + + ReflectionInfo* reflection = new ReflectionInfo; + reflection.name = PM.stringof; + + static if(is(PM: ApiObject)) + reflection.needsInstantiation = true; + + // derivedMembers is changed from allMembers + foreach(member; __traits(derivedMembers, PM)) { + // FIXME: the filthiest of all hacks... + static if(!__traits(compiles, + !is(typeof(__traits(getMember, PM, member)) == function) && + isEnum!(__traits(getMember, PM, member)))) + continue; // must be a data member or something... + else + // DONE WITH FILTHIEST OF ALL HACKS + + //if(member.length == 0) + // continue; + static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isEnum!(__traits(getMember, PM, member)) + && member[0] != '_' + ) { + EnumInfo i; + i.name = member; + foreach(m; __traits(allMembers, __traits(getMember, PM, member))) { + i.names ~= m; + i.values ~= cast(int) __traits(getMember, __traits(getMember, PM, member), m); + } + + reflection.enums[member] = i; + + } else static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isStruct!(__traits(getMember, PM, member)) + && member[0] != '_' + ) { + StructInfo i; + i.name = member; + + typeof(Passthrough!(__traits(getMember, PM, member))) s; + foreach(idx, m; s.tupleof) { + StructMemberInfo mem; + + mem.name = s.tupleof[idx].stringof[2..$]; + mem.staticType = typeof(m).stringof; + + mem.defaultValue = null; // FIXME + + i.members ~= mem; + } + + reflection.structs[member] = i; + } else static if( + is(typeof(__traits(getMember, PM, member)) == function) + && ( + member[0] != '_' && + ( + member.length < 5 || + ( + member[$ - 5 .. $] != "_Page" && + member[$ - 5 .. $] != "_Form") && + !(member.length > 16 && member[$ - 16 .. $] == "_PermissionCheck") + ))) { + FunctionInfo f; + ParameterTypeTuple!(__traits(getMember, PM, member)) fargs; + + f.returnType = ReturnType!(__traits(getMember, PM, member)).stringof; + f.returnTypeIsDocument = is(ReturnType!(__traits(getMember, PM, member)) : Document); + + f.name = toUrlName(member); + f.originalName = member; + + assert(instantiation !is null); + f.dispatcher = generateWrapper!( + generateGetter!(PM, Parent, member, __traits(getMember, PM, member)), + __traits(getMember, PM, member), Parent, member + )(reflection, instantiation); + + //f.uriPath = f.originalName; + + auto names = parameterNamesOf!(__traits(getMember, PM, member)); + + foreach(idx, param; fargs) { + Parameter p; + + p.name = names[idx]; + p.staticType = typeof(fargs[idx]).stringof; + static if( is( typeof(param) == enum )) { + p.type = "select"; + + foreach(opt; __traits(allMembers, typeof(param))) { + p.options ~= opt; + p.optionValues ~= to!string(__traits(getMember, param, opt)); + } + } else static if (is(typeof(param) == bool)) { + p.type = "checkbox"; + } else { + if(p.name.toLower.indexOf("password") != -1) // hack to support common naming convention + p.type = "password"; + else + p.type = "text"; + } + + f.parameters ~= p; + } + + static if(__traits(hasMember, PM, member ~ "_Form")) { + f.createForm = &__traits(getMember, PM, member ~ "_Form"); + } + + reflection.functions[f.name] = f; + // also offer the original name if it doesn't + // conflict + //if(f.originalName !in reflection.functions) + reflection.functions[f.originalName] = f; + } + else static if( + !is(typeof(__traits(getMember, PM, member)) == function) && + isApiObject!(__traits(getMember, PM, member)) && + member[0] != '_' + ) { + reflection.objects[member] = prepareReflectionImpl!( + __traits(getMember, PM, member), Parent) + (cgi, instantiation); + } else static if( // child ApiProviders are like child modules + !is(typeof(__traits(getMember, PM, member)) == function) && + isApiProvider!(__traits(getMember, PM, member)) && + member[0] != '_' + ) { + PassthroughType!(__traits(getMember, PM, member)) i; + i = new typeof(i)(); + auto r = prepareReflection!(__traits(getMember, PM, member))(cgi, i); + reflection.objects[member] = r; + if(toLower(member) !in reflection.objects) // web filenames are often lowercase too + reflection.objects[member.toLower] = r; + } + } + + static if(is(PM: ApiProvider)) { + instantiation.cgi = cgi; + instantiation.reflection = cast(immutable) reflection; + instantiation._initialize(); + } + + return cast(immutable) reflection; +} + +void run(Provider)(Cgi cgi, Provider instantiation, int pathInfoStartingPoint = 0) if(is(Provider : ApiProvider)) { + assert(instantiation !is null); + + immutable(ReflectionInfo)* reflection; + if(instantiation.reflection is null) + prepareReflection!(Provider)(cgi, instantiation); + + reflection = instantiation.reflection; + + instantiation._baseUrl = cgi.scriptName ~ cgi.pathInfo[0 .. pathInfoStartingPoint]; + if(cgi.pathInfo[pathInfoStartingPoint .. $].length <= 1) { + auto document = instantiation._defaultPage(); + if(document !is null) { + instantiation._postProcess(document); + cgi.write(document.toString()); + } + cgi.close(); + return; + } + + string funName = cgi.pathInfo[pathInfoStartingPoint + 1..$]; + + // kinda a hack, but this kind of thing should be available anyway + if(funName == "functions.js") { + cgi.setResponseContentType("text/javascript"); + cgi.write(makeJavascriptApi(reflection, replace(cast(string) cgi.requestUri, "functions.js", ""))); + cgi.close(); + return; + } + + instantiation._initializePerCall(); + + // what about some built in functions? + /* + // Basic integer operations + builtin.opAdd + builtin.opSub + builtin.opMul + builtin.opDiv + + // Basic array operations + builtin.opConcat // use to combine calls easily + builtin.opIndex + builtin.opSlice + builtin.length + + // Basic floating point operations + builtin.round + builtin.floor + builtin.ceil + + // Basic object operations + builtin.getMember + + // Basic functional operations + builtin.filter // use to slice down on stuff to transfer + builtin.map // call a server function on a whole array + builtin.reduce + + // Access to the html items + builtin.getAutomaticForm(method) + */ + + const(FunctionInfo)* fun; + + auto envelopeFormat = cgi.request("envelopeFormat", "document"); + Envelope result; + result.userData = cgi.request("passedThroughUserData"); + + string instantiator; + string objectName; + + try { + // Built-ins + string errorMessage; + if(funName.length > 8 && funName[0..8] == "builtin.") { + funName = funName[8..$]; + switch(funName) { + default: assert(0); + case "getAutomaticForm": + auto mfun = new FunctionInfo; + mfun.returnType = "Form"; + mfun.dispatcher = delegate JSONValue (Cgi cgi, string, in string[][string] sargs, in string format, in string secondaryFormat = null) { + auto rfun = cgi.request("method") in reflection.functions; + if(rfun is null) + throw new NoSuchPageException("no such function " ~ cgi.request("method")); + + auto form = createAutomaticForm(new Document, *rfun); + auto idx = cgi.requestUri.indexOf("builtin.getAutomaticForm"); + form.action = cgi.requestUri[0 .. idx] ~ form.action; // make sure it works across the site + JSONValue v; + v.type = JSON_TYPE.STRING; + v.str = form.toString(); + + return v; + }; + + fun = cast(immutable) mfun; + break; + } + } else { + // User-defined + // FIXME: modules? should be done with dots since slashes is used for api objects + fun = funName in reflection.functions; + if(fun is null) { + auto parts = funName.split("/"); + + const(ReflectionInfo)* currentReflection = reflection; + if(parts.length > 1) + while(parts.length) { + if(parts.length > 1) { + objectName = parts[0]; + auto object = objectName in reflection.objects; + if(object is null) { // || object.instantiate is null) + errorMessage = "no such object: " ~ objectName; + goto noSuchFunction; + } + + currentReflection = *object; + + if(!currentReflection.needsInstantiation) { + parts = parts[1 .. $]; + continue; + } + + auto objectIdentifier = parts[1]; + instantiator = objectIdentifier; + + //obj = object.instantiate(objectIdentifier); + + parts = parts[2 .. $]; + + if(parts.length == 0) { + // gotta run the default function + fun = (to!string(cgi.requestMethod)) in currentReflection.functions; + } + } else { + fun = parts[0] in currentReflection.functions; + if(fun is null) + errorMessage = "no such method in class "~objectName~": " ~ parts[0]; + parts = parts[1 .. $]; + } + } + } + } + + if(fun is null) { + noSuchFunction: + if(errorMessage.length) + throw new NoSuchPageException(errorMessage); + string allFuncs, allObjs; + foreach(n, f; reflection.functions) + allFuncs ~= n ~ "\n"; + foreach(n, f; reflection.objects) + allObjs ~= n ~ "\n"; + throw new NoSuchPageException("no such function " ~ funName ~ "\n functions are:\n" ~ allFuncs ~ "\n\nObjects are:\n" ~ allObjs); + } + + assert(fun !is null); + assert(fun.dispatcher !is null); + assert(cgi !is null); + + result.type = fun.returnType; + + string format = cgi.request("format", reflection.defaultOutputFormat); + string secondaryFormat = cgi.request("secondaryFormat", ""); + if(secondaryFormat.length == 0) secondaryFormat = null; + + JSONValue res; + + if(envelopeFormat == "document" && fun.documentDispatcher !is null) { + res = fun.documentDispatcher(cgi, cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.postArray : cgi.getArray); + envelopeFormat = "html"; + } else + res = fun.dispatcher(cgi, instantiator, cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.postArray : cgi.getArray, format, secondaryFormat); + + //if(cgi) + // cgi.setResponseContentType("application/json"); + result.success = true; + result.result = res; + } + catch (Throwable e) { + result.success = false; + result.errorMessage = e.msg; + result.type = e.classinfo.name; + debug result.dFullString = e.toString(); + + if(envelopeFormat == "document" || envelopeFormat == "html") { + auto ipe = cast(InsufficientParametersException) e; + if(ipe !is null) { + assert(fun !is null); + Form form; + if(0 || fun.createForm !is null) { // FIXME: if 0 + // go ahead and use it to make the form page + auto doc = fun.createForm(cgi.requestMethod == Cgi.RequestMethod.POST ? cgi.post : cgi.get); + } else { + Parameter[] params = fun.parameters.dup; + foreach(i, p; fun.parameters) { + string value = ""; + if(p.name in cgi.get) + value = cgi.get[p.name]; + if(p.name in cgi.post) + value = cgi.post[p.name]; + params[i].value = value; + } + + form = createAutomaticForm(new Document, *fun);// params, beautify(fun.originalName)); + foreach(k, v; cgi.get) + form.setValue(k, v); + form.setValue("envelopeFormat", envelopeFormat); + + auto n = form.getElementById("function-name"); + if(n) + n.innerText = beautify(fun.originalName); + } + + assert(form !is null); + result.result.str = form.toString(); + } else { + if(instantiation._errorFunction !is null) { + auto document = instantiation._errorFunction(e); + if(document is null) + goto gotnull; + result.result.str = (document.toString()); + } else { + gotnull: + auto document = new Document; + auto code = document.createElement("pre"); + code.innerText = e.toString(); + + result.result.str = (code.toString()); + } + } + } + } finally { + switch(envelopeFormat) { + case "redirect": + auto redirect = cgi.request("_arsd_redirect_location", cgi.referrer); + + // FIXME: is this safe? it'd make XSS super easy + // add result to url + + if(!result.success) + goto case "none"; + + cgi.setResponseLocation(redirect, false); + break; + case "json": + // this makes firefox ugly + //cgi.setResponseContentType("application/json"); + auto json = toJsonValue(result); + cgi.write(toJSON(&json)); + break; + case "none": + cgi.setResponseContentType("text/plain"); + + if(result.success) { + if(result.result.type == JSON_TYPE.STRING) { + cgi.write(result.result.str); + } else { + cgi.write(toJSON(&result.result)); + } + } else { + cgi.write(result.errorMessage); + } + break; + case "document": + case "html": + default: + cgi.setResponseContentType("text/html"); + + if(result.result.type == JSON_TYPE.STRING) { + auto returned = result.result.str; + + if((fun !is null) && envelopeFormat != "html") { + Document document; + if(fun.returnTypeIsDocument) { + // probably not super efficient... + document = new TemplatedDocument(returned); + } else { + auto e = instantiation._getGenericContainer(); + document = e.parentDocument; + // FIXME: slow, esp if func return element + e.innerHTML = returned; + } + + if(envelopeFormat == "document") + instantiation._postProcess(document); + + returned = document.toString; + } + + cgi.write(returned); + } else + cgi.write(htmlEntitiesEncode(toJSON(&result.result))); + break; + } + + cgi.close(); + } +} + +mixin template FancyMain(T, Args...) { + void fancyMainFunction(Cgi cgi) { //string[] args) { +// auto cgi = new Cgi; + + // there must be a trailing slash for relative links.. + if(cgi.pathInfo.length == 0) { + cgi.setResponseLocation(cgi.requestUri ~ "/"); + cgi.close(); + return; + } + + // FIXME: won't work for multiple objects + T instantiation = new T(); + auto reflection = prepareReflection!(T)(cgi, instantiation); + + run(cgi, instantiation); +/+ + if(args.length > 1) { + string[string][] namedArgs; + foreach(arg; args[2..$]) { + auto lol = arg.indexOf("="); + if(lol == -1) + throw new Exception("use named args for all params"); + //namedArgs[arg[0..lol]] = arg[lol+1..$]; // FIXME + } + + if(!(args[1] in reflection.functions)) { + throw new Exception("No such function"); + } + + //writefln("%s", reflection.functions[args[1]].dispatcher(null, namedArgs, "string")); + } else { ++/ +// } + } + + mixin GenericMain!(fancyMainFunction, Args); +} + +Form createAutomaticForm(Document document, in FunctionInfo func, string[string] fieldTypes = null) { + return createAutomaticForm(document, func.name, func.parameters, beautify(func.originalName), "POST", fieldTypes); +} + +Form createAutomaticForm(Document document, string action, in Parameter[] parameters, string submitText = "Submit", string method = "POST", string[string] fieldTypes = null) { + assert(document !is null); + auto form = cast(Form) document.createElement("form"); + + form.action = action; + + assert(form !is null); + form.method = method; + + + auto fieldset = document.createElement("fieldset"); + auto legend = document.createElement("legend"); + legend.innerText = submitText; + fieldset.appendChild(legend); + + auto table = cast(Table) document.createElement("table"); + assert(table !is null); + + form.appendChild(fieldset); + fieldset.appendChild(table); + + table.appendChild(document.createElement("tbody")); + + static int count = 0; + + foreach(param; parameters) { + Element input; + + string type = param.type; + if(param.name in fieldTypes) + type = fieldTypes[param.name]; + + if(type == "select") { + input = document.createElement("select"); + + foreach(idx, opt; param.options) { + auto option = document.createElement("option"); + option.name = opt; + option.value = param.optionValues[idx]; + + option.innerText = beautify(opt); + + if(option.value == param.value) + option.selected = "selected"; + + input.appendChild(option); + } + + input.name = param.name; + } else if (type == "radio") { + assert(0, "FIXME"); + } else { + if(type.startsWith("textarea")) { + input = document.createElement("textarea"); + input.name = param.name; + input.innerText = param.value; + + auto idx = type.indexOf("-"); + if(idx != -1) { + idx++; + input.rows = type[idx .. $]; + } + } else { + input = document.createElement("input"); + input.type = type; + input.name = param.name; + input.value = param.value; + } + } + + string n = param.name ~ "_auto-form-" ~ to!string(count); + + input.id = n; + + if(type == "hidden") { + form.appendChild(input); + } else { + auto th = document.createElement("th"); + auto label = document.createElement("label"); + label.setAttribute("for", n); + label.innerText = beautify(param.name) ~ ": "; + th.appendChild(label); + + table.appendRow(th, input); + } + + count++; + }; + + auto fmt = document.createElement("select"); + fmt.name = "format"; + fmt.addChild("option", "html").setAttribute("value", "html"); + fmt.addChild("option", "table").setAttribute("value", "table"); + fmt.addChild("option", "json").setAttribute("value", "json"); + fmt.addChild("option", "string").setAttribute("value", "string"); + auto th = table.th(""); + th.addChild("label", "Format:"); + + table.appendRow(th, fmt).className = "format-row"; + + + auto submit = document.createElement("input"); + submit.value = submitText; + submit.type = "submit"; + + table.appendRow(Html(" "), submit); + +// form.setValue("format", reflection.defaultOutputFormat); + + return form; +} + + +/** + * Returns the parameter names of the given function + * + * Params: + * func = the function alias to get the parameter names of + * + * Returns: an array of strings containing the parameter names + */ +/+ +string parameterNamesOf( alias fn )( ) { + string fullName = typeof(&fn).stringof; + + int pos = fullName.lastIndexOf( ')' ); + int end = pos; + int count = 0; + do { + if ( fullName[pos] == ')' ) { + count++; + } else if ( fullName[pos] == '(' ) { + count--; + } + pos--; + } while ( count > 0 ); + + return fullName[pos+2..end]; +} ++/ + + +template parameterNamesOf (alias func) +{ + const parameterNamesOf = parameterNamesOfImpl!(func); +} + + +int indexOfNew(string s, char a) { + foreach(i, c; s) + if(c == a) + return i; + return -1; +} + +/** + * Returns the parameter names of the given function + * + * Params: + * func = the function alias to get the parameter names of + * + * Returns: an array of strings containing the parameter names + */ +private string[] parameterNamesOfImpl (alias func) () +{ + string funcStr = typeof(&func).stringof; + + auto start = funcStr.indexOfNew('('); + auto end = funcStr.indexOfNew(')'); + + const firstPattern = ' '; + const secondPattern = ','; + + funcStr = funcStr[start + 1 .. end]; + + if (funcStr == "") + return null; + + funcStr ~= secondPattern; + + string token; + string[] arr; + + foreach (c ; funcStr) + { + if (c != firstPattern && c != secondPattern) + token ~= c; + + else + { + if (token) + arr ~= token; + + token = null; + } + } + + if (arr.length == 1) + return arr; + + string[] result; + bool skip = false; + + foreach (str ; arr) + { + skip = !skip; + + if (skip) + continue; + + result ~= str; + } + + return result; +} +///////////////////////////////// + +string toHtml(T)(T a) { + string ret; + + static if(is(T : Document)) + ret = a.toString(); + else + static if(isArray!(T)) { + static if(__traits(compiles, typeof(T[0]).makeHtmlArray(a))) + ret = to!string(typeof(T[0]).makeHtmlArray(a)); + else + foreach(v; a) + ret ~= toHtml(v); + } else static if(is(T : Element)) + ret = a.toString(); + else static if(__traits(compiles, a.makeHtmlElement().toString())) + ret = a.makeHtmlElement().toString(); + else static if(is(T == Html)) + ret = a.source; + else + ret = htmlEntitiesEncode(std.array.replace(to!string(a), "\n", "
\n")); + + return ret; +} + +string toJson(T)(T a) { + auto v = toJsonValue(a); + return toJSON(&v); +} + +// FIXME: are the explicit instantiations of this necessary? +JSONValue toJsonValue(T, R = ApiProvider)(T a, string formatToStringAs = null, R api = null) + if(is(R : ApiProvider)) +{ + JSONValue val; + static if(is(T == JSONValue)) { + val = a; + } else static if(__traits(compiles, val = a.makeJsonValue())) { + val = a.makeJsonValue(); + // FIXME: free function to emulate UFCS? + + // FIXME: should we special case something like struct Html? + } else static if(is(T : Element)) { + if(a is null) { + val.type = JSON_TYPE.NULL; + } else { + val.type = JSON_TYPE.STRING; + val.str = a.toString(); + } + } else static if(isIntegral!(T)) { + val.type = JSON_TYPE.INTEGER; + val.integer = to!long(a); + } else static if(isFloatingPoint!(T)) { + val.type = JSON_TYPE.FLOAT; + val.floating = to!real(a); + static assert(0); + } else static if(is(T == void*)) { + val.type = JSON_TYPE.NULL; + } else static if(isPointer!(T)) { + if(a is null) { + val.type = JSON_TYPE.NULL; + } else { + val = toJsonValue!(typeof(*a), R)(*a, formatToStringAs, api); + } + } else static if(is(T == bool)) { + if(a == true) + val.type = JSON_TYPE.TRUE; + if(a == false) + val.type = JSON_TYPE.FALSE; + } else static if(isSomeString!(T)) { + val.type = JSON_TYPE.STRING; + val.str = to!string(a); + } else static if(isAssociativeArray!(T)) { + val.type = JSON_TYPE.OBJECT; + foreach(k, v; a) { + val.object[to!string(k)] = toJsonValue!(typeof(v), R)(v, formatToStringAs, api); + } + } else static if(isArray!(T)) { + val.type = JSON_TYPE.ARRAY; + val.array.length = a.length; + foreach(i, v; a) { + val.array[i] = toJsonValue!(typeof(v), R)(v, formatToStringAs, api); + } + } else static if(is(T == struct)) { // also can do all members of a struct... + val.type = JSON_TYPE.OBJECT; + + foreach(i, member; a.tupleof) { + string name = a.tupleof[i].stringof[2..$]; + static if(a.tupleof[i].stringof[2] != '_') + val.object[name] = toJsonValue!(typeof(member), R)(member, formatToStringAs, api); + } + // HACK: bug in dmd can give debug members in a non-debug build + //static if(__traits(compiles, __traits(getMember, a, member))) + } else { /* our catch all is to just do strings */ + val.type = JSON_TYPE.STRING; + val.str = to!string(a); + // FIXME: handle enums + } + + + // don't want json because it could recurse + if(val.type == JSON_TYPE.OBJECT && formatToStringAs !is null && formatToStringAs != "json") { + JSONValue formatted; + formatted.type = JSON_TYPE.STRING; + + formatAs!(T, R)(a, api, formatted, formatToStringAs, null /* only doing one level of special formatting */); + assert(formatted.type == JSON_TYPE.STRING); + val.object["formattedSecondarily"] = formatted; + } + + return val; +} + +/+ +Document toXml(T)(T t) { + auto xml = new Document; + xml.parse(emptyTag(T.stringof), true, true); + xml.prolog = `` ~ "\n"; + + xml.root = toXmlElement(xml, t); + return xml; +} + +Element toXmlElement(T)(Document document, T t) { + Element val; + static if(is(T == Document)) { + val = t.root; + //} else static if(__traits(compiles, a.makeJsonValue())) { + // val = a.makeJsonValue(); + } else static if(is(T : Element)) { + if(t is null) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else + val = t; + } else static if(is(T == void*)) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else static if(isPointer!(T)) { + if(t is null) { + val = document.createElement("value"); + val.innerText = "null"; + val.setAttribute("isNull", "true"); + } else { + val = toXmlElement(document, *t); + } + } else static if(isAssociativeArray!(T)) { + val = document.createElement("value"); + foreach(k, v; t) { + auto e = document.createElement(to!string(k)); + e.appendChild(toXmlElement(document, v)); + val.appendChild(e); + } + } else static if(isSomeString!(T)) { + val = document.createTextNode(to!string(t)); + } else static if(isArray!(T)) { + val = document.createElement("array"); + foreach(i, v; t) { + auto e = document.createElement("item"); + e.appendChild(toXmlElement(document, v)); + val.appendChild(e); + } + } else static if(is(T == struct)) { // also can do all members of a struct... + val = document.createElement(T.stringof); + foreach(member; __traits(allMembers, T)) { + if(member[0] == '_') continue; // FIXME: skip member functions + auto e = document.createElement(member); + e.appendChild(toXmlElement(document, __traits(getMember, t, member))); + val.appendChild(e); + } + } else { /* our catch all is to just do strings */ + val = document.createTextNode(to!string(t)); + // FIXME: handle enums + } + + return val; +} ++/ + + + +class InsufficientParametersException : Exception { + this(string functionName, string msg) { + super(functionName ~ ": " ~ msg); + } +} + +class InvalidParameterException : Exception { + this(string param, string value, string expected) { + super("bad param: " ~ param ~ ". got: " ~ value ~ ". Expected: " ~expected); + } +} + +void badParameter(alias T)(string expected = "") { + throw new InvalidParameterException(T.stringof, T, expected); +} + +class PermissionDeniedException : Exception { + this(string msg) { + super(msg); + } +} + +class NoSuchPageException : Exception { + this(string msg) { + super(msg); + } +} + +type fromUrlParam(type)(string[] ofInterest) { + type ret; + + // Arrays in a query string are sent as the name repeating... + static if(isArray!(type) && !isSomeString!(type)) { + foreach(a; ofInterest) { + ret ~= to!(ElementType!(type))(a); + } + } + else static if(is(type : Element)) { + auto doc = new Document(ofInterest[$-1], true, true); + + ret = doc.root; + } + /* + else static if(is(type : struct)) { + static assert(0, "struct not supported yet"); + } + */ + else { + // enum should be handled by this too + ret = to!type(ofInterest[$-1]); + } // FIXME: can we support classes? + + + return ret; +} + +WrapperFunction generateWrapper(alias getInstantiation, alias f, alias group, string funName, R)(ReflectionInfo* reflection, R api) if(is(R: ApiProvider)) { + JSONValue wrapper(Cgi cgi, string instantiationIdentifier, in string[][string] sargs, in string format, in string secondaryFormat = null) { + JSONValue returnValue; + returnValue.type = JSON_TYPE.STRING; + + auto instantiation = getInstantiation(instantiationIdentifier, api); + + ParameterTypeTuple!(f) args; + + Throwable t; // the error we see + + // this permission check thing might be removed. It's just there so you can check before + // doing the automatic form... but I think that would be better done some other way. + static if(__traits(hasMember, group, funName ~ "_PermissionCheck")) { + ParameterTypeTuple!(__traits(getMember, group, funName ~ "_PermissionCheck")) argsperm; + + foreach(i, type; ParameterTypeTuple!(__traits(getMember, group, funName ~ "_PermissionCheck"))) { + string name = parameterNamesOf!(__traits(getMember, group, funName ~ "_PermissionCheck"))[i]; + static if(is(type == bool)) { + if(name in sargs) + args[i] = true; + else + args[i] = false; + } else { + if(!(name in sargs)) { + t = new InsufficientParametersException(funName, "arg " ~ name ~ " is not present for permission check"); + goto maybeThrow; + } + argsperm[i] = to!type(sargs[name][$-1]); + } + } + + __traits(getMember, group, funName ~ "_PermissionCheck")(argsperm); + } + // done with arguably useless permission check + + + // Actually calling the function + foreach(i, type; ParameterTypeTuple!(f)) { + string name = parameterNamesOf!(f)[i]; + + // We want to check the named argument first. If it's not there, + // try the positional arguments + string using = name; + if(name !in sargs) + using = "positional-arg-" ~ to!string(i); + + // FIXME: if it's a struct, we should do it's pieces independently here + + static if(is(type == bool)) { + // bool is special cased because HTML checkboxes don't send anything if it isn't checked + if(using in sargs) + args[i] = true; // FIXME: should try looking at the value + else + args[i] = false; + } else { + if(using !in sargs) { + throw new InsufficientParametersException(funName, "arg " ~ name ~ " is not present"); + } + + // We now check the type reported by the client, if there is one + // Right now, only one type is supported: ServerResult, which means + // it's actually a nested function call + + string[] ofInterest = cast(string[]) sargs[using]; // I'm changing the reference, but not the underlying stuff, so this cast is ok + + if(using ~ "-type" in sargs) { + string reportedType = sargs[using ~ "-type"][$-1]; + if(reportedType == "ServerResult") { + + // FIXME: doesn't handle functions that return + // compound types (structs, arrays, etc) + + ofInterest = null; + + string str = sargs[using][$-1]; + int idx = str.indexOf("?"); + string callingName, callingArguments; + if(idx == -1) { + callingName = str; + } else { + callingName = str[0..idx]; + callingArguments = str[idx + 1 .. $]; + } + + // find it in reflection + ofInterest ~= reflection.functions[callingName]. + dispatcher(cgi, null, decodeVariables(callingArguments), "string").str; + } + } + + + args[i] = fromUrlParam!type(ofInterest); + } + } + + static if(!is(ReturnType!f == void)) + ReturnType!(f) ret; + else + void* ret; + + static if(!is(ReturnType!f == void)) + ret = instantiation(args); // version 1 didn't handle exceptions + else + instantiation(args); + + formatAs(ret, api, returnValue, format, secondaryFormat); + + done: + + return returnValue; + } + + return &wrapper; +} + + +void formatAs(T, R)(T ret, R api, ref JSONValue returnValue, string format, string formatJsonToStringAs = null) if(is(R : ApiProvider)) { + + if(api !is null) { + static if(__traits(compiles, api.customFormat(ret, format))) { + auto customFormatted = api.customFormat(ret, format); + if(customFormatted !is null) { + returnValue.str = customFormatted; + return; + } + } + } + switch(format) { + case "html": + // FIXME: should we actually post process here? + /+ + static if(is(typeof(ret) : Document)) { + instantiation._postProcess(ret); + return ret.toString(); + break; + } + static if(__traits(hasMember, group, funName ~ "_Page")) { + auto doc = __traits(getMember, group, funName ~ "_Page")(ret); + instantiation._postProcess(doc); + return doc.toString(); + break; + } + +/ + + returnValue.str = toHtml(ret); + break; + case "string": + static if(__traits(compiles, to!string(ret))) + returnValue.str = to!string(ret); + else goto badType; + break; + case "json": + returnValue = toJsonValue!(typeof(ret), R)(ret, formatJsonToStringAs, api); + break; + case "table": + auto document = new Document(""); + static if(__traits(compiles, structToTable(document, ret))) + returnValue.str = structToTable(document, ret).toString(); + else + goto badType; + break; + default: + badType: + throw new Exception("Couldn't get result as " ~ format); + } +} + + +private string emptyTag(string rootName) { + return ("<" ~ rootName ~ ">"); +} + + +alias JSONValue delegate(Cgi cgi, string, in string[][string] args, in string format, in string secondaryFormat = null) WrapperFunction; + +string urlToBeauty(string url) { + string u = url.replace("/", ""); + + string ret; + + bool capitalize = true; + foreach(c; u) { + if(capitalize) { + ret ~= ("" ~ c).toUpper; + capitalize = false; + } else { + if(c == '-') { + ret ~= " "; + capitalize = true; + } else + ret ~= c; + } + } + + return ret; +} + +string toUrlName(string name) { + string res; + foreach(c; name) { + if(c >= 'a' && c <= 'z') + res ~= c; + else { + res ~= '-'; + if(c >= 'A' && c <= 'Z') + res ~= c + 0x20; + else + res ~= c; + } + } + return res; +} + +string beautify(string name) { + string n; + n ~= toUpper(name[0..1]); + + dchar last; + foreach(dchar c; name[1..$]) { + if((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + if(last != ' ') + n ~= " "; + } + + if(c == '_') + n ~= " "; + else + n ~= c; + last = c; + } + return n; +} + + + + + + +import std.md5; +import core.stdc.stdlib; +import core.stdc.time; +import std.file; +string getSessionId(Cgi cgi) { + static string token; // FIXME: should this actually be static? it seems wrong + if(token is null) { + if("_sess_id" in cgi.cookies) + token = cgi.cookies["_sess_id"]; + else { + auto tmp = uniform(0, int.max); + token = to!string(tmp); + + cgi.setCookie("_sess_id", token, /*60 * 8 * 1000*/ 0, "/", null, true); + } + } + + return getDigestString(cgi.remoteAddress ~ "\r\n" ~ cgi.userAgent ~ "\r\n" ~ token); +} + +void setLoginCookie(Cgi cgi, string name, string value) { + cgi.setCookie(name, value, 0, "/", null, true); +} + +string htmlTemplateWithData(in string text, in string[string] vars) { + assert(text !is null); + + string newText = text; + + if(vars !is null) + foreach(k, v; vars) { + //assert(k !is null); + //assert(v !is null); + newText = newText.replace("{$" ~ k ~ "}", htmlEntitiesEncode(v).replace("\n", "
")); + } + + return newText; +} + +string htmlTemplate(string filename, string[string] vars) { + return htmlTemplateWithData(readText(filename), vars); +} + +class TemplatedDocument : Document { + const override string toString() { + string s; + if(vars !is null) + s = htmlTemplateWithData(super.toString(), vars); + else + s = super.toString(); + + return s; + } + + public: + string[string] vars; + + this(string src) { + super(); + parse(src, true, true); + } + + this() { } + + void delegate(TemplatedDocument)[] preToStringFilters; + void delegate(ref string)[] postToStringFilters; +} + +void writeDocument(Cgi cgi, TemplatedDocument document) { + foreach(f; document.preToStringFilters) + f(document); + + auto s = document.toString(); + + foreach(f; document.postToStringFilters) + f(s); + + cgi.write(s); +} + +/* Password helpers */ + +string makeSaltedPasswordHash(string userSuppliedPassword, string salt = null) { + if(salt is null) + salt = to!string(uniform(0, int.max)); + + return hashToString(SHA256(salt ~ userSuppliedPassword)) ~ ":" ~ salt; +} + +bool checkPassword(string saltedPasswordHash, string userSuppliedPassword) { + auto parts = saltedPasswordHash.split(":"); + + return makeSaltedPasswordHash(userSuppliedPassword, parts[1]) == saltedPasswordHash; +} + + + +Table structToTable(T)(Document document, T arr, string[] fieldsToSkip = null) if(isArray!(T) && !isAssociativeArray!(T)) { + auto t = cast(Table) document.createElement("table"); + t.border = "1"; + + static if(is(T == string[string][])) { + string[string] allKeys; + foreach(row; arr) { + foreach(k; row.keys) + allKeys[k] = k; + } + + auto sortedKeys = allKeys.keys.sort; + Element tr; + + auto thead = t.addChild("thead"); + auto tbody = t.addChild("tbody"); + + tr = thead.addChild("tr"); + foreach(key; sortedKeys) + tr.addChild("th", key); + + bool odd = true; + foreach(row; arr) { + tr = tbody.addChild("tr"); + foreach(k; sortedKeys) { + tr.addChild("td", k in row ? row[k] : ""); + } + if(odd) + tr.addClass("odd"); + + odd = !odd; + } + } else static if(is(typeof(T[0]) == struct)) { + { + auto thead = t.addChild("thead"); + auto tr = thead.addChild("tr"); + auto s = arr[0]; + foreach(idx, member; s.tupleof) + tr.addChild("th", s.tupleof[idx].stringof[2..$]); + } + + bool odd = true; + auto tbody = t.addChild("tbody"); + foreach(s; arr) { + auto tr = tbody.addChild("tr"); + foreach(member; s.tupleof) { + tr.addChild("td", to!string(member)); + } + + if(odd) + tr.addClass("odd"); + + odd = !odd; + } + } else static assert(0); + + return t; +} + +// this one handles horizontal tables showing just one item +Table structToTable(T)(Document document, T s, string[] fieldsToSkip = null) if(!isArray!(T) || isAssociativeArray!(T)) { + static if(__traits(compiles, s.makeHtmlTable(document))) + return s.makeHtmlTable(document); + else { + + auto t = cast(Table) document.createElement("table"); + + static if(is(T == struct)) { + main: foreach(i, member; s.tupleof) { + string name = s.tupleof[i].stringof[2..$]; + foreach(f; fieldsToSkip) + if(name == f) + continue main; + + string nameS = name.idup; + name = ""; + foreach(idx, c; nameS) { + if(c >= 'A' && c <= 'Z') + name ~= " " ~ c; + else if(c == '_') + name ~= " "; + else + name ~= c; + } + + t.appendRow(t.th(name.capitalize), + to!string(member)); + } + } else static if(is(T == string[string])) { + foreach(k, v; s){ + t.appendRow(t.th(k), v); + } + } else static assert(0); + + return t; + } +} + +debug string javascriptBase = ` + // change this in your script to get fewer error popups + "_debugMode":true,` ~ javascriptBaseImpl; +else string javascriptBase = ` + // change this in your script to get more details in errors + "_debugMode":false,` ~ javascriptBaseImpl; + +enum string javascriptBaseImpl = q{ + "_doRequest": function(url, args, callback, method, async) { + var xmlHttp; + try { + xmlHttp=new XMLHttpRequest(); + } + catch (e) { + try { + xmlHttp=new ActiveXObject("Msxml2.XMLHTTP"); + } + catch (e) { + xmlHttp=new ActiveXObject("Microsoft.XMLHTTP"); + } + } + + if(async) + xmlHttp.onreadystatechange=function() { + if(xmlHttp.readyState==4) { + if(callback) { + callback(xmlHttp.responseText, xmlHttp.responseXML); + } + } + } + + var argString = this._getArgString(args); + if(method == "GET" && url.indexOf("?") == -1) + url = url + "?" + argString; + + xmlHttp.open(method, url, async); + + var a = ""; + + if(method == "POST") { + xmlHttp.setRequestHeader("Content-type","application/x-www-form-urlencoded"); + a = argString; + } else { + xmlHttp.setRequestHeader("Content-type", "text/plain"); + } + + xmlHttp.send(a); + + if(!async && callback) { + return callback(xmlHttp.responseText, xmlHttp.responseXML); + } + return xmlHttp; + }, + + "_raiseError":function(type, message) { + var error = new Error(message); + error.name = type; + throw error; + }, + + "_getUriRelativeToBase":function(name, args) { + var str = name; + var argsStr = this._getArgString(args); + if(argsStr.length) + str += "?" + argsStr; + + return str; + }, + + "_getArgString":function(args) { + var a = ""; + var outputted = false; + var i; // wow Javascript sucks! god damned global loop variables + for(i in args) { + if(outputted) { + a += "&"; + } else outputted = true; + var arg = args[i]; + var argType = ""; + // Make sure the types are all sane + + if(arg && arg._arsdTypeOf && arg._arsdTypeOf == "ServerResult") { + argType = arg._arsdTypeOf; + arg = this._getUriRelativeToBase(arg._serverFunction, arg._serverArguments); + + // this arg is a nested server call + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg); + } else if(arg && arg.length && typeof arg != "string") { + // FIXME: are we sure this is actually an array? + + var outputtedHere = false; + for(var idx = 0; idx < arg.length; idx++) { + if(outputtedHere) { + a += "&"; + } else outputtedHere = true; + + // FIXME: ought to be recursive + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg[idx]); + } + + } else { + // a regular argument + a += encodeURIComponent(i) + "="; + a += encodeURIComponent(arg); + } + // else if: handle arrays and objects too + + if(argType.length > 0) { + a += "&"; + a += encodeURIComponent(i + "-type") + "="; + a += encodeURIComponent(argType); + } + } + + return a; + }, + + "_onError":function(error) { + throw error; + }, + + /// returns an object that can be used to get the actual response from the server + "_serverCall": function (name, passedArgs, returnType) { + var me = this; // this is the Api object + var args = passedArgs; + return { + // type info metadata + "_arsdTypeOf":"ServerResult", + "_staticType":(typeof returnType == "undefined" ? null : returnType), + + // Info about the thing + "_serverFunction":name, + "_serverArguments":args, + + // lower level implementation + "_get":function(callback, onError, async) { + var resObj = this; + if(args == null) + args = {}; + if(!args.format) + args.format = "json"; + args.envelopeFormat = "json"; + return me._doRequest(me._apiBase + name, args, function(t, xml) { + if(me._debugMode) { + try { + var obj = eval("(" + t + ")"); + } catch(e) { + alert("Bad server json: " + e + + "\nOn page: " + (me._apiBase + name) + + "\nGot:\n" + t); + } + } else { + var obj = eval("(" + t + ")"); + } + + if(obj.success) { + if(typeof callback == "function") + callback(obj.result); + else if(typeof resObj.onSuccess == "function") { + resObj.onSuccess(obj.result); + } else if(typeof me.onSuccess == "function") { // do we really want this? + me.onSuccess(obj.result); + } else { + // can we automatically handle it? + // If it's an element, we should replace innerHTML by ID if possible + // if a callback is given and it's a string, that's an id. Return type of element + // should replace that id. return type of string should be appended + // FIXME: meh just do something here. + } + + return obj.result; + } else { + // how should we handle the error? I guess throwing is better than nothing + // but should there be an error callback too? + var error = new Error(obj.errorMessage); + error.name = obj.type; + error.functionUrl = me._apiBase + name; + error.functionArgs = args; + error.errorMessage = obj.errorMessage; + + // myFunction.caller should be available and checked too + // btw arguments.callee is like this for functions + + if(me._debugMode) { + var ourMessage = obj.type + ": " + obj.errorMessage + + "\nOn: " + me._apiBase + name; + if(args.toSource) + ourMessage += args.toSource(); + if(args.stack) + ourMessage += "\n" + args.stack; + + error.message = ourMessage; + + // alert(ourMessage); + } + + if(onError) // local override first... + return onError(error); + else if(resObj.onError) // then this object + return resObj.onError(error); + else if(me._onError) // then the global object + return me._onError(error); + + throw error; // if all else fails... + } + + // assert(0); // not reached + }, (name.indexOf("get") == 0) ? "GET" : "POST", async); // FIXME: hack: naming convention used to figure out method to use + }, + + // should pop open the thing in HTML format + // "popup":null, // FIXME not implemented + + "onError":null, // null means call the global one + + "onSuccess":null, // a generic callback. generally pass something to get instead. + + "formatSet":false, // is the format overridden? + + // gets the result. Works automatically if you don't pass a callback. + // You can also curry arguments to your callback by listing them here. The + // result is put on the end of the arg list to the callback + "get":function(callbackObj) { + var callback = null; + var errorCb = null; + var callbackThis = null; + if(callbackObj) { + if(typeof callbackObj == "function") + callback = callbackObj; + else { + if(callbackObj.length) { + // array + callback = callbackObj[0]; + + if(callbackObj.length >= 2) + errorCb = callbackObj[1]; + } else { + if(callbackObj.onSuccess) + callback = callbackObj.onSuccess; + if(callbackObj.onError) + errorCb = callbackObj.onError; + if(callbackObj.self) + callbackThis = callbackObj.self; + else + callbackThis = callbackObj; + } + } + } + if(arguments.length > 1) { + var ourArguments = []; + for(var a = 1; a < arguments.length; a++) + ourArguments.push(arguments[a]); + + function cb(obj, xml) { + ourArguments.push(obj); + ourArguments.push(xml); + + // that null is the this object inside the function... can + // we make that work? + return callback.apply(callbackThis, ourArguments); + } + + function cberr(err) { + ourArguments.unshift(err); + + // that null is the this object inside the function... can + // we make that work? + return errorCb.apply(callbackThis, ourArguments); + } + + + this._get(cb, errorCb ? cberr : null, true); + } else { + this._get(callback, errorCb, true); + } + }, + + // If you need a particular format, use this. + "getFormat":function(format /* , same args as get... */) { + this.format(format); + var forwardedArgs = []; + for(var a = 1; a < arguments.length; a++) + forwardedArgs.push(arguments[a]); + this.get.apply(this, forwardedArgs); + }, + + // sets the format of the request so normal get uses it + // myapi.someFunction().format('table').get(...); + // see also: getFormat and getHtml + // the secondaryFormat only makes sense if format is json. It + // sets the format returned by object.toString() in the returned objects. + "format":function(format, secondaryFormat) { + if(args == null) + args = {}; + args.format = format; + + if(typeof secondaryFormat == "string" && secondaryFormat) { + if(format != "json") + me._raiseError("AssertError", "secondaryFormat only works if format == json"); + args.secondaryFormat = secondaryFormat; + } + + this.formatSet = true; + return this; + }, + + "getHtml":function(/* args to get... */) { + this.format("html"); + this.get.apply(this, arguments); + }, + + // FIXME: add post aliases + + // don't use unless you're deploying to localhost or something + "getSync":function() { + function cb(obj) { + // no nothing, we're returning the value below + } + + return this._get(cb, null, false); + }, + // takes the result and appends it as html to the given element + + // FIXME: have a type override + "appendTo":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._appendContent(what)); + }, + // use it to replace the content of the given element + "useToReplace":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._replaceContent(what)); + }, + // use to replace the given element altogether + "useToReplaceElement":function(what) { + if(!this.formatSet) + this.format("html"); + this.get(me._replaceElement(what)); + }, + "useToFillForm":function(what) { + this.get(me._fillForm(what)); + } + // runAsScript has been removed, use get(eval) instead + // FIXME: might be nice to have an automatic popin function too + }; + }, + + "getAutomaticForm":function(method) { + return this._serverCall("builtin.getAutomaticForm", {"method":method}, "Form"); + }, + + "_fillForm": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + alert("FIXME: list of forms not implemented"); + else return function(obj) { + if(e.elements && typeof obj == "object") { + for(i in obj) + if(e.elements[i]) + e.elements[i].value = obj[i]; // FIXME: what about checkboxes, selects, etc? + } else + throw new Error("unsupported response"); + }; + }, + + "_getElement": function(what) { + var e; + if(typeof what == "string") + e = document.getElementById(what); + else + e = what; + + return e; + }, + + "_isListOfNodes": function(what) { + // length is on both arrays and lists, but some elements + // have it too. We disambiguate with getAttribute + return (what && (what.length && !what.getAttribute)) + }, + + // These are some convenience functions to use as callbacks + "_replaceContent": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + return function(obj) { + for(var a = 0; a < obj.length; a++) { + if( (e[a].tagName.toLowerCase() == "input" + && + e[a].getAttribute("type") == "text") + || + e[a].tagName.toLowerCase() == "textarea") + { + e[a].value = obj; + } else + e[a].innerHTML = obj; + } + } + else + return function(obj) { + if( (e.tagName.toLowerCase() == "input" + && + e.getAttribute("type") == "text") + || + e.tagName.toLowerCase() == "textarea") + { + e.value = obj; + } else + e.innerHTML = obj; + } + }, + + // note: what must be only a single element, FIXME: could check the static type + "_replaceElement": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) + throw new Error("Can only replace individual elements since removal from a list may be unstable."); + return function(obj) { + var n = document.createElement("div"); + n.innerHTML = obj; + + if(n.firstChild) { + e.parentNode.replaceChild(n.firstChild, e); + } else { + e.parentNode.removeChild(e); + } + } + }, + + "_appendContent": function(what) { + var e = this._getElement(what); + if(this._isListOfNodes(e)) // FIXME: repeating myself... + return function(obj) { + for(var a = 0; a < e.length; a++) + e[a].innerHTML += obj; + } + else + return function(obj) { + e.innerHTML += obj; + } + }, +}; + + +/* + + + +Note for future: dom.d makes working with html easy, since you can +do various forms of post processing on it to make custom formats +among other things. + +I'm considering adding similar stuff for CSS and Javascript. +dom.d now has some more css support - you can apply a stylesheet +to a document and get the computed style and do some minor changes +programmically. StyleSheet : css file :: Document : html file. + +My css lexer/parser is still pretty crappy though. Also, I'm +not sure it's worth going all the way here. + +I'm doing some of it to support my little browser, but for server +side programs, I'm not sure how useful it is to do this kind of +thing. + +A simple textual macro would be more useful for css than a +struct for it.... I kinda want nested declarations and some +functions (the sass thing from ruby is kinda nice in some ways). + +But I'm fairly meh on it anyway. + + +For javascript, I wouldn't mind having a D style foreach in it. +But is it worth it writing a fancy javascript AST thingy just +for that? + +Aside from that, I don't mind the language with how sparingly I +use it though. Besides, writing: + +CoolApi.doSomething("asds").appendTo('element'); + +really isn't bad anyway. + + +The benefit for html was very easy and big. I'm not so sure about +css and js. +*/