change of binary data - POTENTIALLY BREAKING - test your code

database.d

@@ -13,7 +13,24 @@
 	db.query("INSERT INTO people (id, name) VALUES (?, ?)", 5, "Adam");
 	---
 
-	To convert to other types, just use [std.conv.to] since everything comes out of this as simple strings.
+	To convert to other types, just use [std.conv.to] since everything comes out of this as simple strings with the exception of binary data,
+	which you'll want to cast to const(ubyte)[].
+
+	History:
+		Originally written prior to 2011.
+
+		On August 2, 2022, the behavior of BLOB (or BYTEA in postgres) changed significantly.
+		Before, it would convert to strings with `to!string(bytes)` on insert and platform specific
+		on query. It didn't really work at all.
+
+		It now actually stores ubyte[] as a blob and retrieves it without modification. Note you need to
+		cast it.
+
+		This is potentially breaking, but since it didn't work much before I doubt anyone was using it successfully
+		but this might be a problem. I advise you to retest.
+
+		Be aware I don't like this string interface much anymore and want to change it significantly but idk
+		how to work it in without breaking a decade of code.
 +/
 module arsd.database;
 
@@ -49,6 +66,8 @@ interface Database {
 
 	/// Escapes data for inclusion into an sql string literal
 	string escape(string sqlData);
+	/// Escapes binary data for inclusion into a sql string. Note that unlike `escape`, the returned string here SHOULD include the quotes.
+	string escapeBinaryString(const(ubyte)[] sqlData);
 
 	/// query to start a transaction, only here because sqlite is apparently different in syntax...
 	void startTransaction();
@@ -380,8 +399,40 @@ class SelectBuilder : SqlBuilder {
 
 // /////////////////////sql//////////////////////////////////
 
+package string tohexsql(const(ubyte)[] b) {
+	char[] x;
+	x.length = b.length * 2 + 3;
+	int pos = 0;
+	x[pos++] = 'x';
+	x[pos++] = '\'';
+
+	char tohex(ubyte a) {
+		if(a < 10)
+			return cast(char)(a + '0');
+		else
+			return cast(char)(a - 10 + 'A');
+	}
+
+	foreach(item; b) {
+		x[pos++] = tohex(item >> 4);
+		x[pos++] = tohex(item & 0x0f);
+	}
+
+	x[pos++] = '\'';
+
+	return cast(string) x;
+}
+
 // used in the internal placeholder thing
 string toSql(Database db, Variant a) {
+
+	string binary(const(ubyte)[] b) {
+		if(b is null)
+			return "NULL";
+		else
+			return db.escapeBinaryString(b);
+	}
+
 	auto v = a.peek!(void*);
 	if(v && (*v is null)) {
 		return "NULL";
@@ -390,6 +441,10 @@ string toSql(Database db, Variant a) {
 	} else if(auto t = a.peek!(DateTime)) {
 		// FIXME: this might be broken cuz of timezones!
 		return db.sysTimeToValue(cast(SysTime) *t);
+	} else if(auto t = a.peek!(ubyte[])) {
+		return binary(*t);
+	} else if(auto t = a.peek!(immutable(ubyte)[])) {
+		return binary(*t);
 	} else if(auto t = a.peek!string) {
 		auto str = *t;
 		if(str is null)

mssql.d

@@ -79,6 +79,10 @@ class MsSql : Database {
 		//return ret.replace("'", "''");
 	}
 
+	string escapeBinaryString(const(ubyte)[] data) { // FIXME
+		return "'" ~ escape(cast(string) data) ~ "'";
+	}
+
 
 	string error() {
 		return null; // FIXME

mysql.d

@@ -307,6 +307,10 @@ class MySql : Database {
 		return cast(string) buffer;
 	}
 
+	string escapeBinaryString(const(ubyte)[] data) {
+		return tohexsql(b);
+	}
+
 	string escaped(T...)(string sql, T t) {
 		static if(t.length > 0) {
 			string fixedup;

postgres.d

@@ -146,6 +146,24 @@ class PostgreSql : Database {
 		return ret;
 	}
 
+	string escapeBinaryString(const(ubyte)[] data) {
+		// must include '\x ... ' here
+		size_t len;
+		char* buf = PQescapeByteaConn(conn, data.ptr, data.length, &len);
+		if(buf is null)
+			throw new Exception("pgsql out of memory escaping binary string");
+
+		string res;
+		if(len == 0)
+			res = "''";
+		else
+			res = cast(string) ("'" ~ buf[0 .. len - 1] ~ "'"); // gotta cut the zero terminator off
+
+		PQfreemem(buf);
+
+		return res;
+	}
+
 
 	///
 	string error() {
@@ -243,7 +261,26 @@ class PostgresResult : ResultSet {
 				if(PQgetisnull(res, position, i))
 					a = null;
 				else {
-					a = copyCString(PQgetvalue(res, position, i), PQgetlength(res, position, i));
+					switch(PQfformat(res, i)) {
+						case 0: // text representation
+							switch(PQftype(res, i)) {
+								case BYTEAOID:
+									size_t len;
+									char* c = PQunescapeBytea(PQgetvalue(res, position, i), &len);
+
+									a = cast(string) c[0 .. len].idup;
+
+									PQfreemem(c);
+								break;
+								default:
+									a = copyCString(PQgetvalue(res, position, i), PQgetlength(res, position, i));
+							}
+						break;
+						case 1: // binary representation
+							throw new Exception("unexpected format returned by pq");
+						default:
+							throw new Exception("unknown pq format");
+					}
 
 				}
 				row ~= a;
@@ -321,6 +358,19 @@ extern(C) {
 			int row_number,
 			int column_number);
 
+	int PQfformat(const PGresult *res, int column_number);
+
+	alias Oid = int;
+	enum BYTEAOID = 17;
+	Oid PQftype(const PGresult* res, int column_number);
+
+	char *PQescapeByteaConn(PGconn *conn,
+                                 const ubyte *from,
+                                 size_t from_length,
+                                 size_t *to_length);
+	char *PQunescapeBytea(const char *from, size_t *to_length);
+	void PQfreemem(void *ptr);
+
 	char* PQcmdTuples(PGresult *res);
 
 }

sqlite.d

@@ -148,6 +148,10 @@ class Sqlite : Database {
 		return esc;
 	}
 
+	string escapeBinaryString(const(ubyte)[] b) {
+		return tohexsql(b);
+	}
+
 	string error(){
 		import core.stdc.string : strlen;
 		char* mesg = sqlite3_errmsg(db);
@@ -231,7 +235,10 @@ class SqliteResult :  ResultSet {
 		if(rows.length <= position)
 			throw new Exception("Result is empty");
 		foreach(c; rows[position]) {
-			r.row ~= c.coerce!(string);
+			if(auto t = c.peek!(immutable(byte)[]))
+				r.row ~= cast(string) *t;
+			else
+				r.row ~= c.coerce!(string);
 		}
 
 		return r;
@@ -502,6 +509,7 @@ template extract(A, T, R...){
 		void bind (const char[] name, int value){ bind(bindNameLookUp(name), value); }
 		void bind (const char[] name, float value){ bind(bindNameLookUp(name), value); }
 		void bind (const char[] name, const byte[] value){ bind(bindNameLookUp(name), value); }
+		void bind (const char[] name, const ubyte[] value){ bind(bindNameLookUp(name), value); }
 
 	void bind(int col, typeof(null) value){
 		if(sqlite3_bind_null(s, col) != SQLITE_OK)
@@ -526,7 +534,17 @@ template extract(A, T, R...){
 		if(sqlite3_bind_int64(s, col, value) != SQLITE_OK)
 			throw new DatabaseException("bind " ~ db.error());
 	}
-	
+
+	void bind(int col, const ubyte[] value){
+		if(value is null) {
+			if(sqlite3_bind_null(s, col) != SQLITE_OK)
+				throw new DatabaseException("bind " ~ db.error());
+		} else {
+			if(sqlite3_bind_blob(s, col, cast(void*)value.ptr, cast(int) value.length, cast(void*)-1) != SQLITE_OK)
+				throw new DatabaseException("bind " ~ db.error());
+		}
+	}
+
 	void bind(int col, const byte[] value){
 		if(value is null) {
 			if(sqlite3_bind_null(s, col) != SQLITE_OK)
@@ -556,6 +574,10 @@ template extract(A, T, R...){
 			bind(col, v.get!float);
 		else if(v.peek!(byte[]))
 			bind(col, v.get!(byte[]));
+		else if(v.peek!(ubyte[]))
+			bind(col, v.get!(ubyte[]));
+		else if(v.peek!(immutable(ubyte)[]))
+			bind(col, v.get!(immutable(ubyte)[]));
 		else if(v.peek!(void*) && v.get!(void*) is null)
 			bind(col, null);
 		else