arsd/http.d

/++
	OBSOLETE: Old version of my http implementation. Do not use this, instead use [arsd.http2].

	I no longer work on this, use http2.d instead.
+/
/*deprecated*/ module arsd.http; // adrdox apparently loses the comment above with deprecated, i need to fix that over there.

import std.socket;

// FIXME: check Transfer-Encoding: gzip always

version(with_openssl) {
	pragma(lib, "crypto");
	pragma(lib, "ssl");
}

ubyte[] getBinary(string url, string[string] cookies = null) {
	auto hr = httpRequest("GET", url, null, cookies);
	if(hr.code != 200)
		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
	return hr.content;
}

/**
	Gets a textual document, ignoring headers. Throws on non-text or error.
*/
string get(string url, string[string] cookies = null) {
	auto hr = httpRequest("GET", url, null, cookies);
	if(hr.code != 200)
		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
	if(hr.contentType.indexOf("text/") == -1)
		throw new Exception(hr.contentType ~ " is bad content for conversion to string");
	return cast(string) hr.content;

}

static import std.uri;

string post(string url, string[string] args, string[string] cookies = null) {
	string content;

	foreach(name, arg; args) {
		if(content.length)
			content ~= "&";
		content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg);
	}

	auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]);
	if(hr.code != 200)
		throw new Exception(format("HTTP answered %d instead of 200", hr.code));
	if(hr.contentType.indexOf("text/") == -1)
		throw new Exception(hr.contentType ~ " is bad content for conversion to string");

	return cast(string) hr.content;
}

struct HttpResponse {
	int code;
	string contentType;
	string[string] cookies;
	string[] headers;
	ubyte[] content;
}

import std.string;
static import std.algorithm;
import std.conv;

struct UriParts {
	string original;
	string method;
	string host;
	ushort port;
	string path;

	bool useHttps;

	this(string uri) {
		original = uri;

		if(uri[0 .. 8] == "https://")
			useHttps = true;
		else
		if(uri[0..7] != "http://")
			throw new Exception("You must use an absolute, http or https URL.");

		version(with_openssl) {} else
		if(useHttps)
			throw new Exception("openssl support not compiled in try -version=with_openssl");

		int start = useHttps ? 8 : 7;

		auto posSlash = uri[start..$].indexOf("/");
		if(posSlash != -1)
			posSlash += start;

		if(posSlash == -1)
			posSlash = uri.length;

		auto posColon = uri[start..$].indexOf(":");
		if(posColon != -1)
			posColon += start;

		if(useHttps)
			port = 443;
		else
			port = 80;

		if(posColon != -1 && posColon < posSlash) {
			host = uri[start..posColon];
			port = to!ushort(uri[posColon+1..posSlash]);
		} else
			host = uri[start..posSlash];

		path = uri[posSlash..$];
		if(path == "")
			path = "/";
	}
}

HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null) {
	import std.socket;

	auto u = UriParts(uri);
	// auto f = openNetwork(u.host, u.port);
	auto f = new TcpSocket();
	f.connect(new InternetAddress(u.host, u.port));

	void delegate(string) write = (string d) {
		f.send(d);
	};

	char[4096] readBuffer; // rawRead actually blocks until it can fill up the whole buffer... which is broken as far as http goes so one char at a time i guess. slow lol
	char[] delegate() read = () {
		size_t num = f.receive(readBuffer);
		return readBuffer[0..num];
	};

	version(with_openssl) {
		import deimos.openssl.ssl;
		SSL* ssl;
		SSL_CTX* ctx;
		if(u.useHttps) {
			void sslAssert(bool ret){
				if (!ret){
					throw new Exception("SSL_ERROR");
				}
			}
			SSL_library_init();
			OpenSSL_add_all_algorithms();
			SSL_load_error_strings();

			ctx = SSL_CTX_new(SSLv3_client_method());
			sslAssert(!(ctx is null));

			ssl = SSL_new(ctx);
			SSL_set_fd(ssl, f.handle);
			sslAssert(SSL_connect(ssl) != -1);

			write = (string d) {
				SSL_write(ssl, d.ptr, cast(uint)d.length);
			};

			read = () {
				auto len = SSL_read(ssl, readBuffer.ptr, readBuffer.length);
				return readBuffer[0 .. len];
			};
		}
	}


	HttpResponse response = doHttpRequestOnHelpers(write, read, method, uri, content, cookies, headers, u.useHttps);

	version(with_openssl) {
		if(u.useHttps) {
			SSL_free(ssl);
			SSL_CTX_free(ctx);
		}
	}

	return response;
}

/**
	Executes a generic http request, returning the full result. The correct formatting
	of the parameters are the caller's responsibility. Content-Length is added automatically,
	but YOU must give Content-Type!
*/
HttpResponse doHttpRequestOnHelpers(void delegate(string) write, char[] delegate() read, string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null, bool https = false)
	in {
		assert(method == "POST" || method == "GET");
	}
do {
	auto u = UriParts(uri);





	write(format("%s %s HTTP/1.1\r\n", method, u.path));
	write(format("Host: %s\r\n", u.host));
	write(format("Connection: close\r\n"));
	if(content !is null)
		write(format("Content-Length: %d\r\n", content.length));

	if(cookies !is null) {
		string cookieHeader = "Cookie: ";
		bool first = true;
		foreach(k, v; cookies) {
			if(first)
				first = false;
			else
				cookieHeader ~= "; ";
			cookieHeader ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
		}

		write(format("%s\r\n", cookieHeader));
	}

	if(headers !is null)
		foreach(header; headers)
			write(format("%s\r\n", header));
	write("\r\n");
	if(content !is null)
		write(cast(string) content);


	string buffer;

	string readln() {
		auto idx = buffer.indexOf("\r\n");
		if(idx == -1) {
			auto more = read();
			if(more.length == 0) { // end of file or something
				auto ret = buffer;
				buffer = null;
				return ret;
			}
			buffer ~= more;
			return readln();
		}
		auto ret = buffer[0 .. idx + 2]; // + the \r\n
		if(idx + 2 < buffer.length)
			buffer = buffer[idx + 2 .. $];
		else
			buffer = null;
		return ret;
	}

	HttpResponse hr;
 cont:
	string l = readln();
	if(l[0..9] != "HTTP/1.1 ")
		throw new Exception("Not talking to a http server");

	hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK

	if(hr.code == 100) { // continue
		do {
			l = readln();
		} while(l.length > 1);

		goto cont;
	}

	bool chunked = false;

	auto line = readln();
	while(line.length) {
		if(line.strip.length == 0)
			break;
		hr.headers ~= line;
		if(line.startsWith("Content-Type: "))
			hr.contentType = line[14..$-1];
		if(line.startsWith("Set-Cookie: ")) {
			auto hdr = line["Set-Cookie: ".length .. $-1];
			auto semi = hdr.indexOf(";");
			if(semi != -1)
				hdr = hdr[0 .. semi];

			auto equal = hdr.indexOf("=");
			string name, value;
			if(equal == -1) {
				name = hdr;
				// doesn't this mean erase the cookie?
			} else {
				name = hdr[0 .. equal];
				value = hdr[equal + 1 .. $];
			}

			name = std.uri.decodeComponent(name);
			value = std.uri.decodeComponent(value);

			hr.cookies[name] = value;
		}
		if(line.startsWith("Transfer-Encoding: chunked"))
			chunked = true;
		line = readln();
	}

	// there might be leftover stuff in the line buffer
	ubyte[] response = cast(ubyte[]) buffer.dup;
	auto part = read();
	while(part.length) {
		response ~= part;
		part = read();
	}

	if(chunked) {
		// read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character
		// read binary data of that length. it is our content
		// repeat until a zero sized chunk
		// then read footers as headers.

		int state = 0;
		int size;
		int start = 0;
		for(int a = 0; a < response.length; a++) {
			final switch(state) {
				case 0: // reading hex
					char c = response[a];
					if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
						// just keep reading
					} else {
						int power = 1;
						size = 0;
						for(int b = a-1; b >= start; b--) {
							char cc = response[b];
							if(cc >= 'a' && cc <= 'z')
								cc -= 0x20;
							int val = 0;
							if(cc >= '0' && cc <= '9')
								val = cc - '0';
							else
								val = cc - 'A' + 10;

							size += power * val;
							power *= 16;
						}
						state++;
						continue;
					}
				break;
				case 1: // reading until end of line
					char c = response[a];
					if(c == '\n') {
						if(size == 0)
							state = 3;
						else
							state = 2;
					}
				break;
				case 2: // reading data
					hr.content ~= response[a..a+size];
					a += size;
					a+= 1; // skipping a 13 10
					start = a + 1;
					state = 0;
				break;
				case 3: // reading footers
					goto done; // FIXME
			}
		}
	} else
		hr.content = response;
	done:

	return hr;
}


/*
void main(string args[]) {
	write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"]));
}
*/

version(none):

struct Url {
	string url;
}

struct BasicAuth {
	string username;
	string password;
}

/*
	When you send something, it creates a request
	and sends it asynchronously. The request object

	auto request = new HttpRequest();
	// set any properties here

	// synchronous usage
	auto reply = request.perform();

	// async usage, type 1:
	request.send();
	request2.send();

	// wait until the first one is done, with the second one still in-flight
	auto response = request.waitForCompletion();


	// async usage, type 2:
	request.onDataReceived = (HttpRequest hr) {
		if(hr.state == HttpRequest.State.complete) {
			// use hr.responseData
		}
	};
	request.send(); // send, using the callback

	// before terminating, be sure you wait for your requests to finish!

	request.waitForCompletion();

*/

class HttpRequest {
	private static {
		// we manage the actual connections. When a request is made on a particular
		// host, we try to reuse connections. We may open more than one connection per
		// host to do parallel requests.
		//
		// The key is the *domain name*. Multiple domains on the same address will have separate connections.
		Socket[][string] socketsPerHost;

		// only one request can be active on a given socket (at least HTTP < 2.0) so this is that
		HttpRequest[Socket] activeRequestOnSocket;
		HttpRequest[] pending; // and these are the requests that are waiting

		SocketSet readSet;


		void advanceConnections() {
			if(readSet is null)
				readSet = new SocketSet();

			// are there pending requests? let's try to send them

			readSet.reset();

			// active requests need to be read or written to
			foreach(sock, request; activeRequestOnSocket)
				readSet.add(sock);

			// check the other sockets just for EOF, if they close, take them out of our list,
			// we'll reopen if needed upon request.

			auto got = Socket.select(readSet, writeSet, null, 10.seconds /* timeout */);
			if(got == 0) /* timeout */
				{}
			else
			if(got == -1) /* interrupted */
				{}
			else /* ready */
				{}

			// call select(), do what needs to be done
			// no requests are active, send the ones pending connection now
			// we've completed a request, are there any more pending connection? if so, send them now

			auto readSet = new SocketSet();
		}
	}

	this() {
		addConnection(this);
	}

	~this() {
		removeConnection(this);
	}

	HttpResponse responseData;
	HttpRequestParameters parameters;
	private HttpClient parentClient;

	size_t bodyBytesSent;
	size_t bodyBytesReceived;

	State state;
	/// Called when data is received. Check the state to see what data is available.
	void delegate(AsynchronousHttpRequest) onDataReceived;

	enum State {
		/// The request has not yet been sent
		unsent,

		/// The send() method has been called, but no data is
		/// sent on the socket yet because the connection is busy.
		pendingAvailableConnection,

		/// The headers are being sent now
		sendingHeaders,

		/// The body is being sent now
		sendingBody,

		/// The request has been sent but we haven't received any response yet
		waitingForResponse,

		/// We have received some data and are currently receiving headers
		readingHeaders,

		/// All headers are available but we're still waiting on the body
		readingBody,

		/// The request is complete.
		complete,

		/// The request is aborted, either by the abort() method, or as a result of the server disconnecting
		aborted
	}

	/// Sends now and waits for the request to finish, returning the response.
	HttpResponse perform() {
		send();
		return waitForCompletion();
	}

	/// Sends the request asynchronously.
	void send() {
		if(state != State.unsent && state != State.aborted)
			return; // already sent

		responseData = HttpResponse.init;
		bodyBytesSent = 0;
		bodyBytesReceived = 0;
		state = State.pendingAvailableConnection;

		HttpResponse.advanceConnections();
	}


	/// Waits for the request to finish or timeout, whichever comes furst.
	HttpResponse waitForCompletion() {
		while(state != State.aborted && state != State.complete)
			HttpResponse.advanceConnections();
		return responseData;
	}

	/// Aborts this request.
	/// Due to the nature of the HTTP protocol, aborting one request will result in all subsequent requests made on this same connection to be aborted as well.
	void abort() {
		parentClient.close();
	}
}

struct HttpRequestParameters {
	Duration timeout;

	// debugging
	bool useHttp11 = true;
	bool acceptGzip = true;

	// the request itself
	HttpVerb method;
	string host;
	string uri;

	string userAgent;

	string[string] cookies;

	string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property

	string contentType;
	ubyte[] bodyData;
}

interface IHttpClient {

}

enum HttpVerb { GET, HEAD, POST, PUT, DELETE, OPTIONS, TRACE, CONNECT }

/*
	Usage:

	auto client = new HttpClient("localhost", 80);
	// relative links work based on the current url
	client.get("foo/bar");
	client.get("baz"); // gets foo/baz

	auto request = client.get("rofl");
	auto response = request.waitForCompletion();
*/

/// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser.
class HttpClient {
	/* Protocol restrictions, useful to disable when debugging servers */
	bool useHttp11 = true;
	bool useGzip = true;

	/// Automatically follow a redirection?
	bool followLocation = false;

	@property Url location() {
		return currentUrl;
	}

	/// High level function that works similarly to entering a url
	/// into a browser.
	///
	/// Follows locations, updates the current url.
	AsynchronousHttpRequest navigateTo(Url where) {
		currentUrl = where.basedOn(currentUrl);
		assert(0);
	}

	private Url currentUrl;

	this() {

	}

	this(Url url) {
		open(url);
	}

	this(string host, ushort port = 80, bool useSsl = false) {
		open(host, port);
	}

	// FIXME: add proxy
	// FIXME: some kind of caching

	void open(Url url) {

	}

	void open(string host, ushort port = 80, bool useSsl = false) {

	}

	void close() {
		socket.close();
	}

	void setCookie(string name, string value) {

	}

	void clearCookies() {

	}

	HttpResponse sendSynchronously() {
		auto request = sendAsynchronously();
		return request.waitForCompletion();
	}

	AsynchronousHttpRequest sendAsynchronously() {

	}

	string method;
	string host;
	ushort port;
	string uri;

	string[] headers;
	ubyte[] requestBody;

	string userAgent;

	/* inter-request state */
	string[string] cookies;
}

// FIXME: websocket