forks
/
arsd
mirror of https://github.com/adamdruppe/arsd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

initial proxy support

This commit is contained in:
Adam D. Ruppe 2021-04-13 09:20:07 -04:00
parent 3d396dfaa6
commit 5ea0b493c6
1 changed files with 173 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
http2.d
View File

@ -680,6 +680,10 @@ struct BasicAuth {
string password; ///
}
class ProxyException : Exception {
this(string msg) {super(msg); }
}
/**
Represents a HTTP request. You usually create these through a [HttpClient].
@ -720,10 +724,11 @@ class HttpRequest {
}
///
this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds) {
this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) {
populateFromInfo(where, method);
setTimeout(timeout);
this.cache = cache;
this.proxy = proxy;
}
/++
@ -743,6 +748,16 @@ class HttpRequest {
private ICache cache;
/++
Proxy to use for this request. It should be a URL or `null`.
This must be sent before you call `send`.
History:
Added April 12, 2021 (dub v9.5)
+/
string proxy;
/// Final url after any redirections
string finalUrl;
@ -840,7 +855,21 @@ class HttpRequest {
string headers;
headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri;
headers ~= to!string(requestParameters.method);
headers ~= " ";
if(proxy.length && !requestParameters.ssl) {
// if we're doing a http proxy, we need to send a complete, absolute uri
// so reconstruct it
headers ~= "http://";
headers ~= requestParameters.host;
if(requestParameters.port != 80) {
headers ~= ":";
headers ~= to!string(requestParameters.port);
}
}
headers ~= requestParameters.uri;
if(requestParameters.useHttp11)
headers ~= " HTTP/1.1\r\n";
else
@ -927,7 +956,7 @@ class HttpRequest {
this.state = State.aborted;
this.responseData.code = 1;
this.responseData.codeText = "request.abort called";
// FIXME actually cancel it?
// the actual cancellation happens in the event loop
}
HttpRequestParameters requestParameters; ///
@ -967,8 +996,7 @@ class HttpRequest {
}
}
Socket getOpenSocketOnHost(string host, ushort port, bool ssl, string unixSocketPath) {
Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath) {
Socket openNewConnection() {
Socket socket;
if(ssl) {
@ -980,6 +1008,7 @@ class HttpRequest {
} else
socket = new Socket(family(unixSocketPath), SocketType.STREAM);
// FIXME: connect timeout?
if(unixSocketPath) {
import std.stdio; writeln(cast(ubyte[]) unixSocketPath);
socket.connect(new UnixAddress(unixSocketPath));
@ -987,7 +1016,51 @@ class HttpRequest {
// FIXME: i should prolly do ipv6 if available too.
if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough
throw new Exception("No host given for request");
socket.connect(new InternetAddress(host, port));
if(proxy.length) {
if(proxy.indexOf("//") == -1)
proxy = "http://" ~ proxy;
auto proxyurl = Uri(proxy);
// the precise types here are important to help with overload
// resolution of the devirtualized call!
Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80);
debug(arsd_http2) writeln("using proxy ", pa.toString());
// the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection
// using the parent class functions let us bypass the encryption
socket.Socket.connect(pa);
string message;
if(ssl) {
auto hostName = host ~ ":" ~ to!string(port);
message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n";
message ~= "Host: " ~ hostName ~ "\r\n";
if(proxyurl.userinfo.length) {
import std.base64;
message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n";
}
message ~= "\r\n";
// again gotta use plain
// FIXME: what if proxy times out? should be reasonably fast too.
socket.Socket.send(message, SocketFlags.NONE);
ubyte[1024] recvBuffer;
// and last time
auto rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE);
if(rcvGot == -1)
throw new ProxyException("proxy receive error");
auto got = cast(string) recvBuffer[0 .. rcvGot];
auto expect = "HTTP/1.1 200";
if(got[0 .. expect.length] != expect)
throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length]);
(cast(OpenSslSocket)socket).do_ssl_connect();
} else {
}
} else {
socket.connect(new InternetAddress(host, port));
}
}
debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket);
@ -1076,13 +1149,25 @@ class HttpRequest {
Socket socket;
try {
socket = getOpenSocketOnHost(pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath);
socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath);
} catch(ProxyException e) {
// connection refused or timed out (I should disambiguate somehow)...
pc.state = HttpRequest.State.aborted;
pc.responseData.code = 2;
pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy;
hadAbortedRequest = true;
removeFromPending[removeFromPendingCount++] = pc;
continue;
} catch(SocketException e) {
// connection refused or timed out (I should disambiguate somehow)...
pc.state = HttpRequest.State.aborted;
pc.responseData.code = 2;
pc.responseData.codeText = "connection failed";
pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed";
hadAbortedRequest = true;
@ -1103,6 +1188,18 @@ class HttpRequest {
foreach(rp; removeFromPending[0 .. removeFromPendingCount])
pending = pending.remove!((a) => a is rp)();
tryAgain:
Socket[16] inactive;
int inactiveCount = 0;
void killInactives() {
foreach(s; inactive[0 .. inactiveCount]) {
debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s);
activeRequestOnSocket.remove(s);
}
}
readSet.reset();
writeSet.reset();
@ -1113,6 +1210,15 @@ class HttpRequest {
// active requests need to be read or written to
foreach(sock, request; activeRequestOnSocket) {
if(request.state == State.aborted) {
inactive[inactiveCount++] = sock;
sock.close();
loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
hadAbortedRequest = true;
continue;
}
// check the other sockets just for EOF, if they close, take them out of our list,
// we'll reopen if needed upon request.
readSet.add(sock);
@ -1134,21 +1240,11 @@ class HttpRequest {
}
if(!hadOne) {
if(hadAbortedRequest)
if(hadAbortedRequest) {
killInactives();
return 0; // something got aborted, that's progress
return 2; // automatic timeout, nothing to do
}
tryAgain:
Socket[16] inactive;
int inactiveCount = 0;
void killInactives() {
foreach(s; inactive[0 .. inactiveCount]) {
debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s);
activeRequestOnSocket.remove(s);
}
return 2; // automatic timeout, nothing to do
}
auto selectGot = Socket.select(readSet, writeSet, null, minTimeout);
@ -1725,7 +1821,19 @@ class HttpClient {
(but will still save cookies btw... when that is implemented)
+/
HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) {
auto request = new HttpRequest(uri, method, cache, defaultTimeout);
string proxyToUse;
switch(uri.scheme) {
case "http":
proxyToUse = httpProxy;
break;
case "https":
proxyToUse = httpsProxy;
break;
default:
proxyToUse = null;
}
auto request = new HttpRequest(uri, method, cache, defaultTimeout, proxyToUse);
request.requestParameters.userAgent = userAgent;
request.requestParameters.authorization = authorization;
@ -1753,11 +1861,46 @@ class HttpClient {
this(ICache cache = null) {
this.cache = cache;
loadDefaultProxy();
}
// FIXME: add proxy
// FIXME: some kind of caching
/++
Loads the system-default proxy. Note that the constructor does this automatically
so you should rarely need to call this explicitly.
The environment variables are used, if present, on all operating systems.
History:
Added April 12, 2021 (included in dub v9.5)
Bugs:
On Windows, it does NOT currently check the IE settings, but I do intend to
implement that in the future. When I do, it will be classified as a bug fix,
NOT a breaking change.
+/
void loadDefaultProxy() {
import std.process;
httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null));
httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null));
// FIXME: on Windows, I should use the Internet Explorer proxy settings
}
/++
Proxies to use for requests. The [HttpClient] constructor will set these to the system values,
then you can reset it to `null` if you want to override and not use the proxy after all, or you
can set it after construction to whatever.
The proxy from the client will be automatically set to the requests performed through it. You can
also override on a per-request basis by creating the request and setting the `proxy` field there
before sending it.
History:
Added April 12, 2021 (included in dub v9.5)
+/
string httpProxy;
/// ditto
string httpsProxy;
///
void setCookie(string name, string value, string domain = null) {
@ -2254,6 +2397,11 @@ version(use_openssl) {
@trusted
override void connect(Address to) {
super.connect(to);
do_ssl_connect();
}
@trusted
void do_ssl_connect() {
if(SSL_connect(ssl) == -1) {
ERR_print_errors_fp(core.stdc.stdio.stderr);
int i;