This commit is contained in:
Adam D. Ruppe 2020-01-03 10:04:58 -05:00
parent 046d55e880
commit d3506d8b00
7 changed files with 288 additions and 179 deletions

View File

@ -113,7 +113,7 @@ bool processTar(
if(*bytesRemainingOnCurrentFile) {
bool isNew = *bytesRemainingOnCurrentFile == header.size();
if(*bytesRemainingOnCurrentFile < 512) {
handleData(header, isNew, true, dataBuffer[0 .. *bytesRemainingOnCurrentFile]);
handleData(header, isNew, true, dataBuffer[0 .. cast(size_t) *bytesRemainingOnCurrentFile]);
*bytesRemainingOnCurrentFile = 0;
} else {
handleData(header, isNew, false, dataBuffer[]);

6
cgi.d
View File

@ -9069,11 +9069,11 @@ bool apiDispatcher()(Cgi cgi) {
}
+/
/*
Copyright: Adam D. Ruppe, 2008 - 2019
License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
Copyright: Adam D. Ruppe, 2008 - 2020
License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0].
Authors: Adam D. Ruppe
Copyright Adam D. Ruppe 2008 - 2019.
Copyright Adam D. Ruppe 2008 - 2020.
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE_1_0.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)

71
dom.d
View File

@ -33,6 +33,21 @@
If you want it to stand alone, just always use the `Document.parseUtf8`
function or the constructor that takes a string.
Symbol_groups:
core_functionality =
These members provide core functionality. The members on these classes
will provide most your direct interaction.
bonus_functionality =
These provide additional functionality for special use cases.
implementations =
These provide implementations of other functionality.
+/
module arsd.dom;
@ -78,6 +93,7 @@ bool isConvenientAttribute(string name) {
/// The main document interface, including a html parser.
/// Group: core_functionality
class Document : FileResource {
/// Convenience method for web scraping. Requires [arsd.http2] to be
/// included in the build as well as [arsd.characterencodings].
@ -1423,6 +1439,7 @@ class Document : FileResource {
}
/// This represents almost everything in the DOM.
/// Group: core_functionality
class Element {
/// Returns a collection of elements by selector.
/// See: [Document.opIndex]
@ -3474,6 +3491,7 @@ class Element {
// FIXME: since Document loosens the input requirements, it should probably be the sub class...
/// Specializes Document for handling generic XML. (always uses strict mode, uses xml mime type and file header)
/// Group: core_functionality
class XmlDocument : Document {
this(string data) {
contentType = "text/xml; charset=utf-8";
@ -3491,6 +3509,7 @@ import std.string;
/* domconvenience follows { */
/// finds comments that match the given txt. Case insensitive, strips whitespace.
/// Group: core_functionality
Element[] findComments(Document document, string txt) {
return findComments(document.root, txt);
}
@ -3510,6 +3529,7 @@ Element[] findComments(Element element, string txt) {
}
/// An option type that propagates null. See: [Element.optionSelector]
/// Group: implementations
struct MaybeNullElement(SomeElementType) {
this(SomeElementType ele) {
this.element = ele;
@ -3543,6 +3563,7 @@ struct MaybeNullElement(SomeElementType) {
/++
A collection of elements which forwards methods to the children.
+/
/// Group: implementations
struct ElementCollection {
///
this(Element e) {
@ -3641,6 +3662,7 @@ struct ElementCollection {
/// this puts in operators and opDispatch to handle string indexes and properties, forwarding to get and set functions.
/// Group: implementations
mixin template JavascriptStyleDispatch() {
///
string opDispatch(string name)(string v = null) if(name != "popFront") { // popFront will make this look like a range. Do not want.
@ -3668,6 +3690,7 @@ mixin template JavascriptStyleDispatch() {
/// A proxy object to do the Element class' dataset property. See Element.dataset for more info.
///
/// Do not create this object directly.
/// Group: implementations
struct DataSet {
///
this(Element e) {
@ -3691,6 +3714,7 @@ struct DataSet {
}
/// Proxy object for attributes which will replace the main opDispatch eventually
/// Group: implementations
struct AttributeSet {
///
this(Element e) {
@ -3718,6 +3742,7 @@ struct AttributeSet {
/// for style, i want to be able to set it with a string like a plain attribute,
/// but also be able to do properties Javascript style.
/// Group: implementations
struct ElementStyle {
this(Element parent) {
_element = parent;
@ -3874,6 +3899,7 @@ import std.range;
Document implements this interface with type = text/html (see Document.contentType for more info)
and data = document.toString, so you can return Documents anywhere web.d expects FileResources.
+/
/// Group: bonus_functionality
interface FileResource {
/// the content-type of the file. e.g. "text/html; charset=utf-8" or "image/png"
@property string contentType() const;
@ -3885,10 +3911,12 @@ interface FileResource {
///.
/// Group: bonus_functionality
enum NodeType { Text = 3 }
/// You can use this to do an easy null check or a dynamic cast+null check on any element.
/// Group: core_functionality
T require(T = Element, string file = __FILE__, int line = __LINE__)(Element e) if(is(T : Element))
in {}
out(ret) { assert(ret !is null); }
@ -3901,6 +3929,7 @@ body {
///.
/// Group: core_functionality
class DocumentFragment : Element {
///.
this(Document _parentDocument) {
@ -3951,6 +3980,7 @@ class DocumentFragment : Element {
///
/// The output parameter can be given to append to an existing buffer. You don't have to
/// pass one; regardless, the return value will be usable for you, with just the data encoded.
/// Group: core_functionality
string htmlEntitiesEncode(string data, Appender!string output = appender!string(), bool encodeNonAscii = true) {
// if there's no entities, we can save a lot of time by not bothering with the
// decoding loop. This check cuts the net toString time by better than half in my test.
@ -4003,11 +4033,13 @@ string htmlEntitiesEncode(string data, Appender!string output = appender!string(
}
/// An alias for htmlEntitiesEncode; it works for xml too
/// Group: core_functionality
string xmlEntitiesEncode(string data) {
return htmlEntitiesEncode(data);
}
/// This helper function is used for decoding html entities. It has a hard-coded list of entities and characters.
/// Group: core_functionality
dchar parseEntity(in dchar[] entity) {
switch(entity[1..$-1]) {
case "quot":
@ -5505,6 +5537,7 @@ import std.stdio;
/// This takes a string of raw HTML and decodes the entities into a nice D utf-8 string.
/// By default, it uses loose mode - it will try to return a useful string from garbage input too.
/// Set the second parameter to true if you'd prefer it to strictly throw exceptions on garbage input.
/// Group: core_functionality
string htmlEntitiesDecode(string data, bool strict = false) {
// this check makes a *big* difference; about a 50% improvement of parse speed on my test.
if(data.indexOf("&") == -1) // all html entities begin with &
@ -5585,6 +5618,7 @@ string htmlEntitiesDecode(string data, bool strict = false) {
return cast(string) a; // assumeUnique is actually kinda slow, lol
}
/// Group: implementations
abstract class SpecialElement : Element {
this(Document _parentDocument) {
super(_parentDocument);
@ -5602,6 +5636,7 @@ abstract class SpecialElement : Element {
}
///.
/// Group: implementations
class RawSource : SpecialElement {
///.
this(Document _parentDocument, string s) {
@ -5634,6 +5669,7 @@ class RawSource : SpecialElement {
string source;
}
/// Group: implementations
abstract class ServerSideCode : SpecialElement {
this(Document _parentDocument, string type) {
super(_parentDocument);
@ -5663,6 +5699,7 @@ abstract class ServerSideCode : SpecialElement {
}
///.
/// Group: implementations
class PhpCode : ServerSideCode {
///.
this(Document _parentDocument, string s) {
@ -5676,6 +5713,7 @@ class PhpCode : ServerSideCode {
}
///.
/// Group: implementations
class AspCode : ServerSideCode {
///.
this(Document _parentDocument, string s) {
@ -5689,6 +5727,7 @@ class AspCode : ServerSideCode {
}
///.
/// Group: implementations
class BangInstruction : SpecialElement {
///.
this(Document _parentDocument, string s) {
@ -5728,6 +5767,7 @@ class BangInstruction : SpecialElement {
}
///.
/// Group: implementations
class QuestionInstruction : SpecialElement {
///.
this(Document _parentDocument, string s) {
@ -5768,6 +5808,7 @@ class QuestionInstruction : SpecialElement {
}
///.
/// Group: implementations
class HtmlComment : SpecialElement {
///.
this(Document _parentDocument, string s) {
@ -5811,6 +5852,7 @@ class HtmlComment : SpecialElement {
///.
/// Group: implementations
class TextNode : Element {
public:
///.
@ -5926,6 +5968,7 @@ class TextNode : Element {
*/
///.
/// Group: implementations
class Link : Element {
///.
@ -6064,6 +6107,7 @@ class Link : Element {
}
///.
/// Group: implementations
class Form : Element {
///.
@ -6314,6 +6358,7 @@ class Form : Element {
import std.conv;
///.
/// Group: implementations
class Table : Element {
///.
@ -6553,6 +6598,7 @@ class Table : Element {
}
/// Represents a table row element - a <tr>
/// Group: implementations
class TableRow : Element {
///.
this(Document _parentDocument) {
@ -6565,6 +6611,7 @@ class TableRow : Element {
}
/// Represents anything that can be a table cell - <td> or <th> html.
/// Group: implementations
class TableCell : Element {
///.
this(Document _parentDocument, string _tagName) {
@ -6601,6 +6648,7 @@ class TableCell : Element {
///.
/// Group: implementations
class MarkupException : Exception {
///.
@ -6610,6 +6658,7 @@ class MarkupException : Exception {
}
/// This is used when you are using one of the require variants of navigation, and no matching element can be found in the tree.
/// Group: implementations
class ElementNotFoundException : Exception {
/// type == kind of element you were looking for and search == a selector describing the search.
@ -6624,6 +6673,7 @@ class ElementNotFoundException : Exception {
/// The html struct is used to differentiate between regular text nodes and html in certain functions
///
/// Easiest way to construct it is like this: `auto html = Html("<p>hello</p>");`
/// Group: core_functionality
struct Html {
/// This string holds the actual html. Use it to retrieve the contents.
string source;
@ -7294,19 +7344,24 @@ int intFromHex(string hex) {
}
/++
Represents a parsed CSS selector.
Represents a parsed CSS selector. You never have to use this directly, but you can if you know it is going to be reused a lot to avoid a bit of repeat parsing.
See_Also:
[Element.querySelector]
[Element.querySelectorAll]
[Document.querySelector]
[Document.querySelectorAll]
$(LIST
* [Element.querySelector]
* [Element.querySelectorAll]
* [Element.matches]
* [Element.closest]
* [Document.querySelector]
* [Document.querySelectorAll]
)
+/
/// Group: core_functionality
struct Selector {
SelectorComponent[] components;
string original;
/++
Parses the selector string and returns the usable structure.
Parses the selector string and constructs the usable structure.
+/
this(string cssSelector) {
components = parseSelectorString(cssSelector);
@ -8709,11 +8764,11 @@ unittest {
}
/*
Copyright: Adam D. Ruppe, 2010 - 2019
Copyright: Adam D. Ruppe, 2010 - 2020
License: <a href="http://www.boost.org/LICENSE_1_0.txt">Boost License 1.0</a>.
Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others
Copyright Adam D. Ruppe 2010-2019.
Copyright Adam D. Ruppe 2010-2020.
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE_1_0.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)

View File

@ -3,7 +3,7 @@
"targetType": "library",
"importPaths": ["."],
"sourceFiles": ["package.d"],
"description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more.",
"description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more with a commitment to long-term compatibility and stability.",
"authors": ["Adam D. Ruppe"],
"license":"BSL-1.0",
"dependencies": {

338
http2.d
View File

@ -1,4 +1,4 @@
// Copyright 2013-2019, Adam D. Ruppe.
// Copyright 2013-2020, Adam D. Ruppe.
/++
This is version 2 of my http/1.1 client implementation.
@ -20,6 +20,8 @@ debug(arsd_http2_verbose) debug=arsd_http2;
debug(arsd_http2) import std.stdio : writeln;
version=arsd_http_internal_implementation;
version(without_openssl) {}
else {
version=use_openssl;
@ -28,6 +30,12 @@ version(older_openssl) {} else
version=newer_openssl;
}
version(arsd_http_winhttp_implementation) {
pragma(lib, "winhttp")
import core.sys.windows.winhttp;
// FIXME: alter the dub package file too
}
/++
@ -605,6 +613,176 @@ class HttpRequest {
/// Automatically follow a redirection?
bool followLocation = false;
this() {
}
///
this(Uri where, HttpVerb method) {
populateFromInfo(where, method);
}
/// Final url after any redirections
string finalUrl;
void populateFromInfo(Uri where, HttpVerb method) {
auto parts = where;
finalUrl = where.toString();
requestParameters.method = method;
requestParameters.host = parts.host;
requestParameters.port = cast(ushort) parts.port;
requestParameters.ssl = parts.scheme == "https";
if(parts.port == 0)
requestParameters.port = requestParameters.ssl ? 443 : 80;
requestParameters.uri = parts.path.length ? parts.path : "/";
if(parts.query.length) {
requestParameters.uri ~= "?";
requestParameters.uri ~= parts.query;
}
}
~this() {
}
ubyte[] sendBuffer;
HttpResponse responseData;
private HttpClient parentClient;
size_t bodyBytesSent;
size_t bodyBytesReceived;
State state_;
State state() { return state_; }
State state(State s) {
assert(state_ != State.complete);
return state_ = s;
}
/// Called when data is received. Check the state to see what data is available.
void delegate(HttpRequest) onDataReceived;
enum State {
/// The request has not yet been sent
unsent,
/// The send() method has been called, but no data is
/// sent on the socket yet because the connection is busy.
pendingAvailableConnection,
/// The headers are being sent now
sendingHeaders,
/// The body is being sent now
sendingBody,
/// The request has been sent but we haven't received any response yet
waitingForResponse,
/// We have received some data and are currently receiving headers
readingHeaders,
/// All headers are available but we're still waiting on the body
readingBody,
/// The request is complete.
complete,
/// The request is aborted, either by the abort() method, or as a result of the server disconnecting
aborted
}
/// Sends now and waits for the request to finish, returning the response.
HttpResponse perform() {
send();
return waitForCompletion();
}
/// Sends the request asynchronously.
void send() {
sendPrivate(true);
}
private void sendPrivate(bool advance) {
if(state != State.unsent && state != State.aborted)
return; // already sent
string headers;
headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri;
if(requestParameters.useHttp11)
headers ~= " HTTP/1.1\r\n";
else
headers ~= " HTTP/1.0\r\n";
headers ~= "Host: "~requestParameters.host~"\r\n";
if(requestParameters.userAgent.length)
headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n";
if(requestParameters.contentType.length)
headers ~= "Content-Type: "~requestParameters.contentType~"\r\n";
if(requestParameters.authorization.length)
headers ~= "Authorization: "~requestParameters.authorization~"\r\n";
if(requestParameters.bodyData.length)
headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n";
if(requestParameters.acceptGzip)
headers ~= "Accept-Encoding: gzip\r\n";
foreach(header; requestParameters.headers)
headers ~= header ~ "\r\n";
headers ~= "\r\n";
sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData;
// import std.stdio; writeln("******* ", sendBuffer);
responseData = HttpResponse.init;
responseData.requestParameters = requestParameters;
bodyBytesSent = 0;
bodyBytesReceived = 0;
state = State.pendingAvailableConnection;
bool alreadyPending = false;
foreach(req; pending)
if(req is this) {
alreadyPending = true;
break;
}
if(!alreadyPending) {
pending ~= this;
}
if(advance)
HttpRequest.advanceConnections();
}
/// Waits for the request to finish or timeout, whichever comes first.
HttpResponse waitForCompletion() {
while(state != State.aborted && state != State.complete) {
if(state == State.unsent)
send();
if(auto err = HttpRequest.advanceConnections())
throw new Exception("waitForCompletion got err " ~ to!string(err));
}
return responseData;
}
/// Aborts this request.
void abort() {
this.state = State.aborted;
// FIXME
}
HttpRequestParameters requestParameters; ///
version(arsd_http_winhttp_implementation) {
public static void resetInternals() {
}
static assert(0, "implementation not finished");
}
version(arsd_http_internal_implementation) {
private static {
// we manage the actual connections. When a request is made on a particular
// host, we try to reuse connections. We may open more than one connection per
@ -1170,165 +1348,7 @@ class HttpRequest {
return stillAlive;
}
this() {
}
///
this(Uri where, HttpVerb method) {
populateFromInfo(where, method);
}
/// Final url after any redirections
string finalUrl;
void populateFromInfo(Uri where, HttpVerb method) {
auto parts = where;
finalUrl = where.toString();
requestParameters.method = method;
requestParameters.host = parts.host;
requestParameters.port = cast(ushort) parts.port;
requestParameters.ssl = parts.scheme == "https";
if(parts.port == 0)
requestParameters.port = requestParameters.ssl ? 443 : 80;
requestParameters.uri = parts.path.length ? parts.path : "/";
if(parts.query.length) {
requestParameters.uri ~= "?";
requestParameters.uri ~= parts.query;
}
}
~this() {
}
ubyte[] sendBuffer;
HttpResponse responseData;
private HttpClient parentClient;
size_t bodyBytesSent;
size_t bodyBytesReceived;
State state_;
State state() { return state_; }
State state(State s) {
assert(state_ != State.complete);
return state_ = s;
}
/// Called when data is received. Check the state to see what data is available.
void delegate(HttpRequest) onDataReceived;
enum State {
/// The request has not yet been sent
unsent,
/// The send() method has been called, but no data is
/// sent on the socket yet because the connection is busy.
pendingAvailableConnection,
/// The headers are being sent now
sendingHeaders,
/// The body is being sent now
sendingBody,
/// The request has been sent but we haven't received any response yet
waitingForResponse,
/// We have received some data and are currently receiving headers
readingHeaders,
/// All headers are available but we're still waiting on the body
readingBody,
/// The request is complete.
complete,
/// The request is aborted, either by the abort() method, or as a result of the server disconnecting
aborted
}
/// Sends now and waits for the request to finish, returning the response.
HttpResponse perform() {
send();
return waitForCompletion();
}
/// Sends the request asynchronously.
void send() {
sendPrivate(true);
}
private void sendPrivate(bool advance) {
if(state != State.unsent && state != State.aborted)
return; // already sent
string headers;
headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri;
if(requestParameters.useHttp11)
headers ~= " HTTP/1.1\r\n";
else
headers ~= " HTTP/1.0\r\n";
headers ~= "Host: "~requestParameters.host~"\r\n";
if(requestParameters.userAgent.length)
headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n";
if(requestParameters.contentType.length)
headers ~= "Content-Type: "~requestParameters.contentType~"\r\n";
if(requestParameters.authorization.length)
headers ~= "Authorization: "~requestParameters.authorization~"\r\n";
if(requestParameters.bodyData.length)
headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n";
if(requestParameters.acceptGzip)
headers ~= "Accept-Encoding: gzip\r\n";
foreach(header; requestParameters.headers)
headers ~= header ~ "\r\n";
headers ~= "\r\n";
sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData;
// import std.stdio; writeln("******* ", sendBuffer);
responseData = HttpResponse.init;
responseData.requestParameters = requestParameters;
bodyBytesSent = 0;
bodyBytesReceived = 0;
state = State.pendingAvailableConnection;
bool alreadyPending = false;
foreach(req; pending)
if(req is this) {
alreadyPending = true;
break;
}
if(!alreadyPending) {
pending ~= this;
}
if(advance)
HttpRequest.advanceConnections();
}
/// Waits for the request to finish or timeout, whichever comes first.
HttpResponse waitForCompletion() {
while(state != State.aborted && state != State.complete) {
if(state == State.unsent)
send();
if(auto err = HttpRequest.advanceConnections())
throw new Exception("waitForCompletion got err " ~ to!string(err));
}
return responseData;
}
/// Aborts this request.
void abort() {
this.state = State.aborted;
// FIXME
}
HttpRequestParameters requestParameters; ///
}
///

46
jni.d
View File

@ -120,6 +120,24 @@
+/
module arsd.jni;
/*
New Java classes:
class Foo : extends!Bar {
mixin stuff;
}
mixin stuff;
The `extends` template creates a wrapper that calls the nonvirtual
methods, so `super()` just works.
receiving an object should perhaps always give a subclass that is javafied;
calls the virtuals, unless of course it is final.
dynamic downcasts of java objects will probably never work.
*/
/+
For interfaces:
@ -261,6 +279,7 @@ private string getJavaName(alias a)() {
version(WithClassLoadSupport) {
import arsd.declarativeloader;
/// translator.
void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, bool delegate(string className) classFilter = null) {
import std.zip;
import std.file;
@ -278,7 +297,7 @@ void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, Jav
}
}
inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) {
private inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) {
import std.string;
s ~= "."; // lol i suck
s = s.replace(".function.", ".function_.");
@ -286,12 +305,13 @@ inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) {
return s[0 .. $-1]; // god i am such a bad programmer
}
inout(char)[] fixupJavaClassName(inout(char)[] s) {
private inout(char)[] fixupJavaClassName(inout(char)[] s) {
if(s == "Throwable" || s == "Object" || s == "Exception" || s == "Error" || s == "TypeInfo")
s = cast(typeof(s)) "Java" ~ s;
return s;
}
/// For the translator
struct JavaTranslationConfig {
/// List the Java methods, imported to D.
bool doImports;
@ -299,8 +319,11 @@ struct JavaTranslationConfig {
bool doExports;
/// Put implementations inline. If false, this separates interface from impl for quicker builds with dmd -i.
bool inlineImplementations;
/// Treat native functions as imports, otherwise fills in as exports. Make sure doImports == true.
bool nativesAreImports = true;
}
/// translator.
void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) {
import std.file;
import std.path;
@ -370,10 +393,15 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output
dc ~= (isInterface ? "interface " : "final class ") ~ lastClassName ~ " : IJavaObject {\n";
foreach(method; cf.methodsListing) {
bool native = (method.flags & 0x0100) ? true : false;
if(native && !jtc.doExports)
continue;
if(!native && !jtc.doImports)
continue;
if(jtc.nativesAreImports) {
if(!jtc.doImports)
continue;
} else {
if(native && !jtc.doExports)
continue;
if(!native && !jtc.doImports)
continue;
}
auto port = native ? "@Export" : "@Import";
if(method.flags & 1) { // public
@ -2084,3 +2112,9 @@ union jvalue
jdouble d;
jobject l;
}
/*
Copyright 2019-2020, Adam D. Ruppe.
Boost license. or whatever.
Most work done in December 2019.
*/

View File

@ -459,7 +459,7 @@
I live in the eastern United States, so I will most likely not be around at night in
that US east timezone.
License: Copyright Adam D. Ruppe, 2011-2017. Released under the Boost Software License.
License: Copyright Adam D. Ruppe, 2011-2020. Released under the Boost Software License.
Building documentation: You may wish to use the `arsd.ddoc` file from my github with
building the documentation for simpledisplay yourself. It will give it a bit more style.