diff --git a/archive.d b/archive.d index 8d71883..7c3fb87 100644 --- a/archive.d +++ b/archive.d @@ -27,6 +27,12 @@ struct TarFile { } +/ +inout(char)[] upToZero(inout(char)[] a) { + int i = 0; + while(i < a.length && a[i]) i++; + return a[0 .. i]; +} + /++ A header of a file in the archive. This represents the @@ -56,8 +62,8 @@ struct TarFileHeader { const(char)[] filename() { import core.stdc.string; if(filenamePrefix_[0]) - return filenamePrefix_[0 .. strlen(filenamePrefix_.ptr)] ~ fileName_[0 .. strlen(fileName_.ptr)]; - return fileName_[0 .. strlen(fileName_.ptr)]; + return upToZero(filenamePrefix_[]) ~ upToZero(fileName_[]); + return upToZero(fileName_[]); } /// @@ -112,8 +118,8 @@ bool processTar( if(*bytesRemainingOnCurrentFile) { bool isNew = *bytesRemainingOnCurrentFile == header.size(); - if(*bytesRemainingOnCurrentFile < 512) { - handleData(header, isNew, true, dataBuffer[0 .. *bytesRemainingOnCurrentFile]); + if(*bytesRemainingOnCurrentFile <= 512) { + handleData(header, isNew, true, dataBuffer[0 .. cast(size_t) *bytesRemainingOnCurrentFile]); *bytesRemainingOnCurrentFile = 0; } else { handleData(header, isNew, false, dataBuffer[]); @@ -123,6 +129,8 @@ bool processTar( *header = *(cast(TarFileHeader*) dataBuffer.ptr); auto s = header.size(); *bytesRemainingOnCurrentFile = s; + if(header.type() == TarFileType.directory) + handleData(header, true, false, null); if(s == 0 && header.type == TarFileType.normal) return false; } @@ -1877,7 +1885,7 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) { switch(p.state) { - default: assert(0); + default: return ELzma2State.LZMA2_STATE_ERROR; case ELzma2State.LZMA2_STATE_CONTROL: p.control = b; if (p.control == 0) @@ -1928,7 +1936,6 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return ELzma2State.LZMA2_STATE_DATA; } } - return ELzma2State.LZMA2_STATE_ERROR; } static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, Byte *src, SizeT size) diff --git a/cgi.d b/cgi.d index 1d95f69..dcc1dc9 100644 --- a/cgi.d +++ b/cgi.d @@ -1586,6 +1586,7 @@ class Cgi { immutable(ubyte)[] data; void rdo(const(ubyte)[] d) { + //import std.stdio; writeln(d); sendAll(ir.source, d); } @@ -3370,6 +3371,8 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC try { fun(cgi); cgi.close(); + if(cgi.websocketMode) + closeConnection = true; } catch(ConnectionException ce) { closeConnection = true; } catch(Throwable t) { @@ -3665,6 +3668,8 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { try { fun(cgi); cgi.close(); + if(cgi.websocketMode) + closeConnection = true; } catch(ConnectionException ce) { // broken pipe or something, just abort the connection closeConnection = true; @@ -3947,6 +3952,7 @@ class BufferedInputRange { // we might have to grow the buffer if(minBytesToSettleFor > underlyingBuffer.length || view.length == underlyingBuffer.length) { if(allowGrowth) { + import std.stdio; writeln("growth"); auto viewStart = view.ptr - underlyingBuffer.ptr; size_t growth = 4096; // make sure we have enough for what we're being asked for @@ -3959,7 +3965,7 @@ class BufferedInputRange { } do { - auto freeSpace = underlyingBuffer[underlyingBuffer.ptr - view.ptr + view.length .. $]; + auto freeSpace = underlyingBuffer[view.ptr - underlyingBuffer.ptr + view.length .. $]; try_again: auto ret = source.receive(freeSpace); if(ret == Socket.ERROR) { @@ -3981,6 +3987,7 @@ class BufferedInputRange { return; } + //import std.stdio; writeln(view.ptr); writeln(underlyingBuffer.ptr); writeln(view.length, " ", ret, " = ", view.length + ret); view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; } while(view.length < minBytesToSettleFor); } @@ -3992,6 +3999,7 @@ class BufferedInputRange { /// You do not need to call this if you always want to wait for more data when you /// consume some. ubyte[] consume(size_t bytes) { + //import std.stdio; writeln("consuime ", bytes, "/", view.length); view = view[bytes > $ ? $ : bytes .. $]; if(view.length == 0) { view = underlyingBuffer[0 .. 0]; // go ahead and reuse the beginning @@ -4639,18 +4647,13 @@ version(cgi_with_websocket) { // note: this blocks WebSocketMessage recv() { // FIXME: should we automatically handle pings and pongs? - assert(!cgi.idlol.empty()); + if(cgi.idlol.empty()) + throw new Exception("remote side disconnected"); cgi.idlol.popFront(0); WebSocketMessage message; - auto info = cgi.idlol.front(); - - // FIXME: read should prolly take the whole range so it can request more if needed - // read should also go ahead and consume the range - message = WebSocketMessage.read(info); - - cgi.idlol.consume(info.length); + message = WebSocketMessage.read(cgi.idlol); return message; } @@ -4700,7 +4703,7 @@ version(cgi_with_websocket) { WebSocket acceptWebsocket(Cgi cgi) { assert(!cgi.closed); assert(!cgi.outputtedResponseData); - cgi.setResponseStatus("101 Web Socket Protocol Handshake"); + cgi.setResponseStatus("101 Switching Protocols"); cgi.header("Upgrade: WebSocket"); cgi.header("Connection: upgrade"); @@ -4834,7 +4837,15 @@ version(cgi_with_websocket) { cgi.flush(); } - static WebSocketMessage read(ubyte[] d) { + static WebSocketMessage read(BufferedInputRange ir) { + + auto d = ir.front(); + while(d.length < 2) { + ir.popFront(); + d = ir.front(); + } + auto start = d; + WebSocketMessage msg; assert(d.length >= 2); @@ -4882,7 +4893,11 @@ version(cgi_with_websocket) { d = d[4 .. $]; } - msg.data = d[0 .. $]; + //if(d.length < msg.realLength) { + + //} + msg.data = d[0 .. msg.realLength]; + d = d[msg.realLength .. $]; if(msg.masked) { // let's just unmask it now @@ -4896,6 +4911,8 @@ version(cgi_with_websocket) { } } + ir.consume(start.length - d.length); + return msg; } @@ -6427,6 +6444,7 @@ void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoS import core.sys.posix.poll; } + version(linux) eis.epoll_fd = epoll_fd; auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); @@ -9069,11 +9087,11 @@ bool apiDispatcher()(Cgi cgi) { } +/ /* -Copyright: Adam D. Ruppe, 2008 - 2019 -License: Boost License 1.0. +Copyright: Adam D. Ruppe, 2008 - 2020 +License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0]. Authors: Adam D. Ruppe - Copyright Adam D. Ruppe 2008 - 2019. + Copyright Adam D. Ruppe 2008 - 2020. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/declarativeloader.d b/declarativeloader.d index a7861a4..28299c2 100644 --- a/declarativeloader.d +++ b/declarativeloader.d @@ -5,26 +5,27 @@ module arsd.declarativeloader; import std.range; -// @VariableLength indicates the value is saved in a MIDI like format -// @BigEndian, @LittleEndian -// @NumBytes!Field or @NumElements!Field controls length of embedded arrays -// @Tagged!Field indicates a tagged union. Each struct within should have @Tag(X) which is a value of Field -// @MustBe() causes it to throw if not the given value - -// @NotSaved indicates a struct member that is not actually saved in the file - +/// enum BigEndian; +/// enum LittleEndian; +/// @VariableLength indicates the value is saved in a MIDI like format enum VariableLength; +/// @NumBytes!Field or @NumElements!Field controls length of embedded arrays struct NumBytes(alias field) {} +/// ditto struct NumElements(alias field) {} +/// @Tagged!Field indicates a tagged union. Each struct within should have @Tag(X) which is a value of Field struct Tagged(alias field) {} -struct TagStruct(T) { T t; } +/// ditto auto Tag(T)(T t) { return TagStruct!T(t); } -enum NotSaved; +struct TagStruct(T) { T t; } struct MustBeStruct(T) { T t; } +/// The marked field is not in the actual file +enum NotSaved; +/// Insists the field must be a certain value, like for magic numbers auto MustBe(T)(T t) { return MustBeStruct!T(t); } @@ -65,10 +66,18 @@ union N(ty) { ubyte[ty.sizeof] bytes; } -// input range of ubytes... +/// input range of ubytes... int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) { int bytesConsumed; + string currentItem; + + import std.conv; + scope(failure) + throw new Exception(T.stringof ~ "." ~ currentItem ~ " trouble " ~ to!string(t)); + ubyte next() { + if(r.empty) + throw new Exception(T.stringof ~ "." ~ currentItem ~ " trouble " ~ to!string(t)); auto bfr = r.front; r.popFront; bytesConsumed++; @@ -77,6 +86,7 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) bool endianness = bigEndian!T(assumeBigEndian); static foreach(memberName; __traits(allMembers, T)) {{ + currentItem = memberName; static if(is(typeof(__traits(getMember, T, memberName)))) { alias f = __traits(getMember, T, memberName); alias ty = typeof(f); @@ -114,11 +124,17 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) auto tag = __traits(getMember, t, tagField); // find the child of the union matching the tag... + bool found = false; static foreach(um; __traits(allMembers, ty)) { if(tag == getTag!(__traits(getMember, ty, um))) { bytesConsumed += loadFrom(__traits(getMember, __traits(getMember, t, memberName), um), r, endianness); + found = true; } } + if(!found) { + import std.format; + throw new Exception(format("found unknown union tag %s at %s", tag, t)); + } } else static if(is(ty == E[], E)) { static foreach(attr; __traits(getAttributes, f)) { static if(is(attr == NumBytes!Field, alias Field)) @@ -160,9 +176,19 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) } } else { while(numElementsRemaining) { + //import std.stdio; writeln(memberName); E piece; auto by = loadFrom(piece, r, endianness); numElementsRemaining--; + + // such a filthy hack, needed for Java's mistake though :( + static if(__traits(compiles, piece.takesTwoSlots())) { + if(piece.takesTwoSlots()) { + __traits(getMember, t, memberName) ~= piece; + numElementsRemaining--; + } + } + bytesConsumed += by; __traits(getMember, t, memberName) ~= piece; } diff --git a/dom.d b/dom.d index d31c48b..b3b20e8 100644 --- a/dom.d +++ b/dom.d @@ -1,8 +1,7 @@ -// FIXME: add classList +// FIXME: add classList. it is a live list and removes whitespace and duplicates when you use it. // FIXME: xml namespace support??? -// FIXME: add matchesSelector - standard name is `matches`. also `closest` walks up to find the parent that matches // FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML -// FIXME: appendChild should not fail if the thing already has a parent; it should just automatically remove it per standard. +// FIXME: parentElement is parentNode that skips DocumentFragment etc but will be hard to work in with my compatibility... // FIXME: the scriptable list is quite arbitrary @@ -34,6 +33,21 @@ If you want it to stand alone, just always use the `Document.parseUtf8` function or the constructor that takes a string. + + Symbol_groups: + + core_functionality = + + These members provide core functionality. The members on these classes + will provide most your direct interaction. + + bonus_functionality = + + These provide additional functionality for special use cases. + + implementations = + + These provide implementations of other functionality. +/ module arsd.dom; @@ -79,6 +93,7 @@ bool isConvenientAttribute(string name) { /// The main document interface, including a html parser. +/// Group: core_functionality class Document : FileResource { /// Convenience method for web scraping. Requires [arsd.http2] to be /// included in the build as well as [arsd.characterencodings]. @@ -1185,37 +1200,70 @@ class Document : FileResource { if( is(SomeElementType : Element)) out(ret) { assert(ret !is null); } body { - return root.requireSelector!(SomeElementType)(selector, file, line); + auto e = cast(SomeElementType) querySelector(selector); + if(e is null) + throw new ElementNotFoundException(SomeElementType.stringof, selector, this.root, file, line); + return e; } final MaybeNullElement!SomeElementType optionSelector(SomeElementType = Element)(string selector, string file = __FILE__, size_t line = __LINE__) if(is(SomeElementType : Element)) { - return root.optionSelector!(SomeElementType)(selector, file, line); + auto e = cast(SomeElementType) querySelector(selector); + return MaybeNullElement!SomeElementType(e); } - /// ditto + @scriptable Element querySelector(string selector) { - return root.querySelector(selector); + // see comment below on Document.querySelectorAll + auto s = Selector(selector);//, !loose); + foreach(ref comp; s.components) + if(comp.parts.length && comp.parts[0].separation == 0) + comp.parts[0].separation = -1; + foreach(e; s.getMatchingElementsLazy(this.root)) + return e; + return null; + } /// ditto + @scriptable Element[] querySelectorAll(string selector) { - return root.querySelectorAll(selector); + // In standards-compliant code, the document is slightly magical + // in that it is a pseudoelement at top level. It should actually + // match the root as one of its children. + // + // In versions of dom.d before Dec 29 2019, this worked because + // querySelectorAll was willing to return itself. With that bug fix + // (search "arbitrary id asduiwh" in this file for associated unittest) + // this would have failed. Hence adding back the root if it matches the + // selector itself. + // + // I'd love to do this better later. + + auto s = Selector(selector);//, !loose); + foreach(ref comp; s.components) + if(comp.parts.length && comp.parts[0].separation == 0) + comp.parts[0].separation = -1; + return s.getMatchingElements(this.root); } /// ditto + @scriptable + deprecated("use querySelectorAll instead") Element[] getElementsBySelector(string selector) { return root.getElementsBySelector(selector); } /// ditto + @scriptable Element[] getElementsByTagName(string tag) { return root.getElementsByTagName(tag); } /// ditto + @scriptable Element[] getElementsByClassName(string tag) { return root.getElementsByClassName(tag); } @@ -1395,6 +1443,7 @@ class Document : FileResource { } /// This represents almost everything in the DOM. +/// Group: core_functionality class Element { /// Returns a collection of elements by selector. /// See: [Document.opIndex] @@ -2181,10 +2230,37 @@ class Element { } /// a more standards-compliant alias for getElementsBySelector + @scriptable Element[] querySelectorAll(string selector) { return getElementsBySelector(selector); } + /// If the element matches the given selector. Previously known as `matchesSelector`. + @scriptable + bool matches(string selector) { + /+ + bool caseSensitiveTags = true; + if(parentDocument && parentDocument.loose) + caseSensitiveTags = false; + +/ + + Selector s = Selector(selector); + return s.matchesElement(this); + } + + /// Returns itself or the closest parent that matches the given selector, or null if none found + /// See_also: https://developer.mozilla.org/en-US/docs/Web/API/Element/closest + @scriptable + Element closest(string selector) { + Element e = this; + while(e !is null) { + if(e.matches(selector)) + return e; + e = e.parentNode; + } + return null; + } + /** Returns elements that match the given CSS selector @@ -2543,11 +2619,17 @@ class Element { } - /// Appends the given element to this one. The given element must not have a parent already. + /++ + Appends the given element to this one. If it already has a parent, it is removed from that tree and moved to this one. + + See_also: https://developer.mozilla.org/en-US/docs/Web/API/Node/appendChild + + History: + Prior to 1 Jan 2020 (git tag v4.4.1 and below), it required that the given element must not have a parent already. This was in violation of standard, so it changed the behavior to remove it from the existing parent and instead move it here. + +/ Element appendChild(Element e) in { assert(e !is null); - assert(e.parentNode is null, e.parentNode.toString); } out (ret) { assert((cast(DocumentFragment) this !is null) || (e.parentNode is this), e.toString);// e.parentNode ? e.parentNode.toString : "null"); @@ -2555,6 +2637,9 @@ class Element { assert(e is ret); } body { + if(e.parentNode !is null) + e.parentNode.removeChild(e); + selfClosed = false; e.parentNode = this; e.parentDocument = this.parentDocument; @@ -3410,6 +3495,7 @@ class Element { // FIXME: since Document loosens the input requirements, it should probably be the sub class... /// Specializes Document for handling generic XML. (always uses strict mode, uses xml mime type and file header) +/// Group: core_functionality class XmlDocument : Document { this(string data) { contentType = "text/xml; charset=utf-8"; @@ -3427,6 +3513,7 @@ import std.string; /* domconvenience follows { */ /// finds comments that match the given txt. Case insensitive, strips whitespace. +/// Group: core_functionality Element[] findComments(Document document, string txt) { return findComments(document.root, txt); } @@ -3446,6 +3533,7 @@ Element[] findComments(Element element, string txt) { } /// An option type that propagates null. See: [Element.optionSelector] +/// Group: implementations struct MaybeNullElement(SomeElementType) { this(SomeElementType ele) { this.element = ele; @@ -3479,6 +3567,7 @@ struct MaybeNullElement(SomeElementType) { /++ A collection of elements which forwards methods to the children. +/ +/// Group: implementations struct ElementCollection { /// this(Element e) { @@ -3577,6 +3666,7 @@ struct ElementCollection { /// this puts in operators and opDispatch to handle string indexes and properties, forwarding to get and set functions. +/// Group: implementations mixin template JavascriptStyleDispatch() { /// string opDispatch(string name)(string v = null) if(name != "popFront") { // popFront will make this look like a range. Do not want. @@ -3604,6 +3694,7 @@ mixin template JavascriptStyleDispatch() { /// A proxy object to do the Element class' dataset property. See Element.dataset for more info. /// /// Do not create this object directly. +/// Group: implementations struct DataSet { /// this(Element e) { @@ -3627,6 +3718,7 @@ struct DataSet { } /// Proxy object for attributes which will replace the main opDispatch eventually +/// Group: implementations struct AttributeSet { /// this(Element e) { @@ -3654,6 +3746,7 @@ struct AttributeSet { /// for style, i want to be able to set it with a string like a plain attribute, /// but also be able to do properties Javascript style. +/// Group: implementations struct ElementStyle { this(Element parent) { _element = parent; @@ -3810,6 +3903,7 @@ import std.range; Document implements this interface with type = text/html (see Document.contentType for more info) and data = document.toString, so you can return Documents anywhere web.d expects FileResources. +/ +/// Group: bonus_functionality interface FileResource { /// the content-type of the file. e.g. "text/html; charset=utf-8" or "image/png" @property string contentType() const; @@ -3821,10 +3915,12 @@ interface FileResource { ///. +/// Group: bonus_functionality enum NodeType { Text = 3 } /// You can use this to do an easy null check or a dynamic cast+null check on any element. +/// Group: core_functionality T require(T = Element, string file = __FILE__, int line = __LINE__)(Element e) if(is(T : Element)) in {} out(ret) { assert(ret !is null); } @@ -3837,6 +3933,7 @@ body { ///. +/// Group: core_functionality class DocumentFragment : Element { ///. this(Document _parentDocument) { @@ -3887,6 +3984,7 @@ class DocumentFragment : Element { /// /// The output parameter can be given to append to an existing buffer. You don't have to /// pass one; regardless, the return value will be usable for you, with just the data encoded. +/// Group: core_functionality string htmlEntitiesEncode(string data, Appender!string output = appender!string(), bool encodeNonAscii = true) { // if there's no entities, we can save a lot of time by not bothering with the // decoding loop. This check cuts the net toString time by better than half in my test. @@ -3939,11 +4037,13 @@ string htmlEntitiesEncode(string data, Appender!string output = appender!string( } /// An alias for htmlEntitiesEncode; it works for xml too +/// Group: core_functionality string xmlEntitiesEncode(string data) { return htmlEntitiesEncode(data); } /// This helper function is used for decoding html entities. It has a hard-coded list of entities and characters. +/// Group: core_functionality dchar parseEntity(in dchar[] entity) { switch(entity[1..$-1]) { case "quot": @@ -5441,6 +5541,7 @@ import std.stdio; /// This takes a string of raw HTML and decodes the entities into a nice D utf-8 string. /// By default, it uses loose mode - it will try to return a useful string from garbage input too. /// Set the second parameter to true if you'd prefer it to strictly throw exceptions on garbage input. +/// Group: core_functionality string htmlEntitiesDecode(string data, bool strict = false) { // this check makes a *big* difference; about a 50% improvement of parse speed on my test. if(data.indexOf("&") == -1) // all html entities begin with & @@ -5521,6 +5622,7 @@ string htmlEntitiesDecode(string data, bool strict = false) { return cast(string) a; // assumeUnique is actually kinda slow, lol } +/// Group: implementations abstract class SpecialElement : Element { this(Document _parentDocument) { super(_parentDocument); @@ -5538,6 +5640,7 @@ abstract class SpecialElement : Element { } ///. +/// Group: implementations class RawSource : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5570,6 +5673,7 @@ class RawSource : SpecialElement { string source; } +/// Group: implementations abstract class ServerSideCode : SpecialElement { this(Document _parentDocument, string type) { super(_parentDocument); @@ -5599,6 +5703,7 @@ abstract class ServerSideCode : SpecialElement { } ///. +/// Group: implementations class PhpCode : ServerSideCode { ///. this(Document _parentDocument, string s) { @@ -5612,6 +5717,7 @@ class PhpCode : ServerSideCode { } ///. +/// Group: implementations class AspCode : ServerSideCode { ///. this(Document _parentDocument, string s) { @@ -5625,6 +5731,7 @@ class AspCode : ServerSideCode { } ///. +/// Group: implementations class BangInstruction : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5664,6 +5771,7 @@ class BangInstruction : SpecialElement { } ///. +/// Group: implementations class QuestionInstruction : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5704,6 +5812,7 @@ class QuestionInstruction : SpecialElement { } ///. +/// Group: implementations class HtmlComment : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5747,6 +5856,7 @@ class HtmlComment : SpecialElement { ///. +/// Group: implementations class TextNode : Element { public: ///. @@ -5862,6 +5972,7 @@ class TextNode : Element { */ ///. +/// Group: implementations class Link : Element { ///. @@ -6000,6 +6111,7 @@ class Link : Element { } ///. +/// Group: implementations class Form : Element { ///. @@ -6250,6 +6362,7 @@ class Form : Element { import std.conv; ///. +/// Group: implementations class Table : Element { ///. @@ -6489,6 +6602,7 @@ class Table : Element { } /// Represents a table row element - a +/// Group: implementations class TableRow : Element { ///. this(Document _parentDocument) { @@ -6501,6 +6615,7 @@ class TableRow : Element { } /// Represents anything that can be a table cell - or html. +/// Group: implementations class TableCell : Element { ///. this(Document _parentDocument, string _tagName) { @@ -6537,6 +6652,7 @@ class TableCell : Element { ///. +/// Group: implementations class MarkupException : Exception { ///. @@ -6546,6 +6662,7 @@ class MarkupException : Exception { } /// This is used when you are using one of the require variants of navigation, and no matching element can be found in the tree. +/// Group: implementations class ElementNotFoundException : Exception { /// type == kind of element you were looking for and search == a selector describing the search. @@ -6560,6 +6677,7 @@ class ElementNotFoundException : Exception { /// The html struct is used to differentiate between regular text nodes and html in certain functions /// /// Easiest way to construct it is like this: `auto html = Html("

hello

");` +/// Group: core_functionality struct Html { /// This string holds the actual html. Use it to retrieve the contents. string source; @@ -7160,6 +7278,7 @@ int intFromHex(string hex) { } auto part = parts[0]; + //writeln("checking ", part, " against ", start, " with ", part.separation); switch(part.separation) { default: assert(0); case -1: @@ -7229,19 +7348,24 @@ int intFromHex(string hex) { } /++ - Represents a parsed CSS selector. + Represents a parsed CSS selector. You never have to use this directly, but you can if you know it is going to be reused a lot to avoid a bit of repeat parsing. See_Also: - [Element.querySelector] - [Element.querySelectorAll] - [Document.querySelector] - [Document.querySelectorAll] + $(LIST + * [Element.querySelector] + * [Element.querySelectorAll] + * [Element.matches] + * [Element.closest] + * [Document.querySelector] + * [Document.querySelectorAll] + ) +/ + /// Group: core_functionality struct Selector { SelectorComponent[] components; string original; /++ - Parses the selector string and returns the usable structure. + Parses the selector string and constructs the usable structure. +/ this(string cssSelector) { components = parseSelectorString(cssSelector); @@ -7336,9 +7460,25 @@ int intFromHex(string hex) { if(e is null) return false; Element where = e; int lastSeparation = -1; - foreach(part; retro(parts)) { - // writeln("matching ", where, " with ", part, " via ", lastSeparation); + auto lparts = parts; + + if(parts.length && parts[0].separation > 0) { + // if it starts with a non-trivial separator, inject + // a "*" matcher to act as a root. for cases like document.querySelector("> body") + // which implies html + + // there is probably a MUCH better way to do this. + auto dummy = SelectorPart.init; + dummy.tagNameFilter = "*"; + dummy.separation = 0; + lparts = dummy ~ lparts; + } + + foreach(part; retro(lparts)) { + + // writeln("matching ", where, " with ", part, " via ", lastSeparation); + // writeln(parts); if(lastSeparation == -1) { if(!part.matchElement(where)) @@ -7346,6 +7486,7 @@ int intFromHex(string hex) { } else if(lastSeparation == 0) { // generic parent // need to go up the whole chain where = where.parentNode; + while(where !is null) { if(part.matchElement(where)) break; @@ -7476,6 +7617,8 @@ int intFromHex(string hex) { if(current.isCleanSlateExceptSeparation()) { current.tagNameFilter = token; + // default thing, see comment under "*" below + if(current.separation == -1) current.separation = 0; } else { // if it was already set, we must see two thingies // separated by whitespace... @@ -7488,6 +7631,10 @@ int intFromHex(string hex) { switch(token) { case "*": current.tagNameFilter = "*"; + // the idea here is if we haven't actually set a separation + // yet (e.g. the > operator), it should assume the generic + // whitespace (descendant) mode to avoid matching self with -1 + if(current.separation == -1) current.separation = 0; break; case " ": // If some other separation has already been set, @@ -7520,16 +7667,20 @@ int intFromHex(string hex) { break; case "[": state = State.ReadingAttributeSelector; + if(current.separation == -1) current.separation = 0; break; case ".": state = State.ReadingClass; + if(current.separation == -1) current.separation = 0; break; case "#": state = State.ReadingId; + if(current.separation == -1) current.separation = 0; break; case ":": case "::": state = State.ReadingPseudoClass; + if(current.separation == -1) current.separation = 0; break; default: @@ -8542,18 +8693,6 @@ private bool isSimpleWhite(dchar c) { return c == ' ' || c == '\r' || c == '\n' || c == '\t'; } -/* -Copyright: Adam D. Ruppe, 2010 - 2019 -License: Boost License 1.0. -Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others - - Copyright Adam D. Ruppe 2010-2019. -Distributed under the Boost Software License, Version 1.0. - (See accompanying file LICENSE_1_0.txt or copy at - http://www.boost.org/LICENSE_1_0.txt) -*/ - - unittest { // Test for issue #120 string s = ` @@ -8570,3 +8709,73 @@ unittest { s2.indexOf("bubbles") < s2.indexOf("giggles"), "paragraph order incorrect:\n" ~ s2); } + +unittest { + // test for suncarpet email dec 24 2019 + // arbitrary id asduiwh + auto document = new Document(" + + + Element.querySelector Test + + +
+
Foo
+
Bar
+
+ +"); + + auto doc = document; + + assert(doc.querySelectorAll("div div").length == 2); + assert(doc.querySelector("div").querySelectorAll("div").length == 2); + assert(doc.querySelectorAll("> html").length == 0); + assert(doc.querySelector("head").querySelectorAll("> title").length == 1); + assert(doc.querySelector("head").querySelectorAll("> meta[charset]").length == 1); + + + assert(doc.root.matches("html")); + assert(!doc.root.matches("nothtml")); + assert(doc.querySelector("#foo > div").matches("div")); + assert(doc.querySelector("body > #foo").matches("#foo")); + + assert(doc.root.querySelectorAll(":root > body").length == 0); // the root has no CHILD root! + assert(doc.querySelectorAll(":root > body").length == 1); // but the DOCUMENT does + assert(doc.querySelectorAll(" > body").length == 1); // should mean the same thing + assert(doc.root.querySelectorAll(" > body").length == 1); // the root of HTML has this + assert(doc.root.querySelectorAll(" > html").length == 0); // but not this +} + +unittest { + // based on https://developer.mozilla.org/en-US/docs/Web/API/Element/closest example + auto document = new Document(`
+
Here is div-01 +
Here is div-02 +
Here is div-03
+
+
+
`, true, true); + + auto el = document.getElementById("div-03"); + assert(el.closest("#div-02").id == "div-02"); + assert(el.closest("div div").id == "div-03"); + assert(el.closest("article > div").id == "div-01"); + assert(el.closest(":not(div)").tagName == "article"); + + assert(el.closest("p") is null); + assert(el.closest("p, div") is el); +} + +/* +Copyright: Adam D. Ruppe, 2010 - 2020 +License: Boost License 1.0. +Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others + + Copyright Adam D. Ruppe 2010-2020. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + + diff --git a/dub.json b/dub.json index 2070dab..812dd0f 100644 --- a/dub.json +++ b/dub.json @@ -3,7 +3,7 @@ "targetType": "library", "importPaths": ["."], "sourceFiles": ["package.d"], - "description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more.", + "description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more with a commitment to long-term compatibility and stability.", "authors": ["Adam D. Ruppe"], "license":"BSL-1.0", "dependencies": { diff --git a/htmltotext.d b/htmltotext.d index 8410680..3fd98a7 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -468,6 +468,6 @@ class HtmlConverter { /// string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { auto converter = new HtmlConverter(); - return converter.convert(html, true, wrapAmount); + return converter.convert(html, wantWordWrap, wrapAmount); } diff --git a/http2.d b/http2.d index afc23fc..4bd2ada 100644 --- a/http2.d +++ b/http2.d @@ -1,4 +1,4 @@ -// Copyright 2013-2019, Adam D. Ruppe. +// Copyright 2013-2020, Adam D. Ruppe. /++ This is version 2 of my http/1.1 client implementation. @@ -14,12 +14,16 @@ +/ module arsd.http2; +// FIXME: I think I want to disable sigpipe here too. + import std.uri : encodeComponent; debug(arsd_http2_verbose) debug=arsd_http2; debug(arsd_http2) import std.stdio : writeln; +version=arsd_http_internal_implementation; + version(without_openssl) {} else { version=use_openssl; @@ -28,6 +32,12 @@ version(older_openssl) {} else version=newer_openssl; } +version(arsd_http_winhttp_implementation) { + pragma(lib, "winhttp") + import core.sys.windows.winhttp; + // FIXME: alter the dub package file too +} + /++ @@ -605,6 +615,178 @@ class HttpRequest { /// Automatically follow a redirection? bool followLocation = false; + this() { + } + + /// + this(Uri where, HttpVerb method) { + populateFromInfo(where, method); + } + + /// Final url after any redirections + string finalUrl; + + void populateFromInfo(Uri where, HttpVerb method) { + auto parts = where; + finalUrl = where.toString(); + requestParameters.method = method; + requestParameters.host = parts.host; + requestParameters.port = cast(ushort) parts.port; + requestParameters.ssl = parts.scheme == "https"; + if(parts.port == 0) + requestParameters.port = requestParameters.ssl ? 443 : 80; + requestParameters.uri = parts.path.length ? parts.path : "/"; + if(parts.query.length) { + requestParameters.uri ~= "?"; + requestParameters.uri ~= parts.query; + } + } + + ~this() { + } + + ubyte[] sendBuffer; + + HttpResponse responseData; + private HttpClient parentClient; + + size_t bodyBytesSent; + size_t bodyBytesReceived; + + State state_; + State state() { return state_; } + State state(State s) { + assert(state_ != State.complete); + return state_ = s; + } + /// Called when data is received. Check the state to see what data is available. + void delegate(HttpRequest) onDataReceived; + + enum State { + /// The request has not yet been sent + unsent, + + /// The send() method has been called, but no data is + /// sent on the socket yet because the connection is busy. + pendingAvailableConnection, + + /// The headers are being sent now + sendingHeaders, + + /// The body is being sent now + sendingBody, + + /// The request has been sent but we haven't received any response yet + waitingForResponse, + + /// We have received some data and are currently receiving headers + readingHeaders, + + /// All headers are available but we're still waiting on the body + readingBody, + + /// The request is complete. + complete, + + /// The request is aborted, either by the abort() method, or as a result of the server disconnecting + aborted + } + + /// Sends now and waits for the request to finish, returning the response. + HttpResponse perform() { + send(); + return waitForCompletion(); + } + + /// Sends the request asynchronously. + void send() { + sendPrivate(true); + } + + private void sendPrivate(bool advance) { + if(state != State.unsent && state != State.aborted) + return; // already sent + string headers; + + headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; + if(requestParameters.useHttp11) + headers ~= " HTTP/1.1\r\n"; + else + headers ~= " HTTP/1.0\r\n"; + headers ~= "Host: "~requestParameters.host~"\r\n"; + if(requestParameters.userAgent.length) + headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; + if(requestParameters.contentType.length) + headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; + if(requestParameters.authorization.length) + headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; + if(requestParameters.bodyData.length) + headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; + if(requestParameters.acceptGzip) + headers ~= "Accept-Encoding: gzip\r\n"; + if(requestParameters.keepAlive) + headers ~= "Connection: keep-alive\r\n"; + + foreach(header; requestParameters.headers) + headers ~= header ~ "\r\n"; + + headers ~= "\r\n"; + + sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; + + // import std.stdio; writeln("******* ", sendBuffer); + + responseData = HttpResponse.init; + responseData.requestParameters = requestParameters; + bodyBytesSent = 0; + bodyBytesReceived = 0; + state = State.pendingAvailableConnection; + + bool alreadyPending = false; + foreach(req; pending) + if(req is this) { + alreadyPending = true; + break; + } + if(!alreadyPending) { + pending ~= this; + } + + if(advance) + HttpRequest.advanceConnections(); + } + + + /// Waits for the request to finish or timeout, whichever comes first. + HttpResponse waitForCompletion() { + while(state != State.aborted && state != State.complete) { + if(state == State.unsent) + send(); + if(auto err = HttpRequest.advanceConnections()) + throw new Exception("waitForCompletion got err " ~ to!string(err)); + } + + return responseData; + } + + /// Aborts this request. + void abort() { + this.state = State.aborted; + // FIXME + } + + HttpRequestParameters requestParameters; /// + + version(arsd_http_winhttp_implementation) { + public static void resetInternals() { + + } + + static assert(0, "implementation not finished"); + } + + + version(arsd_http_internal_implementation) { private static { // we manage the actual connections. When a request is made on a particular // host, we try to reuse connections. We may open more than one connection per @@ -914,6 +1096,9 @@ class HttpRequest { if(colon == -1) return; auto name = header[0 .. colon]; + if(colon + 1 == header.length) + return; // empty header, idk + assert(colon + 2 < header.length, header); auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space switch(name) { @@ -1170,165 +1355,7 @@ class HttpRequest { return stillAlive; } - this() { } - - /// - this(Uri where, HttpVerb method) { - populateFromInfo(where, method); - } - - /// Final url after any redirections - string finalUrl; - - void populateFromInfo(Uri where, HttpVerb method) { - auto parts = where; - finalUrl = where.toString(); - requestParameters.method = method; - requestParameters.host = parts.host; - requestParameters.port = cast(ushort) parts.port; - requestParameters.ssl = parts.scheme == "https"; - if(parts.port == 0) - requestParameters.port = requestParameters.ssl ? 443 : 80; - requestParameters.uri = parts.path.length ? parts.path : "/"; - if(parts.query.length) { - requestParameters.uri ~= "?"; - requestParameters.uri ~= parts.query; - } - } - - ~this() { - } - - ubyte[] sendBuffer; - - HttpResponse responseData; - private HttpClient parentClient; - - size_t bodyBytesSent; - size_t bodyBytesReceived; - - State state_; - State state() { return state_; } - State state(State s) { - assert(state_ != State.complete); - return state_ = s; - } - /// Called when data is received. Check the state to see what data is available. - void delegate(HttpRequest) onDataReceived; - - enum State { - /// The request has not yet been sent - unsent, - - /// The send() method has been called, but no data is - /// sent on the socket yet because the connection is busy. - pendingAvailableConnection, - - /// The headers are being sent now - sendingHeaders, - - /// The body is being sent now - sendingBody, - - /// The request has been sent but we haven't received any response yet - waitingForResponse, - - /// We have received some data and are currently receiving headers - readingHeaders, - - /// All headers are available but we're still waiting on the body - readingBody, - - /// The request is complete. - complete, - - /// The request is aborted, either by the abort() method, or as a result of the server disconnecting - aborted - } - - /// Sends now and waits for the request to finish, returning the response. - HttpResponse perform() { - send(); - return waitForCompletion(); - } - - /// Sends the request asynchronously. - void send() { - sendPrivate(true); - } - - private void sendPrivate(bool advance) { - if(state != State.unsent && state != State.aborted) - return; // already sent - string headers; - - headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; - if(requestParameters.useHttp11) - headers ~= " HTTP/1.1\r\n"; - else - headers ~= " HTTP/1.0\r\n"; - headers ~= "Host: "~requestParameters.host~"\r\n"; - if(requestParameters.userAgent.length) - headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; - if(requestParameters.contentType.length) - headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; - if(requestParameters.authorization.length) - headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; - if(requestParameters.bodyData.length) - headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; - if(requestParameters.acceptGzip) - headers ~= "Accept-Encoding: gzip\r\n"; - - foreach(header; requestParameters.headers) - headers ~= header ~ "\r\n"; - - headers ~= "\r\n"; - - sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; - - // import std.stdio; writeln("******* ", sendBuffer); - - responseData = HttpResponse.init; - responseData.requestParameters = requestParameters; - bodyBytesSent = 0; - bodyBytesReceived = 0; - state = State.pendingAvailableConnection; - - bool alreadyPending = false; - foreach(req; pending) - if(req is this) { - alreadyPending = true; - break; - } - if(!alreadyPending) { - pending ~= this; - } - - if(advance) - HttpRequest.advanceConnections(); - } - - - /// Waits for the request to finish or timeout, whichever comes first. - HttpResponse waitForCompletion() { - while(state != State.aborted && state != State.complete) { - if(state == State.unsent) - send(); - if(auto err = HttpRequest.advanceConnections()) - throw new Exception("waitForCompletion got err " ~ to!string(err)); - } - - return responseData; - } - - /// Aborts this request. - void abort() { - this.state = State.aborted; - // FIXME - } - - HttpRequestParameters requestParameters; /// } /// @@ -1338,6 +1365,7 @@ struct HttpRequestParameters { // debugging bool useHttp11 = true; /// bool acceptGzip = true; /// + bool keepAlive = true; /// // the request itself HttpVerb method; /// @@ -1403,6 +1431,7 @@ class HttpClient { /* Protocol restrictions, useful to disable when debugging servers */ bool useHttp11 = true; /// bool acceptGzip = true; /// + bool keepAlive = true; /// /// @property Uri location() { @@ -1425,6 +1454,7 @@ class HttpClient { request.requestParameters.useHttp11 = this.useHttp11; request.requestParameters.acceptGzip = this.acceptGzip; + request.requestParameters.keepAlive = this.keepAlive; return request; } @@ -1442,6 +1472,7 @@ class HttpClient { request.requestParameters.useHttp11 = this.useHttp11; request.requestParameters.acceptGzip = this.acceptGzip; + request.requestParameters.keepAlive = this.keepAlive; request.requestParameters.bodyData = bodyData; request.requestParameters.contentType = contentType; @@ -1638,8 +1669,8 @@ version(use_openssl) { if(SSL_connect(ssl) == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl connect"); } } @@ -1651,8 +1682,8 @@ version(use_openssl) { if(retval == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl send"); } return retval; @@ -1667,8 +1698,8 @@ version(use_openssl) { if(retval == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl send"); } return retval; @@ -2014,3 +2045,905 @@ class FormData { } } +private bool bicmp(in ubyte[] item, in char[] search) { + if(item.length != search.length) return false; + + foreach(i; 0 .. item.length) { + ubyte a = item[i]; + ubyte b = search[i]; + if(a >= 'A' && a <= 'Z') + a += 32; + //if(b >= 'A' && b <= 'Z') + //b += 32; + if(a != b) + return false; + } + + return true; +} + +/++ + WebSocket client, based on the browser api, though also with other api options. + + --- + auto ws = new WebSocket(URI("ws://....")); + + ws.onmessage = (in char[] msg) { + ws.send("a reply"); + }; + + ws.connect(); + + WebSocket.eventLoop(); + --- + + Symbol_groups: + foundational = + Used with all API styles. + + browser_api = + API based on the standard in the browser. + + event_loop_integration = + Integrating with external event loops is done through static functions. You should + call these BEFORE doing anything else with the WebSocket module or class. + + $(PITFALL NOT IMPLEMENTED) + --- + WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); + // or something like that. it is not implemented yet. + --- + $(PITFALL NOT IMPLEMENTED) + + blocking_api = + The blocking API is best used when you only need basic functionality with a single connection. + + --- + WebSocketFrame msg; + do { + // FIXME good demo + } while(msg); + --- + + Or to check for blocks before calling: + + --- + try_to_process_more: + while(ws.isMessageBuffered()) { + auto msg = ws.waitForNextMessage(); + // process msg + } + if(ws.isDataPending()) { + ws.lowLevelReceive(); + goto try_to_process_more; + } else { + // nothing ready, you can do other things + // or at least sleep a while before trying + // to process more. + if(ws.readyState == WebSocket.OPEN) { + Thread.sleep(1.seconds); + goto try_to_process_more; + } + } + --- + ++/ +class WebSocket { + private Uri uri; + private string[string] cookies; + private string origin; + + private string host; + private ushort port; + private bool ssl; + + private int readyState_; + + private Socket socket; + private ubyte[] receiveBuffer; + private size_t receiveBufferUsedLength; + + private Config config; + + enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. + enum OPEN = 1; /// The connection is open and ready to communicate. + enum CLOSING = 2; /// The connection is in the process of closing. + enum CLOSED = 3; /// The connection is closed or couldn't be opened. + + /++ + + +/ + /// Group: foundational + static struct Config { + /++ + These control the size of the receive buffer. + + It starts at the initial size, will temporarily + balloon up to the maximum size, and will reuse + a buffer up to the likely size. + + Anything larger than the maximum size will cause + the connection to be aborted and an exception thrown. + This is to protect you against a peer trying to + exhaust your memory, while keeping the user-level + processing simple. + +/ + size_t initialReceiveBufferSize = 4096; + size_t likelyReceiveBufferSize = 4096; /// ditto + size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + + /++ + Maximum combined size of a message. + +/ + size_t maximumMessageSize = 10 * 1024 * 1024; + + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; + string origin; /// Origin URL to send with the handshake, if desired. + string protocol; /// the protocol header, if desired. + + int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping + } + + /++ + Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. + +/ + int readyState() { + return readyState_; + } + + /++ +wss://echo.websocket.org + +/ + /// Group: foundational + this(Uri uri, Config config = Config.init) + in (uri.scheme == "ws" || uri.scheme == "wss") + { + this.uri = uri; + this.config = config; + + this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); + + host = uri.host; + ssl = uri.scheme == "wss"; + port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); + + if(ssl) { + version(with_openssl) + socket = new SslClientSocket(AddressFamily.INET, SocketType.STREAM); + else + throw new Exception("SSL not compiled in"); + } else + socket = new Socket(AddressFamily.INET, SocketType.STREAM); + + } + + /++ + + +/ + /// Group: foundational + void connect() { + socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... + // FIXME: websocket handshake could and really should be async too. + + auto uri = this.uri.path.length ? this.uri.path : "/"; + if(this.uri.query.length) { + uri ~= "?"; + uri ~= this.uri.query; + } + + // the headers really shouldn't be bigger than this, at least + // the chunks i need to process + ubyte[4096] buffer; + size_t pos; + + void append(in char[][] items...) { + foreach(what; items) { + buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; + pos += what.length; + } + } + + append("GET ", uri, " HTTP/1.1\r\n"); + append("Host: ", this.uri.host, "\r\n"); + + append("Upgrade: websocket\r\n"); + append("Connection: Upgrade\r\n"); + append("Sec-WebSocket-Version: 13\r\n"); + + // FIXME: randomize this + append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); + + if(config.protocol.length) + append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); + if(config.origin.length) + append("Origin: ", origin, "\r\n"); + + append("\r\n"); + + auto remaining = buffer[0 .. pos]; + //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); + while(remaining.length) { + auto r = socket.send(remaining); + if(r < 0) + throw new Exception(lastSocketError()); + if(r == 0) + throw new Exception("unexpected connection termination"); + remaining = remaining[r .. $]; + } + + // the response shouldn't be especially large at this point, just + // headers for the most part. gonna try to get it in the stack buffer. + // then copy stuff after headers, if any, to the frame buffer. + ubyte[] used; + + void more() { + auto r = socket.receive(buffer[used.length .. $]); + + if(r < 0) + throw new Exception(lastSocketError()); + if(r == 0) + throw new Exception("unexpected connection termination"); + //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); + + used = buffer[0 .. used.length + r]; + } + + more(); + + import std.algorithm; + if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) + throw new Exception("didn't get a websocket answer"); + // skip the status line + while(used.length && used[0] != '\n') + used = used[1 .. $]; + + if(used.length == 0) + throw new Exception("wtf"); + + if(used.length < 1) + more(); + + used = used[1 .. $]; // skip the \n + + if(used.length == 0) + more(); + + // checks on the protocol from ehaders + bool isWebsocket; + bool isUpgrade; + const(ubyte)[] protocol; + const(ubyte)[] accept; + + while(used.length) { + if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { + used = used[2 .. $]; + break; // all done + } + int idxColon; + while(idxColon < used.length && used[idxColon] != ':') + idxColon++; + if(idxColon == used.length) + more(); + auto idxStart = idxColon + 1; + while(idxStart < used.length && used[idxStart] == ' ') + idxStart++; + if(idxStart == used.length) + more(); + auto idxEnd = idxStart; + while(idxEnd < used.length && used[idxEnd] != '\r') + idxEnd++; + if(idxEnd == used.length) + more(); + + auto headerName = used[0 .. idxColon]; + auto headerValue = used[idxStart .. idxEnd]; + + // move past this header + used = used[idxEnd .. $]; + // and the \r\n + if(2 <= used.length) + used = used[2 .. $]; + + if(headerName.bicmp("upgrade")) { + if(headerValue.bicmp("websocket")) + isWebsocket = true; + } else if(headerName.bicmp("connection")) { + if(headerValue.bicmp("upgrade")) + isUpgrade = true; + } else if(headerName.bicmp("sec-websocket-accept")) { + accept = headerValue; + } else if(headerName.bicmp("sec-websocket-protocol")) { + protocol = headerValue; + } + + if(!used.length) { + more(); + } + } + + + if(!isWebsocket) + throw new Exception("didn't answer as websocket"); + if(!isUpgrade) + throw new Exception("didn't answer as upgrade"); + + + // FIXME: check protocol if config requested one + // FIXME: check accept for the right hash + + receiveBuffer[0 .. used.length] = used[]; + receiveBufferUsedLength = used.length; + + readyState_ = OPEN; + + if(onopen) + onopen(); + + registerActiveSocket(this); + } + + /++ + Closes the connection, sending a graceful teardown message to the other side. + +/ + /// Group: foundational + void close(int code = 0, string reason = null) + in (reason.length < 123) + { + if(readyState_ != OPEN) + return; // it cool, we done + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.close; + wss.data = cast(ubyte[]) reason; + wss.send(&llsend); + + readyState_ = CLOSING; + + socket.shutdown(SocketShutdown.SEND); + } + + /++ + Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. + +/ + /// Group: foundational + void ping() { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.ping; + wss.send(&llsend); + } + + // automatically handled.... + void pong() { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.pong; + wss.send(&llsend); + } + + /++ + Sends a text message through the websocket. + +/ + /// Group: foundational + void send(in char[] textData) { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.text; + wss.data = cast(ubyte[]) textData; + wss.send(&llsend); + } + + /++ + Sends a binary message through the websocket. + +/ + /// Group: foundational + void send(in ubyte[] binaryData) { + WebSocketFrame wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData; + wss.send(&llsend); + } + + + private void llsend(ubyte[] d) { + while(d.length) { + auto r = socket.send(d); + if(r <= 0) throw new Exception("wtf"); + d = d[r .. $]; + } + } + + /++ + Waits for more data off the low-level socket and adds it to the pending buffer. + + Returns `true` if the connection is still active. + +/ + /// Group: blocking_api + public bool lowLevelReceive() { + auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); + if(r == 0) + return false; + if(r <= 0) + throw new Exception("wtf"); + receiveBufferUsedLength += r; + return true; + } + + /++ + Waits for and returns the next complete message on the socket. + + Note that the onmessage function is still called, right before + this returns. + +/ + /// Group: blocking_api + public WebSocketFrame waitForNextMessage() { + do { + auto m = processOnce(); + if(m.populated) + return m; + } while(lowLevelReceive()); + + return WebSocketFrame.init; // FIXME? maybe. + } + + /++ + Tells if [waitForNextMessage] would block. + +/ + /// Group: blocking_api + public bool waitForNextMessageWouldBlock() { + checkAgain: + if(isMessageBuffered()) + return false; + if(!isDataPending()) + return true; + while(isDataPending()) + lowLevelReceive(); + goto checkAgain; + } + + /++ + Is there a message in the buffer already? + If `true`, [waitForNextMessage] is guaranteed to return immediately. + If `false`, check [isDataPending] as the next step. + +/ + /// Group: blocking_api + public bool isMessageBuffered() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + if(d.length) { + auto orig = d; + auto m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d !is orig) + return true; + } + + return false; + } + + /++ + Is data pending on the socket? Also check [isMessageBuffered] to see if there + is already a message in memory too. + + If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] + again. + +/ + /// Group: blocking_api + public bool isDataPending() { + static SocketSet readSet; + if(readSet is null) + readSet = new SocketSet(); + + version(with_openssl) + if(auto s = cast(SslClientSocket) socket) { + // select doesn't handle the case with stuff + // left in the ssl buffer so i'm checking it separately + if(s.dataPending()) { + return true; + } + } + + readSet.add(socket); + + //tryAgain: + auto selectGot = Socket.select(readSet, null, null, 0.seconds /* timeout */); + if(selectGot == 0) { /* timeout */ + // timeout + return false; + } else if(selectGot == -1) { /* interrupted */ + return false; + } else { /* ready */ + if(readSet.isSet(socket)) { + return true; + } + } + + return false; + } + + private ubyte continuingType; + private ubyte[] continuingData; + //private size_t continuingDataLength; + + private WebSocketFrame processOnce() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. + WebSocketFrame m; + if(d.length) { + auto orig = d; + m = WebSocketFrame.read(d); + // that's how it indicates that it needs more data + if(d is orig) + return WebSocketFrame.init; + switch(m.opcode) { + case WebSocketOpcode.continuation: + if(continuingData.length + m.data.length > config.maximumMessageSize) + throw new Exception("message size exceeded"); + + continuingData ~= m.data; + if(m.fin) { + if(ontextmessage) + ontextmessage(cast(char[]) continuingData); + if(onbinarymessage) + onbinarymessage(continuingData); + + continuingData = null; + } + break; + case WebSocketOpcode.text: + if(m.fin) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.binary: + if(m.fin) { + if(onbinarymessage) + onbinarymessage(m.data); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.close: + readyState_ = CLOSED; + if(onclose) + onclose(); + + unregisterActiveSocket(this); + break; + case WebSocketOpcode.ping: + pong(); + break; + case WebSocketOpcode.pong: + // just really references it is still alive, nbd. + break; + default: // ignore though i could and perhaps should throw too + } + } + receiveBufferUsedLength -= s.length - d.length; + + return m; + } + + private void autoprocess() { + // FIXME + do { + processOnce(); + } while(lowLevelReceive()); + } + + + void delegate() onclose; /// + void delegate() onerror; /// + void delegate(in char[]) ontextmessage; /// + void delegate(in ubyte[]) onbinarymessage; /// + void delegate() onopen; /// + + /++ + + +/ + /// Group: browser_api + void onmessage(void delegate(in char[]) dg) { + ontextmessage = dg; + } + + /// ditto + void onmessage(void delegate(in ubyte[]) dg) { + onbinarymessage = dg; + } + + /* + const int bufferedAmount // amount pending + const string extensions + + const string protocol + const string url + */ + + static { + /++ + + +/ + void eventLoop() { + + static SocketSet readSet; + + if(readSet is null) + readSet = new SocketSet(); + + outermost: while(!loopExited) { + readSet.reset(); + + bool hadAny; + foreach(sock; activeSockets) { + readSet.add(sock.socket); + hadAny = true; + } + + if(!hadAny) + return; + + tryAgain: + auto selectGot = Socket.select(readSet, null, null, 10.seconds /* timeout */); + if(selectGot == 0) { /* timeout */ + // timeout + goto tryAgain; + } else if(selectGot == -1) { /* interrupted */ + goto tryAgain; + } else { + foreach(sock; activeSockets) { + if(readSet.isSet(sock.socket)) { + if(!sock.lowLevelReceive()) { + sock.readyState_ = CLOSED; + unregisterActiveSocket(sock); + continue outermost; + } + while(sock.processOnce().populated) {} + selectGot--; + if(selectGot <= 0) + break; + } + } + } + } + } + + private bool loopExited; + /++ + + +/ + void exitEventLoop() { + loopExited = true; + } + + WebSocket[] activeSockets; + void registerActiveSocket(WebSocket s) { + activeSockets ~= s; + } + void unregisterActiveSocket(WebSocket s) { + foreach(i, a; activeSockets) + if(s is a) { + activeSockets[i] = activeSockets[$-1]; + activeSockets = activeSockets[0 .. $-1]; + break; + } + } + } +} + +/* copy/paste from cgi.d */ +private { + enum WebSocketOpcode : ubyte { + continuation = 0, + text = 1, + binary = 2, + // 3, 4, 5, 6, 7 RESERVED + close = 8, + ping = 9, + pong = 10, + // 11,12,13,14,15 RESERVED + } + + public struct WebSocketFrame { + private bool populated; + bool fin; + bool rsv1; + bool rsv2; + bool rsv3; + WebSocketOpcode opcode; // 4 bits + bool masked; + ubyte lengthIndicator; // don't set this when building one to send + ulong realLength; // don't use when sending + ubyte[4] maskingKey; // don't set this when sending + ubyte[] data; + + static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { + WebSocketFrame msg; + msg.fin = true; + msg.opcode = opcode; + msg.data = cast(ubyte[]) data; + + return msg; + } + + private void send(scope void delegate(ubyte[]) llsend) { + ubyte[64] headerScratch; + int headerScratchPos = 0; + + realLength = data.length; + + { + ubyte b1; + b1 |= cast(ubyte) opcode; + b1 |= rsv3 ? (1 << 4) : 0; + b1 |= rsv2 ? (1 << 5) : 0; + b1 |= rsv1 ? (1 << 6) : 0; + b1 |= fin ? (1 << 7) : 0; + + headerScratch[0] = b1; + headerScratchPos++; + } + + { + headerScratchPos++; // we'll set header[1] at the end of this + auto rlc = realLength; + ubyte b2; + b2 |= masked ? (1 << 7) : 0; + + assert(headerScratchPos == 2); + + if(realLength > 65535) { + // use 64 bit length + b2 |= 0x7f; + + // FIXME: double check endinaness + foreach(i; 0 .. 8) { + headerScratch[2 + 7 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 8; + } else if(realLength > 127) { + // use 16 bit length + b2 |= 0x7e; + + // FIXME: double check endinaness + foreach(i; 0 .. 2) { + headerScratch[2 + 1 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 2; + } else { + // use 7 bit length + b2 |= realLength & 0b_0111_1111; + } + + headerScratch[1] = b2; + } + + //assert(!masked, "masking key not properly implemented"); + if(masked) { + // FIXME: randomize this + headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; + headerScratchPos += 4; + + // we'll just mask it in place... + int keyIdx = 0; + foreach(i; 0 .. data.length) { + data[i] = data[i] ^ maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); + llsend(headerScratch[0 .. headerScratchPos]); + llsend(data); + } + + static WebSocketFrame read(ref ubyte[] d) { + WebSocketFrame msg; + + auto orig = d; + + WebSocketFrame needsMoreData() { + d = orig; + return WebSocketFrame.init; + } + + if(d.length < 2) + return needsMoreData(); + + ubyte b = d[0]; + + msg.populated = true; + + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); + b >>= 4; + msg.rsv3 = b & 0x01; + b >>= 1; + msg.rsv2 = b & 0x01; + b >>= 1; + msg.rsv1 = b & 0x01; + b >>= 1; + msg.fin = b & 0x01; + + b = d[1]; + msg.masked = (b & 0b1000_0000) ? true : false; + msg.lengthIndicator = b & 0b0111_1111; + + d = d[2 .. $]; + + if(msg.lengthIndicator == 0x7e) { + // 16 bit length + msg.realLength = 0; + + if(d.length < 2) return needsMoreData(); + + foreach(i; 0 .. 2) { + msg.realLength |= d[0] << ((1-i) * 8); + d = d[1 .. $]; + } + } else if(msg.lengthIndicator == 0x7f) { + // 64 bit length + msg.realLength = 0; + + if(d.length < 8) return needsMoreData(); + + foreach(i; 0 .. 8) { + msg.realLength |= d[0] << ((7-i) * 8); + d = d[1 .. $]; + } + } else { + // 7 bit length + msg.realLength = msg.lengthIndicator; + } + + if(msg.masked) { + + if(d.length < 4) return needsMoreData(); + + msg.maskingKey = d[0 .. 4]; + d = d[4 .. $]; + } + + if(msg.realLength > d.length) { + return needsMoreData(); + } + + msg.data = d[0 .. msg.realLength]; + d = d[msg.realLength .. $]; + + if(msg.masked) { + // let's just unmask it now + int keyIdx = 0; + foreach(i; 0 .. msg.data.length) { + msg.data[i] = msg.data[i] ^ msg.maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + return msg; + } + + char[] textData() { + return cast(char[]) data; + } + } +} diff --git a/jni.d b/jni.d index 6258b37..cd3d3be 100644 --- a/jni.d +++ b/jni.d @@ -120,6 +120,29 @@ +/ module arsd.jni; +// I need to figure out some way that users can set this. maybe. or dynamically fall back from newest to oldest we can handle +__gshared auto JNI_VERSION_DESIRED = JNI_VERSION_1_6; + +// i could perhaps do a struct to bean thingy + +/* + New Java classes: + + class Foo : extends!Bar { + + mixin stuff; + } + mixin stuff; + + The `extends` template creates a wrapper that calls the nonvirtual + methods, so `super()` just works. + + receiving an object should perhaps always give a subclass that is javafied; + calls the virtuals, unless of course it is final. + + dynamic downcasts of java objects will probably never work. +*/ + /+ For interfaces: @@ -148,6 +171,20 @@ module arsd.jni; // it should let them do that for more efficiency // e.g. @Import Manual!(int[]) getJavaArray(); +/+ + So in Java, a lambda expression is turned into an anonymous class + that implements the one abstract method in the required interface. + + In D, they are a different type. And with no implicit construction I + can't convert automatically. + + But I could prolly do something like javaLambda!Interface(x => foo) + but woof that isn't so much different than an anonymous class anymore. ++/ + +/// hack used by the translator for default constructors not really being a default constructor +struct Default {} + /+ final class CharSequence : JavaClass!("java.lang", CharSequence) { @Import string toString(); // this triggers a dmd segfault! whoa. FIXME dmd @@ -178,13 +215,13 @@ interface CharSequence : JavaInterface!("java.lang", CharSequence) { translated = to!wstring(data); } // Java copies the buffer so it is perfectly fine to return here now - return dummyClass((*env).NewString(env, translated.ptr, cast(jsize) translated.length)); + return dummyClass!(typeof(this))((*env).NewString(env, translated.ptr, cast(jsize) translated.length)); } /// static CharSequence fromDString(wstring data) { auto env = activeEnv; assert(env !is null); - return dummyClass((*env).NewString(env, data.ptr, cast(jsize) data.length)); + return dummyClass!(typeof(this))((*env).NewString(env, data.ptr, cast(jsize) data.length)); } } @@ -210,17 +247,19 @@ interface CharSequence : JavaInterface!("java.lang", CharSequence) { +/ interface JavaInterface(string javaPackage, CRTP) : IJavaObject { mixin JavaPackageId!(javaPackage, CRTP); - - /// I may not keep this. But for now if you need a dummy class in D - /// to represent some object that implements this interface in Java, - /// you can use this. The dummy class assumes all interface methods are @Imported. - static CRTP dummyClass(jobject obj) { - return new class CRTP { - jobject getJavaHandle() { return obj; } - }; - } + mixin JavaInterfaceMembers!(null); } +/// I may not keep this. But for now if you need a dummy class in D +/// to represent some object that implements this interface in Java, +/// you can use this. The dummy class assumes all interface methods are @Imported. +static T dummyClass(T)(jobject obj) { + return new class T { + jobject getJavaHandle() { return obj; } + }; +} + + /++ Can be used as a UDA for methods or classes where the D name and the Java name don't match (for example, if it happens to be @@ -243,12 +282,434 @@ private string getJavaName(alias a)() { return name; } +/+ + to benchmark build stats + cd ~/Android/d_android/java_bindings/android/java + /usr/bin/time -f "%E %M" dmd -o- -c `find . | grep -E '\.d$'` ~/arsd/jni.d -I../.. ++/ + /+ Java class file definitions { +/ -// see: https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-4.html#jvms-4.6 +// see: https://docs.oracle.com/javase/specs/jvms/se13/html/jvms-4.html version(WithClassLoadSupport) { import arsd.declarativeloader; +/// translator. +void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, bool delegate(string className) classFilter = null) { + import std.zip; + import std.file; + import std.algorithm; + + auto zip = new ZipArchive(read(jarPath)); + + ClassFile[string] allClasses; + + foreach(name, am; zip.directory) { + if(name.endsWith(".class")) { + zip.expand(am); + + ClassFile cf; + + auto classBytes = cast(ubyte[]) am.expandedData; + auto originalClassBytes = classBytes; + + debug try { + cf.loadFrom!ClassFile(classBytes); + } catch(Exception e) { + std.file.write("spam.bin", originalClassBytes); + throw e; + } else + cf.loadFrom!ClassFile(classBytes); + + string className = cf.className.idup; + + if(classFilter is null || classFilter(className)) + allClasses[className] = cf; + + //rawClassBytesToD(cast(ubyte[]) am.expandedData, dPackagePrefix, outputDirectory, jtc); + //am.expandedData = null; // let the GC take it if it wants + } + } + + foreach(name, cf; allClasses) + rawClassStructToD(cf, dPackagePrefix, outputDirectory, jtc, allClasses); +} + +private inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { + import std.string; + s ~= "."; // lol i suck + s = s.replace(".function.", ".function_."); + s = s.replace(".ref.", ".ref_."); + return s[0 .. $-1]; // god i am such a bad programmer +} + +private inout(char)[] fixupJavaClassName(inout(char)[] s) { + if(s == "Throwable" || s == "Object" || s == "Exception" || s == "Error" || s == "TypeInfo") + s = cast(typeof(s)) "Java" ~ s; + return s; +} + +/// For the translator +struct JavaTranslationConfig { + /// List the Java methods, imported to D. + bool doImports; + /// List the native methods, assuming they should be exported from D + bool doExports; + /// Put implementations inline. If false, this separates interface from impl for quicker builds with dmd -i. + bool inlineImplementations; + /// Treat native functions as imports, otherwise fills in as exports. Make sure doImports == true. + bool nativesAreImports = true; +} + +/// translator +void rawClassBytesToD()(ubyte[] bytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { + ClassFile f; + f.loadFrom(bytes); + rawClassStructToD(f, dPackagePrefix, outputDirectory, jtc, null); +} + +/// translator. +void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, ClassFile[string] allClasses) { + import std.file; + import std.path; + import std.algorithm; + import std.array; + import std.string; + + string importPrefix = "import"; + + const(char)[] javaPackage; + const(char)[] lastClassName; + + const(char)[] originalJavaPackage; + const(char)[] originalClassName; + + const(char)[] cn = cf.className; + auto idx = cn.lastIndexOf("/"); + if(idx != -1) { + javaPackage = cn[0 .. idx].replace("$", "_").replace("/", ".").fixupKeywordsInJavaPackageName; + lastClassName = cn[idx + 1 .. $]; + originalJavaPackage = cn[0 .. idx].replace("/", "."); + originalClassName = lastClassName; + } else { + lastClassName = cn; + originalJavaPackage = ""; + originalClassName = lastClassName; + } + + lastClassName = lastClassName.replace("$", "_"); // NOTE rughs strings in this file + lastClassName = fixupJavaClassName(lastClassName); + + auto filename = (outputDirectory.length ? (outputDirectory ~ "/") : "") + ~ (dPackagePrefix.length ? (dPackagePrefix.replace(".", "/") ~ "/") : "") + ~ javaPackage.replace(".", "/"); + mkdirRecurse(filename); + if(filename.length) + filename ~= "/"; + filename ~= lastClassName ~ ".d"; + + if(filename.indexOf("-") != -1) + return; + + + string dco; + + auto thisModule = cast(string)((dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ javaPackage); + if(thisModule.length && thisModule[$-1] != '.') + thisModule ~= "."; + thisModule ~= lastClassName; + + bool isInterface = (cf.access_flags & 0x0200) ? true : false; + bool isAbstract = (cf.access_flags & ClassFile.ACC_ABSTRACT) ? true : false; + + if(jtc.inlineImplementations) { + dco = "module " ~ thisModule ~ ";\n\n"; + } else { + dco ~= "module " ~ thisModule ~ "_d_interface;\n"; + } + + dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, JavaName, IJavaObject, ImportExportImpl, JavaInterfaceMembers;\n"; + dco ~= "static import arsd.jni;\n\n"; + + string[string] javaPackages; + string[string] javaPackagesReturn; + string[string] javaPackagesArguments; + + string dc; + if(lastClassName != originalClassName) + dc ~= "@JavaName(\""~originalClassName~"\")\n"; + + bool outputMixinTemplate = false; + + string mainThing; + //string helperThing; + + // so overriding Java classes from D is iffy and with separate implementation + // non final leads to linker errors anyway... + //mainThing ~= (isInterface ? "interface " : (jtc.inlineImplementations ? "class " : isAbstract ? "abstract class " : "final class ")) ~ lastClassName ~ " : "; + + mainThing ~= "final class " ~ lastClassName ~ " : IJavaObject {\n"; + mainThing ~= "\tstatic immutable string[] _d_canCastTo = [\n"; + + // not putting super class on inline implementations since that forces vtable... + if(jtc.inlineImplementations) { + auto scn = cf.superclassName; + + if(scn.length) { + mainThing ~= "\t\t\"" ~ scn ~ "\",\n"; + } + + /+ + //if(!scn.startsWith("java/")) { + // superclasses need the implementation too so putting it in the return list lol + if(scn.length && scn != "java/lang/Object") { // && scn in allClasses) { + mainThing ~= javaObjectToDTypeString(scn, javaPackages, javaPackagesReturn, importPrefix); + mainThing ~= ", "; + } + //} + +/ + } + + foreach(name; cf.interfacesNames) { + //if(name.startsWith("java/")) + //continue; // these probably aren't important to D and can really complicate version management + //if(name !in allClasses) + //continue; + //mainThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); + //mainThing ~= ", "; + + mainThing ~= "\t\t\"" ~ name ~ "\",\n"; + } + + mainThing ~= "\t];\n"; + + //helperThing ~= "interface " ~ lastClassName ~ "_d_methods : "; + + + string[string] mentioned; + + string[string] processed; + + void addMethods(ClassFile* current, bool isTopLevel) { + if(current is null) return; + if(current.className in processed) return; + foreach(method; current.methodsListing) { + bool native = (method.flags & 0x0100) ? true : false; + if(jtc.nativesAreImports) { + native = false; // kinda hacky but meh + if(!jtc.doImports) + continue; + } else { + if(native && !jtc.doExports) + continue; + if(!native && !jtc.doImports) + continue; + } + auto port = native ? "@Export" : "@Import"; + if(method.flags & 1) { // public + if(!isTopLevel && method.name == "") + continue; + + bool maybeOverride = false;// !isInterface; + if(method.flags & 0x0008) { + port ~= " static"; + } + if(method.flags & method_info.ACC_ABSTRACT) { + //if(!isInterface) + //port ~= " abstract"; + } else { + // this represents a default implementation in a Java interface + // D cannot express this... so I need to add it to the mixin template + // associated with this interface as well. + //if(isInterface && (!(method.flags & 0x0008))) { + //addToMixinTemplate = true; + //} + } + + //if(maybeOverride && method.isOverride(allClasses)) + //port ~= " override"; + + auto name = method.name; + + // FIXME: maybe check name for other D keywords but since so many overlap with java I think we will be ok most the time for now + if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString" || name == "init" || name == "lazy" || name == "immutable" || name == "is" || name == "function" || name == "delegate" || name == "template") { + // toString is special btw in order to avoid a dmd bug + port ~= " @JavaName(\""~name~"\")"; + name ~= "_"; + } + + // NOTE rughs strings in this file + name = name.replace("$", "_"); + + bool ctor = name == ""; + + auto sig = method.signature; + + auto lidx = sig.lastIndexOf(")"); + assert(lidx != -1); + auto retJava = sig[lidx + 1 .. $]; + auto argsJava = sig[1 .. lidx]; + + string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages, javaPackagesReturn, importPrefix); + string args = javaSignatureToDTypeString(argsJava, javaPackages, javaPackagesArguments, importPrefix); + auto oargs = args; + + if(!jtc.inlineImplementations) { + if(ctor && args.length == 0) + args = "arsd.jni.Default"; + } + + string men = cast(immutable) (name ~ "(" ~ args ~ ")"); + if(men in mentioned) + continue; // avoid duplicate things. idk why this is there though + mentioned[men] = men; + + string proto = cast(string) ("\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " { assert(0); }" : ";")~"\n"); + mainThing ~= proto; + + if(oargs.length == 0 && name == "toString_" && !(method.flags & 0x0008)) + mainThing ~= "\toverride string toString() { return arsd.jni.javaObjectToString(this); }\n"; + } + } + + processed[current.className.idup] = "done"; + if(current.superclassName.length) { + auto c = current.superclassName in allClasses; + addMethods(c, false); + } + foreach(iface; current.interfacesNames) { + auto c = iface in allClasses; + addMethods(c, false); + } + } + + addMethods(&cf, true); + + mainThing ~= "\tmixin IJavaObjectImplementation!(false);\n"; + mainThing ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; + + mainThing ~= "}\n\n"; + dc ~= mainThing; + dc ~= "\n\n"; + + foreach(pkg, prefix; javaPackages) { + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; + // keeping thisModule because of the prefix nonsense + //if(m == thisModule) + //continue; + if(jtc.inlineImplementations) + dco ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; + else + dco ~= "import " ~ prefix ~ " = " ~ m ~ "_d_interface;\n"; + } + if(javaPackages.keys.length) + dco ~= "\n"; + dco ~= dc; + + if(jtc.inlineImplementations) { + dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; + std.file.write(filename, dco); + } else { + string impl; + impl ~= "module " ~ thisModule ~ ";\n"; + impl ~= "public import " ~ thisModule ~ "_d_interface;\n\n"; + + impl ~= "import arsd.jni : ImportExportImpl;\n"; + impl ~= "mixin ImportExportImpl!"~lastClassName~";\n"; + + impl ~= "\n"; + foreach(pkg, prefix; javaPackagesReturn) { + // I also need to import implementations of return values so they just work + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; + impl ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; + } + + std.file.write(filename, impl); + std.file.write(filename[0 .. $-2] ~ "_d_interface.d", dco); + } +} + +string javaObjectToDTypeString(const(char)[] input, ref string[string] javaPackages, ref string[string] detailedPackages, string importPrefix) { + + string ret; + + if(input == "java/lang/String") { + ret = "string"; // or could be wstring... + } else if(input == "java/lang/Object") { + ret = "IJavaObject"; + } else { + // NOTE rughs strings in this file + string type = input.replace("$", "_").idup; + + string jp, cn, dm; + + auto idx = type.lastIndexOf("/"); + if(idx != -1) { + jp = type[0 .. idx].replace("/", ".").fixupKeywordsInJavaPackageName; + cn = type[idx + 1 .. $].fixupJavaClassName; + dm = jp ~ "." ~ cn; + } else { + cn = type; + dm = jp; + } + + string prefix; + if(auto n = dm in javaPackages) { + prefix = *n; + } else { + import std.conv; + // FIXME: this scheme sucks, would prefer something deterministic + prefix = importPrefix ~ to!string(javaPackages.keys.length); + //prefix = dm.replace(".", "0"); + + javaPackages[dm] = prefix; + detailedPackages[dm] = prefix; + } + + ret = prefix ~ (prefix.length ? ".":"") ~ cn; + } + + return ret; +} + +string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaPackages, ref string[string] detailedPackages, string importPrefix) { + string all; + + while(js.length) { + string type; + switch(js[0]) { + case '[': + js = js[1 .. $]; + type = javaSignatureToDTypeString(js, javaPackages, detailedPackages, importPrefix); + type ~= "[]"; + break; + case 'L': + import std.string; + auto idx = js.indexOf(";"); + type = js[1 .. idx].idup; + js = js[idx + 1 .. $]; + + type = javaObjectToDTypeString(type, javaPackages, detailedPackages, importPrefix); + break; + case 'V': js = js[1 .. $]; type = "void"; break; + case 'Z': js = js[1 .. $]; type = "bool"; break; + case 'B': js = js[1 .. $]; type = "byte"; break; + case 'C': js = js[1 .. $]; type = "wchar"; break; + case 'S': js = js[1 .. $]; type = "short"; break; + case 'J': js = js[1 .. $]; type = "long"; break; + case 'F': js = js[1 .. $]; type = "float"; break; + case 'D': js = js[1 .. $]; type = "double"; break; + case 'I': js = js[1 .. $]; type = "int"; break; + default: assert(0, js); + } + + if(all.length) all ~= ", "; + all ~= type; + } + + return all; +} + struct cp_info { enum CONSTANT_Class = 7; // sizeof = 2 @@ -299,7 +760,7 @@ struct cp_info { @BigEndian: double bytes; } - enum CONSTANT_NameAndType = 12; // sizeof = 2 + enum CONSTANT_NameAndType = 12; // sizeof = 4 struct CONSTANT_NameAndType_info { @BigEndian: ushort name_index; @@ -328,6 +789,18 @@ struct cp_info { ushort bootstrap_method_attr_index; ushort name_and_type_index; } + enum CONSTANT_Module = 19; + struct CONSTANT_Module_info { + @BigEndian: + ushort name_index; + } + enum CONSTANT_Package = 20; + struct CONSTANT_Package_info { + @BigEndian: + ushort name_index; + } + + ubyte tag; @Tagged!(tag) @@ -346,8 +819,21 @@ struct cp_info { @Tag(CONSTANT_MethodHandle) CONSTANT_MethodHandle_info methodHandle_info; @Tag(CONSTANT_MethodType) CONSTANT_MethodType_info methodType_info; @Tag(CONSTANT_InvokeDynamic) CONSTANT_InvokeDynamic_info invokeDynamic_info; + @Tag(CONSTANT_Module) CONSTANT_Module_info module_info; + @Tag(CONSTANT_Package) CONSTANT_Package_info package_info; } Info info; + + bool takesTwoSlots() { + return (tag == CONSTANT_Long || tag == CONSTANT_Double); + } + + string toString() { + if(tag == CONSTANT_Utf8) + return cast(string) info.utf8_info.bytes; + import std.format; + return format("cp_info(%s)", tag); + } } struct field_info { @@ -417,7 +903,17 @@ struct ClassFile { } const(char)[] superclassName() { - return this.constant(this.constant(this.super_class).info.class_info.name_index).info.utf8_info.bytes; + if(this.super_class) + return this.constant(this.constant(this.super_class).info.class_info.name_index).info.utf8_info.bytes; + return null; + } + + const(char)[][] interfacesNames() { + typeof(return) ret; + foreach(iface; interfaces) { + ret ~= this.constant(this.constant(iface).info.class_info.name_index).info.utf8_info.bytes; + } + return ret; } Method[] methodsListing() { @@ -427,15 +923,40 @@ struct ClassFile { m.name = this.constant(met.name_index).info.utf8_info.bytes; m.signature = this.constant(met.descriptor_index).info.utf8_info.bytes; m.flags = met.access_flags; + m.cf = &this; ms ~= m; } return ms; } + bool hasConcreteMethod(const(char)[] name, const(char)[] signature, ClassFile[string] allClasses) { + // I don't really care cuz I don't use the same root in D + if(this.className == "java/lang/Object") + return false; + + foreach(m; this.methodsListing) { + if(m.name == name)// && m.signature == signature) + return true; + //return (m.flags & method_info.ACC_ABSTRACT) ? false : true; // abstract impls do not count as methods as far as overrides are concerend... + } + + if(auto s = this.superclassName in allClasses) + return s.hasConcreteMethod(name, signature, allClasses); + return false; + } + static struct Method { const(char)[] name; const(char)[] signature; ushort flags; + ClassFile* cf; + bool isOverride(ClassFile[string] allClasses) { + if(name == "") + return false; + if(auto s = cf.superclassName in allClasses) + return s.hasConcreteMethod(name, signature, allClasses); + return false; + } } @@ -517,14 +1038,14 @@ export jint JNI_OnLoad(JavaVM* vm, void* reserved) { activeJvm = vm; JNIEnv* env; - if ((*vm).GetEnv(vm, cast(void**) &env, JNI_VERSION_1_6) != JNI_OK) { + if ((*vm).GetEnv(vm, cast(void**) &env, JNI_VERSION_DESIRED) != JNI_OK) { return JNI_ERR; } try { foreach(init; classInitializers_) if(init(env) != 0) - return JNI_ERR; + {}//return JNI_ERR; foreach(init; newClassInitializers_) if(init(env) != 0) return JNI_ERR; @@ -534,7 +1055,7 @@ export jint JNI_OnLoad(JavaVM* vm, void* reserved) { return JNI_ERR; } - return JNI_VERSION_1_6; + return JNI_VERSION_DESIRED; } extern(System) export void JNI_OnUnload(JavaVM* vm, void* reserved) { @@ -640,7 +1161,7 @@ auto createJvm()() { //options[1].optionString = `-Djava.class.path=c:\Users\me\program\jni\`; /* user classes */ //options[2].optionString = `-Djava.library.path=c:\Users\me\program\jdk-13.0.1\lib\`; /* set native library path */ - vm_args.version_ = JNI_VERSION_1_6; + vm_args.version_ = JNI_VERSION_DESIRED; vm_args.options = options.ptr; vm_args.nOptions = cast(int) options.length; vm_args.ignoreUnrecognized = true; @@ -737,10 +1258,14 @@ private enum ImportImplementationString = q{ auto ret = (*env).CallSTATICIntMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); return ret; + } else static if(is(typeof(return) == short)) { + auto ret = (*env).CallSTATICShortMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); + exceptionCheck(env); + return ret; } else static if(is(typeof(return) : IJavaObject)) { auto ret = (*env).CallSTATICObjectMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); - return typeof(return).fromExistingJavaObject(ret); + return fromExistingJavaObject!(typeof(return))(ret); } else static if(is(typeof(return) == long)) { auto ret = (*env).CallSTATICLongMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); @@ -803,18 +1328,31 @@ private enum ImportImplementationString = q{ auto eles = (*env).GetByteArrayElements(env, jarr, null); auto res = eles[0 .. len]; (*env).ReleaseByteArrayElements(env, jarr, eles, 0); + } else static if(is(E == string)) { + /* + auto eles = (*env).GetByteArrayElements(env, jarr, null); + auto res = eles[0 .. len]; + (*env).ReleaseByteArrayElements(env, jarr, eles, 0); + */ + string[] res; // FIXME } else static if(is(E : IJavaObject)) { // FIXME: implement this typeof(return) res = null; + } else static if(true) { + E[] res; // FIXME FIXME } else static assert(0, E.stringof ~ " not supported array element type yet"); // FIXME handle object arrays too. which would also prolly include arrays of arrays. return res; } else { - static assert(0, "Unsupported return type for JNI " ~ typeof(return).stringof); + static assert(0, "Unsupported return type for JNI: " ~ typeof(return).stringof); //return DDataToJni(env, __traits(getMember, dobj, __traits(identifier, method))(JavaParamsToD!(Parameters!method)(env, args).args)); } }; +import std.string; +static immutable ImportImplementationString_static = ImportImplementationString.replace("STATIC", "Static"); +static immutable ImportImplementationString_not = ImportImplementationString.replace("STATIC", ""); + private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { import std.traits; @@ -829,25 +1367,35 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { if(!_jmethodID) { jclass jc; - if(!internalJavaClassHandle_) { + if(!T.internalJavaClassHandle_) { jc = (*env).FindClass(env, (T._javaParameterString[1 .. $-1] ~ "\0").ptr); if(!jc) throw new Exception("Cannot find Java class " ~ T._javaParameterString[1 .. $-1]); - internalJavaClassHandle_ = jc; + T.internalJavaClassHandle_ = jc; } else { - jc = internalJavaClassHandle_; + jc = T.internalJavaClassHandle_; } - _jmethodID = (*env).GetMethodID(env, jc, - "", - // java method string is (args)ret - ("(" ~ DTypesToJniString!(typeof(args)) ~ ")V\0").ptr - ); + static if(args.length == 1 && is(typeof(args[0]) == arsd.jni.Default)) + _jmethodID = (*env).GetMethodID(env, jc, + "", + // java method string is (args)ret + ("()V\0").ptr + ); + else + _jmethodID = (*env).GetMethodID(env, jc, + "", + // java method string is (args)ret + ("(" ~ DTypesToJniString!(typeof(args)) ~ ")V\0").ptr + ); if(!_jmethodID) throw new Exception("Cannot find static Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - auto o = (*env).NewObject(env, internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); + static if(args.length == 1 && is(typeof(args[0]) == arsd.jni.Default)) + auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID); + else + auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); this_.internalJavaHandle_ = o; return this_; } @@ -860,13 +1408,13 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { if(!_jmethodID) { jclass jc; - if(!internalJavaClassHandle_) { + if(!T.internalJavaClassHandle_) { jc = (*env).FindClass(env, (T._javaParameterString[1 .. $-1] ~ "\0").ptr); if(!jc) throw new Exception("Cannot find Java class " ~ T._javaParameterString[1 .. $-1]); - internalJavaClassHandle_ = jc; + T.internalJavaClassHandle_ = jc; } else { - jc = internalJavaClassHandle_; + jc = T.internalJavaClassHandle_; } _jmethodID = (*env).GetStaticMethodID(env, jc, getJavaName!method.ptr, @@ -878,10 +1426,9 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { throw new Exception("Cannot find static Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - auto jobj = internalJavaClassHandle_; + auto jobj = T.internalJavaClassHandle_; - import std.string; - mixin(ImportImplementationString.replace("STATIC", "Static")); + mixin(ImportImplementationString_static); } else pragma(mangle, method.mangleof) @@ -905,8 +1452,7 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { throw new Exception("Cannot find Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - import std.string; - mixin(ImportImplementationString.replace("STATIC", "")); + mixin(ImportImplementationString_not); } } @@ -1009,7 +1555,11 @@ private template DTypesToJni(Types...) { alias DTypesToJni = jfloatArray; else static if(is(T == double[])) alias DTypesToJni = jdoubleArray; - else static assert(0, "Unsupported type for JNI " ~ T.stringof); + else static if(is(T == string[])) // prolly FIXME + alias DTypesToJni = jobjectArray; + else static if(is(T == E[], E)) // FIXME!!!!!!! + alias DTypesToJni = jobjectArray; + else static assert(0, "Unsupported type for JNI: " ~ T.stringof); } else { import std.meta; // FIXME: write about this later if you forget the ! on the final DTypesToJni, dmd @@ -1059,8 +1609,25 @@ auto DDatumToJni(T)(JNIEnv* env, T data) { else static if(is(T == long)) return data; else static if(is(T == float)) return data; else static if(is(T == double)) return data; + else static if(is(T == size_t)) return cast(int) data; else static if(is(T : IJavaObject)) return data is null ? null : data.getJavaHandle(); + + + else static if(is(T == string[])) return null; // FIXME!!! + + else static if(is(T == IJavaObject[])) return null; // FIXME!!! + else static if(is(T == bool[])) return null; // FIXME!!! + else static if(is(T == byte[])) return null; // FIXME!!! + else static if(is(T == wchar[])) return null; // FIXME!!! + else static if(is(T == short[])) return null; // FIXME!!! + else static if(is(T == int[])) return null; // FIXME!!! + else static if(is(T == long[])) return null; // FIXME!!! + else static if(is(T == float[])) return null; // FIXME!!! + else static if(is(T == double[])) return null; // FIXME!!! + else static if(is(T == E[], E)) return null; // FIXME!!! + + else static assert(0, "Unsupported type " ~ T.stringof); /* // FIXME: finish these. else static if(is(T == IJavaObject[])) @@ -1155,9 +1722,7 @@ private struct JavaParamsToD(Spec...) { } // FIXME other types of arrays } else static if(is(T : IJavaObject)) { - auto dobj = new T(); - dobj.internalJavaHandle_ = jarg; - arg = dobj; + arg = fromExistingJavaObject!T(jarg); } else static assert(0, "Unimplemented/unsupported type " ~ T.stringof); @@ -1180,15 +1745,6 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { static if(__traits(identifier, method) == "__ctor") static assert(0, "Cannot export D constructors"); - /+ - static private string JniMangle() { - // this actually breaks with -betterC though so does a lot more so meh. - static if(is(T : JavaClass!(JP, P), string JP, P)) - return "Java_" ~replace(JP, ".", "_") ~ (JP.length ? "_" : "") ~ P.stringof ~ "_" ~ __traits(identifier, method); - else static assert(0); - } - +/ - extern(System) private static DTypesToJni!(ReturnType!method) privateJniImplementation(JNIEnv* env, jobject obj, DTypesToJni!(Parameters!method) args) { // set it up in the thread for future calls @@ -1230,7 +1786,7 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { shared static this() { - nativeMethodsData_ ~= JNINativeMethod( + T.nativeMethodsData_ ~= JNINativeMethod( getJavaName!method.ptr, ("(" ~ DTypesToJniString!(Parameters!method) ~ ")" ~ DTypesToJniString!(ReturnType!method) ~ "\0").ptr, &privateJniImplementation @@ -1247,8 +1803,98 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { interface IJavaObject { /// Remember the returned object is a TEMPORARY local reference! protected jobject getJavaHandle(); + + enum Import; /// UDA to indicate you are importing the method from Java. Do NOT put a body on these methods. Only put these on implementation classes, not interfaces. + enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. Only put these on implementation classes, not interfaces. } +string javaObjectToString(IJavaObject i) { + return "FIXME"; +} + +T as(T, R)(R obj) { + // FIXME: this will have to do downcasts to interfaces + return T.init; +} + + +static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == interface)) { + import core.memory; + auto ptr = GC.malloc(__traits(classInstanceSize, T)); + ptr[0 .. __traits(classInstanceSize, T)] = typeid(T).initializer[]; + auto obj = cast(T) ptr; + obj.internalJavaHandle_ = o; + return obj; +} + +static auto fromExistingJavaObject(T)(jobject o) if(is(T == interface)) { + import std.traits; + static class Dummy : T { + static foreach(memberName; __traits(allMembers, T)) { + static foreach(idx, overload; __traits(getOverloads, T, memberName)) + static if(!__traits(isStaticFunction, overload)) + static foreach(attr; __traits(getAttributes, overload)) { + //static if(!__traits(isStaticFunction, __traits(getMember, T, memberName))) + //static foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) { + static if(is(attr == IJavaObject.Import)) { + //mixin("@Import override ReturnType!(__traits(getMember, T, memberName)) " ~ memberName ~ "(Parameters!(__traits(getMember, T, memberName)));"); + mixin("@Import override ReturnType!overload " ~ memberName ~ "(Parameters!overload);"); + } + } + } + + mixin IJavaObjectImplementation!(false); + + static if(!__traits(compiles, T._javaParameterString)) + mixin JavaPackageId!("java.lang", "Object"); + } + JavaBridge!Dummy bridge; // just to instantiate the impl template + return fromExistingJavaObject!Dummy(o); +} + + +mixin template ImportExportImpl(Class) if(is(Class == class)) { + static import arsd.jni; + private static arsd.jni.JavaBridge!(Class) _javaDBridge; +} + +mixin template ImportExportImpl(Interface) if(is(Interface == interface)) { + static import arsd.jni; + private static arsd.jni.JavaBridgeForInterface!(Interface) _javaDBridge; +} + +final class JavaBridgeForInterface(Interface) { + // for interfaces, we do need to implement static members, but nothing else + static foreach(memberName; __traits(derivedMembers, Interface)) { + static foreach(oi, overload; __traits(getOverloads, Interface, memberName)) + static if(__traits(isStaticFunction, overload)) + static foreach(attr; __traits(getAttributes, overload)) { + static if(is(attr == IJavaObject.Import)) + mixin JavaImportImpl!(Interface, overload, oi); + } + } +} + +final class JavaBridge(Class) { + static foreach(memberName; __traits(derivedMembers, Class)) { + // validations + static if(is(typeof(__traits(getMember, Class, memberName).offsetof))) + static assert(1, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); // FIXME + else static if(memberName == "__ctor") + static assert("JavaClasses can only be constructed by Java. Try making a constructor in Java, then make an @Import this(args); here."); + + // implementations + static foreach(oi, overload; __traits(getOverloads, Class, memberName)) + static foreach(attr; __traits(getAttributes, overload)) { + static if(is(attr == IJavaObject.Import)) + mixin JavaImportImpl!(Class, overload, oi); + else static if(is(attr == IJavaObject.Export)) + mixin JavaExportImpl!(Class, overload, oi); + } + } +} + + /++ This is the base class you inherit from in D classes that represent Java classes. You can then mark your methods @Import if they are implemented in Java and you want @@ -1264,18 +1910,6 @@ class JavaClass(string javaPackage, CRTP, Parent = void, bool isNewClass = false static assert(__traits(isFinalClass, CRTP), "Java classes must be final on the D side and " ~ CRTP.stringof ~ " is not"); - enum Import; /// UDA to indicate you are importing the method from Java. Do NOT put a body on these methods. - enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. - - static CRTP fromExistingJavaObject(jobject o) { - import core.memory; - auto ptr = GC.malloc(__traits(classInstanceSize, CRTP)); - ptr[0 .. __traits(classInstanceSize, CRTP)] = typeid(CRTP).initializer[]; - auto obj = cast(CRTP) ptr; - obj.internalJavaHandle_ = o; - return obj; - } - /+ /++ D constructors on Java objects don't work right, so this is disabled to ensure @@ -1285,69 +1919,89 @@ class JavaClass(string javaPackage, CRTP, Parent = void, bool isNewClass = false @disable this(){} +/ - static foreach(memberName; __traits(derivedMembers, CRTP)) { - // validations - static if(is(typeof(__traits(getMember, CRTP, memberName).offsetof))) - static assert(0, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); - else static if(memberName == "__ctor") - static assert("JavaClasses can only be constructed by Java. Try making a constructor in Java, then make an @Import this(args); here."); + mixin ImportExportImpl!CRTP; + mixin IJavaObjectImplementation!(isNewClass); + mixin JavaPackageId!(javaPackage, CRTP); +} - // implementations - static foreach(oi, overload; __traits(getOverloads, CRTP, memberName)) - static foreach(attr; __traits(getAttributes, overload)) { - static if(is(attr == Import)) - mixin JavaImportImpl!(CRTP, overload, oi); - else static if(is(attr == Export)) - mixin JavaExportImpl!(CRTP, overload, oi); - } +mixin template JavaInterfaceMembers(string javaName) { + static import arsd.jni; + /*protected*/ static arsd.jni.jclass internalJavaClassHandle_; + static if(javaName !is null) { + static assert(javaName[0] == 'L' && javaName[$-1] == ';'); + static immutable string _javaParameterString = javaName; } +} - protected jobject internalJavaHandle_; - protected jobject getJavaHandle() { return internalJavaHandle_; } +mixin template IJavaObjectImplementation(bool isNewClass) { + static import arsd.jni; - __gshared static protected /*immutable*/ JNINativeMethod[] nativeMethodsData_; - protected static jclass internalJavaClassHandle_; - protected static int initializeInJvm_(JNIEnv* env) { + /+ + import arsd.jni : IJavaObjectSeperate; // WTF the FQN in the is expression didn't work + static if(is(typeof(this) : IJavaObjectSeperate!(ImplInterface), ImplInterface)) { + ImplInterface _d_helper_; + override ImplInterface _d_helper() { return _d_helper_; } + override void _d_helper(ImplInterface i) { _d_helper_ = i; } + } + +/ + + /+ + static if(is(typeof(this) S == super)) + static foreach(_superInterface; S) + static if(is(_superInterface == interface)) + static if(__traits(compiles, _superInterface.JavaDefaultImplementations)) { + //pragma(msg, "here"); + mixin _superInterface.JavaDefaultImplementations; + } + +/ + + /*protected*/ arsd.jni.jobject internalJavaHandle_; + /*protected*/ override arsd.jni.jobject getJavaHandle() { return internalJavaHandle_; } + + /*protected*/ static arsd.jni.jclass internalJavaClassHandle_; + __gshared static /*protected*/ /*immutable*/ arsd.jni.JNINativeMethod[] nativeMethodsData_; + protected static int initializeInJvm_(arsd.jni.JNIEnv* env) { import core.stdc.stdio; static if(isNewClass) { - ActivateJniEnv aje = ActivateJniEnv(env); + static assert(0, "not really implemented"); + auto aje = arsd.jni.ActivateJniEnv(env); import std.file; auto bytes = cast(byte[]) read("Test2.class"); import std.array; bytes = bytes.replace(cast(byte[]) "Test2", cast(byte[]) "Test3"); - auto loader = ClassLoader.getSystemClassLoader().getJavaHandle(); + auto loader = arsd.jni.ClassLoader.getSystemClassLoader().getJavaHandle(); - auto internalJavaClassHandle_ = (*env).DefineClass(env, "wtf/Test3", loader, bytes.ptr, cast(int) bytes.length); + // doesn't actually work on Android, they didn't implement this function :( :( :( + internalJavaClassHandle_ = (*env).DefineClass(env, "wtf/Test3", loader, bytes.ptr, cast(int) bytes.length); } else { - auto internalJavaClassHandle_ = (*env).FindClass(env, (_javaParameterString[1 .. $-1] ~ "\0").ptr); + internalJavaClassHandle_ = (*env).FindClass(env, (_javaParameterString[1 .. $-1] ~ "\0").ptr); } if(!internalJavaClassHandle_) { (*env).ExceptionDescribe(env); (*env).ExceptionClear(env); - fprintf(stderr, "Cannot %s Java class for %s\n", isNewClass ? "create".ptr : "find".ptr, CRTP.stringof.ptr); + fprintf(stderr, "Cannot %s Java class for %s [%s]\n", isNewClass ? "create".ptr : "find".ptr, typeof(this).stringof.ptr, (_javaParameterString[1 .. $-1] ~ "\0").ptr); return 1; } + if(nativeMethodsData_.length) if((*env).RegisterNatives(env, internalJavaClassHandle_, nativeMethodsData_.ptr, cast(int) nativeMethodsData_.length)) { (*env).ExceptionDescribe(env); (*env).ExceptionClear(env); - fprintf(stderr, ("RegisterNatives failed for " ~ CRTP.stringof)); + fprintf(stderr, ("RegisterNatives failed for " ~ typeof(this).stringof)); return 1; } return 0; } shared static this() { static if(isNewClass) - newClassInitializers_ ~= &initializeInJvm_; + arsd.jni.newClassInitializers_ ~= &initializeInJvm_; else - classInitializers_ ~= &initializeInJvm_; + arsd.jni.classInitializers_ ~= &initializeInJvm_; } - - mixin JavaPackageId!(javaPackage, CRTP); } mixin template JavaPackageId(string javaPackage, CRTP) { @@ -1358,6 +2012,15 @@ mixin template JavaPackageId(string javaPackage, CRTP) { public static immutable string _javaParameterString = "L" ~ getJavaName!CRTP ~ ";"; } +mixin template JavaPackageId(string javaPackage, string javaClassName) { + static import std.string; + static if(javaPackage.length) + public static immutable string _javaParameterString = "L" ~ std.string.replace(javaPackage, ".", "/") ~ "/" ~ javaClassName ~ ";"; + else + public static immutable string _javaParameterString = "L" ~ javaClassName ~ ";"; +} + + __gshared /* immutable */ int function(JNIEnv* env)[] classInitializers_; __gshared /* immutable */ int function(JNIEnv* env)[] newClassInitializers_; @@ -1751,3 +2414,9 @@ union jvalue jdouble d; jobject l; } + +/* + Copyright 2019-2020, Adam D. Ruppe. + Boost license. or whatever. + Most work done in December 2019. +*/ diff --git a/rpc.d b/rpc.d index c74f0b9..334c514 100644 --- a/rpc.d +++ b/rpc.d @@ -6,6 +6,11 @@ module arsd.rpc; 1) integrate with arsd.eventloop 2) make it easy to use with other processes; pipe to a process and talk to it that way. perhaps with shared memory too? 3) extend the serialization capabilities + + + @Throws!(List, Of, Exceptions) + classes are also RPC proxied + stdin/out/err also redirected */ ///+ //example usage diff --git a/simpledisplay.d b/simpledisplay.d index 1dc5700..4d0e970 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -459,7 +459,7 @@ I live in the eastern United States, so I will most likely not be around at night in that US east timezone. - License: Copyright Adam D. Ruppe, 2011-2017. Released under the Boost Software License. + License: Copyright Adam D. Ruppe, 2011-2020. Released under the Boost Software License. Building documentation: You may wish to use the `arsd.ddoc` file from my github with building the documentation for simpledisplay yourself. It will give it a bit more style.