From e12c1a20ea9b64c91084c8e3c026028e974730ba Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 30 Dec 2019 22:47:13 -0500 Subject: [PATCH 01/24] D code generator from Java --- jni.d | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) diff --git a/jni.d b/jni.d index 912f843..f69741a 100644 --- a/jni.d +++ b/jni.d @@ -124,6 +124,17 @@ module arsd.jni; // it should let them do that for more efficiency // e.g. @Import Manual!(int[]) getJavaArray(); +/+ + So in Java, a lambda expression is turned into an anonymous class + that implements the one abstract method in the required interface. + + In D, they are a different type. And with no implicit construction I + can't convert automatically. + + But I could prolly do something like javaLambda!Interface(x => foo) + but woof that isn't so much different than an anonymous class anymore. ++/ + /+ final class CharSequence : JavaClass!("java.lang", CharSequence) { @Import string toString(); // this triggers a dmd segfault! whoa. FIXME dmd @@ -197,6 +208,208 @@ private string getJavaName(alias a)() { version(WithClassLoadSupport) { import arsd.declarativeloader; +void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, bool delegate(string className) classFilter = null) { + import std.zip; + import std.file; + import std.algorithm; + + auto zip = new ZipArchive(read(jarPath)); + + foreach(name, am; zip.directory) { + if(name.endsWith(".class")) { + // FIXME: use classFilter + zip.expand(am); + rawClassBytesToD(cast(ubyte[]) am.expandedData, dPackagePrefix, outputDirectory, jtc); + am.expandedData = null; // let the GC take it if it wants + } + } +} + +inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { + import std.string; + s = s.replace(".function.", ".function_."); + s = s.replace(".ref.", ".ref_."); + return s; +} + +inout(char)[] fixupJavaClassName(inout(char)[] s) { + if(s == "Throwable" || s == "Object" || s == "Exception" || s == "Error" || s == "TypeInfo") + s = cast(typeof(s)) "Java" ~ s; + return s; +} + +struct JavaTranslationConfig { + bool doImports; + bool doExports; +} + +void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { + import std.file; + import std.path; + import std.algorithm; + import std.array; + import std.string; + + ClassFile cf; + cf.loadFrom!ClassFile(classBytes); + + const(char)[] javaPackage; + const(char)[] lastClassName; + + const(char)[] cn = cf.className; + auto idx = cn.lastIndexOf("/"); + if(idx != -1) { + javaPackage = cn[0 .. idx]; + lastClassName = cn[idx + 1 .. $]; + } else { + lastClassName = cn; + } + + auto originalClassName = lastClassName; + lastClassName = lastClassName.replace("$", "_"); // NOTE rughs strings in this file + lastClassName = fixupJavaClassName(lastClassName); + + auto filename = (outputDirectory.length ? (outputDirectory ~ "/") : "") ~ (dPackagePrefix.length ? (dPackagePrefix.replace(".", "/") ~ "/") : "") ~ javaPackage; + mkdirRecurse(filename); + if(filename.length) + filename ~= "/"; + filename ~= lastClassName ~ ".d"; + + + string dco; + + auto thisModule = cast(string)((dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ cn.replace("$", "_").replace("/", ".").fixupKeywordsInJavaPackageName); + + dco = "module " ~ thisModule ~ ";\n\n"; + dco ~= "import arsd.jni : JavaClass, JavaName, IJavaObject;\n\n"; + + string[string] javaPackages; + + string dc; + if(lastClassName != originalClassName) + dc ~= "@JavaName(\""~originalClassName~"\")\n"; + + // FIXME: what if it is an interface? + + dc ~= "final class " ~ lastClassName ~ " : JavaClass!(\""~javaPackage.replace("/", ".")~"\", "~lastClassName~") {\n"; + foreach(method; cf.methodsListing) { + bool native = (method.flags & 0x0100) ? true : false; + if(native && !jtc.doExports) + continue; + if(!native && !jtc.doImports) + continue; + auto port = native ? "@Export" : "@Import"; + if(method.flags & 1) { // public + + if(method.flags & 0x0008) + port ~= " static"; + + auto name = method.name; + + // FIXME: maybe check name for other D keywords but since so many overlap with java I think we will be ok most the time for now + if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString") { + // toString is special btw in order to avoid a dmd bug + port ~= " @JavaName(\""~name~"\")"; + name ~= "_"; + } + + // NOTE rughs strings in this file + name = name.replace("$", "_"); + + bool ctor = name == ""; + + auto sig = method.signature; + + auto lidx = sig.lastIndexOf(")"); + assert(lidx != -1); + auto retJava = sig[lidx + 1 .. $]; + auto argsJava = sig[1 .. lidx]; + + string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages); + string args = javaSignatureToDTypeString(argsJava, javaPackages); + + dc ~= "\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " {}" : ";")~"\n"; + } + } + + // FIXME what if there is a name conflict? the prefix kinda handles it but i dont like how ugly it is + foreach(pkg, prefix; javaPackages) { + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg.fixupKeywordsInJavaPackageName; + // keeping thisModule because of the prefix nonsense + //if(m == thisModule) + //continue; + dco ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; + } + if(javaPackages.keys.length) + dco ~= "\n"; + dco ~= dc; + dco ~= "}\n"; + + std.file.write(filename, dco); +} + +string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaPackages) { + string all; + + while(js.length) { + string type; + switch(js[0]) { + case '[': + js = js[1 .. $]; + type = javaSignatureToDTypeString(js, javaPackages); + type ~= "[]"; + break; + case 'L': + import std.string; + auto idx = js.indexOf(";"); + type = js[1 .. idx].idup; + js = js[idx + 1 .. $]; + + if(type == "java/lang/String") { + type = "string"; // or could be wstring... + } else { + // NOTE rughs strings in this file + type = type.replace("$", "_"); + + string jp = type.replace("/", "."); + + string prefix; + if(auto n = jp in javaPackages) { + prefix = *n; + } else { + import std.conv; + // FIXME: this scheme sucks, would prefer something deterministic + prefix = "import" ~ to!string(javaPackages.keys.length); + + javaPackages[jp] = prefix; + } + + idx = type.lastIndexOf("/"); + if(idx != -1) + type = type[idx + 1 .. $]; + + type = prefix ~ (prefix.length ? ".":"") ~ fixupJavaClassName(type); + } + break; + case 'V': js = js[1 .. $]; type = "void"; break; + case 'Z': js = js[1 .. $]; type = "bool"; break; + case 'B': js = js[1 .. $]; type = "byte"; break; + case 'C': js = js[1 .. $]; type = "wchar"; break; + case 'S': js = js[1 .. $]; type = "short"; break; + case 'J': js = js[1 .. $]; type = "long"; break; + case 'F': js = js[1 .. $]; type = "float"; break; + case 'D': js = js[1 .. $]; type = "double"; break; + case 'I': js = js[1 .. $]; type = "int"; break; + default: assert(0); + } + + if(all.length) all ~= ", "; + all ~= type; + } + + return all; +} + struct cp_info { enum CONSTANT_Class = 7; // sizeof = 2 @@ -296,6 +509,10 @@ struct cp_info { @Tag(CONSTANT_InvokeDynamic) CONSTANT_InvokeDynamic_info invokeDynamic_info; } Info info; + + bool takesTwoSlots() { + return (tag == CONSTANT_Long || tag == CONSTANT_Double); + } } struct field_info { @@ -1223,6 +1440,7 @@ class JavaClass(string javaPackage, CRTP) : IJavaObject { @disable this(){} +/ + //version(none) static foreach(memberName; __traits(derivedMembers, CRTP)) { // validations static if(is(typeof(__traits(getMember, CRTP, memberName).offsetof))) From f877d60831d1ff74b611ac59977410405b3260ed Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Tue, 31 Dec 2019 10:55:57 -0500 Subject: [PATCH 02/24] moving stuff around --- jni.d | 65 +++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/jni.d b/jni.d index f69741a..0380d55 100644 --- a/jni.d +++ b/jni.d @@ -896,7 +896,7 @@ private enum ImportImplementationString = q{ } else static if(is(typeof(return) : IJavaObject)) { auto ret = (*env).CallSTATICObjectMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); - return typeof(return).fromExistingJavaObject(ret); + return fromExistingJavaObject!(typeof(return))(ret); } else static if(is(typeof(return) == long)) { auto ret = (*env).CallSTATICLongMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); @@ -1402,8 +1402,41 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { interface IJavaObject { /// Remember the returned object is a TEMPORARY local reference! protected jobject getJavaHandle(); + + enum Import; /// UDA to indicate you are importing the method from Java. Do NOT put a body on these methods. Only put these on implementation classes, not interfaces. + enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. Only put these on implementation classes, not interfaces. + } +static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject)) { + import core.memory; + auto ptr = GC.malloc(__traits(classInstanceSize, T)); + ptr[0 .. __traits(classInstanceSize, T)] = typeid(T).initializer[]; + auto obj = cast(T) ptr; + obj.internalJavaHandle_ = o; + return obj; +} + +mixin template ImportExportImpl(Class) { + static foreach(memberName; __traits(derivedMembers, Class)) { + // validations + static if(is(typeof(__traits(getMember, Class, memberName).offsetof))) + static assert(0, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); + else static if(memberName == "__ctor") + static assert("JavaClasses can only be constructed by Java. Try making a constructor in Java, then make an @Import this(args); here."); + + // implementations + static foreach(oi, overload; __traits(getOverloads, Class, memberName)) + static foreach(attr; __traits(getAttributes, overload)) { + static if(is(attr == Import)) + mixin JavaImportImpl!(Class, overload, oi); + else static if(is(attr == Export)) + mixin JavaExportImpl!(Class, overload, oi); + } + } +} + + /++ This is the base class you inherit from in D classes that represent Java classes. You can then mark your methods @Import if they are implemented in Java and you want @@ -1419,18 +1452,6 @@ class JavaClass(string javaPackage, CRTP) : IJavaObject { static assert(__traits(isFinalClass, CRTP), "Java classes must be final on the D side and " ~ CRTP.stringof ~ " is not"); - enum Import; /// UDA to indicate you are importing the method from Java. Do NOT put a body on these methods. - enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. - - static CRTP fromExistingJavaObject(jobject o) { - import core.memory; - auto ptr = GC.malloc(__traits(classInstanceSize, CRTP)); - ptr[0 .. __traits(classInstanceSize, CRTP)] = typeid(CRTP).initializer[]; - auto obj = cast(CRTP) ptr; - obj.internalJavaHandle_ = o; - return obj; - } - /+ /++ D constructors on Java objects don't work right, so this is disabled to ensure @@ -1440,23 +1461,8 @@ class JavaClass(string javaPackage, CRTP) : IJavaObject { @disable this(){} +/ - //version(none) - static foreach(memberName; __traits(derivedMembers, CRTP)) { - // validations - static if(is(typeof(__traits(getMember, CRTP, memberName).offsetof))) - static assert(0, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); - else static if(memberName == "__ctor") - static assert("JavaClasses can only be constructed by Java. Try making a constructor in Java, then make an @Import this(args); here."); + mixin ImportExportImpl!CRTP; - // implementations - static foreach(oi, overload; __traits(getOverloads, CRTP, memberName)) - static foreach(attr; __traits(getAttributes, overload)) { - static if(is(attr == Import)) - mixin JavaImportImpl!(CRTP, overload, oi); - else static if(is(attr == Export)) - mixin JavaExportImpl!(CRTP, overload, oi); - } - } protected jobject internalJavaHandle_; protected jobject getJavaHandle() { return internalJavaHandle_; } @@ -1484,6 +1490,7 @@ class JavaClass(string javaPackage, CRTP) : IJavaObject { return 1; } + if(nativeMethodsData_.length) if((*env).RegisterNatives(env, internalJavaClassHandle_, nativeMethodsData_.ptr, cast(int) nativeMethodsData_.length)) { fprintf(stderr, ("RegisterNatives failed for " ~ CRTP.stringof)); return 1; From db25eb3e7119407508eb59892a76db141b500338 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Tue, 31 Dec 2019 14:04:22 -0500 Subject: [PATCH 03/24] generator and reorganization to make generation less buggy --- jni.d | 214 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 145 insertions(+), 69 deletions(-) diff --git a/jni.d b/jni.d index cfe4d75..c11561e 100644 --- a/jni.d +++ b/jni.d @@ -189,13 +189,13 @@ interface CharSequence : JavaInterface!("java.lang", CharSequence) { translated = to!wstring(data); } // Java copies the buffer so it is perfectly fine to return here now - return dummyClass((*env).NewString(env, translated.ptr, cast(jsize) translated.length)); + return dummyClass!(typeof(this))((*env).NewString(env, translated.ptr, cast(jsize) translated.length)); } /// static CharSequence fromDString(wstring data) { auto env = activeEnv; assert(env !is null); - return dummyClass((*env).NewString(env, data.ptr, cast(jsize) data.length)); + return dummyClass!(typeof(this))((*env).NewString(env, data.ptr, cast(jsize) data.length)); } } @@ -221,17 +221,18 @@ interface CharSequence : JavaInterface!("java.lang", CharSequence) { +/ interface JavaInterface(string javaPackage, CRTP) : IJavaObject { mixin JavaPackageId!(javaPackage, CRTP); - - /// I may not keep this. But for now if you need a dummy class in D - /// to represent some object that implements this interface in Java, - /// you can use this. The dummy class assumes all interface methods are @Imported. - static CRTP dummyClass(jobject obj) { - return new class CRTP { - jobject getJavaHandle() { return obj; } - }; - } } +/// I may not keep this. But for now if you need a dummy class in D +/// to represent some object that implements this interface in Java, +/// you can use this. The dummy class assumes all interface methods are @Imported. +static T dummyClass(T)(jobject obj) { + return new class T { + jobject getJavaHandle() { return obj; } + }; +} + + /++ Can be used as a UDA for methods or classes where the D name and the Java name don't match (for example, if it happens to be @@ -279,9 +280,10 @@ void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, Jav inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { import std.string; + s ~= "."; // lol i suck s = s.replace(".function.", ".function_."); s = s.replace(".ref.", ".ref_."); - return s; + return s[0 .. $-1]; // god i am such a bad programmer } inout(char)[] fixupJavaClassName(inout(char)[] s) { @@ -308,20 +310,28 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output const(char)[] javaPackage; const(char)[] lastClassName; + const(char)[] originalJavaPackage; + const(char)[] originalClassName; + const(char)[] cn = cf.className; auto idx = cn.lastIndexOf("/"); if(idx != -1) { - javaPackage = cn[0 .. idx]; + javaPackage = cn[0 .. idx].replace("$", "_").replace("/", ".").fixupKeywordsInJavaPackageName; lastClassName = cn[idx + 1 .. $]; + originalJavaPackage = cn[0 .. idx].replace("/", "."); + originalClassName = lastClassName; } else { lastClassName = cn; + originalJavaPackage = ""; + originalClassName = lastClassName; } - auto originalClassName = lastClassName; lastClassName = lastClassName.replace("$", "_"); // NOTE rughs strings in this file lastClassName = fixupJavaClassName(lastClassName); - auto filename = (outputDirectory.length ? (outputDirectory ~ "/") : "") ~ (dPackagePrefix.length ? (dPackagePrefix.replace(".", "/") ~ "/") : "") ~ javaPackage; + auto filename = (outputDirectory.length ? (outputDirectory ~ "/") : "") + ~ (dPackagePrefix.length ? (dPackagePrefix.replace(".", "/") ~ "/") : "") + ~ javaPackage.replace(".", "/"); mkdirRecurse(filename); if(filename.length) filename ~= "/"; @@ -330,10 +340,13 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output string dco; - auto thisModule = cast(string)((dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ cn.replace("$", "_").replace("/", ".").fixupKeywordsInJavaPackageName); + auto thisModule = cast(string)((dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ javaPackage); + if(thisModule.length && thisModule[$-1] != '.') + thisModule ~= "."; + thisModule ~= lastClassName; dco = "module " ~ thisModule ~ ";\n\n"; - dco ~= "import arsd.jni : JavaClass, JavaName, IJavaObject;\n\n"; + dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, ImportExportImpl, JavaName, IJavaObject;\n\n"; string[string] javaPackages; @@ -343,7 +356,7 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output // FIXME: what if it is an interface? - dc ~= "final class " ~ lastClassName ~ " : JavaClass!(\""~javaPackage.replace("/", ".")~"\", "~lastClassName~") {\n"; + dc ~= "final class " ~ lastClassName ~ " : IJavaObject {\n";// JavaClass!(\""~javaPackage.replace("/", ".")~"\", "~lastClassName~") {\n"; foreach(method; cf.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; if(native && !jtc.doExports) @@ -386,7 +399,7 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output // FIXME what if there is a name conflict? the prefix kinda handles it but i dont like how ugly it is foreach(pkg, prefix; javaPackages) { - auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg.fixupKeywordsInJavaPackageName; + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; // keeping thisModule because of the prefix nonsense //if(m == thisModule) //continue; @@ -395,8 +408,14 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output if(javaPackages.keys.length) dco ~= "\n"; dco ~= dc; + + dco ~= "\tmixin IJavaObjectImplementation!(false);\n"; + dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; + dco ~= "}\n"; + dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; + std.file.write(filename, dco); } @@ -419,28 +438,36 @@ string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaP if(type == "java/lang/String") { type = "string"; // or could be wstring... + } else if(type == "java/lang/Object") { + type = "IJavaObject"; // or could be wstring... } else { // NOTE rughs strings in this file type = type.replace("$", "_"); - string jp = type.replace("/", "."); + string jp, cn, dm; + + idx = type.lastIndexOf("/"); + if(idx != -1) { + jp = type[0 .. idx].replace("/", ".").fixupKeywordsInJavaPackageName; + cn = type[idx + 1 .. $].fixupJavaClassName; + dm = jp ~ "." ~ cn; + } else { + cn = type; + dm = jp; + } string prefix; - if(auto n = jp in javaPackages) { + if(auto n = dm in javaPackages) { prefix = *n; } else { import std.conv; // FIXME: this scheme sucks, would prefer something deterministic prefix = "import" ~ to!string(javaPackages.keys.length); - javaPackages[jp] = prefix; + javaPackages[dm] = prefix; } - idx = type.lastIndexOf("/"); - if(idx != -1) - type = type[idx + 1 .. $]; - - type = prefix ~ (prefix.length ? ".":"") ~ fixupJavaClassName(type); + type = prefix ~ (prefix.length ? ".":"") ~ cn; } break; case 'V': js = js[1 .. $]; type = "void"; break; @@ -954,6 +981,10 @@ private enum ImportImplementationString = q{ auto ret = (*env).CallSTATICIntMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); return ret; + } else static if(is(typeof(return) == short)) { + auto ret = (*env).CallSTATICShortMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); + exceptionCheck(env); + return ret; } else static if(is(typeof(return) : IJavaObject)) { auto ret = (*env).CallSTATICObjectMethod(env, jobj, _jmethodID, DDataToJni(env, args).args); exceptionCheck(env); @@ -1020,14 +1051,23 @@ private enum ImportImplementationString = q{ auto eles = (*env).GetByteArrayElements(env, jarr, null); auto res = eles[0 .. len]; (*env).ReleaseByteArrayElements(env, jarr, eles, 0); + } else static if(is(E == string)) { + /* + auto eles = (*env).GetByteArrayElements(env, jarr, null); + auto res = eles[0 .. len]; + (*env).ReleaseByteArrayElements(env, jarr, eles, 0); + */ + string[] res; // FIXME } else static if(is(E : IJavaObject)) { // FIXME: implement this typeof(return) res = null; + } else static if(true) { + E[] res; // FIXME FIXME } else static assert(0, E.stringof ~ " not supported array element type yet"); // FIXME handle object arrays too. which would also prolly include arrays of arrays. return res; } else { - static assert(0, "Unsupported return type for JNI " ~ typeof(return).stringof); + static assert(0, "Unsupported return type for JNI: " ~ typeof(return).stringof); //return DDataToJni(env, __traits(getMember, dobj, __traits(identifier, method))(JavaParamsToD!(Parameters!method)(env, args).args)); } }; @@ -1046,13 +1086,13 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { if(!_jmethodID) { jclass jc; - if(!internalJavaClassHandle_) { + if(!T.internalJavaClassHandle_) { jc = (*env).FindClass(env, (T._javaParameterString[1 .. $-1] ~ "\0").ptr); if(!jc) throw new Exception("Cannot find Java class " ~ T._javaParameterString[1 .. $-1]); - internalJavaClassHandle_ = jc; + T.internalJavaClassHandle_ = jc; } else { - jc = internalJavaClassHandle_; + jc = T.internalJavaClassHandle_; } _jmethodID = (*env).GetMethodID(env, jc, "", @@ -1064,7 +1104,7 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { throw new Exception("Cannot find static Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - auto o = (*env).NewObject(env, internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); + auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); this_.internalJavaHandle_ = o; return this_; } @@ -1077,13 +1117,13 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { if(!_jmethodID) { jclass jc; - if(!internalJavaClassHandle_) { + if(!T.internalJavaClassHandle_) { jc = (*env).FindClass(env, (T._javaParameterString[1 .. $-1] ~ "\0").ptr); if(!jc) throw new Exception("Cannot find Java class " ~ T._javaParameterString[1 .. $-1]); - internalJavaClassHandle_ = jc; + T.internalJavaClassHandle_ = jc; } else { - jc = internalJavaClassHandle_; + jc = T.internalJavaClassHandle_; } _jmethodID = (*env).GetStaticMethodID(env, jc, getJavaName!method.ptr, @@ -1095,7 +1135,7 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { throw new Exception("Cannot find static Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - auto jobj = internalJavaClassHandle_; + auto jobj = T.internalJavaClassHandle_; import std.string; mixin(ImportImplementationString.replace("STATIC", "Static")); @@ -1226,7 +1266,11 @@ private template DTypesToJni(Types...) { alias DTypesToJni = jfloatArray; else static if(is(T == double[])) alias DTypesToJni = jdoubleArray; - else static assert(0, "Unsupported type for JNI " ~ T.stringof); + else static if(is(T == string[])) // prolly FIXME + alias DTypesToJni = jobjectArray; + else static if(is(T == E[], E)) // FIXME!!!!!!! + alias DTypesToJni = jobjectArray; + else static assert(0, "Unsupported type for JNI: " ~ T.stringof); } else { import std.meta; // FIXME: write about this later if you forget the ! on the final DTypesToJni, dmd @@ -1276,8 +1320,25 @@ auto DDatumToJni(T)(JNIEnv* env, T data) { else static if(is(T == long)) return data; else static if(is(T == float)) return data; else static if(is(T == double)) return data; + else static if(is(T == size_t)) return cast(int) data; else static if(is(T : IJavaObject)) return data is null ? null : data.getJavaHandle(); + + + else static if(is(T == string[])) return null; // FIXME!!! + + else static if(is(T == IJavaObject[])) return null; // FIXME!!! + else static if(is(T == bool[])) return null; // FIXME!!! + else static if(is(T == byte[])) return null; // FIXME!!! + else static if(is(T == wchar[])) return null; // FIXME!!! + else static if(is(T == short[])) return null; // FIXME!!! + else static if(is(T == int[])) return null; // FIXME!!! + else static if(is(T == long[])) return null; // FIXME!!! + else static if(is(T == float[])) return null; // FIXME!!! + else static if(is(T == double[])) return null; // FIXME!!! + else static if(is(T == E[], E)) return null; // FIXME!!! + + else static assert(0, "Unsupported type " ~ T.stringof); /* // FIXME: finish these. else static if(is(T == IJavaObject[])) @@ -1372,9 +1433,7 @@ private struct JavaParamsToD(Spec...) { } // FIXME other types of arrays } else static if(is(T : IJavaObject)) { - auto dobj = new T(); - dobj.internalJavaHandle_ = jarg; - arg = dobj; + arg = fromExistingJavaObject!T(jarg); } else static assert(0, "Unimplemented/unsupported type " ~ T.stringof); @@ -1397,15 +1456,6 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { static if(__traits(identifier, method) == "__ctor") static assert(0, "Cannot export D constructors"); - /+ - static private string JniMangle() { - // this actually breaks with -betterC though so does a lot more so meh. - static if(is(T : JavaClass!(JP, P), string JP, P)) - return "Java_" ~replace(JP, ".", "_") ~ (JP.length ? "_" : "") ~ P.stringof ~ "_" ~ __traits(identifier, method); - else static assert(0); - } - +/ - extern(System) private static DTypesToJni!(ReturnType!method) privateJniImplementation(JNIEnv* env, jobject obj, DTypesToJni!(Parameters!method) args) { // set it up in the thread for future calls @@ -1447,7 +1497,7 @@ private mixin template JavaExportImpl(T, alias method, size_t overloadIndex) { shared static this() { - nativeMethodsData_ ~= JNINativeMethod( + T.nativeMethodsData_ ~= JNINativeMethod( getJavaName!method.ptr, ("(" ~ DTypesToJniString!(Parameters!method) ~ ")" ~ DTypesToJniString!(ReturnType!method) ~ "\0").ptr, &privateJniImplementation @@ -1470,7 +1520,7 @@ interface IJavaObject { } -static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject)) { +static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == IJavaObject)) { import core.memory; auto ptr = GC.malloc(__traits(classInstanceSize, T)); ptr[0 .. __traits(classInstanceSize, T)] = typeid(T).initializer[]; @@ -1479,20 +1529,34 @@ static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject)) { return obj; } +static auto fromExistingJavaObject(T)(jobject o) if(is(T == IJavaObject)) { + static class Dummy : IJavaObject { + mixin IJavaObjectImplementation!(false); + mixin JavaPackageId!("java.lang", "Object"); + } + return fromExistingJavaObject!Dummy(o); +} + + mixin template ImportExportImpl(Class) { + static import arsd.jni; + private static arsd.jni.JavaBridge!(Class) _javaDBridge; +} + +class JavaBridge(Class) { static foreach(memberName; __traits(derivedMembers, Class)) { // validations static if(is(typeof(__traits(getMember, Class, memberName).offsetof))) - static assert(0, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); + static assert(1, "Data members in D on Java classes are not reliable because they cannot be consistently associated back to their corresponding Java classes through JNI without major runtime expense."); // FIXME else static if(memberName == "__ctor") static assert("JavaClasses can only be constructed by Java. Try making a constructor in Java, then make an @Import this(args); here."); // implementations static foreach(oi, overload; __traits(getOverloads, Class, memberName)) static foreach(attr; __traits(getAttributes, overload)) { - static if(is(attr == Import)) + static if(is(attr == IJavaObject.Import)) mixin JavaImportImpl!(Class, overload, oi); - else static if(is(attr == Export)) + else static if(is(attr == IJavaObject.Export)) mixin JavaExportImpl!(Class, overload, oi); } } @@ -1524,35 +1588,40 @@ class JavaClass(string javaPackage, CRTP, Parent = void, bool isNewClass = false +/ mixin ImportExportImpl!CRTP; + mixin IJavaObjectImplementation!(isNewClass); + mixin JavaPackageId!(javaPackage, CRTP); +} +mixin template IJavaObjectImplementation(bool isNewClass) { + static import arsd.jni; - protected jobject internalJavaHandle_; - protected jobject getJavaHandle() { return internalJavaHandle_; } + /*protected*/ arsd.jni.jobject internalJavaHandle_; + /*protected*/ arsd.jni.jobject getJavaHandle() { return internalJavaHandle_; } - __gshared static protected /*immutable*/ JNINativeMethod[] nativeMethodsData_; - protected static jclass internalJavaClassHandle_; - protected static int initializeInJvm_(JNIEnv* env) { + __gshared static /*protected*/ /*immutable*/ arsd.jni.JNINativeMethod[] nativeMethodsData_; + /*protected*/ static arsd.jni.jclass internalJavaClassHandle_; + protected static int initializeInJvm_(arsd.jni.JNIEnv* env) { import core.stdc.stdio; static if(isNewClass) { - ActivateJniEnv aje = ActivateJniEnv(env); + auto aje = arsd.jni.ActivateJniEnv(env); import std.file; auto bytes = cast(byte[]) read("Test2.class"); import std.array; bytes = bytes.replace(cast(byte[]) "Test2", cast(byte[]) "Test3"); - auto loader = ClassLoader.getSystemClassLoader().getJavaHandle(); + auto loader = arsd.jni.ClassLoader.getSystemClassLoader().getJavaHandle(); - auto internalJavaClassHandle_ = (*env).DefineClass(env, "wtf/Test3", loader, bytes.ptr, cast(int) bytes.length); + internalJavaClassHandle_ = (*env).DefineClass(env, "wtf/Test3", loader, bytes.ptr, cast(int) bytes.length); } else { - auto internalJavaClassHandle_ = (*env).FindClass(env, (_javaParameterString[1 .. $-1] ~ "\0").ptr); + internalJavaClassHandle_ = (*env).FindClass(env, (_javaParameterString[1 .. $-1] ~ "\0").ptr); } if(!internalJavaClassHandle_) { (*env).ExceptionDescribe(env); (*env).ExceptionClear(env); - fprintf(stderr, "Cannot %s Java class for %s\n", isNewClass ? "create".ptr : "find".ptr, CRTP.stringof.ptr); + fprintf(stderr, "Cannot %s Java class for %s\n", isNewClass ? "create".ptr : "find".ptr, typeof(this).stringof.ptr); return 1; } @@ -1560,19 +1629,17 @@ class JavaClass(string javaPackage, CRTP, Parent = void, bool isNewClass = false if((*env).RegisterNatives(env, internalJavaClassHandle_, nativeMethodsData_.ptr, cast(int) nativeMethodsData_.length)) { (*env).ExceptionDescribe(env); (*env).ExceptionClear(env); - fprintf(stderr, ("RegisterNatives failed for " ~ CRTP.stringof)); + fprintf(stderr, ("RegisterNatives failed for " ~ typeof(this).stringof)); return 1; } return 0; } shared static this() { static if(isNewClass) - newClassInitializers_ ~= &initializeInJvm_; + arsd.jni.newClassInitializers_ ~= &initializeInJvm_; else - classInitializers_ ~= &initializeInJvm_; + arsd.jni.classInitializers_ ~= &initializeInJvm_; } - - mixin JavaPackageId!(javaPackage, CRTP); } mixin template JavaPackageId(string javaPackage, CRTP) { @@ -1583,6 +1650,15 @@ mixin template JavaPackageId(string javaPackage, CRTP) { public static immutable string _javaParameterString = "L" ~ getJavaName!CRTP ~ ";"; } +mixin template JavaPackageId(string javaPackage, string javaClassName) { + static import std.string; + static if(javaPackage.length) + public static immutable string _javaParameterString = "L" ~ std.string.replace(javaPackage, ".", "/") ~ "/" ~ javaClassName ~ ";"; + else + public static immutable string _javaParameterString = "L" ~ javaClassName ~ ";"; +} + + __gshared /* immutable */ int function(JNIEnv* env)[] classInitializers_; __gshared /* immutable */ int function(JNIEnv* env)[] newClassInitializers_; From 3316e500a56c71537aba179da1f6aa50cacac81f Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 1 Jan 2020 10:24:38 -0500 Subject: [PATCH 04/24] various bug fixes --- dom.d | 194 +++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 172 insertions(+), 22 deletions(-) diff --git a/dom.d b/dom.d index d31c48b..d038095 100644 --- a/dom.d +++ b/dom.d @@ -1,8 +1,7 @@ -// FIXME: add classList +// FIXME: add classList. it is a live list and removes whitespace and duplicates when you use it. // FIXME: xml namespace support??? -// FIXME: add matchesSelector - standard name is `matches`. also `closest` walks up to find the parent that matches // FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML -// FIXME: appendChild should not fail if the thing already has a parent; it should just automatically remove it per standard. +// FIXME: parentElement is parentNode that skips DocumentFragment etc but will be hard to work in with my compatibility... // FIXME: the scriptable list is quite arbitrary @@ -1194,28 +1193,57 @@ class Document : FileResource { return root.optionSelector!(SomeElementType)(selector, file, line); } - /// ditto + @scriptable Element querySelector(string selector) { - return root.querySelector(selector); + // see comment below on Document.querySelectorAll + auto s = Selector(selector);//, !loose); + foreach(ref comp; s.components) + if(comp.parts.length && comp.parts[0].separation == 0) + comp.parts[0].separation = -1; + foreach(e; s.getMatchingElementsLazy(this.root)) + return e; + return null; + } /// ditto + @scriptable Element[] querySelectorAll(string selector) { - return root.querySelectorAll(selector); + // In standards-compliant code, the document is slightly magical + // in that it is a pseudoelement at top level. It should actually + // match the root as one of its children. + // + // In versions of dom.d before Dec 29 2019, this worked because + // querySelectorAll was willing to return itself. With that bug fix + // (search "arbitrary id asduiwh" in this file for associated unittest) + // this would have failed. Hence adding back the root if it matches the + // selector itself. + // + // I'd love to do this better later. + + auto s = Selector(selector);//, !loose); + foreach(ref comp; s.components) + if(comp.parts.length && comp.parts[0].separation == 0) + comp.parts[0].separation = -1; + return s.getMatchingElements(this.root); } /// ditto + @scriptable + deprecated("use querySelectorAll instead") Element[] getElementsBySelector(string selector) { return root.getElementsBySelector(selector); } /// ditto + @scriptable Element[] getElementsByTagName(string tag) { return root.getElementsByTagName(tag); } /// ditto + @scriptable Element[] getElementsByClassName(string tag) { return root.getElementsByClassName(tag); } @@ -2181,10 +2209,37 @@ class Element { } /// a more standards-compliant alias for getElementsBySelector + @scriptable Element[] querySelectorAll(string selector) { return getElementsBySelector(selector); } + /// If the element matches the given selector. Previously known as `matchesSelector`. + @scriptable + bool matches(string selector) { + /+ + bool caseSensitiveTags = true; + if(parentDocument && parentDocument.loose) + caseSensitiveTags = false; + +/ + + Selector s = Selector(selector); + return s.matchesElement(this); + } + + /// Returns itself or the closest parent that matches the given selector, or null if none found + /// See_also: https://developer.mozilla.org/en-US/docs/Web/API/Element/closest + @scriptable + Element closest(string selector) { + Element e = this; + while(e !is null) { + if(e.matches(selector)) + return e; + e = e.parentNode; + } + return null; + } + /** Returns elements that match the given CSS selector @@ -2543,11 +2598,17 @@ class Element { } - /// Appends the given element to this one. The given element must not have a parent already. + /++ + Appends the given element to this one. If it already has a parent, it is removed from that tree and moved to this one. + + See_also: https://developer.mozilla.org/en-US/docs/Web/API/Node/appendChild + + History: + Prior to 1 Jan 2020 (git tag v4.4.1 and below), it required that the given element must not have a parent already. This was in violation of standard, so it changed the behavior to remove it from the existing parent and instead move it here. + +/ Element appendChild(Element e) in { assert(e !is null); - assert(e.parentNode is null, e.parentNode.toString); } out (ret) { assert((cast(DocumentFragment) this !is null) || (e.parentNode is this), e.toString);// e.parentNode ? e.parentNode.toString : "null"); @@ -2555,6 +2616,9 @@ class Element { assert(e is ret); } body { + if(e.parentNode !is null) + e.parentNode.removeChild(e); + selfClosed = false; e.parentNode = this; e.parentDocument = this.parentDocument; @@ -7160,6 +7224,7 @@ int intFromHex(string hex) { } auto part = parts[0]; + //writeln("checking ", part, " against ", start, " with ", part.separation); switch(part.separation) { default: assert(0); case -1: @@ -7336,9 +7401,25 @@ int intFromHex(string hex) { if(e is null) return false; Element where = e; int lastSeparation = -1; - foreach(part; retro(parts)) { - // writeln("matching ", where, " with ", part, " via ", lastSeparation); + auto lparts = parts; + + if(parts.length && parts[0].separation > 0) { + // if it starts with a non-trivial separator, inject + // a "*" matcher to act as a root. for cases like document.querySelector("> body") + // which implies html + + // there is probably a MUCH better way to do this. + auto dummy = SelectorPart.init; + dummy.tagNameFilter = "*"; + dummy.separation = 0; + lparts = dummy ~ lparts; + } + + foreach(part; retro(lparts)) { + + // writeln("matching ", where, " with ", part, " via ", lastSeparation); + // writeln(parts); if(lastSeparation == -1) { if(!part.matchElement(where)) @@ -7346,6 +7427,7 @@ int intFromHex(string hex) { } else if(lastSeparation == 0) { // generic parent // need to go up the whole chain where = where.parentNode; + while(where !is null) { if(part.matchElement(where)) break; @@ -7476,6 +7558,8 @@ int intFromHex(string hex) { if(current.isCleanSlateExceptSeparation()) { current.tagNameFilter = token; + // default thing, see comment under "*" below + if(current.separation == -1) current.separation = 0; } else { // if it was already set, we must see two thingies // separated by whitespace... @@ -7488,6 +7572,10 @@ int intFromHex(string hex) { switch(token) { case "*": current.tagNameFilter = "*"; + // the idea here is if we haven't actually set a separation + // yet (e.g. the > operator), it should assume the generic + // whitespace (descendant) mode to avoid matching self with -1 + if(current.separation == -1) current.separation = 0; break; case " ": // If some other separation has already been set, @@ -7520,16 +7608,20 @@ int intFromHex(string hex) { break; case "[": state = State.ReadingAttributeSelector; + if(current.separation == -1) current.separation = 0; break; case ".": state = State.ReadingClass; + if(current.separation == -1) current.separation = 0; break; case "#": state = State.ReadingId; + if(current.separation == -1) current.separation = 0; break; case ":": case "::": state = State.ReadingPseudoClass; + if(current.separation == -1) current.separation = 0; break; default: @@ -8542,18 +8634,6 @@ private bool isSimpleWhite(dchar c) { return c == ' ' || c == '\r' || c == '\n' || c == '\t'; } -/* -Copyright: Adam D. Ruppe, 2010 - 2019 -License: Boost License 1.0. -Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others - - Copyright Adam D. Ruppe 2010-2019. -Distributed under the Boost Software License, Version 1.0. - (See accompanying file LICENSE_1_0.txt or copy at - http://www.boost.org/LICENSE_1_0.txt) -*/ - - unittest { // Test for issue #120 string s = ` @@ -8570,3 +8650,73 @@ unittest { s2.indexOf("bubbles") < s2.indexOf("giggles"), "paragraph order incorrect:\n" ~ s2); } + +unittest { + // test for suncarpet email dec 24 2019 + // arbitrary id asduiwh + auto document = new Document(" + + + Element.querySelector Test + + +
+
Foo
+
Bar
+
+ +"); + + auto doc = document; + + assert(doc.querySelectorAll("div div").length == 2); + assert(doc.querySelector("div").querySelectorAll("div").length == 2); + assert(doc.querySelectorAll("> html").length == 0); + assert(doc.querySelector("head").querySelectorAll("> title").length == 1); + assert(doc.querySelector("head").querySelectorAll("> meta[charset]").length == 1); + + + assert(doc.root.matches("html")); + assert(!doc.root.matches("nothtml")); + assert(doc.querySelector("#foo > div").matches("div")); + assert(doc.querySelector("body > #foo").matches("#foo")); + + assert(doc.root.querySelectorAll(":root > body").length == 0); // the root has no CHILD root! + assert(doc.querySelectorAll(":root > body").length == 1); // but the DOCUMENT does + assert(doc.querySelectorAll(" > body").length == 1); // should mean the same thing + assert(doc.root.querySelectorAll(" > body").length == 1); // the root of HTML has this + assert(doc.root.querySelectorAll(" > html").length == 0); // but not this +} + +unittest { + // based on https://developer.mozilla.org/en-US/docs/Web/API/Element/closest example + auto document = new Document(`
+
Here is div-01 +
Here is div-02 +
Here is div-03
+
+
+
`, true, true); + + auto el = document.getElementById("div-03"); + assert(el.closest("#div-02").id == "div-02"); + assert(el.closest("div div").id == "div-03"); + assert(el.closest("article > div").id == "div-01"); + assert(el.closest(":not(div)").tagName == "article"); + + assert(el.closest("p") is null); + assert(el.closest("p, div") is el); +} + +/* +Copyright: Adam D. Ruppe, 2010 - 2019 +License: Boost License 1.0. +Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others + + Copyright Adam D. Ruppe 2010-2019. +Distributed under the Boost Software License, Version 1.0. + (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + + From 046d55e880df80414a606ccee918a2053d3a33cc Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Wed, 1 Jan 2020 11:51:20 -0500 Subject: [PATCH 05/24] omg so much scary stuff --- declarativeloader.d | 33 ++++++++++++++++--------- jni.d | 60 ++++++++++++++++++++++++++++++++++----------- rpc.d | 5 ++++ 3 files changed, 73 insertions(+), 25 deletions(-) diff --git a/declarativeloader.d b/declarativeloader.d index a7861a4..139b806 100644 --- a/declarativeloader.d +++ b/declarativeloader.d @@ -5,26 +5,27 @@ module arsd.declarativeloader; import std.range; -// @VariableLength indicates the value is saved in a MIDI like format -// @BigEndian, @LittleEndian -// @NumBytes!Field or @NumElements!Field controls length of embedded arrays -// @Tagged!Field indicates a tagged union. Each struct within should have @Tag(X) which is a value of Field -// @MustBe() causes it to throw if not the given value - -// @NotSaved indicates a struct member that is not actually saved in the file - +/// enum BigEndian; +/// enum LittleEndian; +/// @VariableLength indicates the value is saved in a MIDI like format enum VariableLength; +/// @NumBytes!Field or @NumElements!Field controls length of embedded arrays struct NumBytes(alias field) {} +/// ditto struct NumElements(alias field) {} +/// @Tagged!Field indicates a tagged union. Each struct within should have @Tag(X) which is a value of Field struct Tagged(alias field) {} -struct TagStruct(T) { T t; } +/// ditto auto Tag(T)(T t) { return TagStruct!T(t); } -enum NotSaved; +struct TagStruct(T) { T t; } struct MustBeStruct(T) { T t; } +/// The marked field is not in the actual file +enum NotSaved; +/// Insists the field must be a certain value, like for magic numbers auto MustBe(T)(T t) { return MustBeStruct!T(t); } @@ -65,7 +66,7 @@ union N(ty) { ubyte[ty.sizeof] bytes; } -// input range of ubytes... +/// input range of ubytes... int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) { int bytesConsumed; ubyte next() { @@ -160,9 +161,19 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) } } else { while(numElementsRemaining) { + //import std.stdio; writeln(memberName); E piece; auto by = loadFrom(piece, r, endianness); numElementsRemaining--; + + // such a filthy hack, needed for Java's mistake though :( + static if(__traits(compiles, piece.takesTwoSlots())) { + if(piece.takesTwoSlots()) { + __traits(getMember, t, memberName) ~= piece; + numElementsRemaining--; + } + } + bytesConsumed += by; __traits(getMember, t, memberName) ~= piece; } diff --git a/jni.d b/jni.d index c11561e..e385c4d 100644 --- a/jni.d +++ b/jni.d @@ -293,8 +293,12 @@ inout(char)[] fixupJavaClassName(inout(char)[] s) { } struct JavaTranslationConfig { + /// List the Java methods, imported to D. bool doImports; + /// List the native methods, assuming they should be exported from D bool doExports; + /// Put implementations inline. If false, this separates interface from impl for quicker builds with dmd -i. + bool inlineImplementations; } void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { @@ -345,8 +349,15 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output thisModule ~= "."; thisModule ~= lastClassName; - dco = "module " ~ thisModule ~ ";\n\n"; - dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, ImportExportImpl, JavaName, IJavaObject;\n\n"; + bool isInterface = (cf.access_flags & 0x0200) ? true : false; + + if(jtc.inlineImplementations) { + dco = "module " ~ thisModule ~ ";\n\n"; + } else { + dco ~= "module " ~ thisModule ~ "_d_interface;\n"; + } + + dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, JavaName, IJavaObject, ImportExportImpl;\n\n"; string[string] javaPackages; @@ -354,9 +365,9 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output if(lastClassName != originalClassName) dc ~= "@JavaName(\""~originalClassName~"\")\n"; - // FIXME: what if it is an interface? - - dc ~= "final class " ~ lastClassName ~ " : IJavaObject {\n";// JavaClass!(\""~javaPackage.replace("/", ".")~"\", "~lastClassName~") {\n"; + // so overriding Java classes from D is iffy and with separate implementation + // non final leads to linker errors anyway... + dc ~= (isInterface ? "interface " : "final class ") ~ lastClassName ~ " : IJavaObject {\n"; foreach(method; cf.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; if(native && !jtc.doExports) @@ -393,30 +404,51 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages); string args = javaSignatureToDTypeString(argsJava, javaPackages); + if(!jtc.inlineImplementations) { + if(ctor && args.length == 0) + continue; // FIXME skipping default ctor to avoid factory from trying to get to it in separate compilation + } + dc ~= "\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " {}" : ";")~"\n"; } } - // FIXME what if there is a name conflict? the prefix kinda handles it but i dont like how ugly it is foreach(pkg, prefix; javaPackages) { auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; // keeping thisModule because of the prefix nonsense //if(m == thisModule) //continue; - dco ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; + if(jtc.inlineImplementations) + dco ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; + else + dco ~= "import " ~ prefix ~ " = " ~ m ~ "_d_interface;\n"; } if(javaPackages.keys.length) dco ~= "\n"; dco ~= dc; - dco ~= "\tmixin IJavaObjectImplementation!(false);\n"; + if(!isInterface) + dco ~= "\tmixin IJavaObjectImplementation!(false);\n"; dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; dco ~= "}\n"; - dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; + if(jtc.inlineImplementations) { + if(!isInterface) + dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; + std.file.write(filename, dco); + } else { + string impl; + impl ~= "module " ~ thisModule ~ ";\n"; + impl ~= "public import " ~ thisModule ~ "_d_interface;\n\n"; + if(!isInterface) { + impl ~= "import arsd.jni : ImportExportImpl;\n"; + impl ~= "mixin ImportExportImpl!"~lastClassName~";\n"; + } - std.file.write(filename, dco); + std.file.write(filename, impl); + std.file.write(filename[0 .. $-2] ~ "_d_interface.d", dco); + } } string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaPackages) { @@ -439,7 +471,7 @@ string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaP if(type == "java/lang/String") { type = "string"; // or could be wstring... } else if(type == "java/lang/Object") { - type = "IJavaObject"; // or could be wstring... + type = "IJavaObject"; } else { // NOTE rughs strings in this file type = type.replace("$", "_"); @@ -1520,7 +1552,7 @@ interface IJavaObject { } -static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == IJavaObject)) { +static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == interface)) { import core.memory; auto ptr = GC.malloc(__traits(classInstanceSize, T)); ptr[0 .. __traits(classInstanceSize, T)] = typeid(T).initializer[]; @@ -1529,12 +1561,12 @@ static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == return obj; } -static auto fromExistingJavaObject(T)(jobject o) if(is(T == IJavaObject)) { +static auto fromExistingJavaObject(T)(jobject o) if(is(T == interface)) { static class Dummy : IJavaObject { mixin IJavaObjectImplementation!(false); mixin JavaPackageId!("java.lang", "Object"); } - return fromExistingJavaObject!Dummy(o); + return cast(T) cast(void*) fromExistingJavaObject!Dummy(o); // FIXME this is so wrong } diff --git a/rpc.d b/rpc.d index c74f0b9..334c514 100644 --- a/rpc.d +++ b/rpc.d @@ -6,6 +6,11 @@ module arsd.rpc; 1) integrate with arsd.eventloop 2) make it easy to use with other processes; pipe to a process and talk to it that way. perhaps with shared memory too? 3) extend the serialization capabilities + + + @Throws!(List, Of, Exceptions) + classes are also RPC proxied + stdin/out/err also redirected */ ///+ //example usage From d3506d8b007514df9d46c7930e0c5b0f2a19d10e Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 3 Jan 2020 10:04:58 -0500 Subject: [PATCH 06/24] catchup --- archive.d | 2 +- cgi.d | 6 +- dom.d | 71 ++++++++-- dub.json | 2 +- http2.d | 338 +++++++++++++++++++++++++----------------------- jni.d | 46 ++++++- simpledisplay.d | 2 +- 7 files changed, 288 insertions(+), 179 deletions(-) diff --git a/archive.d b/archive.d index 8d71883..b7c02c2 100644 --- a/archive.d +++ b/archive.d @@ -113,7 +113,7 @@ bool processTar( if(*bytesRemainingOnCurrentFile) { bool isNew = *bytesRemainingOnCurrentFile == header.size(); if(*bytesRemainingOnCurrentFile < 512) { - handleData(header, isNew, true, dataBuffer[0 .. *bytesRemainingOnCurrentFile]); + handleData(header, isNew, true, dataBuffer[0 .. cast(size_t) *bytesRemainingOnCurrentFile]); *bytesRemainingOnCurrentFile = 0; } else { handleData(header, isNew, false, dataBuffer[]); diff --git a/cgi.d b/cgi.d index 1d95f69..697049f 100644 --- a/cgi.d +++ b/cgi.d @@ -9069,11 +9069,11 @@ bool apiDispatcher()(Cgi cgi) { } +/ /* -Copyright: Adam D. Ruppe, 2008 - 2019 -License: Boost License 1.0. +Copyright: Adam D. Ruppe, 2008 - 2020 +License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0]. Authors: Adam D. Ruppe - Copyright Adam D. Ruppe 2008 - 2019. + Copyright Adam D. Ruppe 2008 - 2020. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/dom.d b/dom.d index d038095..b7e9567 100644 --- a/dom.d +++ b/dom.d @@ -33,6 +33,21 @@ If you want it to stand alone, just always use the `Document.parseUtf8` function or the constructor that takes a string. + + Symbol_groups: + + core_functionality = + + These members provide core functionality. The members on these classes + will provide most your direct interaction. + + bonus_functionality = + + These provide additional functionality for special use cases. + + implementations = + + These provide implementations of other functionality. +/ module arsd.dom; @@ -78,6 +93,7 @@ bool isConvenientAttribute(string name) { /// The main document interface, including a html parser. +/// Group: core_functionality class Document : FileResource { /// Convenience method for web scraping. Requires [arsd.http2] to be /// included in the build as well as [arsd.characterencodings]. @@ -1423,6 +1439,7 @@ class Document : FileResource { } /// This represents almost everything in the DOM. +/// Group: core_functionality class Element { /// Returns a collection of elements by selector. /// See: [Document.opIndex] @@ -3474,6 +3491,7 @@ class Element { // FIXME: since Document loosens the input requirements, it should probably be the sub class... /// Specializes Document for handling generic XML. (always uses strict mode, uses xml mime type and file header) +/// Group: core_functionality class XmlDocument : Document { this(string data) { contentType = "text/xml; charset=utf-8"; @@ -3491,6 +3509,7 @@ import std.string; /* domconvenience follows { */ /// finds comments that match the given txt. Case insensitive, strips whitespace. +/// Group: core_functionality Element[] findComments(Document document, string txt) { return findComments(document.root, txt); } @@ -3510,6 +3529,7 @@ Element[] findComments(Element element, string txt) { } /// An option type that propagates null. See: [Element.optionSelector] +/// Group: implementations struct MaybeNullElement(SomeElementType) { this(SomeElementType ele) { this.element = ele; @@ -3543,6 +3563,7 @@ struct MaybeNullElement(SomeElementType) { /++ A collection of elements which forwards methods to the children. +/ +/// Group: implementations struct ElementCollection { /// this(Element e) { @@ -3641,6 +3662,7 @@ struct ElementCollection { /// this puts in operators and opDispatch to handle string indexes and properties, forwarding to get and set functions. +/// Group: implementations mixin template JavascriptStyleDispatch() { /// string opDispatch(string name)(string v = null) if(name != "popFront") { // popFront will make this look like a range. Do not want. @@ -3668,6 +3690,7 @@ mixin template JavascriptStyleDispatch() { /// A proxy object to do the Element class' dataset property. See Element.dataset for more info. /// /// Do not create this object directly. +/// Group: implementations struct DataSet { /// this(Element e) { @@ -3691,6 +3714,7 @@ struct DataSet { } /// Proxy object for attributes which will replace the main opDispatch eventually +/// Group: implementations struct AttributeSet { /// this(Element e) { @@ -3718,6 +3742,7 @@ struct AttributeSet { /// for style, i want to be able to set it with a string like a plain attribute, /// but also be able to do properties Javascript style. +/// Group: implementations struct ElementStyle { this(Element parent) { _element = parent; @@ -3874,6 +3899,7 @@ import std.range; Document implements this interface with type = text/html (see Document.contentType for more info) and data = document.toString, so you can return Documents anywhere web.d expects FileResources. +/ +/// Group: bonus_functionality interface FileResource { /// the content-type of the file. e.g. "text/html; charset=utf-8" or "image/png" @property string contentType() const; @@ -3885,10 +3911,12 @@ interface FileResource { ///. +/// Group: bonus_functionality enum NodeType { Text = 3 } /// You can use this to do an easy null check or a dynamic cast+null check on any element. +/// Group: core_functionality T require(T = Element, string file = __FILE__, int line = __LINE__)(Element e) if(is(T : Element)) in {} out(ret) { assert(ret !is null); } @@ -3901,6 +3929,7 @@ body { ///. +/// Group: core_functionality class DocumentFragment : Element { ///. this(Document _parentDocument) { @@ -3951,6 +3980,7 @@ class DocumentFragment : Element { /// /// The output parameter can be given to append to an existing buffer. You don't have to /// pass one; regardless, the return value will be usable for you, with just the data encoded. +/// Group: core_functionality string htmlEntitiesEncode(string data, Appender!string output = appender!string(), bool encodeNonAscii = true) { // if there's no entities, we can save a lot of time by not bothering with the // decoding loop. This check cuts the net toString time by better than half in my test. @@ -4003,11 +4033,13 @@ string htmlEntitiesEncode(string data, Appender!string output = appender!string( } /// An alias for htmlEntitiesEncode; it works for xml too +/// Group: core_functionality string xmlEntitiesEncode(string data) { return htmlEntitiesEncode(data); } /// This helper function is used for decoding html entities. It has a hard-coded list of entities and characters. +/// Group: core_functionality dchar parseEntity(in dchar[] entity) { switch(entity[1..$-1]) { case "quot": @@ -5505,6 +5537,7 @@ import std.stdio; /// This takes a string of raw HTML and decodes the entities into a nice D utf-8 string. /// By default, it uses loose mode - it will try to return a useful string from garbage input too. /// Set the second parameter to true if you'd prefer it to strictly throw exceptions on garbage input. +/// Group: core_functionality string htmlEntitiesDecode(string data, bool strict = false) { // this check makes a *big* difference; about a 50% improvement of parse speed on my test. if(data.indexOf("&") == -1) // all html entities begin with & @@ -5585,6 +5618,7 @@ string htmlEntitiesDecode(string data, bool strict = false) { return cast(string) a; // assumeUnique is actually kinda slow, lol } +/// Group: implementations abstract class SpecialElement : Element { this(Document _parentDocument) { super(_parentDocument); @@ -5602,6 +5636,7 @@ abstract class SpecialElement : Element { } ///. +/// Group: implementations class RawSource : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5634,6 +5669,7 @@ class RawSource : SpecialElement { string source; } +/// Group: implementations abstract class ServerSideCode : SpecialElement { this(Document _parentDocument, string type) { super(_parentDocument); @@ -5663,6 +5699,7 @@ abstract class ServerSideCode : SpecialElement { } ///. +/// Group: implementations class PhpCode : ServerSideCode { ///. this(Document _parentDocument, string s) { @@ -5676,6 +5713,7 @@ class PhpCode : ServerSideCode { } ///. +/// Group: implementations class AspCode : ServerSideCode { ///. this(Document _parentDocument, string s) { @@ -5689,6 +5727,7 @@ class AspCode : ServerSideCode { } ///. +/// Group: implementations class BangInstruction : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5728,6 +5767,7 @@ class BangInstruction : SpecialElement { } ///. +/// Group: implementations class QuestionInstruction : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5768,6 +5808,7 @@ class QuestionInstruction : SpecialElement { } ///. +/// Group: implementations class HtmlComment : SpecialElement { ///. this(Document _parentDocument, string s) { @@ -5811,6 +5852,7 @@ class HtmlComment : SpecialElement { ///. +/// Group: implementations class TextNode : Element { public: ///. @@ -5926,6 +5968,7 @@ class TextNode : Element { */ ///. +/// Group: implementations class Link : Element { ///. @@ -6064,6 +6107,7 @@ class Link : Element { } ///. +/// Group: implementations class Form : Element { ///. @@ -6314,6 +6358,7 @@ class Form : Element { import std.conv; ///. +/// Group: implementations class Table : Element { ///. @@ -6553,6 +6598,7 @@ class Table : Element { } /// Represents a table row element - a +/// Group: implementations class TableRow : Element { ///. this(Document _parentDocument) { @@ -6565,6 +6611,7 @@ class TableRow : Element { } /// Represents anything that can be a table cell - or html. +/// Group: implementations class TableCell : Element { ///. this(Document _parentDocument, string _tagName) { @@ -6601,6 +6648,7 @@ class TableCell : Element { ///. +/// Group: implementations class MarkupException : Exception { ///. @@ -6610,6 +6658,7 @@ class MarkupException : Exception { } /// This is used when you are using one of the require variants of navigation, and no matching element can be found in the tree. +/// Group: implementations class ElementNotFoundException : Exception { /// type == kind of element you were looking for and search == a selector describing the search. @@ -6624,6 +6673,7 @@ class ElementNotFoundException : Exception { /// The html struct is used to differentiate between regular text nodes and html in certain functions /// /// Easiest way to construct it is like this: `auto html = Html("

hello

");` +/// Group: core_functionality struct Html { /// This string holds the actual html. Use it to retrieve the contents. string source; @@ -7294,19 +7344,24 @@ int intFromHex(string hex) { } /++ - Represents a parsed CSS selector. + Represents a parsed CSS selector. You never have to use this directly, but you can if you know it is going to be reused a lot to avoid a bit of repeat parsing. See_Also: - [Element.querySelector] - [Element.querySelectorAll] - [Document.querySelector] - [Document.querySelectorAll] + $(LIST + * [Element.querySelector] + * [Element.querySelectorAll] + * [Element.matches] + * [Element.closest] + * [Document.querySelector] + * [Document.querySelectorAll] + ) +/ + /// Group: core_functionality struct Selector { SelectorComponent[] components; string original; /++ - Parses the selector string and returns the usable structure. + Parses the selector string and constructs the usable structure. +/ this(string cssSelector) { components = parseSelectorString(cssSelector); @@ -8709,11 +8764,11 @@ unittest { } /* -Copyright: Adam D. Ruppe, 2010 - 2019 +Copyright: Adam D. Ruppe, 2010 - 2020 License: Boost License 1.0. Authors: Adam D. Ruppe, with contributions by Nick Sabalausky, Trass3r, and ketmar among others - Copyright Adam D. Ruppe 2010-2019. + Copyright Adam D. Ruppe 2010-2020. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/dub.json b/dub.json index 2070dab..812dd0f 100644 --- a/dub.json +++ b/dub.json @@ -3,7 +3,7 @@ "targetType": "library", "importPaths": ["."], "sourceFiles": ["package.d"], - "description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more.", + "description": "Subpackage collection for web, database, terminal ui, gui, scripting, and more with a commitment to long-term compatibility and stability.", "authors": ["Adam D. Ruppe"], "license":"BSL-1.0", "dependencies": { diff --git a/http2.d b/http2.d index afc23fc..e141d1a 100644 --- a/http2.d +++ b/http2.d @@ -1,4 +1,4 @@ -// Copyright 2013-2019, Adam D. Ruppe. +// Copyright 2013-2020, Adam D. Ruppe. /++ This is version 2 of my http/1.1 client implementation. @@ -20,6 +20,8 @@ debug(arsd_http2_verbose) debug=arsd_http2; debug(arsd_http2) import std.stdio : writeln; +version=arsd_http_internal_implementation; + version(without_openssl) {} else { version=use_openssl; @@ -28,6 +30,12 @@ version(older_openssl) {} else version=newer_openssl; } +version(arsd_http_winhttp_implementation) { + pragma(lib, "winhttp") + import core.sys.windows.winhttp; + // FIXME: alter the dub package file too +} + /++ @@ -605,6 +613,176 @@ class HttpRequest { /// Automatically follow a redirection? bool followLocation = false; + this() { + } + + /// + this(Uri where, HttpVerb method) { + populateFromInfo(where, method); + } + + /// Final url after any redirections + string finalUrl; + + void populateFromInfo(Uri where, HttpVerb method) { + auto parts = where; + finalUrl = where.toString(); + requestParameters.method = method; + requestParameters.host = parts.host; + requestParameters.port = cast(ushort) parts.port; + requestParameters.ssl = parts.scheme == "https"; + if(parts.port == 0) + requestParameters.port = requestParameters.ssl ? 443 : 80; + requestParameters.uri = parts.path.length ? parts.path : "/"; + if(parts.query.length) { + requestParameters.uri ~= "?"; + requestParameters.uri ~= parts.query; + } + } + + ~this() { + } + + ubyte[] sendBuffer; + + HttpResponse responseData; + private HttpClient parentClient; + + size_t bodyBytesSent; + size_t bodyBytesReceived; + + State state_; + State state() { return state_; } + State state(State s) { + assert(state_ != State.complete); + return state_ = s; + } + /// Called when data is received. Check the state to see what data is available. + void delegate(HttpRequest) onDataReceived; + + enum State { + /// The request has not yet been sent + unsent, + + /// The send() method has been called, but no data is + /// sent on the socket yet because the connection is busy. + pendingAvailableConnection, + + /// The headers are being sent now + sendingHeaders, + + /// The body is being sent now + sendingBody, + + /// The request has been sent but we haven't received any response yet + waitingForResponse, + + /// We have received some data and are currently receiving headers + readingHeaders, + + /// All headers are available but we're still waiting on the body + readingBody, + + /// The request is complete. + complete, + + /// The request is aborted, either by the abort() method, or as a result of the server disconnecting + aborted + } + + /// Sends now and waits for the request to finish, returning the response. + HttpResponse perform() { + send(); + return waitForCompletion(); + } + + /// Sends the request asynchronously. + void send() { + sendPrivate(true); + } + + private void sendPrivate(bool advance) { + if(state != State.unsent && state != State.aborted) + return; // already sent + string headers; + + headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; + if(requestParameters.useHttp11) + headers ~= " HTTP/1.1\r\n"; + else + headers ~= " HTTP/1.0\r\n"; + headers ~= "Host: "~requestParameters.host~"\r\n"; + if(requestParameters.userAgent.length) + headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; + if(requestParameters.contentType.length) + headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; + if(requestParameters.authorization.length) + headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; + if(requestParameters.bodyData.length) + headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; + if(requestParameters.acceptGzip) + headers ~= "Accept-Encoding: gzip\r\n"; + + foreach(header; requestParameters.headers) + headers ~= header ~ "\r\n"; + + headers ~= "\r\n"; + + sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; + + // import std.stdio; writeln("******* ", sendBuffer); + + responseData = HttpResponse.init; + responseData.requestParameters = requestParameters; + bodyBytesSent = 0; + bodyBytesReceived = 0; + state = State.pendingAvailableConnection; + + bool alreadyPending = false; + foreach(req; pending) + if(req is this) { + alreadyPending = true; + break; + } + if(!alreadyPending) { + pending ~= this; + } + + if(advance) + HttpRequest.advanceConnections(); + } + + + /// Waits for the request to finish or timeout, whichever comes first. + HttpResponse waitForCompletion() { + while(state != State.aborted && state != State.complete) { + if(state == State.unsent) + send(); + if(auto err = HttpRequest.advanceConnections()) + throw new Exception("waitForCompletion got err " ~ to!string(err)); + } + + return responseData; + } + + /// Aborts this request. + void abort() { + this.state = State.aborted; + // FIXME + } + + HttpRequestParameters requestParameters; /// + + version(arsd_http_winhttp_implementation) { + public static void resetInternals() { + + } + + static assert(0, "implementation not finished"); + } + + + version(arsd_http_internal_implementation) { private static { // we manage the actual connections. When a request is made on a particular // host, we try to reuse connections. We may open more than one connection per @@ -1170,165 +1348,7 @@ class HttpRequest { return stillAlive; } - this() { } - - /// - this(Uri where, HttpVerb method) { - populateFromInfo(where, method); - } - - /// Final url after any redirections - string finalUrl; - - void populateFromInfo(Uri where, HttpVerb method) { - auto parts = where; - finalUrl = where.toString(); - requestParameters.method = method; - requestParameters.host = parts.host; - requestParameters.port = cast(ushort) parts.port; - requestParameters.ssl = parts.scheme == "https"; - if(parts.port == 0) - requestParameters.port = requestParameters.ssl ? 443 : 80; - requestParameters.uri = parts.path.length ? parts.path : "/"; - if(parts.query.length) { - requestParameters.uri ~= "?"; - requestParameters.uri ~= parts.query; - } - } - - ~this() { - } - - ubyte[] sendBuffer; - - HttpResponse responseData; - private HttpClient parentClient; - - size_t bodyBytesSent; - size_t bodyBytesReceived; - - State state_; - State state() { return state_; } - State state(State s) { - assert(state_ != State.complete); - return state_ = s; - } - /// Called when data is received. Check the state to see what data is available. - void delegate(HttpRequest) onDataReceived; - - enum State { - /// The request has not yet been sent - unsent, - - /// The send() method has been called, but no data is - /// sent on the socket yet because the connection is busy. - pendingAvailableConnection, - - /// The headers are being sent now - sendingHeaders, - - /// The body is being sent now - sendingBody, - - /// The request has been sent but we haven't received any response yet - waitingForResponse, - - /// We have received some data and are currently receiving headers - readingHeaders, - - /// All headers are available but we're still waiting on the body - readingBody, - - /// The request is complete. - complete, - - /// The request is aborted, either by the abort() method, or as a result of the server disconnecting - aborted - } - - /// Sends now and waits for the request to finish, returning the response. - HttpResponse perform() { - send(); - return waitForCompletion(); - } - - /// Sends the request asynchronously. - void send() { - sendPrivate(true); - } - - private void sendPrivate(bool advance) { - if(state != State.unsent && state != State.aborted) - return; // already sent - string headers; - - headers ~= to!string(requestParameters.method) ~ " "~requestParameters.uri; - if(requestParameters.useHttp11) - headers ~= " HTTP/1.1\r\n"; - else - headers ~= " HTTP/1.0\r\n"; - headers ~= "Host: "~requestParameters.host~"\r\n"; - if(requestParameters.userAgent.length) - headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; - if(requestParameters.contentType.length) - headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; - if(requestParameters.authorization.length) - headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; - if(requestParameters.bodyData.length) - headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; - if(requestParameters.acceptGzip) - headers ~= "Accept-Encoding: gzip\r\n"; - - foreach(header; requestParameters.headers) - headers ~= header ~ "\r\n"; - - headers ~= "\r\n"; - - sendBuffer = cast(ubyte[]) headers ~ requestParameters.bodyData; - - // import std.stdio; writeln("******* ", sendBuffer); - - responseData = HttpResponse.init; - responseData.requestParameters = requestParameters; - bodyBytesSent = 0; - bodyBytesReceived = 0; - state = State.pendingAvailableConnection; - - bool alreadyPending = false; - foreach(req; pending) - if(req is this) { - alreadyPending = true; - break; - } - if(!alreadyPending) { - pending ~= this; - } - - if(advance) - HttpRequest.advanceConnections(); - } - - - /// Waits for the request to finish or timeout, whichever comes first. - HttpResponse waitForCompletion() { - while(state != State.aborted && state != State.complete) { - if(state == State.unsent) - send(); - if(auto err = HttpRequest.advanceConnections()) - throw new Exception("waitForCompletion got err " ~ to!string(err)); - } - - return responseData; - } - - /// Aborts this request. - void abort() { - this.state = State.aborted; - // FIXME - } - - HttpRequestParameters requestParameters; /// } /// diff --git a/jni.d b/jni.d index e385c4d..0583865 100644 --- a/jni.d +++ b/jni.d @@ -120,6 +120,24 @@ +/ module arsd.jni; +/* + New Java classes: + + class Foo : extends!Bar { + + mixin stuff; + } + mixin stuff; + + The `extends` template creates a wrapper that calls the nonvirtual + methods, so `super()` just works. + + receiving an object should perhaps always give a subclass that is javafied; + calls the virtuals, unless of course it is final. + + dynamic downcasts of java objects will probably never work. +*/ + /+ For interfaces: @@ -261,6 +279,7 @@ private string getJavaName(alias a)() { version(WithClassLoadSupport) { import arsd.declarativeloader; +/// translator. void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, bool delegate(string className) classFilter = null) { import std.zip; import std.file; @@ -278,7 +297,7 @@ void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, Jav } } -inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { +private inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { import std.string; s ~= "."; // lol i suck s = s.replace(".function.", ".function_."); @@ -286,12 +305,13 @@ inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { return s[0 .. $-1]; // god i am such a bad programmer } -inout(char)[] fixupJavaClassName(inout(char)[] s) { +private inout(char)[] fixupJavaClassName(inout(char)[] s) { if(s == "Throwable" || s == "Object" || s == "Exception" || s == "Error" || s == "TypeInfo") s = cast(typeof(s)) "Java" ~ s; return s; } +/// For the translator struct JavaTranslationConfig { /// List the Java methods, imported to D. bool doImports; @@ -299,8 +319,11 @@ struct JavaTranslationConfig { bool doExports; /// Put implementations inline. If false, this separates interface from impl for quicker builds with dmd -i. bool inlineImplementations; + /// Treat native functions as imports, otherwise fills in as exports. Make sure doImports == true. + bool nativesAreImports = true; } +/// translator. void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { import std.file; import std.path; @@ -370,10 +393,15 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output dc ~= (isInterface ? "interface " : "final class ") ~ lastClassName ~ " : IJavaObject {\n"; foreach(method; cf.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; - if(native && !jtc.doExports) - continue; - if(!native && !jtc.doImports) - continue; + if(jtc.nativesAreImports) { + if(!jtc.doImports) + continue; + } else { + if(native && !jtc.doExports) + continue; + if(!native && !jtc.doImports) + continue; + } auto port = native ? "@Export" : "@Import"; if(method.flags & 1) { // public @@ -2084,3 +2112,9 @@ union jvalue jdouble d; jobject l; } + +/* + Copyright 2019-2020, Adam D. Ruppe. + Boost license. or whatever. + Most work done in December 2019. +*/ diff --git a/simpledisplay.d b/simpledisplay.d index 1dc5700..4d0e970 100644 --- a/simpledisplay.d +++ b/simpledisplay.d @@ -459,7 +459,7 @@ I live in the eastern United States, so I will most likely not be around at night in that US east timezone. - License: Copyright Adam D. Ruppe, 2011-2017. Released under the Boost Software License. + License: Copyright Adam D. Ruppe, 2011-2020. Released under the Boost Software License. Building documentation: You may wish to use the `arsd.ddoc` file from my github with building the documentation for simpledisplay yourself. It will give it a bit more style. From a0367f644ce840b045a7b1fbfe26464096398f86 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 3 Jan 2020 10:06:18 -0500 Subject: [PATCH 07/24] this little change made it succeed building 9s, 3 GB whereas the old failed after a couple minute wait eating > 18 GB --- jni.d | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jni.d b/jni.d index 0583865..00eba8b 100644 --- a/jni.d +++ b/jni.d @@ -1132,6 +1132,10 @@ private enum ImportImplementationString = q{ } }; +import std.string; +static immutable ImportImplementationString_static = ImportImplementationString.replace("STATIC", "Static"); +static immutable ImportImplementationString_not = ImportImplementationString.replace("STATIC", ""); + private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { import std.traits; @@ -1197,8 +1201,7 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { auto jobj = T.internalJavaClassHandle_; - import std.string; - mixin(ImportImplementationString.replace("STATIC", "Static")); + mixin(ImportImplementationString_static); } else pragma(mangle, method.mangleof) @@ -1222,8 +1225,7 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { throw new Exception("Cannot find Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - import std.string; - mixin(ImportImplementationString.replace("STATIC", "")); + mixin(ImportImplementationString_not); } } From ee3b087f3e15507c767d8a80397b0f770d52dfb3 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Fri, 3 Jan 2020 12:28:32 -0500 Subject: [PATCH 08/24] improved --- jni.d | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jni.d b/jni.d index 00eba8b..1c01c97 100644 --- a/jni.d +++ b/jni.d @@ -394,6 +394,7 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output foreach(method; cf.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; if(jtc.nativesAreImports) { + native = false; // kinda hacky but meh if(!jtc.doImports) continue; } else { @@ -411,7 +412,7 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output auto name = method.name; // FIXME: maybe check name for other D keywords but since so many overlap with java I think we will be ok most the time for now - if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString") { + if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString" || name == "init") { // toString is special btw in order to avoid a dmd bug port ~= " @JavaName(\""~name~"\")"; name ~= "_"; @@ -457,7 +458,9 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output if(!isInterface) dco ~= "\tmixin IJavaObjectImplementation!(false);\n"; - dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; + //dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; + // the following saves some compile time of the bindings; might as well do some calculations ahead of time + dco ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ "\";\n"; dco ~= "}\n"; @@ -1605,7 +1608,7 @@ mixin template ImportExportImpl(Class) { private static arsd.jni.JavaBridge!(Class) _javaDBridge; } -class JavaBridge(Class) { +final class JavaBridge(Class) { static foreach(memberName; __traits(derivedMembers, Class)) { // validations static if(is(typeof(__traits(getMember, Class, memberName).offsetof))) From b116c0e5a39d1a119015ff6fcd1a2fa80bb327c3 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 4 Jan 2020 21:49:20 -0500 Subject: [PATCH 09/24] initial websocket client code, wip --- http2.d | 581 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 575 insertions(+), 6 deletions(-) diff --git a/http2.d b/http2.d index e141d1a..64120d5 100644 --- a/http2.d +++ b/http2.d @@ -1658,8 +1658,8 @@ version(use_openssl) { if(SSL_connect(ssl) == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl connect"); } } @@ -1671,8 +1671,8 @@ version(use_openssl) { if(retval == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl send"); } return retval; @@ -1687,8 +1687,8 @@ version(use_openssl) { if(retval == -1) { ERR_print_errors_fp(core.stdc.stdio.stderr); int i; - printf("wtf\n"); - scanf("%d\n", i); + //printf("wtf\n"); + //scanf("%d\n", i); throw new Exception("ssl send"); } return retval; @@ -2034,3 +2034,572 @@ class FormData { } } +private bool bicmp(in ubyte[] item, in char[] search) { + if(item.length != search.length) return false; + + foreach(i; 0 .. item.length) { + ubyte a = item[i]; + ubyte b = search[i]; + if(a >= 'A' && a <= 'Z') + a += 32; + //if(b >= 'A' && b <= 'Z') + //b += 32; + if(a != b) + return false; + } + + return true; +} + +/++ + WebSocket client, based on the browser api. + + on receive + on frame + on message ++/ +class WebSocket { + private Uri uri; + private string[string] cookies; + private string origin; + + private string host; + private ushort port; + private bool ssl; + + private int readyState_; + + private Socket socket; + private ubyte[] receiveBuffer; + private size_t receiveBufferUsedLength; + + private Config config; + + enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. + enum OPEN = 1; /// The connection is open and ready to communicate. + enum CLOSING = 2; /// The connection is in the process of closing. + enum CLOSED = 3; /// The connection is closed or couldn't be opened. + + /++ + + +/ + static struct Config { + /++ + These control the size of the receive buffer. + + It starts at the initial size, will temporarily + balloon up to the maximum size, and will reuse + a buffer up to the likely size. + + Anything larger than the maximum size will cause + the connection to be aborted and an exception thrown. + This is to protect you against a peer trying to + exhaust your memory, while keeping the user-level + processing simple. + +/ + size_t initialReceiveBufferSize = 4096; + size_t likelyReceiveBufferSize = 4096; /// ditto + size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; + string origin; /// Origin URL to send with the handshake, if desired. + string protocol; /// the protocol header, if desired. + } + + /++ + Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. + +/ + int readyState() { + return readyState_; + } + + /++ +wss://echo.websocket.org + +/ + this(Uri uri, Config config = Config.init) + in (uri.scheme == "ws" || uri.scheme == "wss") + { + this.uri = uri; + this.config = config; + + this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); + + host = uri.host; + ssl = uri.scheme == "wss"; + port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); + + if(ssl) { + version(with_openssl) + socket = new SslClientSocket(AddressFamily.INET, SocketType.STREAM); + else + throw new Exception("SSL not compiled in"); + } else + socket = new Socket(AddressFamily.INET, SocketType.STREAM); + + } + + /++ + + +/ + void connect() { + socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... + // FIXME: websocket handshake could and really should be async too. + + auto uri = this.uri.path.length ? this.uri.path : "/"; + if(this.uri.query.length) { + uri ~= "?"; + uri ~= this.uri.query; + } + + // the headers really shouldn't be bigger than this, at least + // the chunks i need to process + ubyte[4096] buffer; + size_t pos; + + void append(in char[][] items...) { + foreach(what; items) { + buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; + pos += what.length; + } + } + + append("GET ", uri, " HTTP/1.1\r\n"); + append("Host: ", this.uri.host, "\r\n"); + + append("Upgrade: websocket\r\n"); + append("Connection: Upgrade\r\n"); + append("Sec-WebSocket-Version: 13\r\n"); + + // FIXME: randomize this + append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); + + if(config.protocol.length) + append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); + if(config.origin.length) + append("Origin: ", origin, "\r\n"); + + append("\r\n"); + + auto remaining = buffer[0 .. pos]; + //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); + while(remaining.length) { + auto r = socket.send(remaining); + if(r < 0) + throw new Exception(lastSocketError()); + if(r == 0) + throw new Exception("unexpected connection termination"); + remaining = remaining[r .. $]; + } + + // the response shouldn't be especially large at this point, just + // headers for the most part. gonna try to get it in the stack buffer. + // then copy stuff after headers, if any, to the frame buffer. + ubyte[] used; + + void more() { + auto r = socket.receive(buffer[used.length .. $]); + + if(r < 0) + throw new Exception(lastSocketError()); + if(r == 0) + throw new Exception("unexpected connection termination"); + //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); + + used = buffer[0 .. used.length + r]; + } + + more(); + + import std.algorithm; + if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) + throw new Exception("didn't get a websocket answer"); + // skip the status line + while(used.length && used[0] != '\n') + used = used[1 .. $]; + + if(used.length == 0) + throw new Exception("wtf"); + + if(used.length < 1) + more(); + + used = used[1 .. $]; // skip the \n + + if(used.length == 0) + more(); + + // checks on the protocol from ehaders + bool isWebsocket; + bool isUpgrade; + const(ubyte)[] protocol; + const(ubyte)[] accept; + + while(used.length) { + if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { + used = used[2 .. $]; + break; // all done + } + int idxColon; + while(idxColon < used.length && used[idxColon] != ':') + idxColon++; + if(idxColon == used.length) + more(); + auto idxStart = idxColon + 1; + while(idxStart < used.length && used[idxStart] == ' ') + idxStart++; + if(idxStart == used.length) + more(); + auto idxEnd = idxStart; + while(idxEnd < used.length && used[idxEnd] != '\r') + idxEnd++; + if(idxEnd == used.length) + more(); + + auto headerName = used[0 .. idxColon]; + auto headerValue = used[idxStart .. idxEnd]; + + // move past this header + used = used[idxEnd .. $]; + // and the \r\n + if(2 <= used.length) + used = used[2 .. $]; + + if(headerName.bicmp("upgrade")) { + if(headerValue.bicmp("websocket")) + isWebsocket = true; + } else if(headerName.bicmp("connection")) { + if(headerValue.bicmp("upgrade")) + isUpgrade = true; + } else if(headerName.bicmp("sec-websocket-accept")) { + accept = headerValue; + } else if(headerName.bicmp("sec-websocket-protocol")) { + protocol = headerValue; + } + + if(!used.length) { + more(); + } + } + + + if(!isWebsocket) + throw new Exception("didn't answer as websocket"); + if(!isUpgrade) + throw new Exception("didn't answer as upgrade"); + + + // FIXME: check protocol if config requested one + // FIXME: check accept + + receiveBuffer[0 .. used.length] = used[]; + receiveBufferUsedLength = used.length; + + readyState_ = OPEN; + + if(onopen) + onopen(); + } + + /++ + + +/ + void close(int code = 0, string reason = null) + in (reason.length < 123) + { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.close; + wss.data = cast(ubyte[]) reason; + wss.send(&llsend); + + readyState_ = CLOSING; + + socket.shutdown(SocketShutdown.SEND); + } + + /++ + + +/ + void ping() { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.ping; + wss.send(&llsend); + } + + // automatically handled.... + void pong() { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.pong; + wss.send(&llsend); + } + + /++ + + +/ + void send(in char[] textData) { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.text; + wss.data = cast(ubyte[]) textData; + wss.send(&llsend); + } + + private void llsend(ubyte[] d) { + while(d.length) { + auto r = socket.send(d); + if(r <= 0) throw new Exception("wtf"); + d = d[r .. $]; + } + } + + /*private*/ bool llreceive() { + auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); + if(r == 0) + return false; + if(r <= 0) + throw new Exception("wtf"); + receiveBufferUsedLength += r; + return true; + } + + public void autoprocess() { + do { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + while(d.length) { + auto orig = d; + auto m = WebSocketMessage.read(d); + if(d is orig) + break; + if(m.opcode == WebSocketOpcode.close) { + readyState_ = CLOSED; + if(onclose) + onclose(); + } else if(m.opcode == WebSocketOpcode.text) { + if(onmessage) + onmessage(m.textData); + } else { + // FIXME + } + } + receiveBufferUsedLength -= s.length - d.length; + } while(llreceive()); + } + + + /++ + + +/ + void send(in ubyte[] binaryData) { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData; + wss.send(&llsend); + } + + void delegate() onclose; /// + void delegate() onerror; /// + void delegate(in char[]) onmessage; /// + void delegate() onopen; /// + + /* + const int bufferedAmount // amount pending + const string extensions + + const string protocol + const string url + */ +} + +/* copy/paste from cgi.d */ +private { + enum WebSocketOpcode : ubyte { + text = 1, + binary = 2, + // 3, 4, 5, 6, 7 RESERVED + close = 8, + ping = 9, + pong = 10, + // 11,12,13,14,15 RESERVED + } + + struct WebSocketMessage { + bool fin; + bool rsv1; + bool rsv2; + bool rsv3; + WebSocketOpcode opcode; // 4 bits + bool masked; + ubyte lengthIndicator; // don't set this when building one to send + ulong realLength; // don't use when sending + ubyte[4] maskingKey; // don't set this when sending + ubyte[] data; + + static WebSocketMessage simpleMessage(WebSocketOpcode opcode, void[] data) { + WebSocketMessage msg; + msg.fin = true; + msg.opcode = opcode; + msg.data = cast(ubyte[]) data; + + return msg; + } + + private void send(scope void delegate(ubyte[]) llsend) { + ubyte[64] headerScratch; + int headerScratchPos = 0; + + realLength = data.length; + + { + ubyte b1; + b1 |= cast(ubyte) opcode; + b1 |= rsv3 ? (1 << 4) : 0; + b1 |= rsv2 ? (1 << 5) : 0; + b1 |= rsv1 ? (1 << 6) : 0; + b1 |= fin ? (1 << 7) : 0; + + headerScratch[0] = b1; + headerScratchPos++; + } + + { + headerScratchPos++; // we'll set header[1] at the end of this + auto rlc = realLength; + ubyte b2; + b2 |= masked ? (1 << 7) : 0; + + assert(headerScratchPos == 2); + + if(realLength > 65535) { + // use 64 bit length + b2 |= 0x7f; + + // FIXME: double check endinaness + foreach(i; 0 .. 8) { + headerScratch[2 + 7 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 8; + } else if(realLength > 127) { + // use 16 bit length + b2 |= 0x7e; + + // FIXME: double check endinaness + foreach(i; 0 .. 2) { + headerScratch[2 + 1 - i] = rlc & 0x0ff; + rlc >>>= 8; + } + + headerScratchPos += 2; + } else { + // use 7 bit length + b2 |= realLength & 0b_0111_1111; + } + + headerScratch[1] = b2; + } + + //assert(!masked, "masking key not properly implemented"); + if(masked) { + // FIXME: randomize this + headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; + headerScratchPos += 4; + + // we'll just mask it in place... + int keyIdx = 0; + foreach(i; 0 .. data.length) { + data[i] = data[i] ^ maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); + llsend(headerScratch[0 .. headerScratchPos]); + llsend(data); + } + + static WebSocketMessage read(ref ubyte[] d) { + WebSocketMessage msg; + assert(d.length >= 2); + + auto orig = d; + + ubyte b = d[0]; + + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); + b >>= 4; + msg.rsv3 = b & 0x01; + b >>= 1; + msg.rsv2 = b & 0x01; + b >>= 1; + msg.rsv1 = b & 0x01; + b >>= 1; + msg.fin = b & 0x01; + + b = d[1]; + msg.masked = (b & 0b1000_0000) ? true : false; + msg.lengthIndicator = b & 0b0111_1111; + + d = d[2 .. $]; + + if(msg.lengthIndicator == 0x7e) { + // 16 bit length + msg.realLength = 0; + + foreach(i; 0 .. 2) { + msg.realLength |= d[0] << ((1-i) * 8); + d = d[1 .. $]; + } + } else if(msg.lengthIndicator == 0x7f) { + // 64 bit length + msg.realLength = 0; + + foreach(i; 0 .. 8) { + msg.realLength |= d[0] << ((7-i) * 8); + d = d[1 .. $]; + } + } else { + // 7 bit length + msg.realLength = msg.lengthIndicator; + } + + if(msg.masked) { + msg.maskingKey = d[0 .. 4]; + d = d[4 .. $]; + } + + if(msg.realLength > d.length) { + d = orig; + return WebSocketMessage.init; + } + + msg.data = d[0 .. msg.realLength]; + d = d[msg.realLength .. $]; + + if(msg.masked) { + // let's just unmask it now + int keyIdx = 0; + foreach(i; 0 .. msg.data.length) { + msg.data[i] = msg.data[i] ^ msg.maskingKey[keyIdx]; + if(keyIdx == 3) + keyIdx = 0; + else + keyIdx++; + } + } + + return msg; + } + + char[] textData() { + return cast(char[]) data; + } + } +} From a0236624f260a77525c89eb350f34c8eefc2fa8a Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 4 Jan 2020 21:59:18 -0500 Subject: [PATCH 10/24] the old websocket server was pure trash. and even the core one was iffy in certain cases --- cgi.d | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/cgi.d b/cgi.d index 697049f..d653110 100644 --- a/cgi.d +++ b/cgi.d @@ -1586,6 +1586,7 @@ class Cgi { immutable(ubyte)[] data; void rdo(const(ubyte)[] d) { + //import std.stdio; writeln(d); sendAll(ir.source, d); } @@ -3370,6 +3371,8 @@ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxC try { fun(cgi); cgi.close(); + if(cgi.websocketMode) + closeConnection = true; } catch(ConnectionException ce) { closeConnection = true; } catch(Throwable t) { @@ -3665,6 +3668,8 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) { try { fun(cgi); cgi.close(); + if(cgi.websocketMode) + closeConnection = true; } catch(ConnectionException ce) { // broken pipe or something, just abort the connection closeConnection = true; @@ -3947,6 +3952,7 @@ class BufferedInputRange { // we might have to grow the buffer if(minBytesToSettleFor > underlyingBuffer.length || view.length == underlyingBuffer.length) { if(allowGrowth) { + import std.stdio; writeln("growth"); auto viewStart = view.ptr - underlyingBuffer.ptr; size_t growth = 4096; // make sure we have enough for what we're being asked for @@ -3959,7 +3965,7 @@ class BufferedInputRange { } do { - auto freeSpace = underlyingBuffer[underlyingBuffer.ptr - view.ptr + view.length .. $]; + auto freeSpace = underlyingBuffer[view.ptr - underlyingBuffer.ptr + view.length .. $]; try_again: auto ret = source.receive(freeSpace); if(ret == Socket.ERROR) { @@ -3981,6 +3987,7 @@ class BufferedInputRange { return; } + //import std.stdio; writeln(view.ptr); writeln(underlyingBuffer.ptr); writeln(view.length, " ", ret, " = ", view.length + ret); view = underlyingBuffer[view.ptr - underlyingBuffer.ptr .. view.length + ret]; } while(view.length < minBytesToSettleFor); } @@ -3992,6 +3999,7 @@ class BufferedInputRange { /// You do not need to call this if you always want to wait for more data when you /// consume some. ubyte[] consume(size_t bytes) { + //import std.stdio; writeln("consuime ", bytes, "/", view.length); view = view[bytes > $ ? $ : bytes .. $]; if(view.length == 0) { view = underlyingBuffer[0 .. 0]; // go ahead and reuse the beginning @@ -4639,18 +4647,13 @@ version(cgi_with_websocket) { // note: this blocks WebSocketMessage recv() { // FIXME: should we automatically handle pings and pongs? - assert(!cgi.idlol.empty()); + if(cgi.idlol.empty()) + throw new Exception("remote side disconnected"); cgi.idlol.popFront(0); WebSocketMessage message; - auto info = cgi.idlol.front(); - - // FIXME: read should prolly take the whole range so it can request more if needed - // read should also go ahead and consume the range - message = WebSocketMessage.read(info); - - cgi.idlol.consume(info.length); + message = WebSocketMessage.read(cgi.idlol); return message; } @@ -4700,7 +4703,7 @@ version(cgi_with_websocket) { WebSocket acceptWebsocket(Cgi cgi) { assert(!cgi.closed); assert(!cgi.outputtedResponseData); - cgi.setResponseStatus("101 Web Socket Protocol Handshake"); + cgi.setResponseStatus("101 Switching Protocols"); cgi.header("Upgrade: WebSocket"); cgi.header("Connection: upgrade"); @@ -4834,7 +4837,15 @@ version(cgi_with_websocket) { cgi.flush(); } - static WebSocketMessage read(ubyte[] d) { + static WebSocketMessage read(BufferedInputRange ir) { + + auto d = ir.front(); + while(d.length < 2) { + ir.popFront(); + d = ir.front(); + } + auto start = d; + WebSocketMessage msg; assert(d.length >= 2); @@ -4882,7 +4893,11 @@ version(cgi_with_websocket) { d = d[4 .. $]; } - msg.data = d[0 .. $]; + //if(d.length < msg.realLength) { + + //} + msg.data = d[0 .. msg.realLength]; + d = d[msg.realLength .. $]; if(msg.masked) { // let's just unmask it now @@ -4896,6 +4911,8 @@ version(cgi_with_websocket) { } } + ir.consume(start.length - d.length); + return msg; } From 9ad7b08ce096cda72181104a8e73e6197330caad Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 5 Jan 2020 09:40:07 -0500 Subject: [PATCH 11/24] thanks BorisCarvajal for spotting bug in issue 233 --- archive.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive.d b/archive.d index b7c02c2..8d14c26 100644 --- a/archive.d +++ b/archive.d @@ -112,7 +112,7 @@ bool processTar( if(*bytesRemainingOnCurrentFile) { bool isNew = *bytesRemainingOnCurrentFile == header.size(); - if(*bytesRemainingOnCurrentFile < 512) { + if(*bytesRemainingOnCurrentFile <= 512) { handleData(header, isNew, true, dataBuffer[0 .. cast(size_t) *bytesRemainingOnCurrentFile]); *bytesRemainingOnCurrentFile = 0; } else { From acb5825c7c0b1c666eba19ec711a57b7467c1b49 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 5 Jan 2020 22:43:47 -0500 Subject: [PATCH 12/24] websocket polling/blocking api --- http2.d | 273 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 231 insertions(+), 42 deletions(-) diff --git a/http2.d b/http2.d index 64120d5..1d78fe5 100644 --- a/http2.d +++ b/http2.d @@ -2052,11 +2052,70 @@ private bool bicmp(in ubyte[] item, in char[] search) { } /++ - WebSocket client, based on the browser api. + WebSocket client, based on the browser api, though also with other api options. - on receive - on frame - on message + --- + auto ws = new WebSocket(URI("ws://....")); + + ws.onmessage = (in char[] msg) { + ws.send("a reply"); + }; + + ws.connect(); + + WebSocket.eventLoop(); + --- + + Symbol_groups: + foundational = + Used with all API styles. + + browser_api = + API based on the standard in the browser. + + event_loop_integration = + Integrating with external event loops is done through static functions. You should + call these BEFORE doing anything else with the WebSocket module or class. + + $(PITFALL NOT IMPLEMENTED) + --- + WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); + // or something like that. it is not implemented yet. + --- + $(PITFALL NOT IMPLEMENTED) + + blocking_api = + The blocking API is best used when you only need basic functionality with a single connection. + + --- + WebSocketMessage msg; + do { + // FIXME good demo + } while(msg); + --- + + Or to check for blocks before calling: + + --- + try_to_process_more: + while(ws.isMessageBuffered()) { + auto msg = ws.waitForNextMessage(); + // process msg + } + if(ws.isDataPending()) { + ws.lowLevelReceive(); + goto try_to_process_more; + } else { + // nothing ready, you can do other things + // or at least sleep a while before trying + // to process more. + if(ws.readyState == WebSocket.OPEN) { + Thread.sleep(1.seconds); + goto try_to_process_more; + } + } + --- + +/ class WebSocket { private Uri uri; @@ -2083,6 +2142,7 @@ class WebSocket { /++ +/ + /// Group: foundational static struct Config { /++ These control the size of the receive buffer. @@ -2104,6 +2164,8 @@ class WebSocket { string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; string origin; /// Origin URL to send with the handshake, if desired. string protocol; /// the protocol header, if desired. + + int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping } /++ @@ -2116,6 +2178,7 @@ class WebSocket { /++ wss://echo.websocket.org +/ + /// Group: foundational this(Uri uri, Config config = Config.init) in (uri.scheme == "ws" || uri.scheme == "wss") { @@ -2141,6 +2204,7 @@ wss://echo.websocket.org /++ +/ + /// Group: foundational void connect() { socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... // FIXME: websocket handshake could and really should be async too. @@ -2289,7 +2353,7 @@ wss://echo.websocket.org // FIXME: check protocol if config requested one - // FIXME: check accept + // FIXME: check accept for the right hash receiveBuffer[0 .. used.length] = used[]; receiveBufferUsedLength = used.length; @@ -2301,8 +2365,9 @@ wss://echo.websocket.org } /++ - + Closes the connection, sending a graceful teardown message to the other side. +/ + /// Group: foundational void close(int code = 0, string reason = null) in (reason.length < 123) { @@ -2318,8 +2383,9 @@ wss://echo.websocket.org } /++ - + Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. +/ + /// Group: foundational void ping() { WebSocketMessage wss; wss.fin = true; @@ -2336,8 +2402,9 @@ wss://echo.websocket.org } /++ - + Sends a text message through the websocket. +/ + /// Group: foundational void send(in char[] textData) { WebSocketMessage wss; wss.fin = true; @@ -2346,6 +2413,19 @@ wss://echo.websocket.org wss.send(&llsend); } + /++ + Sends a binary message through the websocket. + +/ + /// Group: foundational + void send(in ubyte[] binaryData) { + WebSocketMessage wss; + wss.fin = true; + wss.opcode = WebSocketOpcode.binary; + wss.data = cast(ubyte[]) binaryData; + wss.send(&llsend); + } + + private void llsend(ubyte[] d) { while(d.length) { auto r = socket.send(d); @@ -2354,7 +2434,13 @@ wss://echo.websocket.org } } - /*private*/ bool llreceive() { + /++ + Waits for more data off the low-level socket and adds it to the pending buffer. + + Returns `true` if the connection is still active. + +/ + /// Group: blocking_api + public bool lowLevelReceive() { auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); if(r == 0) return false; @@ -2364,47 +2450,146 @@ wss://echo.websocket.org return true; } - public void autoprocess() { + /++ + Waits for and returns the next complete message on the socket. + + Note that the onmessage function is still called, right before + this returns. + +/ + /// Group: blocking_api + public WebSocketMessage waitForNextMessage() { do { - ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; - auto s = d; - while(d.length) { - auto orig = d; - auto m = WebSocketMessage.read(d); - if(d is orig) - break; - if(m.opcode == WebSocketOpcode.close) { - readyState_ = CLOSED; - if(onclose) - onclose(); - } else if(m.opcode == WebSocketOpcode.text) { - if(onmessage) - onmessage(m.textData); - } else { - // FIXME - } - } - receiveBufferUsedLength -= s.length - d.length; - } while(llreceive()); + auto m = processOnce(); + if(m.populated) + return m; + } while(lowLevelReceive()); + + return WebSocketMessage.init; // FIXME? maybe. } + /++ + Tells if [waitForNextMessage] would block. + +/ + /// Group: blocking_api + public bool waitForNextMessageWouldBlock() { + checkAgain: + if(isMessageBuffered()) + return false; + if(!isDataPending()) + return true; + while(isDataPending()) + lowLevelReceive(); + goto checkAgain; + } + + /++ + Is there a message in the buffer already? + If `true`, [waitForNextMessage] is guaranteed to return immediately. + If `false`, check [isDataPending] as the next step. + +/ + /// Group: blocking_api + public bool isMessageBuffered() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + if(d.length) { + auto orig = d; + auto m = WebSocketMessage.read(d); + // that's how it indicates that it needs more data + if(d !is orig) + return true; + } + + return false; + } + + /++ + Is data pending on the socket? Also check [isMessageBuffered] to see if there + is already a message in memory too. + + If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] + again. + +/ + /// Group: blocking_api + public bool isDataPending() { + static SocketSet readSet; + if(readSet is null) + readSet = new SocketSet(); + + version(with_openssl) + if(auto s = cast(SslClientSocket) socket) { + // select doesn't handle the case with stuff + // left in the ssl buffer so i'm checking it separately + if(s.dataPending()) { + return true; + } + } + + readSet.add(socket); + + //tryAgain: + auto selectGot = Socket.select(readSet, null, null, 0.seconds /* timeout */); + if(selectGot == 0) { /* timeout */ + // timeout + return false; + } else if(selectGot == -1) { /* interrupted */ + return false; + } else { /* ready */ + if(readSet.isSet(socket)) { + return true; + } + } + + return false; + } + + private WebSocketMessage processOnce() { + ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; + auto s = d; + // FIXME: handle continuation frames + WebSocketMessage m; + if(d.length) { + auto orig = d; + m = WebSocketMessage.read(d); + // that's how it indicates that it needs more data + if(d is orig) + return WebSocketMessage.init; + if(m.opcode == WebSocketOpcode.close) { + readyState_ = CLOSED; + if(onclose) + onclose(); + } else if(m.opcode == WebSocketOpcode.text) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + // FIXME + } + } + receiveBufferUsedLength -= s.length - d.length; + + return m; + } + + private void autoprocess() { + // FIXME + do { + processOnce(); + } while(lowLevelReceive()); + } + + + void delegate() onclose; /// + void delegate() onerror; /// + void delegate(in char[]) ontextmessage; /// + void delegate() onopen; /// /++ +/ - void send(in ubyte[] binaryData) { - WebSocketMessage wss; - wss.fin = true; - wss.opcode = WebSocketOpcode.binary; - wss.data = cast(ubyte[]) binaryData; - wss.send(&llsend); + /// Group: browser_api + void onmessage(void delegate(in char[]) dg) { + ontextmessage = dg; } - void delegate() onclose; /// - void delegate() onerror; /// - void delegate(in char[]) onmessage; /// - void delegate() onopen; /// - /* const int bufferedAmount // amount pending const string extensions @@ -2417,6 +2602,7 @@ wss://echo.websocket.org /* copy/paste from cgi.d */ private { enum WebSocketOpcode : ubyte { + continuation = 0, text = 1, binary = 2, // 3, 4, 5, 6, 7 RESERVED @@ -2426,7 +2612,8 @@ private { // 11,12,13,14,15 RESERVED } - struct WebSocketMessage { + public struct WebSocketMessage { + private bool populated; bool fin; bool rsv1; bool rsv2; @@ -2533,6 +2720,8 @@ private { ubyte b = d[0]; + msg.populated = true; + msg.opcode = cast(WebSocketOpcode) (b & 0x0f); b >>= 4; msg.rsv3 = b & 0x01; From 07f73fec9791d2bb84a08e86f2f8726d9ab5dc64 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 9 Jan 2020 18:50:01 -0500 Subject: [PATCH 13/24] failed but keep it for later --- jni.d | 523 +++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 441 insertions(+), 82 deletions(-) diff --git a/jni.d b/jni.d index 1c01c97..6bc457c 100644 --- a/jni.d +++ b/jni.d @@ -120,6 +120,11 @@ +/ module arsd.jni; +// I need to figure out some way that users can set this. maybe. or dynamically fall back from newest to oldest we can handle +__gshared auto JNI_VERSION_DESIRED = JNI_VERSION_1_6; + +// i could perhaps do a struct to bean thingy + /* New Java classes: @@ -177,6 +182,9 @@ module arsd.jni; but woof that isn't so much different than an anonymous class anymore. +/ +/// hack used by the translator for default constructors not really being a default constructor +struct Default {} + /+ final class CharSequence : JavaClass!("java.lang", CharSequence) { @Import string toString(); // this triggers a dmd segfault! whoa. FIXME dmd @@ -239,6 +247,7 @@ interface CharSequence : JavaInterface!("java.lang", CharSequence) { +/ interface JavaInterface(string javaPackage, CRTP) : IJavaObject { mixin JavaPackageId!(javaPackage, CRTP); + mixin JavaInterfaceMembers!(null); } /// I may not keep this. But for now if you need a dummy class in D @@ -273,8 +282,14 @@ private string getJavaName(alias a)() { return name; } +/+ + to benchmark build stats + cd ~/Android/d_android/java_bindings/android/java + /usr/bin/time -f "%E %M" dmd -o- -c `find . | grep -E '\.d$'` ~/arsd/jni.d -I../.. ++/ + /+ Java class file definitions { +/ -// see: https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-4.html#jvms-4.6 +// see: https://docs.oracle.com/javase/specs/jvms/se13/html/jvms-4.html version(WithClassLoadSupport) { import arsd.declarativeloader; @@ -287,14 +302,37 @@ void jarToD()(string jarPath, string dPackagePrefix, string outputDirectory, Jav auto zip = new ZipArchive(read(jarPath)); + ClassFile[string] allClasses; + foreach(name, am; zip.directory) { if(name.endsWith(".class")) { - // FIXME: use classFilter zip.expand(am); - rawClassBytesToD(cast(ubyte[]) am.expandedData, dPackagePrefix, outputDirectory, jtc); - am.expandedData = null; // let the GC take it if it wants + + ClassFile cf; + + auto classBytes = cast(ubyte[]) am.expandedData; + auto originalClassBytes = classBytes; + + debug try { + cf.loadFrom!ClassFile(classBytes); + } catch(Exception e) { + std.file.write("spam.bin", originalClassBytes); + throw e; + } else + cf.loadFrom!ClassFile(classBytes); + + string className = cf.className.idup; + + if(classFilter is null || classFilter(className)) + allClasses[className] = cf; + + //rawClassBytesToD(cast(ubyte[]) am.expandedData, dPackagePrefix, outputDirectory, jtc); + //am.expandedData = null; // let the GC take it if it wants } } + + foreach(name, cf; allClasses) + rawClassStructToD(cf, dPackagePrefix, outputDirectory, jtc, allClasses); } private inout(char)[] fixupKeywordsInJavaPackageName(inout(char)[] s) { @@ -323,16 +361,22 @@ struct JavaTranslationConfig { bool nativesAreImports = true; } +/// translator +void rawClassBytesToD()(ubyte[] bytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { + ClassFile f; + f.loadFrom(bytes); + rawClassStructToD(f, dPackagePrefix, outputDirectory, jtc, null); +} + /// translator. -void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc) { +void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputDirectory, JavaTranslationConfig jtc, ClassFile[string] allClasses) { import std.file; import std.path; import std.algorithm; import std.array; import std.string; - ClassFile cf; - cf.loadFrom!ClassFile(classBytes); + string importPrefix = "import"; const(char)[] javaPackage; const(char)[] lastClassName; @@ -364,6 +408,9 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output filename ~= "/"; filename ~= lastClassName ~ ".d"; + if(filename.indexOf("-") != -1) + return; + string dco; @@ -373,6 +420,7 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output thisModule ~= lastClassName; bool isInterface = (cf.access_flags & 0x0200) ? true : false; + bool isAbstract = (cf.access_flags & ClassFile.ACC_ABSTRACT) ? true : false; if(jtc.inlineImplementations) { dco = "module " ~ thisModule ~ ";\n\n"; @@ -380,17 +428,109 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output dco ~= "module " ~ thisModule ~ "_d_interface;\n"; } - dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, JavaName, IJavaObject, ImportExportImpl;\n\n"; + dco ~= "import arsd.jni : IJavaObjectImplementation, JavaPackageId, JavaName, IJavaObject, ImportExportImpl, JavaInterfaceMembers;\n"; + dco ~= "static import arsd.jni;\n\n"; string[string] javaPackages; + string[string] javaPackagesReturn; + string[string] javaPackagesArguments; string dc; if(lastClassName != originalClassName) dc ~= "@JavaName(\""~originalClassName~"\")\n"; + bool outputMixinTemplate = false; + + string mainThing; + string helperThing; + // so overriding Java classes from D is iffy and with separate implementation // non final leads to linker errors anyway... - dc ~= (isInterface ? "interface " : "final class ") ~ lastClassName ~ " : IJavaObject {\n"; + //mainThing ~= (isInterface ? "interface " : (jtc.inlineImplementations ? "class " : isAbstract ? "abstract class " : "final class ")) ~ lastClassName ~ " : "; + + // not putting super class on inline implementations since that forces vtable... + if(jtc.inlineImplementations) { + auto scn = cf.superclassName; + //if(!scn.startsWith("java/")) { + // superclasses need the implementation too so putting it in the return list lol + if(scn.length && scn != "java/lang/Object") { // && scn in allClasses) { + mainThing ~= javaObjectToDTypeString(scn, javaPackages, javaPackagesReturn, importPrefix); + mainThing ~= ", "; + } + //} + } + + foreach(name; cf.interfacesNames) { + //if(name.startsWith("java/")) + //continue; // these probably aren't important to D and can really complicate version management + //if(name !in allClasses) + //continue; + mainThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); + mainThing ~= ", "; + } + + mainThing ~= "arsd.jni.IJavaObject"; + + mainThing ~= " {\n"; + + //mainThing ~= "\talias _d_helper this;\n"; + mainThing ~= "\tfinal auto opDispatch(string name, Args...)(Args args) if(name != \"opCall\") { auto local = _d_helper(); return __traits(getMember, local, name)(args); }\n"; + if(isInterface) + mainThing ~= "\tfinal auto _d_helper()() { return getDProxy(); }\n"; + else { + mainThing ~= "\t" ~ lastClassName ~ "_d_methods _d_proxy_;\n"; + mainThing ~= "\tfinal auto _d_helper()() {\n"; + mainThing ~= "\t\tif(getDProxy() is null) _d_proxy_ = arsd.jni.createDProxy!("~lastClassName~"_d_methods)(this);\n"; + mainThing ~= "\t\treturn getDProxy();\n"; + mainThing ~= "\t}\n"; + mainThing ~= "\toverride " ~ lastClassName ~ "_d_methods getDProxy() { return _d_proxy_; }\n"; + } + + + helperThing ~= "interface " ~ lastClassName ~ "_d_methods : "; + + // not putting super class on inline implementations since that forces vtable... + if(jtc.inlineImplementations) { + auto scn = cf.superclassName; + //if(!scn.startsWith("java/")) { + // superclasses need the implementation too so putting it in the return list lol + if(scn.length && scn != "java/lang/Object") { // && scn in allClasses) { + helperThing ~= javaObjectToDTypeString(scn, javaPackages, javaPackagesReturn, importPrefix); + helperThing ~= "_d_methods, "; + } + //} + } + + foreach(name; cf.interfacesNames) { + //if(name.startsWith("java/")) + //continue; // these probably aren't important to D and can really complicate version management + //if(name !in allClasses) + //continue; + helperThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); + helperThing ~= "_d_methods, "; + } + + helperThing ~= "IJavaObject"; + + helperThing ~= " {\n"; + + helperThing ~= "\tmixin JavaInterfaceMembers!(\"L" ~ cn ~ ";\");\n"; + + + + + + + + + + string tm; + string[string] tmpackages; + string[string] tmpackagesr; + string[string] tmpackagesa; + + + string[string] mentioned; foreach(method; cf.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; if(jtc.nativesAreImports) { @@ -406,13 +546,36 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output auto port = native ? "@Export" : "@Import"; if(method.flags & 1) { // public - if(method.flags & 0x0008) + bool addToMixinTemplate; + + bool wasStatic; + + bool maybeOverride = false;// !isInterface; + if(method.flags & 0x0008) { port ~= " static"; + maybeOverride = false; + wasStatic = true; + } + if(method.flags & method_info.ACC_ABSTRACT) { + maybeOverride = false; + //if(!isInterface) + port ~= " abstract"; + } else { + // this represents a default implementation in a Java interface + // D cannot express this... so I need to add it to the mixin template + // associated with this interface as well. + if(isInterface && (!(method.flags & 0x0008))) { + addToMixinTemplate = true; + } + } + + if(maybeOverride && method.isOverride(allClasses)) + port ~= " override"; auto name = method.name; // FIXME: maybe check name for other D keywords but since so many overlap with java I think we will be ok most the time for now - if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString" || name == "init") { + if(name == "debug" || name == "delete" || name == "with" || name == "version" || name == "cast" || name == "union" || name == "align" || name == "alias" || name == "in" || name == "out" || name == "toString" || name == "init" || name == "lazy" || name == "immutable" || name == "is" || name == "function" || name == "delegate" || name == "template") { // toString is special btw in order to avoid a dmd bug port ~= " @JavaName(\""~name~"\")"; name ~= "_"; @@ -422,6 +585,10 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output name = name.replace("$", "_"); bool ctor = name == ""; + if(ctor && isInterface) { + ctor = false; + name = "CONSTRUCTOR"; + } auto sig = method.signature; @@ -429,19 +596,70 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output assert(lidx != -1); auto retJava = sig[lidx + 1 .. $]; auto argsJava = sig[1 .. lidx]; + auto retJava2 = retJava; + auto argsJava2 = argsJava; - string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages); - string args = javaSignatureToDTypeString(argsJava, javaPackages); + string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages, javaPackagesReturn, importPrefix); + string args = javaSignatureToDTypeString(argsJava, javaPackages, javaPackagesArguments, importPrefix); if(!jtc.inlineImplementations) { if(ctor && args.length == 0) - continue; // FIXME skipping default ctor to avoid factory from trying to get to it in separate compilation + args = "arsd.jni.Default"; } - dc ~= "\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " {}" : ";")~"\n"; + string men = cast(immutable) (name ~ "(" ~ args ~ ")"); + if(men in mentioned) + continue; // avoid duplicate things. idk why this is there though + mentioned[men] = men; + + string proto = cast(string) ("\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " { assert(0); }" : ";")~"\n"); + if(wasStatic || ctor) + mainThing ~= proto; + else + helperThing ~= proto; + + if(addToMixinTemplate) { + string pfx = "tmimport"; + string ret1 = ctor ? "" : javaSignatureToDTypeString(retJava2, tmpackages, tmpackagesr, pfx); + string args1 = javaSignatureToDTypeString(argsJava2, tmpackages, tmpackagesa, pfx); + tm ~= "\t\t"~port~" " ~ ret1 ~ (ret1.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args1~"){ assert(0); }\n"; + } } } + if(!isInterface) { + mainThing ~= "\tmixin IJavaObjectImplementation!(false);\n"; + mainThing ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; + } else { + mainThing ~= "\tmixin JavaInterfaceMembers!(\"L" ~ cn ~ ";\");\n"; + if(outputMixinTemplate && tm.length) { + mainThing ~= "\tmixin template JavaDefaultImplementations() {\n"; + + foreach(pkg, prefix; tmpackages) { + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; + if(jtc.inlineImplementations) + tm ~= "\t\timport " ~ prefix ~ " = " ~ m ~ ";\n"; + else + tm ~= "\t\timport " ~ prefix ~ " = " ~ m ~ "_d_interface;\n"; + } + if(!jtc.inlineImplementations) + foreach(pkg, prefix; tmpackagesr) { + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; + tm ~= "\t\timport impl_" ~ prefix ~ " = " ~ m ~ ";\n"; + } + + mainThing ~= tm; + mainThing ~= "\t}\n"; + } + } + + + mainThing ~= "}\n\n"; + helperThing ~= "}\n\n"; + dc ~= mainThing; + dc ~= "\n\n"; + dc ~= helperThing; + foreach(pkg, prefix; javaPackages) { auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; // keeping thisModule because of the prefix nonsense @@ -456,25 +674,26 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output dco ~= "\n"; dco ~= dc; - if(!isInterface) - dco ~= "\tmixin IJavaObjectImplementation!(false);\n"; //dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; // the following saves some compile time of the bindings; might as well do some calculations ahead of time - dco ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ "\";\n"; - - dco ~= "}\n"; + //dco ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; if(jtc.inlineImplementations) { - if(!isInterface) - dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; + dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; std.file.write(filename, dco); } else { string impl; impl ~= "module " ~ thisModule ~ ";\n"; impl ~= "public import " ~ thisModule ~ "_d_interface;\n\n"; - if(!isInterface) { - impl ~= "import arsd.jni : ImportExportImpl;\n"; - impl ~= "mixin ImportExportImpl!"~lastClassName~";\n"; + + impl ~= "import arsd.jni : ImportExportImpl;\n"; + impl ~= "mixin ImportExportImpl!"~lastClassName~";\n"; + + impl ~= "\n"; + foreach(pkg, prefix; javaPackagesReturn) { + // I also need to import implementations of return values so they just work + auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; + impl ~= "import " ~ prefix ~ " = " ~ m ~ ";\n"; } std.file.write(filename, impl); @@ -482,7 +701,50 @@ void rawClassBytesToD()(ubyte[] classBytes, string dPackagePrefix, string output } } -string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaPackages) { +string javaObjectToDTypeString(const(char)[] input, ref string[string] javaPackages, ref string[string] detailedPackages, string importPrefix) { + + string ret; + + if(input == "java/lang/String") { + ret = "string"; // or could be wstring... + } else if(input == "java/lang/Object") { + ret = "IJavaObject"; + } else { + // NOTE rughs strings in this file + string type = input.replace("$", "_").idup; + + string jp, cn, dm; + + auto idx = type.lastIndexOf("/"); + if(idx != -1) { + jp = type[0 .. idx].replace("/", ".").fixupKeywordsInJavaPackageName; + cn = type[idx + 1 .. $].fixupJavaClassName; + dm = jp ~ "." ~ cn; + } else { + cn = type; + dm = jp; + } + + string prefix; + if(auto n = dm in javaPackages) { + prefix = *n; + } else { + import std.conv; + // FIXME: this scheme sucks, would prefer something deterministic + prefix = importPrefix ~ to!string(javaPackages.keys.length); + //prefix = dm.replace(".", "0"); + + javaPackages[dm] = prefix; + detailedPackages[dm] = prefix; + } + + ret = prefix ~ (prefix.length ? ".":"") ~ cn; + } + + return ret; +} + +string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaPackages, ref string[string] detailedPackages, string importPrefix) { string all; while(js.length) { @@ -490,7 +752,7 @@ string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaP switch(js[0]) { case '[': js = js[1 .. $]; - type = javaSignatureToDTypeString(js, javaPackages); + type = javaSignatureToDTypeString(js, javaPackages, detailedPackages, importPrefix); type ~= "[]"; break; case 'L': @@ -499,39 +761,7 @@ string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaP type = js[1 .. idx].idup; js = js[idx + 1 .. $]; - if(type == "java/lang/String") { - type = "string"; // or could be wstring... - } else if(type == "java/lang/Object") { - type = "IJavaObject"; - } else { - // NOTE rughs strings in this file - type = type.replace("$", "_"); - - string jp, cn, dm; - - idx = type.lastIndexOf("/"); - if(idx != -1) { - jp = type[0 .. idx].replace("/", ".").fixupKeywordsInJavaPackageName; - cn = type[idx + 1 .. $].fixupJavaClassName; - dm = jp ~ "." ~ cn; - } else { - cn = type; - dm = jp; - } - - string prefix; - if(auto n = dm in javaPackages) { - prefix = *n; - } else { - import std.conv; - // FIXME: this scheme sucks, would prefer something deterministic - prefix = "import" ~ to!string(javaPackages.keys.length); - - javaPackages[dm] = prefix; - } - - type = prefix ~ (prefix.length ? ".":"") ~ cn; - } + type = javaObjectToDTypeString(type, javaPackages, detailedPackages, importPrefix); break; case 'V': js = js[1 .. $]; type = "void"; break; case 'Z': js = js[1 .. $]; type = "bool"; break; @@ -542,7 +772,7 @@ string javaSignatureToDTypeString(ref const(char)[] js, ref string[string] javaP case 'F': js = js[1 .. $]; type = "float"; break; case 'D': js = js[1 .. $]; type = "double"; break; case 'I': js = js[1 .. $]; type = "int"; break; - default: assert(0); + default: assert(0, js); } if(all.length) all ~= ", "; @@ -602,7 +832,7 @@ struct cp_info { @BigEndian: double bytes; } - enum CONSTANT_NameAndType = 12; // sizeof = 2 + enum CONSTANT_NameAndType = 12; // sizeof = 4 struct CONSTANT_NameAndType_info { @BigEndian: ushort name_index; @@ -631,6 +861,18 @@ struct cp_info { ushort bootstrap_method_attr_index; ushort name_and_type_index; } + enum CONSTANT_Module = 19; + struct CONSTANT_Module_info { + @BigEndian: + ushort name_index; + } + enum CONSTANT_Package = 20; + struct CONSTANT_Package_info { + @BigEndian: + ushort name_index; + } + + ubyte tag; @Tagged!(tag) @@ -649,12 +891,21 @@ struct cp_info { @Tag(CONSTANT_MethodHandle) CONSTANT_MethodHandle_info methodHandle_info; @Tag(CONSTANT_MethodType) CONSTANT_MethodType_info methodType_info; @Tag(CONSTANT_InvokeDynamic) CONSTANT_InvokeDynamic_info invokeDynamic_info; + @Tag(CONSTANT_Module) CONSTANT_Module_info module_info; + @Tag(CONSTANT_Package) CONSTANT_Package_info package_info; } Info info; bool takesTwoSlots() { return (tag == CONSTANT_Long || tag == CONSTANT_Double); } + + string toString() { + if(tag == CONSTANT_Utf8) + return cast(string) info.utf8_info.bytes; + import std.format; + return format("cp_info(%s)", tag); + } } struct field_info { @@ -724,7 +975,17 @@ struct ClassFile { } const(char)[] superclassName() { - return this.constant(this.constant(this.super_class).info.class_info.name_index).info.utf8_info.bytes; + if(this.super_class) + return this.constant(this.constant(this.super_class).info.class_info.name_index).info.utf8_info.bytes; + return null; + } + + const(char)[][] interfacesNames() { + typeof(return) ret; + foreach(iface; interfaces) { + ret ~= this.constant(this.constant(iface).info.class_info.name_index).info.utf8_info.bytes; + } + return ret; } Method[] methodsListing() { @@ -734,15 +995,40 @@ struct ClassFile { m.name = this.constant(met.name_index).info.utf8_info.bytes; m.signature = this.constant(met.descriptor_index).info.utf8_info.bytes; m.flags = met.access_flags; + m.cf = &this; ms ~= m; } return ms; } + bool hasConcreteMethod(const(char)[] name, const(char)[] signature, ClassFile[string] allClasses) { + // I don't really care cuz I don't use the same root in D + if(this.className == "java/lang/Object") + return false; + + foreach(m; this.methodsListing) { + if(m.name == name)// && m.signature == signature) + return true; + //return (m.flags & method_info.ACC_ABSTRACT) ? false : true; // abstract impls do not count as methods as far as overrides are concerend... + } + + if(auto s = this.superclassName in allClasses) + return s.hasConcreteMethod(name, signature, allClasses); + return false; + } + static struct Method { const(char)[] name; const(char)[] signature; ushort flags; + ClassFile* cf; + bool isOverride(ClassFile[string] allClasses) { + if(name == "") + return false; + if(auto s = cf.superclassName in allClasses) + return s.hasConcreteMethod(name, signature, allClasses); + return false; + } } @@ -824,14 +1110,14 @@ export jint JNI_OnLoad(JavaVM* vm, void* reserved) { activeJvm = vm; JNIEnv* env; - if ((*vm).GetEnv(vm, cast(void**) &env, JNI_VERSION_1_6) != JNI_OK) { + if ((*vm).GetEnv(vm, cast(void**) &env, JNI_VERSION_DESIRED) != JNI_OK) { return JNI_ERR; } try { foreach(init; classInitializers_) if(init(env) != 0) - return JNI_ERR; + {}//return JNI_ERR; foreach(init; newClassInitializers_) if(init(env) != 0) return JNI_ERR; @@ -841,7 +1127,7 @@ export jint JNI_OnLoad(JavaVM* vm, void* reserved) { return JNI_ERR; } - return JNI_VERSION_1_6; + return JNI_VERSION_DESIRED; } extern(System) export void JNI_OnUnload(JavaVM* vm, void* reserved) { @@ -947,7 +1233,7 @@ auto createJvm()() { //options[1].optionString = `-Djava.class.path=c:\Users\me\program\jni\`; /* user classes */ //options[2].optionString = `-Djava.library.path=c:\Users\me\program\jdk-13.0.1\lib\`; /* set native library path */ - vm_args.version_ = JNI_VERSION_1_6; + vm_args.version_ = JNI_VERSION_DESIRED; vm_args.options = options.ptr; vm_args.nOptions = cast(int) options.length; vm_args.ignoreUnrecognized = true; @@ -1161,17 +1447,27 @@ private mixin template JavaImportImpl(T, alias method, size_t overloadIndex) { } else { jc = T.internalJavaClassHandle_; } - _jmethodID = (*env).GetMethodID(env, jc, - "", - // java method string is (args)ret - ("(" ~ DTypesToJniString!(typeof(args)) ~ ")V\0").ptr - ); + static if(args.length == 1 && is(typeof(args[0]) == arsd.jni.Default)) + _jmethodID = (*env).GetMethodID(env, jc, + "", + // java method string is (args)ret + ("()V\0").ptr + ); + else + _jmethodID = (*env).GetMethodID(env, jc, + "", + // java method string is (args)ret + ("(" ~ DTypesToJniString!(typeof(args)) ~ ")V\0").ptr + ); if(!_jmethodID) throw new Exception("Cannot find static Java method " ~ T.stringof ~ "." ~ __traits(identifier, method)); } - auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); + static if(args.length == 1 && is(typeof(args[0]) == arsd.jni.Default)) + auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID); + else + auto o = (*env).NewObject(env, T.internalJavaClassHandle_, _jmethodID, DDataToJni(env, args).args); this_.internalJavaHandle_ = o; return this_; } @@ -1582,7 +1878,6 @@ interface IJavaObject { enum Import; /// UDA to indicate you are importing the method from Java. Do NOT put a body on these methods. Only put these on implementation classes, not interfaces. enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. Only put these on implementation classes, not interfaces. - } static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == interface)) { @@ -1595,19 +1890,53 @@ static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == } static auto fromExistingJavaObject(T)(jobject o) if(is(T == interface)) { - static class Dummy : IJavaObject { + import std.traits; + static class Dummy : T { + static foreach(memberName; __traits(allMembers, T)) { + static foreach(idx, overload; __traits(getOverloads, T, memberName)) + static if(!__traits(isStaticFunction, overload)) + static foreach(attr; __traits(getAttributes, overload)) { + //static if(!__traits(isStaticFunction, __traits(getMember, T, memberName))) + //static foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) { + static if(is(attr == IJavaObject.Import)) { + //mixin("@Import override ReturnType!(__traits(getMember, T, memberName)) " ~ memberName ~ "(Parameters!(__traits(getMember, T, memberName)));"); + mixin("@Import override ReturnType!overload " ~ memberName ~ "(Parameters!overload);"); + } + } + } + mixin IJavaObjectImplementation!(false); - mixin JavaPackageId!("java.lang", "Object"); + + static if(!__traits(compiles, T._javaParameterString)) + mixin JavaPackageId!("java.lang", "Object"); } - return cast(T) cast(void*) fromExistingJavaObject!Dummy(o); // FIXME this is so wrong + JavaBridge!Dummy bridge; // just to instantiate the impl template + return fromExistingJavaObject!Dummy(o); } -mixin template ImportExportImpl(Class) { +mixin template ImportExportImpl(Class) if(is(Class == class)) { static import arsd.jni; private static arsd.jni.JavaBridge!(Class) _javaDBridge; } +mixin template ImportExportImpl(Interface) if(is(Interface == interface)) { + static import arsd.jni; + private static arsd.jni.JavaBridgeForInterface!(Interface) _javaDBridge; +} + +final class JavaBridgeForInterface(Interface) { + // for interfaces, we do need to implement static members, but nothing else + static foreach(memberName; __traits(derivedMembers, Interface)) { + static foreach(oi, overload; __traits(getOverloads, Interface, memberName)) + static if(__traits(isStaticFunction, overload)) + static foreach(attr; __traits(getAttributes, overload)) { + static if(is(attr == IJavaObject.Import)) + mixin JavaImportImpl!(Interface, overload, oi); + } + } +} + final class JavaBridge(Class) { static foreach(memberName; __traits(derivedMembers, Class)) { // validations @@ -1657,19 +1986,48 @@ class JavaClass(string javaPackage, CRTP, Parent = void, bool isNewClass = false mixin JavaPackageId!(javaPackage, CRTP); } +mixin template JavaInterfaceMembers(string javaName) { + static import arsd.jni; + /*protected*/ static arsd.jni.jclass internalJavaClassHandle_; + static if(javaName !is null) { + static assert(javaName[0] == 'L' && javaName[$-1] == ';'); + static immutable string _javaParameterString = javaName; + } +} + mixin template IJavaObjectImplementation(bool isNewClass) { static import arsd.jni; - /*protected*/ arsd.jni.jobject internalJavaHandle_; - /*protected*/ arsd.jni.jobject getJavaHandle() { return internalJavaHandle_; } + /+ + import arsd.jni : IJavaObjectSeperate; // WTF the FQN in the is expression didn't work + static if(is(typeof(this) : IJavaObjectSeperate!(ImplInterface), ImplInterface)) { + ImplInterface _d_helper_; + override ImplInterface _d_helper() { return _d_helper_; } + override void _d_helper(ImplInterface i) { _d_helper_ = i; } + } + +/ + + /+ + static if(is(typeof(this) S == super)) + static foreach(_superInterface; S) + static if(is(_superInterface == interface)) + static if(__traits(compiles, _superInterface.JavaDefaultImplementations)) { + //pragma(msg, "here"); + mixin _superInterface.JavaDefaultImplementations; + } + +/ + + /*protected*/ arsd.jni.jobject internalJavaHandle_; + /*protected*/ override arsd.jni.jobject getJavaHandle() { return internalJavaHandle_; } - __gshared static /*protected*/ /*immutable*/ arsd.jni.JNINativeMethod[] nativeMethodsData_; /*protected*/ static arsd.jni.jclass internalJavaClassHandle_; + __gshared static /*protected*/ /*immutable*/ arsd.jni.JNINativeMethod[] nativeMethodsData_; protected static int initializeInJvm_(arsd.jni.JNIEnv* env) { import core.stdc.stdio; static if(isNewClass) { + static assert(0, "not really implemented"); auto aje = arsd.jni.ActivateJniEnv(env); import std.file; @@ -1678,6 +2036,7 @@ mixin template IJavaObjectImplementation(bool isNewClass) { bytes = bytes.replace(cast(byte[]) "Test2", cast(byte[]) "Test3"); auto loader = arsd.jni.ClassLoader.getSystemClassLoader().getJavaHandle(); + // doesn't actually work on Android, they didn't implement this function :( :( :( internalJavaClassHandle_ = (*env).DefineClass(env, "wtf/Test3", loader, bytes.ptr, cast(int) bytes.length); } else { internalJavaClassHandle_ = (*env).FindClass(env, (_javaParameterString[1 .. $-1] ~ "\0").ptr); @@ -1686,7 +2045,7 @@ mixin template IJavaObjectImplementation(bool isNewClass) { if(!internalJavaClassHandle_) { (*env).ExceptionDescribe(env); (*env).ExceptionClear(env); - fprintf(stderr, "Cannot %s Java class for %s\n", isNewClass ? "create".ptr : "find".ptr, typeof(this).stringof.ptr); + fprintf(stderr, "Cannot %s Java class for %s [%s]\n", isNewClass ? "create".ptr : "find".ptr, typeof(this).stringof.ptr, (_javaParameterString[1 .. $-1] ~ "\0").ptr); return 1; } From 09a6e315b1517dc80fbb38aeb0aeac170f80574b Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 9 Jan 2020 19:46:30 -0500 Subject: [PATCH 14/24] cool stuff --- declarativeloader.d | 15 ++++ jni.d | 168 +++++++++++++------------------------------- 2 files changed, 63 insertions(+), 120 deletions(-) diff --git a/declarativeloader.d b/declarativeloader.d index 139b806..28299c2 100644 --- a/declarativeloader.d +++ b/declarativeloader.d @@ -69,7 +69,15 @@ union N(ty) { /// input range of ubytes... int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) { int bytesConsumed; + string currentItem; + + import std.conv; + scope(failure) + throw new Exception(T.stringof ~ "." ~ currentItem ~ " trouble " ~ to!string(t)); + ubyte next() { + if(r.empty) + throw new Exception(T.stringof ~ "." ~ currentItem ~ " trouble " ~ to!string(t)); auto bfr = r.front; r.popFront; bytesConsumed++; @@ -78,6 +86,7 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) bool endianness = bigEndian!T(assumeBigEndian); static foreach(memberName; __traits(allMembers, T)) {{ + currentItem = memberName; static if(is(typeof(__traits(getMember, T, memberName)))) { alias f = __traits(getMember, T, memberName); alias ty = typeof(f); @@ -115,11 +124,17 @@ int loadFrom(T, Range)(ref T t, auto ref Range r, bool assumeBigEndian = false) auto tag = __traits(getMember, t, tagField); // find the child of the union matching the tag... + bool found = false; static foreach(um; __traits(allMembers, ty)) { if(tag == getTag!(__traits(getMember, ty, um))) { bytesConsumed += loadFrom(__traits(getMember, __traits(getMember, t, memberName), um), r, endianness); + found = true; } } + if(!found) { + import std.format; + throw new Exception(format("found unknown union tag %s at %s", tag, t)); + } } else static if(is(ty == E[], E)) { static foreach(attr; __traits(getAttributes, f)) { static if(is(attr == NumBytes!Field, alias Field)) diff --git a/jni.d b/jni.d index 6bc457c..6da2309 100644 --- a/jni.d +++ b/jni.d @@ -442,15 +442,24 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD bool outputMixinTemplate = false; string mainThing; - string helperThing; + //string helperThing; // so overriding Java classes from D is iffy and with separate implementation // non final leads to linker errors anyway... //mainThing ~= (isInterface ? "interface " : (jtc.inlineImplementations ? "class " : isAbstract ? "abstract class " : "final class ")) ~ lastClassName ~ " : "; + mainThing ~= "final class " ~ lastClassName ~ " : IJavaObject {\n"; + mainThing ~= "\tstatic immutable string[] _d_canCastTo = [\n"; + // not putting super class on inline implementations since that forces vtable... if(jtc.inlineImplementations) { auto scn = cf.superclassName; + + if(scn.length) { + mainThing ~= "\t\t\"" ~ scn ~ "\",\n"; + } + + /+ //if(!scn.startsWith("java/")) { // superclasses need the implementation too so putting it in the return list lol if(scn.length && scn != "java/lang/Object") { // && scn in allClasses) { @@ -458,6 +467,7 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD mainThing ~= ", "; } //} + +/ } foreach(name; cf.interfacesNames) { @@ -465,73 +475,25 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD //continue; // these probably aren't important to D and can really complicate version management //if(name !in allClasses) //continue; - mainThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); - mainThing ~= ", "; + //mainThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); + //mainThing ~= ", "; + + mainThing ~= "\t\t\"" ~ name ~ "\",\n"; } - mainThing ~= "arsd.jni.IJavaObject"; + mainThing ~= "\t];\n"; - mainThing ~= " {\n"; - - //mainThing ~= "\talias _d_helper this;\n"; - mainThing ~= "\tfinal auto opDispatch(string name, Args...)(Args args) if(name != \"opCall\") { auto local = _d_helper(); return __traits(getMember, local, name)(args); }\n"; - if(isInterface) - mainThing ~= "\tfinal auto _d_helper()() { return getDProxy(); }\n"; - else { - mainThing ~= "\t" ~ lastClassName ~ "_d_methods _d_proxy_;\n"; - mainThing ~= "\tfinal auto _d_helper()() {\n"; - mainThing ~= "\t\tif(getDProxy() is null) _d_proxy_ = arsd.jni.createDProxy!("~lastClassName~"_d_methods)(this);\n"; - mainThing ~= "\t\treturn getDProxy();\n"; - mainThing ~= "\t}\n"; - mainThing ~= "\toverride " ~ lastClassName ~ "_d_methods getDProxy() { return _d_proxy_; }\n"; - } - - - helperThing ~= "interface " ~ lastClassName ~ "_d_methods : "; - - // not putting super class on inline implementations since that forces vtable... - if(jtc.inlineImplementations) { - auto scn = cf.superclassName; - //if(!scn.startsWith("java/")) { - // superclasses need the implementation too so putting it in the return list lol - if(scn.length && scn != "java/lang/Object") { // && scn in allClasses) { - helperThing ~= javaObjectToDTypeString(scn, javaPackages, javaPackagesReturn, importPrefix); - helperThing ~= "_d_methods, "; - } - //} - } - - foreach(name; cf.interfacesNames) { - //if(name.startsWith("java/")) - //continue; // these probably aren't important to D and can really complicate version management - //if(name !in allClasses) - //continue; - helperThing ~= javaObjectToDTypeString(name, javaPackages, javaPackagesReturn, importPrefix); - helperThing ~= "_d_methods, "; - } - - helperThing ~= "IJavaObject"; - - helperThing ~= " {\n"; - - helperThing ~= "\tmixin JavaInterfaceMembers!(\"L" ~ cn ~ ";\");\n"; - - - - - - - - - - string tm; - string[string] tmpackages; - string[string] tmpackagesr; - string[string] tmpackagesa; + //helperThing ~= "interface " ~ lastClassName ~ "_d_methods : "; string[string] mentioned; - foreach(method; cf.methodsListing) { + + string[string] processed; + + void addMethods(ClassFile* current, bool isTopLevel) { + if(current is null) return; + if(current.className in processed) return; + foreach(method; current.methodsListing) { bool native = (method.flags & 0x0100) ? true : false; if(jtc.nativesAreImports) { native = false; // kinda hacky but meh @@ -545,32 +507,27 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD } auto port = native ? "@Export" : "@Import"; if(method.flags & 1) { // public - - bool addToMixinTemplate; - - bool wasStatic; + if(!isTopLevel && method.name == "") + continue; bool maybeOverride = false;// !isInterface; if(method.flags & 0x0008) { port ~= " static"; - maybeOverride = false; - wasStatic = true; } if(method.flags & method_info.ACC_ABSTRACT) { - maybeOverride = false; //if(!isInterface) - port ~= " abstract"; + //port ~= " abstract"; } else { // this represents a default implementation in a Java interface // D cannot express this... so I need to add it to the mixin template // associated with this interface as well. - if(isInterface && (!(method.flags & 0x0008))) { - addToMixinTemplate = true; - } + //if(isInterface && (!(method.flags & 0x0008))) { + //addToMixinTemplate = true; + //} } - if(maybeOverride && method.isOverride(allClasses)) - port ~= " override"; + //if(maybeOverride && method.isOverride(allClasses)) + //port ~= " override"; auto name = method.name; @@ -585,10 +542,6 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD name = name.replace("$", "_"); bool ctor = name == ""; - if(ctor && isInterface) { - ctor = false; - name = "CONSTRUCTOR"; - } auto sig = method.signature; @@ -596,11 +549,10 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD assert(lidx != -1); auto retJava = sig[lidx + 1 .. $]; auto argsJava = sig[1 .. lidx]; - auto retJava2 = retJava; - auto argsJava2 = argsJava; string ret = ctor ? "" : javaSignatureToDTypeString(retJava, javaPackages, javaPackagesReturn, importPrefix); string args = javaSignatureToDTypeString(argsJava, javaPackages, javaPackagesArguments, importPrefix); + auto oargs = args; if(!jtc.inlineImplementations) { if(ctor && args.length == 0) @@ -613,52 +565,32 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD mentioned[men] = men; string proto = cast(string) ("\t"~port~" " ~ ret ~ (ret.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args~")"~(native ? " { assert(0); }" : ";")~"\n"); - if(wasStatic || ctor) - mainThing ~= proto; - else - helperThing ~= proto; + mainThing ~= proto; - if(addToMixinTemplate) { - string pfx = "tmimport"; - string ret1 = ctor ? "" : javaSignatureToDTypeString(retJava2, tmpackages, tmpackagesr, pfx); - string args1 = javaSignatureToDTypeString(argsJava2, tmpackages, tmpackagesa, pfx); - tm ~= "\t\t"~port~" " ~ ret1 ~ (ret1.length ? " " : "") ~ (ctor ? "this" : name) ~ "("~args1~"){ assert(0); }\n"; - } + if(oargs.length == 0 && name == "toString_" && !(method.flags & 0x0008)) + mainThing ~= "\toverride string toString() { return toString_(); }\n"; } } - if(!isInterface) { - mainThing ~= "\tmixin IJavaObjectImplementation!(false);\n"; - mainThing ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; - } else { - mainThing ~= "\tmixin JavaInterfaceMembers!(\"L" ~ cn ~ ";\");\n"; - if(outputMixinTemplate && tm.length) { - mainThing ~= "\tmixin template JavaDefaultImplementations() {\n"; - - foreach(pkg, prefix; tmpackages) { - auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; - if(jtc.inlineImplementations) - tm ~= "\t\timport " ~ prefix ~ " = " ~ m ~ ";\n"; - else - tm ~= "\t\timport " ~ prefix ~ " = " ~ m ~ "_d_interface;\n"; - } - if(!jtc.inlineImplementations) - foreach(pkg, prefix; tmpackagesr) { - auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; - tm ~= "\t\timport impl_" ~ prefix ~ " = " ~ m ~ ";\n"; - } - - mainThing ~= tm; - mainThing ~= "\t}\n"; + processed[current.className.idup] = "done"; + if(current.superclassName.length) { + auto c = current.superclassName in allClasses; + addMethods(c, false); + } + foreach(iface; current.interfacesNames) { + auto c = iface in allClasses; + addMethods(c, false); } } + addMethods(&cf, true); + + mainThing ~= "\tmixin IJavaObjectImplementation!(false);\n"; + mainThing ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; mainThing ~= "}\n\n"; - helperThing ~= "}\n\n"; dc ~= mainThing; dc ~= "\n\n"; - dc ~= helperThing; foreach(pkg, prefix; javaPackages) { auto m = (dPackagePrefix.length ? (dPackagePrefix ~ ".") : "") ~ pkg; @@ -674,10 +606,6 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD dco ~= "\n"; dco ~= dc; - //dco ~= "\tmixin JavaPackageId!(\""~originalJavaPackage.replace("/", ".")~"\", \""~originalClassName~"\");\n"; - // the following saves some compile time of the bindings; might as well do some calculations ahead of time - //dco ~= "\tpublic static immutable string _javaParameterString = \"L" ~ cn ~ ";\";\n"; - if(jtc.inlineImplementations) { dco ~= "\nmixin ImportExportImpl!"~lastClassName~";\n"; std.file.write(filename, dco); From 0fed9f70deee232191e34b674eeb6bf08c226e8c Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 9 Jan 2020 19:47:17 -0500 Subject: [PATCH 15/24] oops --- cgi.d | 1 + 1 file changed, 1 insertion(+) diff --git a/cgi.d b/cgi.d index d653110..dcc1dc9 100644 --- a/cgi.d +++ b/cgi.d @@ -6444,6 +6444,7 @@ void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoS import core.sys.posix.poll; } + version(linux) eis.epoll_fd = epoll_fd; auto acceptOp = allocateIoOp(sock, IoOp.Read, 0, null); From acf94a0fefdceaa4cc65520cc1c95510b79bf1dd Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Thu, 9 Jan 2020 20:15:51 -0500 Subject: [PATCH 16/24] better --- jni.d | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jni.d b/jni.d index 6da2309..a191538 100644 --- a/jni.d +++ b/jni.d @@ -568,7 +568,7 @@ void rawClassStructToD()(ref ClassFile cf, string dPackagePrefix, string outputD mainThing ~= proto; if(oargs.length == 0 && name == "toString_" && !(method.flags & 0x0008)) - mainThing ~= "\toverride string toString() { return toString_(); }\n"; + mainThing ~= "\toverride string toString() { return arsd.jni.javaObjectToString(this); }\n"; } } @@ -1808,6 +1808,11 @@ interface IJavaObject { enum Export; /// UDA to indicate you are exporting the method to Java. Put a D implementation body on these. Only put these on implementation classes, not interfaces. } +string javaObjectToString(IJavaObject i) { + return "FIXME"; +} + + static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == interface)) { import core.memory; auto ptr = GC.malloc(__traits(classInstanceSize, T)); From 135ff31bc210c4ec4837c618f5e261d151c7d64d Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 11 Jan 2020 14:17:49 -0500 Subject: [PATCH 17/24] keep alive --- http2.d | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/http2.d b/http2.d index 1d78fe5..509d314 100644 --- a/http2.d +++ b/http2.d @@ -722,6 +722,8 @@ class HttpRequest { headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; if(requestParameters.acceptGzip) headers ~= "Accept-Encoding: gzip\r\n"; + if(requestParameters.keepAlive) + headers ~= "Connection: keep-alive\r\n"; foreach(header; requestParameters.headers) headers ~= header ~ "\r\n"; @@ -1358,6 +1360,7 @@ struct HttpRequestParameters { // debugging bool useHttp11 = true; /// bool acceptGzip = true; /// + bool keepAlive = true; /// // the request itself HttpVerb method; /// @@ -1423,6 +1426,7 @@ class HttpClient { /* Protocol restrictions, useful to disable when debugging servers */ bool useHttp11 = true; /// bool acceptGzip = true; /// + bool keepAlive = true; /// /// @property Uri location() { @@ -1445,6 +1449,7 @@ class HttpClient { request.requestParameters.useHttp11 = this.useHttp11; request.requestParameters.acceptGzip = this.acceptGzip; + request.requestParameters.keepAlive = this.keepAlive; return request; } @@ -1462,6 +1467,7 @@ class HttpClient { request.requestParameters.useHttp11 = this.useHttp11; request.requestParameters.acceptGzip = this.acceptGzip; + request.requestParameters.keepAlive = this.keepAlive; request.requestParameters.bodyData = bodyData; request.requestParameters.contentType = contentType; From f6189fe119b5a1b5fed1261ae72e6ae5e6f840c3 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 11 Jan 2020 14:21:10 -0500 Subject: [PATCH 18/24] silence warnings --- archive.d | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/archive.d b/archive.d index 8d14c26..453cab9 100644 --- a/archive.d +++ b/archive.d @@ -1877,7 +1877,7 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) { switch(p.state) { - default: assert(0); + default: return ELzma2State.LZMA2_STATE_ERROR; case ELzma2State.LZMA2_STATE_CONTROL: p.control = b; if (p.control == 0) @@ -1928,7 +1928,6 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b) return ELzma2State.LZMA2_STATE_DATA; } } - return ELzma2State.LZMA2_STATE_ERROR; } static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, Byte *src, SizeT size) From 4702c672cab96408fce637c8df0764d28966c76d Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 11 Jan 2020 19:47:41 -0500 Subject: [PATCH 19/24] better websocket stuff --- http2.d | 194 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 165 insertions(+), 29 deletions(-) diff --git a/http2.d b/http2.d index 509d314..7283611 100644 --- a/http2.d +++ b/http2.d @@ -14,6 +14,8 @@ +/ module arsd.http2; +// FIXME: I think I want to disable sigpipe here too. + import std.uri : encodeComponent; debug(arsd_http2_verbose) debug=arsd_http2; @@ -1094,6 +1096,9 @@ class HttpRequest { if(colon == -1) return; auto name = header[0 .. colon]; + if(colon + 1 == header.length) + return; // empty header, idk + assert(colon + 2 < header.length, header); auto value = header[colon + 2 .. $]; // skipping the colon itself and the following space switch(name) { @@ -2094,7 +2099,7 @@ private bool bicmp(in ubyte[] item, in char[] search) { The blocking API is best used when you only need basic functionality with a single connection. --- - WebSocketMessage msg; + WebSocketFrame msg; do { // FIXME good demo } while(msg); @@ -2167,6 +2172,11 @@ class WebSocket { size_t likelyReceiveBufferSize = 4096; /// ditto size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto + /++ + Maximum combined size of a message. + +/ + size_t maximumMessageSize = 10 * 1024 * 1024; + string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; string origin; /// Origin URL to send with the handshake, if desired. string protocol; /// the protocol header, if desired. @@ -2368,6 +2378,8 @@ wss://echo.websocket.org if(onopen) onopen(); + + registerActiveSocket(this); } /++ @@ -2377,7 +2389,9 @@ wss://echo.websocket.org void close(int code = 0, string reason = null) in (reason.length < 123) { - WebSocketMessage wss; + if(readyState_ != OPEN) + return; // it cool, we done + WebSocketFrame wss; wss.fin = true; wss.opcode = WebSocketOpcode.close; wss.data = cast(ubyte[]) reason; @@ -2393,7 +2407,7 @@ wss://echo.websocket.org +/ /// Group: foundational void ping() { - WebSocketMessage wss; + WebSocketFrame wss; wss.fin = true; wss.opcode = WebSocketOpcode.ping; wss.send(&llsend); @@ -2401,7 +2415,7 @@ wss://echo.websocket.org // automatically handled.... void pong() { - WebSocketMessage wss; + WebSocketFrame wss; wss.fin = true; wss.opcode = WebSocketOpcode.pong; wss.send(&llsend); @@ -2412,7 +2426,7 @@ wss://echo.websocket.org +/ /// Group: foundational void send(in char[] textData) { - WebSocketMessage wss; + WebSocketFrame wss; wss.fin = true; wss.opcode = WebSocketOpcode.text; wss.data = cast(ubyte[]) textData; @@ -2424,7 +2438,7 @@ wss://echo.websocket.org +/ /// Group: foundational void send(in ubyte[] binaryData) { - WebSocketMessage wss; + WebSocketFrame wss; wss.fin = true; wss.opcode = WebSocketOpcode.binary; wss.data = cast(ubyte[]) binaryData; @@ -2463,14 +2477,14 @@ wss://echo.websocket.org this returns. +/ /// Group: blocking_api - public WebSocketMessage waitForNextMessage() { + public WebSocketFrame waitForNextMessage() { do { auto m = processOnce(); if(m.populated) return m; } while(lowLevelReceive()); - return WebSocketMessage.init; // FIXME? maybe. + return WebSocketFrame.init; // FIXME? maybe. } /++ @@ -2499,7 +2513,7 @@ wss://echo.websocket.org auto s = d; if(d.length) { auto orig = d; - auto m = WebSocketMessage.read(d); + auto m = WebSocketFrame.read(d); // that's how it indicates that it needs more data if(d !is orig) return true; @@ -2548,26 +2562,72 @@ wss://echo.websocket.org return false; } - private WebSocketMessage processOnce() { + private ubyte continuingType; + private ubyte[] continuingData; + //private size_t continuingDataLength; + + private WebSocketFrame processOnce() { ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; auto s = d; - // FIXME: handle continuation frames - WebSocketMessage m; + // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. + WebSocketFrame m; if(d.length) { auto orig = d; - m = WebSocketMessage.read(d); + m = WebSocketFrame.read(d); // that's how it indicates that it needs more data if(d is orig) - return WebSocketMessage.init; - if(m.opcode == WebSocketOpcode.close) { - readyState_ = CLOSED; - if(onclose) - onclose(); - } else if(m.opcode == WebSocketOpcode.text) { - if(ontextmessage) - ontextmessage(m.textData); - } else { - // FIXME + return WebSocketFrame.init; + switch(m.opcode) { + case WebSocketOpcode.continuation: + if(continuingData.length + m.data.length > config.maximumMessageSize) + throw new Exception("message size exceeded"); + + continuingData ~= m.data; + if(fin) { + if(ontextmessage) + ontextmessage(cast(char[]) continuingData); + if(onbinarymessage) + onbinarymessage(continuingData); + + continuingData = null; + } + break; + case WebSocketOpcode.text: + if(m.fin) { + if(ontextmessage) + ontextmessage(m.textData); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.binary: + if(m.fin) { + if(onbinarymessage) + onbinarymessage(m.data); + } else { + continuingType = m.opcode; + //continuingDataLength = 0; + continuingData = null; + continuingData ~= m.data; + } + break; + case WebSocketOpcode.close: + readyState_ = CLOSED; + if(onclose) + onclose(); + + unregisterActiveSocket(this); + break; + case WebSocketOpcode.ping: + pong(); + break; + case WebSocketOpcode.pong: + // just really references it is still alive, nbd. + break; + default: // ignore though i could and perhaps should throw too } } receiveBufferUsedLength -= s.length - d.length; @@ -2586,6 +2646,7 @@ wss://echo.websocket.org void delegate() onclose; /// void delegate() onerror; /// void delegate(in char[]) ontextmessage; /// + void delegate(in ubyte[]) onbinarymessage; /// void delegate() onopen; /// /++ @@ -2596,6 +2657,11 @@ wss://echo.websocket.org ontextmessage = dg; } + /// ditto + void onmessage(void delegate(in ubyte[]) dg) { + onbinarymessage = dg; + } + /* const int bufferedAmount // amount pending const string extensions @@ -2603,6 +2669,76 @@ wss://echo.websocket.org const string protocol const string url */ + + static { + /++ + + +/ + void eventLoop() { + + static SocketSet readSet; + + if(readSet is null) + readSet = new SocketSet(); + + outermost: while(!loopExited) { + readSet.reset(); + + bool hadAny; + foreach(sock; activeSockets) { + readSet.add(sock.socket); + hadAny = true; + } + + if(!hadAny) + return; + + tryAgain: + auto selectGot = Socket.select(readSet, null, null, 10.seconds /* timeout */); + if(selectGot == 0) { /* timeout */ + // timeout + goto tryAgain; + } else if(selectGot == -1) { /* interrupted */ + goto tryAgain; + } else { + foreach(sock; activeSockets) { + if(readSet.isSet(sock.socket)) { + if(!sock.lowLevelReceive()) { + sock.readyState_ = CLOSED; + unregisterActiveSocket(sock); + continue outermost; + } + while(sock.processOnce().populated) {} + selectGot--; + if(selectGot <= 0) + break; + } + } + } + } + } + + private bool loopExited; + /++ + + +/ + void exitEventLoop() { + loopExited = true; + } + + WebSocket[] activeSockets; + void registerActiveSocket(WebSocket s) { + activeSockets ~= s; + } + void unregisterActiveSocket(WebSocket s) { + foreach(i, a; activeSockets) + if(s is a) { + activeSockets[i] = activeSockets[$-1]; + activeSockets = activeSockets[0 .. $-1]; + break; + } + } + } } /* copy/paste from cgi.d */ @@ -2618,7 +2754,7 @@ private { // 11,12,13,14,15 RESERVED } - public struct WebSocketMessage { + public struct WebSocketFrame { private bool populated; bool fin; bool rsv1; @@ -2631,8 +2767,8 @@ private { ubyte[4] maskingKey; // don't set this when sending ubyte[] data; - static WebSocketMessage simpleMessage(WebSocketOpcode opcode, void[] data) { - WebSocketMessage msg; + static WebSocketFrame simpleMessage(WebSocketOpcode opcode, void[] data) { + WebSocketFrame msg; msg.fin = true; msg.opcode = opcode; msg.data = cast(ubyte[]) data; @@ -2718,8 +2854,8 @@ private { llsend(data); } - static WebSocketMessage read(ref ubyte[] d) { - WebSocketMessage msg; + static WebSocketFrame read(ref ubyte[] d) { + WebSocketFrame msg; assert(d.length >= 2); auto orig = d; @@ -2772,7 +2908,7 @@ private { if(msg.realLength > d.length) { d = orig; - return WebSocketMessage.init; + return WebSocketFrame.init; } msg.data = d[0 .. msg.realLength]; From 1a95ea72021614bee24ab1ba35533e9177d01b9a Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 11 Jan 2020 23:08:55 -0500 Subject: [PATCH 20/24] handle bug from #233 comment about long name --- archive.d | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/archive.d b/archive.d index 453cab9..2e772a2 100644 --- a/archive.d +++ b/archive.d @@ -27,6 +27,12 @@ struct TarFile { } +/ +inout(char)[] upToZero(inout(char)[] a) { + int i = 0; + while(i < a.length && a[i]) i++; + return a[0 .. i]; +} + /++ A header of a file in the archive. This represents the @@ -56,8 +62,8 @@ struct TarFileHeader { const(char)[] filename() { import core.stdc.string; if(filenamePrefix_[0]) - return filenamePrefix_[0 .. strlen(filenamePrefix_.ptr)] ~ fileName_[0 .. strlen(fileName_.ptr)]; - return fileName_[0 .. strlen(fileName_.ptr)]; + return upToZero(filenamePrefix_[]) ~ upToZero(fileName_[]); + return upToZero(fileName_[]); } /// From 7b5e7eb2c4484aae6ed6cec56ff3a8e1be3c0dcd Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sat, 11 Jan 2020 23:10:33 -0500 Subject: [PATCH 21/24] directory thanks to BorisCarvajal in issue #233 comment --- archive.d | 2 ++ jni.d | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/archive.d b/archive.d index 2e772a2..7c3fb87 100644 --- a/archive.d +++ b/archive.d @@ -129,6 +129,8 @@ bool processTar( *header = *(cast(TarFileHeader*) dataBuffer.ptr); auto s = header.size(); *bytesRemainingOnCurrentFile = s; + if(header.type() == TarFileType.directory) + handleData(header, true, false, null); if(s == 0 && header.type == TarFileType.normal) return false; } diff --git a/jni.d b/jni.d index a191538..cd3d3be 100644 --- a/jni.d +++ b/jni.d @@ -1812,6 +1812,11 @@ string javaObjectToString(IJavaObject i) { return "FIXME"; } +T as(T, R)(R obj) { + // FIXME: this will have to do downcasts to interfaces + return T.init; +} + static T fromExistingJavaObject(T)(jobject o) if(is(T : IJavaObject) && !is(T == interface)) { import core.memory; From 059481bff2e87d752ba3df4320022b0c923b3fd4 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Sun, 12 Jan 2020 09:31:55 -0500 Subject: [PATCH 22/24] more bugs in websocket --- http2.d | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/http2.d b/http2.d index 7283611..4bd2ada 100644 --- a/http2.d +++ b/http2.d @@ -2583,7 +2583,7 @@ wss://echo.websocket.org throw new Exception("message size exceeded"); continuingData ~= m.data; - if(fin) { + if(m.fin) { if(ontextmessage) ontextmessage(cast(char[]) continuingData); if(onbinarymessage) @@ -2856,10 +2856,17 @@ private { static WebSocketFrame read(ref ubyte[] d) { WebSocketFrame msg; - assert(d.length >= 2); auto orig = d; + WebSocketFrame needsMoreData() { + d = orig; + return WebSocketFrame.init; + } + + if(d.length < 2) + return needsMoreData(); + ubyte b = d[0]; msg.populated = true; @@ -2884,6 +2891,8 @@ private { // 16 bit length msg.realLength = 0; + if(d.length < 2) return needsMoreData(); + foreach(i; 0 .. 2) { msg.realLength |= d[0] << ((1-i) * 8); d = d[1 .. $]; @@ -2892,6 +2901,8 @@ private { // 64 bit length msg.realLength = 0; + if(d.length < 8) return needsMoreData(); + foreach(i; 0 .. 8) { msg.realLength |= d[0] << ((7-i) * 8); d = d[1 .. $]; @@ -2902,13 +2913,15 @@ private { } if(msg.masked) { + + if(d.length < 4) return needsMoreData(); + msg.maskingKey = d[0 .. 4]; d = d[4 .. $]; } if(msg.realLength > d.length) { - d = orig; - return WebSocketFrame.init; + return needsMoreData(); } msg.data = d[0 .. msg.realLength]; From b8660ee8bc93b3a656b1670aa0f6bb6fed43d86e Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 13 Jan 2020 18:08:17 -0500 Subject: [PATCH 23/24] forgot to forward --- htmltotext.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/htmltotext.d b/htmltotext.d index 8410680..3fd98a7 100644 --- a/htmltotext.d +++ b/htmltotext.d @@ -468,6 +468,6 @@ class HtmlConverter { /// string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) { auto converter = new HtmlConverter(); - return converter.convert(html, true, wrapAmount); + return converter.convert(html, wantWordWrap, wrapAmount); } From 2123bf0db4e057615bee6ac483550004c76a6ef0 Mon Sep 17 00:00:00 2001 From: "Adam D. Ruppe" Date: Mon, 13 Jan 2020 19:50:30 -0500 Subject: [PATCH 24/24] fix utility selectors on document --- dom.d | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dom.d b/dom.d index b7e9567..b3b20e8 100644 --- a/dom.d +++ b/dom.d @@ -1200,13 +1200,17 @@ class Document : FileResource { if( is(SomeElementType : Element)) out(ret) { assert(ret !is null); } body { - return root.requireSelector!(SomeElementType)(selector, file, line); + auto e = cast(SomeElementType) querySelector(selector); + if(e is null) + throw new ElementNotFoundException(SomeElementType.stringof, selector, this.root, file, line); + return e; } final MaybeNullElement!SomeElementType optionSelector(SomeElementType = Element)(string selector, string file = __FILE__, size_t line = __LINE__) if(is(SomeElementType : Element)) { - return root.optionSelector!(SomeElementType)(selector, file, line); + auto e = cast(SomeElementType) querySelector(selector); + return MaybeNullElement!SomeElementType(e); } /// ditto