fix compile

cgi.d

@@ -4698,6 +4698,99 @@ interface EventIoServer {
 	void fileClosed(int fd);
 }
 
+// the sink should buffer it
+private void serialize(T)(scope void delegate(ubyte[]) sink, T t) {
+	static if(is(T == struct)) {
+		foreach(member; __traits(allMembers, T))
+			serialize(sink, __traits(getMember, t, member));
+	} else static if(is(T : int)) {
+		// no need to think of endianness just because this is only used
+		// for local, same-machine stuff anyway. thanks private lol
+		sink((cast(ubyte*) &t)[0 .. t.sizeof]);
+	} else static if(is(T == string) || is(T : const(ubyte)[])) {
+		// these are common enough to optimize
+		int len = cast(int) t.length; // want length consistent size tho, in case 32 bit program sends to 64 bit server, etc.
+		sink((cast(ubyte*) &len)[0 .. int.sizeof]);
+		sink(cast(ubyte[]) t[]);
+	} else static if(is(T : A[], A)) {
+		// generic array is less optimal but still prolly ok
+		static assert(0, T.stringof);
+		int len = cast(int) t.length;
+		sink((cast(ubyte*) &len)[0 .. int.sizeof]);
+		foreach(item; t)
+			serialize(item);
+	} else static assert(0, T.stringof);
+}
+
+private void deserialize(T)(scope ubyte[] delegate(int sz) get, scope void delegate(T) dg) {
+	static if(is(T == struct)) {
+		T t;
+		foreach(member; __traits(allMembers, T))
+			deserialize(get, (T mbr) { __traits(getMember, t, member) = mbr; });
+		dg(t);
+	} else static if(is(T : int)) {
+		// no need to think of endianness just because this is only used
+		// for local, same-machine stuff anyway. thanks private lol
+		T t;
+		auto data = get(t.sizeof);
+		t = (cast(T[]) data)[0];
+		dg(t);
+	} else static if(is(T == string) || is(T : const(ubyte)[])) {
+		// these are common enough to optimize
+		int len;
+		auto data = get(len.sizeof);
+		len = (cast(int[]) data)[0];
+
+		/*
+		typeof(T[0])[2000] stackBuffer;
+		T buffer;
+
+		if(len < stackBuffer.length)
+			buffer = stackBuffer[0 .. len];
+		else
+			buffer = new T(len);
+
+		data = get(len * typeof(T[0]).sizeof);
+		*/
+
+		T t = cast(T) get(len * typeof(T[0]).sizeof);
+
+		dg(t);
+
+	} else static assert(0, T.stringof);
+}
+
+unittest {
+	serialize((ubyte[] b) { assert(b == [0, 0, 0, 1]); }, 1);
+}
+
+
+interface MyLocalServer {}
+
+MyLocalServer connectToLocalServer() { return null; } // return an auto-generated version that RPCs to the interface
+
+class MyLocalServerImpl : MyLocalServer {} // handle the calls.
+
+
+/*
+	FIXME:
+		add a version command line arg
+		version data in the library
+		management gui as external program
+
+		at server with event_fd for each run
+		use .mangleof in the at function name
+
+		i think the at server will have to:
+			pipe args to the child
+			collect child output for logging
+			get child return value for logging
+
+			on windows timers work differently. idk how to best combine with the io stuff.
+
+			will have to have dump and restore too, so i can restart without losing stuff.
+*/
+
 final class BasicDataServer : EventIoServer {
 	static struct ClientConnection {
 	/+
@@ -4714,7 +4807,7 @@ final class BasicDataServer : EventIoServer {
 
 	protected:
 
-	void handleLocalConnectionData(IoOp* op, int receivedFd);
+	void handleLocalConnectionData(IoOp* op, int receivedFd) {}
 
 	void handleLocalConnectionClose(IoOp* op) {} // doesn't really matter, this is a fairly stateless go
 	void handleLocalConnectionComplete(IoOp* op) {} // again, irrelevant
@@ -4742,6 +4835,14 @@ final class BasicDataServer : EventIoServer {
 		char[16] sessionId;
 		Operation operation;
 
+		ushort dataType;
+		/*
+			int
+			float
+			string
+			ubyte[]
+		*/
+
 		int keyLength;
 		char[128] keyBuffer;
 

characterencodings.d

@@ -18,22 +18,26 @@
 	ought to just work.
 
 	Example:
+		---
 		auto data = cast(immutable(ubyte)[])
 			std.file.read("my-windows-file.txt");
 		string utf8String = convertToUtf8(data, "windows-1252");
 		// utf8String can now be used
+		---
 
 
 	The encodings currently implemented for decoding are:
-		UTF-8 (a no-op; it simply casts the array to string)
+		$(LIST
-		UTF-16,
+			* UTF-8 (a no-op; it simply casts the array to string)
-		UTF-32,
+			* UTF-16,
-		Windows-1252,
+			* UTF-32,
-		ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16.
+			* Windows-1252,
+			* ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16.
+			* KOI8-R
+		)
 
 	It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since
-	those labels are pretty much de-facto the same thing in wild documents.
+	those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic).
-
 
 	This module currently makes no attempt to look at control characters.
 */

dom.d

@@ -71,7 +71,7 @@ bool isConvenientAttribute(string name) {
 /// The main document interface, including a html parser.
 class Document : FileResource {
 	/// Convenience method for web scraping. Requires [arsd.http2] to be
-	/// included in the build.
+	/// included in the build as well as [arsd.characterencodings].
 	static Document fromUrl()(string url) {
 		import arsd.http2;
 		auto client = new HttpClient();
@@ -79,7 +79,10 @@ class Document : FileResource {
 		auto req = client.navigateTo(Uri(url), HttpVerb.GET);
 		auto res = req.waitForCompletion();
 
-		return new Document(cast(string) res.content);
+		auto document = new Document();
+		document.parseGarbage(cast(string) res.content);
+
+		return document;
 	}
 
 	///.

htmltotext.d

@@ -7,10 +7,47 @@ import std.string;
 
 import std.uni : isWhite;
 
+///
 class HtmlConverter {
 	int width;
 
+	/++
+		Will enable color output using VT codes. Determines color through dom.d's css support, which means you need to apply a stylesheet first.
+
+		---
+		import arsd.dom;
+
+		auto document = new Document(source_code_for_html);
+		auto stylesheet = new Stylesheet(source_code_for_css);
+		stylesheet.apply(document);
+		---
+	+/
+	bool enableVtOutput;
+
+
+	string color;
+	string backgroundColor;
+
+	///
 	void htmlToText(Element element, bool preformatted, int width) {
+		string color, backgroundColor;
+		if(enableVtOutput) {
+			color = element.computedStyle.getValue("color");
+			backgroundColor = element.computedStyle.getValue("background-color");
+		}
+
+		string originalColor = this.color, originalBackgroundColor = this.backgroundColor;
+
+		this.color = color.length ? color : this.color;
+		this.backgroundColor = backgroundColor.length ? backgroundColor : this.backgroundColor;
+
+		scope(exit) {
+			// the idea is as we pop working back up the tree, it restores what it was here
+			this.color = originalColor;
+			this.backgroundColor = originalBackgroundColor;
+		}
+
+
 		this.width = width;
 		if(auto tn = cast(TextNode) element) {
 			foreach(dchar ch; tn.nodeValue) {
@@ -115,19 +152,27 @@ class HtmlConverter {
 					}
 				break;
 				case "span":
-					/*
+					if(enableVtOutput) {
-					auto csc = element.computedStyle.getValue("color");
+						auto csc = color; // element.computedStyle.getValue("color");
-					if(csc.length) {
+						if(csc.length) {
-						auto c = Color.fromString(csc);
+							auto c = Color.fromString(csc);
-						s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b);
+							s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b);
+						}
+
+						bool bold = element.computedStyle.getValue("font-weight") == "bold";
+
+						if(bold)
+							s ~= "\033[1m";
+
+						sinkChildren();
+
+						if(bold)
+							s ~= "\033[0m";
+						if(csc.length)
+							s ~= "\033[39m";
+					} else {
+						sinkChildren();
 					}
-					sinkChildren();
-
-					if(csc.length)
-						s ~= "\033[39m";
-					*/
-
-					sinkChildren();
 				break;
 				case "p":
 					startBlock();
@@ -138,9 +183,15 @@ class HtmlConverter {
 				case "em", "i":
 					if(element.innerText.length == 0)
 						break;
-					sink('*', false);
+					if(enableVtOutput) {
-					sinkChildren();
+						s ~= "\033[1m";
-					sink('*', false);
+						sinkChildren();
+						s ~= "\033[0m";
+					} else {
+						sink('*', false);
+						sinkChildren();
+						sink('*', false);
+					}
 				break;
 				case "u":
 					if(element.innerText.length == 0)
@@ -258,6 +309,7 @@ class HtmlConverter {
 	int olDepth;
 	int ulDepth;
 
+	///
 	string convert(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 		Document document = new Document;
 
@@ -277,11 +329,13 @@ class HtmlConverter {
 		return convert(start, wantWordWrap, wrapAmount);
 	}
 
+	///
 	string convert(Element start, bool wantWordWrap = true, int wrapAmount = 74) {
 		htmlToText(start, false, wrapAmount);
 		return s;
 	}
 
+	///
 	void reset() {
 		s = null;
 		justOutputWhitespace = true;
@@ -289,6 +343,7 @@ class HtmlConverter {
 		justOutputMargin = true;
 	}
 
+	///
 	string s;
 	bool justOutputWhitespace = true;
 	bool justOutputBlock = true;
@@ -406,140 +461,9 @@ class HtmlConverter {
 	}
 }
 
+///
 string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 	auto converter = new HtmlConverter();
 	return converter.convert(html, true, wrapAmount);
 }
 
-string repeat(string s, ulong num) {
-	string ret;
-	foreach(i; 0 .. num)
-		ret ~= s;
-	return ret;
-}
-
-import std.stdio;
-version(none)
-void penis() {
-
-    again:
-    	string result = "";
-	foreach(ele; start.tree) {
-		if(ele is start) continue;
-		if(ele.nodeType != 1) continue;
-
-		switch(ele.tagName) {
-				goto again;
-			case "h1":
-				ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("=", ele.innerText.length) ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "h2":
-				ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("-", ele.innerText.length) ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "h3":
-				ele.innerText = "\r" ~ ele.innerText.toUpper ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "td":
-			case "p":
-			/*
-				if(ele.innerHTML.length > 1)
-					ele.innerHTML = "\r" ~ wrap(ele.innerHTML) ~ "\r";
-				ele.stripOut();
-				goto again;
-			*/
-			break;
-			case "a":
-				string href = ele.getAttribute("href");
-				if(href && !ele.hasClass("no-brackets")) {
-					if(ele.hasClass("href-text"))
-						ele.innerText = href;
-					else {
-						if(ele.innerText != href)
-							ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
-					}
-				}
-				ele.stripOut();
-				goto again;
-			case "ol":
-			case "ul":
-				ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r";
-			break;
-			case "li":
-				if(!ele.innerHTML.startsWith("* "))
-					ele.innerHTML = "* " ~ ele.innerHTML ~ "\r";
-				// ele.stripOut();
-			break;
-			case "sup":
-				ele.innerText = "^" ~ ele.innerText;
-				ele.stripOut();
-			break;
-			case "img":
-				string alt = ele.getAttribute("alt");
-				if(alt)
-					result ~= ele.alt;
-			break;
-			default:
-				ele.stripOut();
-				goto again;
-		}
-	}
-
-    again2:
-	//start.innerHTML = start.innerHTML().replace("\u0001", "\n");
-
-	foreach(ele; start.tree) {
-		if(ele.tagName == "td") {
-			if(ele.directText().strip().length) {
-				ele.prependText("\r");
-				ele.appendText("\r");
-			}
-			ele.stripOut();
-			goto again2;
-		} else if(ele.tagName == "p") {
-			if(strip(ele.innerText()).length > 1) {
-				string res = "";
-				string all = ele.innerText().replace("\n \n", "\n\n");
-				foreach(part; all.split("\n\n"))
-					res ~= "\r" ~ strip( wantWordWrap ? wrap(part, /*74*/ wrapAmount) : part ) ~ "\r";
-				ele.innerText = res;
-			} else
-				ele.innerText = strip(ele.innerText);
-			ele.stripOut();
-			goto again2;
-		} else if(ele.tagName == "li") {
-			auto part = ele.innerText;
-			part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part );
-			part = "  " ~ part.replace("\n", "\n\v") ~ "\r";
-			ele.innerText = part;
-			ele.stripOut();
-			goto again2;
-		}
-	}
-
-	result = start.innerText();
-	result = squeeze(result, " ");
-
-	result = result.replace("\r ", "\r");
-	result = result.replace(" \r", "\r");
-
-	//result = result.replace("\u00a0", " ");
-
-
-	result = squeeze(result, "\r");
-	result = result.replace("\r", "\n\n");
-
-	result = result.replace("\v", "  ");
-
-	result = result.replace("&#33303;", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
-	result = result.replace("&quot;", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
-	//result = htmlEntitiesDecode(result);  // for special chars mainly
-
-	result = result.replace("\u0001 ", "\n");
-	result = result.replace("\u0001", "\n");
-
-	//a = std.regex.replace(a, std.regex.regex("(\n\t)+", "g"), "\n"); //\t");
-	return result.strip;
-}

terminal.d

@@ -3606,7 +3606,7 @@ struct ScrollbackBuffer {
 	}
 
 	void clear() {
-		lines = null;
+		lines.clear();
 		clickRegions = null;
 		scrollbackPosition = 0;
 	}
@@ -3708,9 +3708,87 @@ struct ScrollbackBuffer {
 		}
 	}
 
-	// FIXME: limit scrollback lines.length
+	static struct CircularBuffer(T) {
+		T[] backing;
 
-	Line[] lines;
+		enum maxScrollback = 8192; // as a power of 2, i hope the compiler optimizes the % below to a simple bit mask...
+
+		int start;
+		int length_;
+
+		void clear() {
+			backing = null;
+			start = 0;
+			length_ = 0;
+		}
+
+		size_t length() {
+			return length_;
+		}
+
+		void opOpAssign(string op : "~")(T line) {
+			if(length_ < maxScrollback) {
+				backing.assumeSafeAppend();
+				backing ~= line;
+				length_++;
+			} else {
+				backing[start] = line;
+				start++;
+				if(start == maxScrollback)
+					start = 0;
+			}
+		}
+
+		T opIndex(int idx) {
+			return backing[(start + idx) % maxScrollback];
+		}
+		T opIndex(Dollar idx) {
+			return backing[(start + (length + idx.offsetFromEnd)) % maxScrollback];
+		}
+
+		CircularBufferRange opSlice(int startOfIteration, Dollar end) {
+			return CircularBufferRange(&this, startOfIteration, cast(int) length - startOfIteration + end.offsetFromEnd);
+		}
+		CircularBufferRange opSlice(int startOfIteration, int end) {
+			return CircularBufferRange(&this, startOfIteration, end - startOfIteration);
+		}
+		CircularBufferRange opSlice() {
+			return CircularBufferRange(&this, 0, cast(int) length);
+		}
+
+		static struct CircularBufferRange {
+			CircularBuffer* item;
+			int position;
+			int remaining;
+			this(CircularBuffer* item, int startOfIteration, int count) {
+				this.item = item;
+				position = startOfIteration;
+				remaining = count;
+			}
+
+			T front() { return (*item)[position]; }
+			bool empty() { return remaining <= 0; }
+			void popFront() {
+				position++;
+				remaining--;
+			}
+
+			T back() { return (*item)[remaining - 1 - position]; }
+			void popBack() {
+				remaining--;
+			}
+		}
+
+		static struct Dollar {
+			int offsetFromEnd;
+			Dollar opBinary(string op : "-")(int rhs) {
+				return Dollar(offsetFromEnd - rhs);
+			}
+		}
+		Dollar opDollar() { return Dollar(0); }
+	}
+
+	CircularBuffer!Line lines;
 	string name;
 
 	int x, y, width, height;
@@ -3840,7 +3918,7 @@ struct ScrollbackBuffer {
 		// second pass: actually draw it
 		int linePos = remaining;
 
-		foreach(idx, line; lines[start .. start + howMany]) {
+		foreach(line; lines[start .. start + howMany]) {
 			int written = 0;
 
 			if(linePos < 0) {