merge

characterencodings.d

@@ -18,22 +18,26 @@
 	ought to just work.
 
 	Example:
+		---
 		auto data = cast(immutable(ubyte)[])
 			std.file.read("my-windows-file.txt");
 		string utf8String = convertToUtf8(data, "windows-1252");
 		// utf8String can now be used
+		---
 
 
 	The encodings currently implemented for decoding are:
-		UTF-8 (a no-op; it simply casts the array to string)
+		$(LIST
-		UTF-16,
+			* UTF-8 (a no-op; it simply casts the array to string)
-		UTF-32,
+			* UTF-16,
-		Windows-1252,
+			* UTF-32,
-		ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16.
+			* Windows-1252,
+			* ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16.
+			* KOI8-R
+		)
 
 	It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since
-	those labels are pretty much de-facto the same thing in wild documents.
+	those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic).
-
 
 	This module currently makes no attempt to look at control characters.
 */

color.d

@@ -1081,6 +1081,79 @@ class TrueColorImage : MemoryImage {
 	}
 }
 
+/+
+/// An RGB array of image data.
+class TrueColorImageWithoutAlpha : MemoryImage {
+	struct Data {
+		ubyte[] bytes; // the data as rgba bytes. Stored left to right, top to bottom, no padding.
+	}
+
+	/// .
+	Data imageData;
+
+	int _width;
+	int _height;
+
+	override void clearInternal () nothrow @system {// @nogc {
+		import core.memory : GC;
+		// it is safe to call [GC.free] with `null` pointer.
+		GC.free(imageData.bytes.ptr); imageData.bytes = null;
+		_width = _height = 0;
+	}
+
+	/// .
+	override TrueColorImageWithoutAlpha clone() const pure nothrow @trusted {
+		auto n = new TrueColorImageWithoutAlpha(width, height);
+		n.imageData.bytes[] = this.imageData.bytes[]; // copy into existing array ctor allocated
+		return n;
+	}
+
+	/// .
+	override int width() const pure nothrow @trusted @nogc { return _width; }
+	///.
+	override int height() const pure nothrow @trusted @nogc { return _height; }
+
+	override Color getPixel(int x, int y) const pure nothrow @trusted @nogc {
+		if (x >= 0 && y >= 0 && x < _width && y < _height) {
+			uint pos = (y*_width+x) * 3;
+			return Color(imageData.bytes[0], imageData.bytes[1], imageData.bytes[2], 255);
+		} else {
+			return Color(0, 0, 0, 0);
+		}
+	}
+
+	override void setPixel(int x, int y, in Color clr) nothrow @trusted {
+		if (x >= 0 && y >= 0 && x < _width && y < _height) {
+			uint pos = y*_width+x;
+			//if (pos < imageData.bytes.length/4) imageData.colors.ptr[pos] = clr;
+			// FIXME
+		}
+	}
+
+	/// .
+	this(int w, int h) pure nothrow @safe {
+		_width = w;
+		_height = h;
+		imageData.bytes = new ubyte[w*h*3];
+	}
+
+	/// Creates with existing data. The data pointer is stored here.
+	this(int w, int h, ubyte[] data) pure nothrow @safe {
+		_width = w;
+		_height = h;
+		assert(data.length == w * h * 3);
+		imageData.bytes = data;
+	}
+
+	///
+	override TrueColorImage getAsTrueColorImage() pure nothrow @safe {
+		// FIXME
+		//return this;
+	}
+}
++/
+
+
 alias extern(C) int function(const void*, const void*) @system Comparator;
 @trusted void nonPhobosSort(T)(T[] obj,  Comparator comparator) {
 	import core.stdc.stdlib;

csv.d

@@ -6,9 +6,10 @@ import std.array;
 
 ///
 string[][] readCsv(string data) {
+	data = data.replace("\r\n", "\n");
 	data = data.replace("\r", "");
 
-	auto idx = data.indexOf("\n");
+	//auto idx = data.indexOf("\n");
 	//data = data[idx + 1 .. $]; // skip headers
 
 	string[] fields;
@@ -39,9 +40,9 @@ string[][] readCsv(string data) {
 					field ~= c;
 			break;
 			case 1: // in quote
-				if(c == '"')
+				if(c == '"') {
 					state = 2;
-				else
+				} else
 					field ~= c;
 			break;
 			case 2: // is it a closing quote or an escaped one?
@@ -55,6 +56,11 @@ string[][] readCsv(string data) {
 		}
 	}
 
+	if(field !is null)
+		current ~= field;
+	if(current !is null)
+		records ~= current;
+
 
 	return records;
 }

dom.d

@@ -1,4 +1,8 @@
 // FIXME: add classList
+// FIXME: add matchesSelector
+// FIXME: https://developer.mozilla.org/en-US/docs/Web/API/Element/insertAdjacentHTML
+// FIXME: appendChild should not fail if the thing already has a parent; it should just automatically remove it per standard.
+
 /++
 	This is an html DOM implementation, started with cloning
 	what the browser offers in Javascript, but going well beyond
@@ -71,7 +75,7 @@ bool isConvenientAttribute(string name) {
 /// The main document interface, including a html parser.
 class Document : FileResource {
 	/// Convenience method for web scraping. Requires [arsd.http2] to be
-	/// included in the build.
+	/// included in the build as well as [arsd.characterencodings].
 	static Document fromUrl()(string url) {
 		import arsd.http2;
 		auto client = new HttpClient();
@@ -79,7 +83,10 @@ class Document : FileResource {
 		auto req = client.navigateTo(Uri(url), HttpVerb.GET);
 		auto res = req.waitForCompletion();
 
-		return new Document(cast(string) res.content);
+		auto document = new Document();
+		document.parseGarbage(cast(string) res.content);
+
+		return document;
 	}
 
 	///.
@@ -821,6 +828,12 @@ class Document : FileResource {
 
 							while(pos < data.length && data[pos] != '>')
 								pos++;
+
+							if(pos >= data.length) {
+								// the tag never closed
+								assert(data.length != 0);
+								pos = data.length - 1; // rewinding so it hits the end at the bottom..
+							}
 						}
 
 						auto whereThisTagStarted = pos; // for better error messages
@@ -3494,8 +3507,14 @@ struct ElementCollection {
 		return !elements.length;
 	}
 
-	/// Collects strings from the collection, concatenating them together
+	/++
-	/// Kinda like running reduce and ~= on it.
+		Collects strings from the collection, concatenating them together
+		Kinda like running reduce and ~= on it.
+
+		---
+		document["p"].collect!"innerText";
+		---
+	+/
 	string collect(string method)(string separator = "") {
 		string text;
 		foreach(e; elements) {
@@ -3514,6 +3533,17 @@ struct ElementCollection {
 		return this;
 	}
 
+	/++
+		Calls [Element.wrapIn] on each member of the collection, but clones the argument `what` for each one.
+	+/
+	ElementCollection wrapIn(Element what) {
+		foreach(e; elements) {
+			e.wrapIn(what.cloneNode(false));
+		}
+
+		return this;
+	}
+
 	/// Concatenates two ElementCollection together.
 	ElementCollection opBinary(string op : "~")(ElementCollection rhs) {
 		return ElementCollection(this.elements ~ rhs.elements);
@@ -5042,7 +5072,7 @@ class Table : Element {
 			return position;
 		}
 
-		foreach(int i, rowElement; rows) {
+		foreach(i, rowElement; rows) {
 			auto row = cast(TableRow) rowElement;
 			assert(row !is null);
 			assert(i < ret.length);
@@ -5059,7 +5089,7 @@ class Table : Element {
 				foreach(int j; 0 .. cell.colspan) {
 					foreach(int k; 0 .. cell.rowspan)
 						// if the first row, always append.
-						insertCell(k + i, k == 0 ? -1 : position, cell);
+						insertCell(k + cast(int) i, k == 0 ? -1 : position, cell);
 					position++;
 				}
 			}

dub.json

@@ -95,6 +95,8 @@
 			"description": "MySQL client library. Wraps the official C library with my database.d interface.",
 			"targetType": "sourceLibrary",
 			"dependencies": {"arsd-official:database_base":"*"},
+			"libs-posix": ["mysqlclient"],
+			"libs-windows": ["libmysql"],
 			"sourceFiles": ["mysql.d"]
 		},
 		{
@@ -102,6 +104,7 @@
 			"description": "Postgresql client library. Wraps the libpq C library with my database.d interface.",
 			"targetType": "sourceLibrary",
 			"dependencies": {"arsd-official:database_base":"*"},
+			"libs": ["pq"],
 			"sourceFiles": ["postgres.d"]
 		},
 
@@ -110,6 +113,8 @@
 			"description": "sqlite wrapper. Wraps the official C library with my database.d interface.",
 			"targetType": "sourceLibrary",
 			"dependencies": {"arsd-official:database_base":"*"},
+			"libs": ["sqlite3"],
+			"libs-posix": ["dl"],
 			"sourceFiles": ["sqlite.d"]
 		},
 
@@ -118,6 +123,7 @@
 			"description": "Microsoft SQL Server client library. Wraps the official ODBC library with my database.d interface.",
 			"targetType": "sourceLibrary",
 			"dependencies": {"arsd-official:database_base":"*"},
+			"libs-windows": ["odbc32"],
 			"sourceFiles": ["mssql.d"]
 		},
 

htmltotext.d

@@ -7,10 +7,47 @@ import std.string;
 
 import std.uni : isWhite;
 
+///
 class HtmlConverter {
 	int width;
 
+	/++
+		Will enable color output using VT codes. Determines color through dom.d's css support, which means you need to apply a stylesheet first.
+
+		---
+		import arsd.dom;
+
+		auto document = new Document(source_code_for_html);
+		auto stylesheet = new Stylesheet(source_code_for_css);
+		stylesheet.apply(document);
+		---
+	+/
+	bool enableVtOutput;
+
+
+	string color;
+	string backgroundColor;
+
+	///
 	void htmlToText(Element element, bool preformatted, int width) {
+		string color, backgroundColor;
+		if(enableVtOutput) {
+			color = element.computedStyle.getValue("color");
+			backgroundColor = element.computedStyle.getValue("background-color");
+		}
+
+		string originalColor = this.color, originalBackgroundColor = this.backgroundColor;
+
+		this.color = color.length ? color : this.color;
+		this.backgroundColor = backgroundColor.length ? backgroundColor : this.backgroundColor;
+
+		scope(exit) {
+			// the idea is as we pop working back up the tree, it restores what it was here
+			this.color = originalColor;
+			this.backgroundColor = originalBackgroundColor;
+		}
+
+
 		this.width = width;
 		if(auto tn = cast(TextNode) element) {
 			foreach(dchar ch; tn.nodeValue) {
@@ -28,7 +65,7 @@ class HtmlConverter {
 				// The table stuff is removed right now because while it looks
 				// ok for test tables, it isn't working well for the emails I have
 				// - it handles data ok but not really nested layouts.
-				case "trfixme":
+				case "trlol":
 					auto children = element.childElements;
 
 					auto tdWidth = (width - cast(int)(children.length)*3) / cast(int)(children.length);
@@ -91,6 +128,16 @@ class HtmlConverter {
 						s ~= "\n";
 					}
 				break;
+				case "tr":
+					startBlock(2);
+					sinkChildren();
+					endBlock();
+				break;
+				case "td":
+					startBlock(0);
+					sinkChildren();
+					endBlock();
+				break;
 				case "a":
 					sinkChildren();
 					if(element.href != element.innerText) {
@@ -105,17 +152,27 @@ class HtmlConverter {
 					}
 				break;
 				case "span":
-					/*
+					if(enableVtOutput) {
-					auto csc = element.computedStyle.getValue("color");
+						auto csc = color; // element.computedStyle.getValue("color");
 						if(csc.length) {
 							auto c = Color.fromString(csc);
 							s ~= format("\033[38;2;%d;%d;%dm", c.r, c.g, c.b);
 						}
+
+						bool bold = element.computedStyle.getValue("font-weight") == "bold";
+
+						if(bold)
+							s ~= "\033[1m";
+
 						sinkChildren();
 
+						if(bold)
+							s ~= "\033[0m";
 						if(csc.length)
 							s ~= "\033[39m";
-					*/
+					} else {
+						sinkChildren();
+					}
 				break;
 				case "p":
 					startBlock();
@@ -126,9 +183,15 @@ class HtmlConverter {
 				case "em", "i":
 					if(element.innerText.length == 0)
 						break;
+					if(enableVtOutput) {
+						s ~= "\033[1m";
+						sinkChildren();
+						s ~= "\033[0m";
+					} else {
 						sink('*', false);
 						sinkChildren();
 						sink('*', false);
+					}
 				break;
 				case "u":
 					if(element.innerText.length == 0)
@@ -139,20 +202,28 @@ class HtmlConverter {
 				break;
 				case "ul":
 					ulDepth++;
+					startBlock(2);
 					sinkChildren();
+					endBlock();
 					ulDepth--;
 				break;
 				case "ol":
 					olDepth++;
+					startBlock(2);
 					sinkChildren();
+					endBlock();
 					olDepth--;
 				break;
 				case "li":
 					startBlock();
 
 					//sink('\t', true);
+					/*
+					foreach(cnt; 0 .. olDepth + ulDepth) {
 						sink(' ', true);
 						sink(' ', true);
+					}
+					*/
 					if(olDepth)
 						sink('*', false);
 					if(ulDepth)
@@ -164,15 +235,33 @@ class HtmlConverter {
 					endBlock();
 				break;
 
-				case "h1", "h2":
+				case "dl":
+				case "dt":
+				case "dd":
+					startBlock(element.tagName == "dd" ? 2 : 0);
+					sinkChildren();
+					endBlock();
+				break;
+
+				case "h1":
+					startBlock();
+					sink('#', true);
+					sink('#', true);
+					sink(' ', true);
+					sinkChildren();
+					sink(' ', true);
+					sink('#', true);
+					sink('#', true);
+					endBlock();
+				break;
+				case "h2", "h3":
 					startBlock();
 					sinkChildren();
 					sink('\n', true);
 					foreach(dchar ch; element.innerText)
-						sink(element.tagName == "h1" ? '=' : '-', false);
+						sink(element.tagName == "h2" ? '=' : '-', false);
 					endBlock();
 				break;
-
 				case "hr":
 					startBlock();
 					foreach(i; 0 .. width / 4)
@@ -185,7 +274,6 @@ class HtmlConverter {
 				case "br":
 					sink('\n', true);
 				break;
-				case "tr":
 				case "div":
 					startBlock();
 
@@ -207,7 +295,7 @@ class HtmlConverter {
 					endBlock();
 				break;
 				case "pre":
-					startBlock();
+					startBlock(4);
 					foreach(child; element.childNodes)
 						htmlToText(child, true, width);
 					endBlock();
@@ -221,6 +309,7 @@ class HtmlConverter {
 	int olDepth;
 	int ulDepth;
 
+	///
 	string convert(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 		Document document = new Document;
 
@@ -237,10 +326,16 @@ class HtmlConverter {
 		//auto stylesheet = new StyleSheet(readText("/var/www/dpldocs.info/experimental-docs/style.css"));
 		//stylesheet.apply(document);
 
+		return convert(start, wantWordWrap, wrapAmount);
+	}
+
+	///
+	string convert(Element start, bool wantWordWrap = true, int wrapAmount = 74) {
 		htmlToText(start, false, wrapAmount);
 		return s;
 	}
 
+	///
 	void reset() {
 		s = null;
 		justOutputWhitespace = true;
@@ -248,6 +343,7 @@ class HtmlConverter {
 		justOutputMargin = true;
 	}
 
+	///
 	string s;
 	bool justOutputWhitespace = true;
 	bool justOutputBlock = true;
@@ -255,6 +351,12 @@ class HtmlConverter {
 	int lineLength;
 
 	void sink(dchar item, bool preformatted, int lineWidthOverride = int.min) {
+
+		if(needsIndent && item != '\n') {
+			lineLength += doIndent();
+			needsIndent = false;
+		}
+
 		int width = lineWidthOverride == int.min ? this.width : lineWidthOverride;
 		if(!preformatted && isWhite(item)) {
 			if(!justOutputWhitespace) {
@@ -282,8 +384,9 @@ class HtmlConverter {
 					auto os = s;
 					s = os[0 .. idx];
 					s ~= '\n';
-					s ~= os[idx + 1 .. $];
 					lineLength = cast(int)(os[idx+1..$].length);
+					lineLength += doIndent();
+					s ~= os[idx + 1 .. $];
 					broken = true;
 					break;
 				}
@@ -295,15 +398,17 @@ class HtmlConverter {
 			if(!broken) {
 				s ~= '\n';
 				lineLength = 0;
+				needsIndent = true;
 				justOutputWhitespace = true;
 			}
 
 		}
 
 
-		if(item == '\n')
+		if(item == '\n') {
 			lineLength = 0;
-		else
+			needsIndent = true;
+		} else
 			lineLength ++;
 
 
@@ -312,163 +417,53 @@ class HtmlConverter {
 			justOutputMargin = false;
 		}
 	}
-	void startBlock() {
+
+	int doIndent() {
+		int cnt = 0;
+		foreach(i; indentStack)
+			foreach(lol; 0 .. i) {
+				s ~= ' ';
+				cnt++;
+			}
+		return cnt;
+	}
+
+	int[] indentStack;
+	bool needsIndent = false;
+
+	void startBlock(int indent = 0) {
+
+		indentStack ~= indent;
+
 		if(!justOutputBlock) {
 			s ~= "\n";
 			lineLength = 0;
+			needsIndent = true;
 			justOutputBlock = true;
 		}
 		if(!justOutputMargin) {
 			s ~= "\n";
 			lineLength = 0;
+			needsIndent = true;
 			justOutputMargin = true;
 		}
 	}
 	void endBlock() {
+		if(indentStack.length)
+			indentStack = indentStack[0 .. $ - 1];
+
 		if(!justOutputMargin) {
 			s ~= "\n";
 			lineLength = 0;
+			needsIndent = true;
 			justOutputMargin = true;
 		}
 	}
 }
 
+///
 string htmlToText(string html, bool wantWordWrap = true, int wrapAmount = 74) {
 	auto converter = new HtmlConverter();
 	return converter.convert(html, true, wrapAmount);
 }
 
-string repeat(string s, ulong num) {
-	string ret;
-	foreach(i; 0 .. num)
-		ret ~= s;
-	return ret;
-}
-
-import std.stdio;
-version(none)
-void penis() {
-
-    again:
-    	string result = "";
-	foreach(ele; start.tree) {
-		if(ele is start) continue;
-		if(ele.nodeType != 1) continue;
-
-		switch(ele.tagName) {
-				goto again;
-			case "h1":
-				ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("=", ele.innerText.length) ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "h2":
-				ele.innerText = "\r" ~ ele.innerText ~ "\n" ~ repeat("-", ele.innerText.length) ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "h3":
-				ele.innerText = "\r" ~ ele.innerText.toUpper ~ "\r";
-				ele.stripOut();
-				goto again;
-			case "td":
-			case "p":
-			/*
-				if(ele.innerHTML.length > 1)
-					ele.innerHTML = "\r" ~ wrap(ele.innerHTML) ~ "\r";
-				ele.stripOut();
-				goto again;
-			*/
-			break;
-			case "a":
-				string href = ele.getAttribute("href");
-				if(href && !ele.hasClass("no-brackets")) {
-					if(ele.hasClass("href-text"))
-						ele.innerText = href;
-					else {
-						if(ele.innerText != href)
-							ele.innerText = ele.innerText ~ " <" ~ href ~ "> ";
-					}
-				}
-				ele.stripOut();
-				goto again;
-			case "ol":
-			case "ul":
-				ele.innerHTML = "\r" ~ ele.innerHTML ~ "\r";
-			break;
-			case "li":
-				if(!ele.innerHTML.startsWith("* "))
-					ele.innerHTML = "* " ~ ele.innerHTML ~ "\r";
-				// ele.stripOut();
-			break;
-			case "sup":
-				ele.innerText = "^" ~ ele.innerText;
-				ele.stripOut();
-			break;
-			/*
-			case "img":
-				string alt = ele.getAttribute("alt");
-				if(alt)
-					result ~= ele.alt;
-			break;
-			*/
-			default:
-				ele.stripOut();
-				goto again;
-		}
-	}
-
-    again2:
-	//start.innerHTML = start.innerHTML().replace("\u0001", "\n");
-
-	foreach(ele; start.tree) {
-		if(ele.tagName == "td") {
-			if(ele.directText().strip().length) {
-				ele.prependText("\r");
-				ele.appendText("\r");
-			}
-			ele.stripOut();
-			goto again2;
-		} else if(ele.tagName == "p") {
-			if(strip(ele.innerText()).length > 1) {
-				string res = "";
-				string all = ele.innerText().replace("\n \n", "\n\n");
-				foreach(part; all.split("\n\n"))
-					res ~= "\r" ~ strip( wantWordWrap ? wrap(part, /*74*/ wrapAmount) : part ) ~ "\r";
-				ele.innerText = res;
-			} else
-				ele.innerText = strip(ele.innerText);
-			ele.stripOut();
-			goto again2;
-		} else if(ele.tagName == "li") {
-			auto part = ele.innerText;
-			part = strip( wantWordWrap ? wrap(part, wrapAmount - 2) : part );
-			part = "  " ~ part.replace("\n", "\n\v") ~ "\r";
-			ele.innerText = part;
-			ele.stripOut();
-			goto again2;
-		}
-	}
-
-	result = start.innerText();
-	result = squeeze(result, " ");
-
-	result = result.replace("\r ", "\r");
-	result = result.replace(" \r", "\r");
-
-	//result = result.replace("\u00a0", " ");
-
-
-	result = squeeze(result, "\r");
-	result = result.replace("\r", "\n\n");
-
-	result = result.replace("\v", "  ");
-
-	result = result.replace("&#33303;", "'"); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
-	result = result.replace("&quot;", "\""); // HACK: this shouldn't be needed, but apparently is in practice surely due to a bug elsewhere
-	//result = htmlEntitiesDecode(result);  // for special chars mainly
-
-	result = result.replace("\u0001 ", "\n");
-	result = result.replace("\u0001", "\n");
-
-	//a = std.regex.replace(a, std.regex.regex("(\n\t)+", "g"), "\n"); //\t");
-	return result.strip;
-}

http.d

@@ -3,7 +3,7 @@
 
 	I no longer work on this, use http2.d instead.
 +/
-module arsd.http;
+deprecated module arsd.http;
 
 import std.socket;
 

http2.d

@@ -75,6 +75,20 @@ HttpRequest get(string url) {
 	return request;
 }
 
+HttpRequest post(string url, string[string] req) {
+	auto client = new HttpClient();
+	ubyte[] bdata;
+	foreach(k, v; req) {
+		if(bdata.length)
+			bdata ~= cast(ubyte[]) "&";
+		bdata ~= cast(ubyte[]) encodeComponent(k);
+		bdata ~= cast(ubyte[]) "=";
+		bdata ~= cast(ubyte[]) encodeComponent(v);
+	}
+	auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded");
+	return request;
+}
+
 /// gets the text off a url. basic operation only.
 string getText(string url) {
 	auto request = get(url);
@@ -1090,7 +1104,6 @@ class HttpRequest {
 	ubyte[] sendBuffer;
 
 	HttpResponse responseData;
-	HttpRequestParameters parameters;
 	private HttpClient parentClient;
 
 	size_t bodyBytesSent;

jpeg.d

@@ -2971,9 +2971,10 @@ public bool detect_jpeg_image_from_file (const(char)[] filename, out int width,
   bool m_eof_flag, m_error_flag;
 
   if (filename.length == 0) throw new Exception("cannot open unnamed file");
-  if (filename.length < 2048) {
+  if (filename.length < 512) {
-    import core.stdc.stdlib : alloca;
+    char[513] buffer;
-    auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1];
+    //import core.stdc.stdlib : alloca;
+    auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1];
     tfn[0..filename.length] = filename[];
     tfn[filename.length] = 0;
     m_pFile = fopen(tfn.ptr, "rb");
@@ -3147,9 +3148,10 @@ public ubyte[] decompress_jpeg_image_from_file(bool useMalloc=false) (const(char
   bool m_eof_flag, m_error_flag;
 
   if (filename.length == 0) throw new Exception("cannot open unnamed file");
-  if (filename.length < 2048) {
+  if (filename.length < 512) {
-    import core.stdc.stdlib : alloca;
+	char[513] buffer;
-    auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1];
+    //import core.stdc.stdlib : alloca;
+    auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1];
     tfn[0..filename.length] = filename[];
     tfn[filename.length] = 0;
     m_pFile = fopen(tfn.ptr, "rb");
@@ -3338,9 +3340,10 @@ public MemoryImage readJpeg (const(char)[] filename) {
   bool m_eof_flag, m_error_flag;
 
   if (filename.length == 0) throw new Exception("cannot open unnamed file");
-  if (filename.length < 2048) {
+  if (filename.length < 512) {
-    import core.stdc.stdlib : alloca;
+	char[513] buffer;
-    auto tfn = (cast(char*)alloca(filename.length+1))[0..filename.length+1];
+    //import core.stdc.stdlib : alloca;
+    auto tfn = buffer[0 .. filename.length + 1]; // (cast(char*)alloca(filename.length+1))[0..filename.length+1];
     tfn[0..filename.length] = filename[];
     tfn[filename.length] = 0;
     m_pFile = fopen(tfn.ptr, "rb");

jsvar.d

@@ -709,7 +709,7 @@ struct var {
 			case Type.String:
 			case Type.Function:
 				assert(0); // FIXME
-			break;
+			//break;
 			case Type.Integral:
 				return var(-this.get!long);
 			case Type.Floating:

minigui.d

@@ -1,5 +1,16 @@
 // http://msdn.microsoft.com/en-us/library/windows/desktop/bb775498%28v=vs.85%29.aspx
 
+// So a window needs to have a selection, and that can be represented by a type. This is manipulated by various
+// functions like cut, copy, paste. Widgets can have a selection and that would assert teh selection ownership for
+// the window.
+
+// so what about context menus?
+
+// https://docs.microsoft.com/en-us/windows/desktop/Controls/about-custom-draw
+
+// FIXME: add a command search thingy built in and implement tip.
+// FIXME: omg omg what if menu functions have arguments and it can pop up a gui or command line script them?!
+
 // On Windows:
 // FIXME: various labels look broken in high contrast mode
 // FIXME: changing themes while the program is upen doesn't trigger a redraw
@@ -3910,8 +3921,10 @@ class MainWindow : Window {
 				.toolbar toolbar;
 				bool separator;
 				.accelerator accelerator;
+				.hotkey hotkey;
 				.icon icon;
 				string label;
+				string tip;
 				foreach(attr; __traits(getAttributes, __traits(getMember, T, memberName))) {
 					static if(is(typeof(attr) == .menu))
 						menu = attr;
@@ -3921,10 +3934,14 @@ class MainWindow : Window {
 						separator = true;
 					else static if(is(typeof(attr) == .accelerator))
 						accelerator = attr;
+					else static if(is(typeof(attr) == .hotkey))
+						hotkey = attr;
 					else static if(is(typeof(attr) == .icon))
 						icon = attr;
 					else static if(is(typeof(attr) == .label))
 						label = attr.label;
+					else static if(is(typeof(attr) == .tip))
+						tip = attr.tip;
 				}
 
 				if(menu !is .menu.init || toolbar !is .toolbar.init) {
@@ -5504,6 +5521,15 @@ abstract class EditableTextWidget : EditableTextWidgetParent {
 		super(parent);
 	}
 
+	bool wordWrapEnabled_ = false;
+	void wordWrapEnabled(bool enabled) {
+		version(win32_widgets) {
+			SendMessageW(hwnd, EM_FMTLINES, enabled ? 1 : 0, 0);
+		} else version(custom_widgets) {
+			wordWrapEnabled_ = enabled; // FIXME
+		} else static assert(false);
+	}
+
 	override int minWidth() { return 16; }
 	override int minHeight() { return Window.lineHeight + 0; } // the +0 is to leave room for the padding
 	override int widthStretchiness() { return 7; }
@@ -5563,6 +5589,7 @@ abstract class EditableTextWidget : EditableTextWidgetParent {
 
 	void addText(string txt) {
 		version(custom_widgets) {
+
 			textLayout.addText(txt);
 
 			{
@@ -5579,12 +5606,12 @@ abstract class EditableTextWidget : EditableTextWidgetParent {
 			SendMessageW( hwnd, EM_GETSEL, cast(WPARAM)(&StartPos), cast(WPARAM)(&EndPos) );
 
 			// move the caret to the end of the text
-			int outLength = GetWindowTextLengthW( hwndOutput );
+			int outLength = GetWindowTextLengthW(hwnd);
 			SendMessageW( hwnd, EM_SETSEL, outLength, outLength );
 
 			// insert the text at the new caret position
 			WCharzBuffer bfr = WCharzBuffer(txt, WindowsStringConversionFlags.convertNewLines);
-			SendMessageW( hwnd, EM_REPLACESEL, TRUE, txt );
+			SendMessageW( hwnd, EM_REPLACESEL, TRUE, cast(int) bfr.ptr );
 
 			// restore the previous selection
 			SendMessageW( hwnd, EM_SETSEL, StartPos, EndPos );
@@ -6721,6 +6748,12 @@ struct icon { ushort id; }
 ///
 /// Group: generating_from_code
 struct label { string label; }
+///
+/// Group: generating_from_code
+struct hotkey { dchar ch; }
+///
+/// Group: generating_from_code
+struct tip { string tip; }
 
 
 /++

mssql.d

@@ -155,16 +155,16 @@ class MsSqlResult : ResultSet {
 					string a;
 
 					more:
-				        SQLCHAR[255] buf;
+				        SQLCHAR[1024] buf;
-					if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 255, &ptr) != SQL_SUCCESS)
+					if(SQLGetData(statement, cast(ushort)(i+1), SQL_CHAR, buf.ptr, 1024, &ptr) != SQL_SUCCESS)
 						throw new DatabaseException("get data: " ~ getSQLError(SQL_HANDLE_STMT, statement));
 
 					assert(ptr != SQL_NO_TOTAL);
 					if(ptr == SQL_NULL_DATA)
 						a = null;
 					else {
-						a ~= cast(string) buf[0 .. ptr > 255 ? 255 : ptr].idup;
+						a ~= cast(string) buf[0 .. ptr > 1024 ? 1024 : ptr].idup;
-						ptr -= ptr > 255 ? 255 : ptr;
+						ptr -= ptr > 1024 ? 1024 : ptr;
 						if(ptr)
 							goto more;
 					}
@@ -181,11 +181,11 @@ class MsSqlResult : ResultSet {
 		void makeFieldMapping() {
 			for(int i = 0; i < numFields; i++) {
 				SQLSMALLINT len;
-				SQLCHAR[255] buf;
+				SQLCHAR[1024] buf;
 				auto ret = SQLDescribeCol(statement,
 					cast(ushort)(i+1),
 					cast(ubyte*)buf.ptr,
-					255,
+					1024,
 					&len,
 					null, null, null, null);
 				if (ret != SQL_SUCCESS)

nanovega.d

@@ -1,4 +1,7 @@
 //
+// Would be nice: way to take output of the canvas to an image file (raster and/or svg)
+//
+//
 // Copyright (c) 2013 Mikko Mononen memon@inside.org
 //
 // This software is provided 'as-is', without any express or implied

png.d

@@ -22,6 +22,31 @@ void writePng(string filename, MemoryImage mi) {
 	std.file.write(filename, writePng(png));
 }
 
+///
+enum PngType {
+	greyscale = 0, /// The data must be `depth` bits per pixel
+	truecolor = 2, /// The data will be RGB triples, so `depth * 3` bits per pixel. Depth must be 8 or 16.
+	indexed = 3, /// The data must be `depth` bits per pixel, with a palette attached. Use [writePng] with [IndexedImage] for this mode. Depth must be <= 8.
+	greyscale_with_alpha = 4, /// The data must be (grey, alpha) byte pairs for each pixel. Thus `depth * 2` bits per pixel. Depth must be 8 or 16.
+	truecolor_with_alpha = 6 /// The data must be RGBA quads for each pixel. Thus, `depth * 4` bits per pixel. Depth must be 8 or 16.
+}
+
+/// Saves an image from an existing array. Note that depth other than 8 may not be implemented yet. Also note depth of 16 must be stored big endian
+void writePng(string filename, const ubyte[] data, int width, int height, PngType type, ubyte depth = 8) {
+	PngHeader h;
+	h.width = width;
+	h.height = height;
+	h.type = cast(ubyte) type;
+	h.depth = depth;
+
+	auto png = blankPNG(h);
+	addImageDatastreamToPng(data, png);
+
+	import std.file;
+	std.file.write(filename, writePng(png));
+}
+
+
 /*
 //Here's a simple test program that shows how to write a quick image viewer with simpledisplay:
 
@@ -629,9 +654,28 @@ void addImageDatastreamToPng(const(ubyte)[] data, PNG* png) {
 
 	PngHeader h = getHeader(png);
 
-	auto bytesPerLine = h.width * 4;
+	int bytesPerLine;
-	if(h.type == 3)
+	switch(h.type) {
-		bytesPerLine = h.width * h.depth / 8;
+		case 0:
+			// FIXME: < 8 depth not supported here but should be
+			bytesPerLine = h.width * 1 * h.depth / 8;
+		break;
+		case 2:
+			bytesPerLine = h.width * 3 * h.depth / 8;
+		break;
+		case 3:
+			bytesPerLine = h.width * 1 * h.depth / 8;
+		break;
+		case 4:
+			// FIXME: < 8 depth not supported here but should be
+			bytesPerLine = h.width * 2 * h.depth / 8;
+		break;
+		case 6:
+			bytesPerLine = h.width * 4 * h.depth / 8;
+		break;
+		default: assert(0);
+	
+	}
 	Chunk dat;
 	dat.type = ['I', 'D', 'A', 'T'];
 	int pos = 0;

script.d

@@ -90,6 +90,8 @@
 		obj.__prop("name", value); // bypasses operator overloading, useful for use inside the opIndexAssign especially
 
 		Note: if opIndex is not overloaded, getting a non-existent member will actually add it to the member. This might be a bug but is needed right now in the D impl for nice chaining. Or is it? FIXME
+
+		FIXME: it doesn't do opIndex with multiple args.
 	* if/else
 	* array slicing, but note that slices are rvalues currently
 	* variables must start with A-Z, a-z, _, or $, then must be [A-Za-z0-9_]*.
@@ -2304,7 +2306,9 @@ Expression parseExpression(MyTokenStreamHere)(ref MyTokenStreamHere tokens, bool
 					auto ident = tokens.requireNextToken(ScriptToken.Type.identifier);
 
 					tokens.requireNextToken(ScriptToken.Type.symbol, "(");
-					auto args = parseVariableDeclaration(tokens, ")");
+					VariableDeclaration args;
+					if(!tokens.peekNextToken(ScriptToken.Type.symbol, ")"))
+						args = parseVariableDeclaration(tokens, ")");
 					tokens.requireNextToken(ScriptToken.Type.symbol, ")");
 					auto bod = parseExpression(tokens);
 
@@ -2538,7 +2542,7 @@ Expression parseStatement(MyTokenStreamHere)(ref MyTokenStreamHere tokens, strin
 
 					return parseFunctionCall(tokens, new AssertKeyword(token));
 
-				break;
+				//break;
 				// declarations
 				case "var":
 					return parseVariableDeclaration(tokens, ";");

simpledisplay.d

@@ -1158,6 +1158,64 @@ string sdpyWindowClass () {
 	return null;
 }
 
+/++
+	Returns the DPI of the default monitor. [0] is width, [1] is height (they are usually the same though). You may wish to round the numbers off.
++/
+float[2] getDpi() {
+	float[2] dpi;
+	version(Windows) {
+		HDC screen = GetDC(null);
+		dpi[0] = GetDeviceCaps(screen, LOGPIXELSX);
+		dpi[1] = GetDeviceCaps(screen, LOGPIXELSY);
+	} else version(X11) {
+		auto display = XDisplayConnection.get;
+		auto screen = DefaultScreen(display);
+
+		void fallback() {
+			// 25.4 millimeters in an inch...
+			dpi[0] = cast(float) DisplayWidth(display, screen) / DisplayWidthMM(display, screen) * 25.4;
+			dpi[1] = cast(float) DisplayHeight(display, screen) / DisplayHeightMM(display, screen) * 25.4;
+		}
+
+		char* resourceString = XResourceManagerString(display);
+		XrmInitialize();
+
+		auto db = XrmGetStringDatabase(resourceString);
+
+		if (resourceString) {
+			XrmValue value;
+			char* type;
+			if (XrmGetResource(db, "Xft.dpi", "String", &type, &value) == true) {
+				if (value.addr) {
+					import core.stdc.stdlib;
+					dpi[0] = atof(cast(char*) value.addr);
+					dpi[1] = dpi[0];
+				} else {
+					fallback();
+				}
+			} else {
+				fallback();
+			}
+		} else {
+			fallback();
+		}
+	}
+
+	return dpi;
+}
+
+version(X11) {
+	extern(C) char* XResourceManagerString(Display*);
+	extern(C) void XrmInitialize();
+	extern(C) XrmDatabase XrmGetStringDatabase(char* data);
+	extern(C) bool XrmGetResource(XrmDatabase, const char*, const char*, char**, XrmValue*);
+	alias XrmDatabase = void*;
+	struct XrmValue {
+		uint size;
+		void* addr;
+	}
+}
+
 TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int width, int height) {
 	throw new Exception("not implemented");
 	version(none) {
@@ -1165,6 +1223,8 @@ TrueColorImage trueColorImageFromNativeHandle(NativeWindowHandle handle, int wid
 		auto display = XDisplayConnection.get;
 		auto image = XGetImage(display, handle, 0, 0, width, height, (cast(c_ulong) ~0) /*AllPlanes*/, ZPixmap);
 
+		// https://github.com/adamdruppe/arsd/issues/98
+
 		// FIXME: copy that shit
 
 		XDestroyImage(image);
@@ -11899,6 +11959,8 @@ struct Visual
 	int DefaultDepth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).root_depth; }
 	int DisplayWidth(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).width; }
 	int DisplayHeight(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).height; }
+	int DisplayWidthMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mwidth; }
+	int DisplayHeightMM(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).mheight; }
 	auto DefaultColormap(Display* dpy, int scr) { return ScreenOfDisplay(dpy, scr).cmap; }
 
 	int ConnectionNumber(Display* dpy) { return dpy.fd; }

terminal.d

@@ -1,4 +1,3 @@
-// FIXME: have a simple function that integrates with sdpy event loop. it can be a template
 // for optional dependency
 /++
 	Module for interacting with the user's terminal, including color output, cursor manipulation, and full-featured real-time mouse and keyboard input. Also includes high-level convenience methods, like [Terminal.getline], which gives the user a line editor with history, completion, etc. See the [#examples].
@@ -78,6 +77,8 @@ unittest {
 		string line = terminal.getline();
 		terminal.writeln("You wrote: ", line);
 	}
+
+	main; // exclude from docs
 }
 
 /++
@@ -88,6 +89,7 @@ unittest {
 +/
 unittest {
 	import arsd.terminal;
+
 	void main() {
 		auto terminal = Terminal(ConsoleOutputType.linear);
 		terminal.color(Color.green, Color.black);
@@ -95,6 +97,8 @@ unittest {
 		terminal.color(Color.DEFAULT, Color.DEFAULT);
 		terminal.writeln("And back to normal.");
 	}
+
+	main; // exclude from docs
 }
 
 /++
@@ -105,6 +109,7 @@ unittest {
 +/
 unittest {
 	import arsd.terminal;
+
 	void main() {
 		auto terminal = Terminal(ConsoleOutputType.linear);
 		auto input = RealTimeConsoleInput(&terminal, ConsoleInputFlags.raw);
@@ -113,6 +118,8 @@ unittest {
 		auto ch = input.getch();
 		terminal.writeln("You pressed ", ch);
 	}
+
+	main; // exclude from docs
 }
 
 /*
@@ -427,6 +434,25 @@ struct Terminal {
 	@disable this(this);
 	private ConsoleOutputType type;
 
+	/++
+		Terminal is only valid to use on an actual console device or terminal
+		handle. You should not attempt to construct a Terminal instance if this
+		returns false;
+	+/
+	static bool stdoutIsTerminal() {
+		version(Posix) {
+			import core.sys.posix.unistd;
+			return cast(bool) isatty(1);
+		} else version(Windows) {
+			auto hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
+			CONSOLE_SCREEN_BUFFER_INFO originalSbi;
+			if(GetConsoleScreenBufferInfo(hConsole, &originalSbi) == 0)
+				return false;
+			else
+				return true;
+		} else static assert(0);
+	}
+
 	version(Posix) {
 		private int fdOut;
 		private int fdIn;
@@ -797,11 +823,19 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as
 
 		defaultForegroundColor = cast(Color) (originalSbi.wAttributes & 0x0f);
 		defaultBackgroundColor = cast(Color) ((originalSbi.wAttributes >> 4) & 0x0f);
+
+		oldCp = GetConsoleOutputCP();
+		SetConsoleOutputCP(65001); // UTF-8
+
+		oldCpIn = GetConsoleCP();
+		SetConsoleCP(65001); // UTF-8
 	}
 
 	version(Windows) {
 		private Color defaultBackgroundColor = Color.black;
 		private Color defaultForegroundColor = Color.white;
+		UINT oldCp;
+		UINT oldCpIn;
 	}
 
 	// only use this if you are sure you know what you want, since the terminal is a shared resource you generally really want to reset it to normal when you leave...
@@ -836,6 +870,10 @@ http://msdn.microsoft.com/en-us/library/windows/desktop/ms683193%28v=vs.85%29.as
 		if(lineGetter !is null)
 			lineGetter.dispose();
 
+
+		SetConsoleOutputCP(oldCp);
+		SetConsoleCP(oldCpIn);
+
 		auto stdo = GetStdHandle(STD_OUTPUT_HANDLE);
 		SetConsoleActiveScreenBuffer(stdo);
 		if(hConsole !is stdo)
@@ -1587,6 +1625,16 @@ struct RealTimeConsoleInput {
 
 	void delegate(InputEvent) userEventHandler;
 
+	/++
+		If you are using [arsd.simpledisplay] and want terminal interop too, you can call
+		this function to add it to the sdpy event loop and get the callback called on new
+		input.
+
+		Note that you will probably need to call `terminal.flush()` when you are doing doing
+		output, as the sdpy event loop doesn't know to do that (yet). I will probably change
+		that in a future version, but it doesn't hurt to call it twice anyway, so I recommend
+		calling flush yourself in any code you write using this.
+	+/
 	void integrateWithSimpleDisplayEventLoop()(void delegate(InputEvent) userEventHandler) {
 		this.userEventHandler = userEventHandler;
 		import arsd.simpledisplay;
@@ -3577,7 +3625,7 @@ struct ScrollbackBuffer {
 	}
 
 	void clear() {
-		lines = null;
+		lines.clear();
 		clickRegions = null;
 		scrollbackPosition = 0;
 	}
@@ -3679,9 +3727,87 @@ struct ScrollbackBuffer {
 		}
 	}
 
-	// FIXME: limit scrollback lines.length
+	static struct CircularBuffer(T) {
+		T[] backing;
 
-	Line[] lines;
+		enum maxScrollback = 8192; // as a power of 2, i hope the compiler optimizes the % below to a simple bit mask...
+
+		int start;
+		int length_;
+
+		void clear() {
+			backing = null;
+			start = 0;
+			length_ = 0;
+		}
+
+		size_t length() {
+			return length_;
+		}
+
+		void opOpAssign(string op : "~")(T line) {
+			if(length_ < maxScrollback) {
+				backing.assumeSafeAppend();
+				backing ~= line;
+				length_++;
+			} else {
+				backing[start] = line;
+				start++;
+				if(start == maxScrollback)
+					start = 0;
+			}
+		}
+
+		T opIndex(int idx) {
+			return backing[(start + idx) % maxScrollback];
+		}
+		T opIndex(Dollar idx) {
+			return backing[(start + (length + idx.offsetFromEnd)) % maxScrollback];
+		}
+
+		CircularBufferRange opSlice(int startOfIteration, Dollar end) {
+			return CircularBufferRange(&this, startOfIteration, cast(int) length - startOfIteration + end.offsetFromEnd);
+		}
+		CircularBufferRange opSlice(int startOfIteration, int end) {
+			return CircularBufferRange(&this, startOfIteration, end - startOfIteration);
+		}
+		CircularBufferRange opSlice() {
+			return CircularBufferRange(&this, 0, cast(int) length);
+		}
+
+		static struct CircularBufferRange {
+			CircularBuffer* item;
+			int position;
+			int remaining;
+			this(CircularBuffer* item, int startOfIteration, int count) {
+				this.item = item;
+				position = startOfIteration;
+				remaining = count;
+			}
+
+			T front() { return (*item)[position]; }
+			bool empty() { return remaining <= 0; }
+			void popFront() {
+				position++;
+				remaining--;
+			}
+
+			T back() { return (*item)[remaining - 1 - position]; }
+			void popBack() {
+				remaining--;
+			}
+		}
+
+		static struct Dollar {
+			int offsetFromEnd;
+			Dollar opBinary(string op : "-")(int rhs) {
+				return Dollar(offsetFromEnd - rhs);
+			}
+		}
+		Dollar opDollar() { return Dollar(0); }
+	}
+
+	CircularBuffer!Line lines;
 	string name;
 
 	int x, y, width, height;
@@ -3811,7 +3937,7 @@ struct ScrollbackBuffer {
 		// second pass: actually draw it
 		int linePos = remaining;
 
-		foreach(idx, line; lines[start .. start + howMany]) {
+		foreach(line; lines[start .. start + howMany]) {
 			int written = 0;
 
 			if(linePos < 0) {