// helper program is in ~me/encodings.d to make more tables from wikipedia
/**
This is meant to help get data from the wild into utf8 strings
so you can work with them easily inside D.
The main function is convertToUtf8(), which takes a byte array
of your raw data (a byte array because it isn't really a D string
yet until it is utf8), and a runtime string telling it's current
encoding.
The current encoding argument is meant to come from the data's
metadata, and is flexible on exact format - it is case insensitive
and takes several variations on the names.
This way, you should be able to send it the encoding string directly
from an XML document, a HTTP header, or whatever you have, and it
ought to just work.
Example:
---
auto data = cast(immutable(ubyte)[])
std.file.read("my-windows-file.txt");
string utf8String = convertToUtf8(data, "windows-1252");
// utf8String can now be used
---
The encodings currently implemented for decoding are:
$(LIST
* UTF-8 (a no-op; it simply casts the array to string)
* UTF-16,
* UTF-32,
* Windows-1252,
* ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16.
* KOI8-R
)
It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since
those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic).
This module currently makes no attempt to look at control characters.
*/
module arsd.characterencodings;
import std.string;
import std.array;
import std.conv;
/// Like convertToUtf8, but if the encoding is unknown, it just strips all chars > 127 and calls it done instead of throwing
string convertToUtf8Lossy(immutable(ubyte)[] data, string dataCharacterEncoding) {
try {
auto ret = convertToUtf8(data, dataCharacterEncoding);
import std.utf;
validate(ret);
return ret;
} catch(Exception e) {
string ret;
foreach(b; data)
if(b < 128)
ret ~= b;
return ret;
}
}
/// Takes data from a given character encoding and returns it as UTF-8
string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) {
// just to normalize the passed string...
auto encoding = dataCharacterEncoding.toLower();
encoding = encoding.replace(" ", "");
encoding = encoding.replace("-", "");
encoding = encoding.replace("_", "");
// should be good enough.
switch(encoding) {
default:
throw new Exception("I don't know how to convert " ~ dataCharacterEncoding ~ " to UTF-8");
// since the input is immutable, these are ok too.
// just want to cover all the bases with one runtime function.
case "utf16":
case "utf16le":
return to!string(cast(wstring) data);
case "utf32":
case "utf32le":
return to!string(cast(dstring) data);
// FIXME: does the big endian to little endian conversion work?
case "ascii":
case "usascii": // utf-8 is a superset of ascii
case "utf8":
return cast(string) data;
// and now the various 8 bit encodings we support.
case "windows1252":
return decodeImpl(data, ISO_8859_1, Windows_1252);
case "windows1251":
return decodeImpl(data, Windows_1251, Windows_1251_Lower);
case "koi8r":
return decodeImpl(data, KOI8_R, KOI8_R_Lower);
case "latin1":
case "iso88591":
// Why am I putting Windows_1252 here? A lot of
// stuff in the wild is mislabeled, so this will
// do some good in the Just Works department.
// Regardless, I don't handle the
// control char set in that zone anyway right now.
return decodeImpl(data, ISO_8859_1, Windows_1252);
case "iso88592":
return decodeImpl(data, ISO_8859_2);
case "iso88593":
return decodeImpl(data, ISO_8859_3);
case "iso88594":
return decodeImpl(data, ISO_8859_4);
case "iso88595":
return decodeImpl(data, ISO_8859_5);
case "iso88596":
return decodeImpl(data, ISO_8859_6);
case "iso88597":
return decodeImpl(data, ISO_8859_7);
case "iso88598":
return decodeImpl(data, ISO_8859_8);
case "iso88599":
return decodeImpl(data, ISO_8859_9);
case "iso885910":
return decodeImpl(data, ISO_8859_10);
case "iso885911":
return decodeImpl(data, ISO_8859_11);
case "iso885913":
return decodeImpl(data, ISO_8859_13);
case "iso885914":
return decodeImpl(data, ISO_8859_14);
case "iso885915":
return decodeImpl(data, ISO_8859_15);
case "iso885916":
return decodeImpl(data, ISO_8859_16);
}
assert(0);
}
/// Tries to determine the current encoding based on the content.
/// Only really helps with the UTF variants.
/// Returns null if it can't be reasonably sure.
string tryToDetermineEncoding(in ubyte[] rawdata) {
import std.utf;
try {
validate!string(cast(string) rawdata);
// the odds of non stuff validating as utf-8 are pretty low
return "UTF-8";
} catch(UTFException t) {
// it's definitely not UTF-8!
// we'll look at the first few characters. If there's a
// BOM, it's probably UTF-16 or UTF-32
if(rawdata.length > 4) {
// not checking for utf8 bom; if it was that, we
// wouldn't be here.
if(rawdata[0] == 0xff && rawdata[1] == 0xfe)
return "UTF-16 LE";
else if(rawdata[0] == 0xfe && rawdata[1] == 0xff)
return "UTF-16 BE";
else if(rawdata[0] == 0x00 && rawdata[1] == 0x00
&& rawdata[2] == 0xfe && rawdata[3] == 0xff)
return "UTF-32 BE";
else if(rawdata[0] == 0xff && rawdata[1] == 0xfe
&& rawdata[2] == 0x00 && rawdata[3] == 0x00)
return "UTF-32 LE";
else {
// this space is intentionally left blank
}
}
}
// we don't know with enough confidence. The app will have to find another way.
return null;
}
// this function actually does the work, using the translation tables
// below.
string decodeImpl(in ubyte[] data, in dchar[] chars160to255, in dchar[] chars128to159 = null, in dchar[] chars0to127 = null)
in {
assert(chars160to255.length == 256 - 160);
assert(chars128to159 is null || chars128to159.length == 160 - 128);
assert(chars0to127 is null || chars0to127.length == 128 - 0);
}
out(ret) {
import std.utf;
validate(ret);
}
body {
string utf8;
/// I'm sure this could be a lot more efficient, but whatever, it
/// works.
foreach(octet; data) {
if(octet < 128) {
if(chars0to127 !is null)
utf8 ~= chars0to127[octet];
else
utf8 ~= cast(char) octet; // ascii is the same
} else if(octet < 160) {
if(chars128to159 !is null)
utf8 ~= chars128to159[octet - 128];
else
utf8 ~= " ";
} else {
utf8 ~= chars160to255[octet - 160];
}
}
return utf8;
}
// Here come the translation tables.
// this table gives characters for decimal 128 through 159.
// the < 128 characters are the same as ascii, and > 159 the same as
// iso 8859 1, seen below.
immutable dchar[] Windows_1252 = [
'€', ' ', '‚', 'ƒ', '„', '…', '†', '‡',
'ˆ', '‰', 'Š', '‹', 'Œ', ' ', 'Ž', ' ',
' ', '‘', '’', '“', '”', '•', '–', '—',
'˜', '™', 'š', '›', 'œ', ' ', 'ž', 'Ÿ'];
// the following tables give the characters from decimal 160 up to 255
// in the given encodings.
immutable dchar[] ISO_8859_1 = [
' ', '¡', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', 'ª', '«', '¬', '', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·',
'¸', '¹', 'º', '»', '¼', '½', '¾', '¿',
'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
'Ð', 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×',
'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß',
'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
'ð', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ'];
immutable dchar[] ISO_8859_2 = [
' ', 'Ą', '˘', 'Ł', '¤', 'Ľ', 'Ś', '§',
'¨', 'Š', 'Ş', 'Ť', 'Ź', '', 'Ž', 'Ż',
'°', 'ą', '˛', 'ł', '´', 'ľ', 'ś', 'ˇ',
'¸', 'š', 'ş', 'ť', 'ź', '˝', 'ž', 'ż',
'Ŕ', 'Á', 'Â', 'Ă', 'Ä', 'Ĺ', 'Ć', 'Ç',
'Č', 'É', 'Ę', 'Ë', 'Ě', 'Í', 'Î', 'Ď',
'Đ', 'Ń', 'Ň', 'Ó', 'Ô', 'Ő', 'Ö', '×',
'Ř', 'Ů', 'Ú', 'Ű', 'Ü', 'Ý', 'Ţ', 'ß',
'ŕ', 'á', 'â', 'ă', 'ä', 'ĺ', 'ć', 'ç',
'č', 'é', 'ę', 'ë', 'ě', 'í', 'î', 'ď',
'đ', 'ń', 'ň', 'ó', 'ô', 'ő', 'ö', '÷',
'ř', 'ů', 'ú', 'ű', 'ü', 'ý', 'ţ', '˙'];
immutable dchar[] ISO_8859_3 = [
' ', 'Ħ', '˘', '£', '¤', ' ', 'Ĥ', '§',
'¨', 'İ', 'Ş', 'Ğ', 'Ĵ', '', ' ', 'Ż',
'°', 'ħ', '²', '³', '´', 'µ', 'ĥ', '·',
'¸', 'ı', 'ş', 'ğ', 'ĵ', '½', ' ', 'ż',
'À', 'Á', 'Â', ' ', 'Ä', 'Ċ', 'Ĉ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
' ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ġ', 'Ö', '×',
'Ĝ', 'Ù', 'Ú', 'Û', 'Ü', 'Ŭ', 'Ŝ', 'ß',
'à', 'á', 'â', ' ', 'ä', 'ċ', 'ĉ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
' ', 'ñ', 'ò', 'ó', 'ô', 'ġ', 'ö', '÷',
'ĝ', 'ù', 'ú', 'û', 'ü', 'ŭ', 'ŝ', '˙'];
immutable dchar[] ISO_8859_4 = [
' ', 'Ą', 'ĸ', 'Ŗ', '¤', 'Ĩ', 'Ļ', '§',
'¨', 'Š', 'Ē', 'Ģ', 'Ŧ', '', 'Ž', '¯',
'°', 'ą', '˛', 'ŗ', '´', 'ĩ', 'ļ', 'ˇ',
'¸', 'š', 'ē', 'ģ', 'ŧ', 'Ŋ', 'ž', 'ŋ',
'Ā', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Į',
'Č', 'É', 'Ę', 'Ë', 'Ė', 'Í', 'Î', 'Ī',
'Đ', 'Ņ', 'Ō', 'Ķ', 'Ô', 'Õ', 'Ö', '×',
'Ø', 'Ų', 'Ú', 'Û', 'Ü', 'Ũ', 'Ū', 'ß',
'ā', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'į',
'č', 'é', 'ę', 'ë', 'ė', 'í', 'î', 'ī',
'đ', 'ņ', 'ō', 'ķ', 'ô', 'õ', 'ö', '÷',
'ø', 'ų', 'ú', 'û', 'ü', 'ũ', 'ū', '˙'];
immutable dchar[] ISO_8859_5 = [
' ', 'Ё', 'Ђ', 'Ѓ', 'Є', 'Ѕ', 'І', 'Ї',
'Ј', 'Љ', 'Њ', 'Ћ', 'Ќ', '', 'Ў', 'Џ',
'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З',
'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П',
'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч',
'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я',
'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з',
'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п',
'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч',
'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я',
'№', 'ё', 'ђ', 'ѓ', 'є', 'ѕ', 'і', 'ї',
'ј', 'љ', 'њ', 'ћ', 'ќ', '§', 'ў', 'џ'];
immutable dchar[] ISO_8859_6 = [
' ', ' ', ' ', ' ', '¤', ' ', ' ', ' ',
' ', ' ', ' ', ' ', '،', '', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', '؛', ' ', ' ', ' ', '؟',
' ', 'ء', 'آ', 'أ', 'ؤ', 'إ', 'ئ', 'ا',
'ب', 'ة', 'ت', 'ث', 'ج', 'ح', 'خ', 'د',
'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط',
'ظ', 'ع', 'غ', ' ', ' ', ' ', ' ', ' ',
'ـ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه',
'و', 'ى', 'ي', 'ً', 'ٌ', 'ٍ', 'َ', 'ُ',
'ِ', 'ّ', 'ْ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '];
immutable dchar[] ISO_8859_7 = [
' ', '‘', '’', '£', '€', '₯', '¦', '§',
'¨', '©', 'ͺ', '«', '¬', '', ' ', '―',
'°', '±', '²', '³', '΄', '΅', 'Ά', '·',
'Έ', 'Ή', 'Ί', '»', 'Ό', '½', 'Ύ', 'Ώ',
'ΐ', 'Α', 'Β', 'Γ', 'Δ', 'Ε', 'Ζ', 'Η',
'Θ', 'Ι', 'Κ', 'Λ', 'Μ', 'Ν', 'Ξ', 'Ο',
'Π', 'Ρ', ' ', 'Σ', 'Τ', 'Υ', 'Φ', 'Χ',
'Ψ', 'Ω', 'Ϊ', 'Ϋ', 'ά', 'έ', 'ή', 'ί',
'ΰ', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο',
'π', 'ρ', 'ς', 'σ', 'τ', 'υ', 'φ', 'χ',
'ψ', 'ω', 'ϊ', 'ϋ', 'ό', 'ύ', 'ώ', ' '];
immutable dchar[] ISO_8859_8 = [
' ', ' ', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', '×', '«', '¬', '', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·',
'¸', '¹', '÷', '»', '¼', '½', '¾', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', '‗',
'א', 'ב', 'ג', 'ד', 'ה', 'ו', 'ז', 'ח',
'ט', 'י', 'ך', 'כ', 'ל', 'ם', 'מ', 'ן',
'נ', 'ס', 'ע', 'ף', 'פ', 'ץ', 'צ', 'ק',
// v v those are wrong
'ר', 'ש', 'ת', ' ', ' ', ' ', ' ', ' ']; // FIXME: those ones marked wrong are supposed to be left to right and right to left markers, not spaces. lol maybe it isn't wrong
immutable dchar[] ISO_8859_9 = [
' ', '¡', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', 'ª', '«', '¬', '', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·',
'¸', '¹', 'º', '»', '¼', '½', '¾', '¿',
'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
'Ğ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×',
'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'İ', 'Ş', 'ß',
'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
'ğ', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ı', 'ş', 'ÿ'];
immutable dchar[] ISO_8859_10 = [
' ', 'Ą', 'Ē', 'Ģ', 'Ī', 'Ĩ', 'Ķ', '§',
'Ļ', 'Đ', 'Š', 'Ŧ', 'Ž', '', 'Ū', 'Ŋ',
'°', 'ą', 'ē', 'ģ', 'ī', 'ĩ', 'ķ', '·',
'ļ', 'đ', 'š', 'ŧ', 'ž', '―', 'ū', 'ŋ',
'Ā', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Į',
'Č', 'É', 'Ę', 'Ë', 'Ė', 'Í', 'Î', 'Ï',
'Ð', 'Ņ', 'Ō', 'Ó', 'Ô', 'Õ', 'Ö', 'Ũ',
'Ø', 'Ų', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß',
'ā', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'į',
'č', 'é', 'ę', 'ë', 'ė', 'í', 'î', 'ï',
'ð', 'ņ', 'ō', 'ó', 'ô', 'õ', 'ö', 'ũ',
'ø', 'ų', 'ú', 'û', 'ü', 'ý', 'þ', 'ĸ'];
immutable dchar[] ISO_8859_11 = [
' ', 'ก', 'ข', 'ฃ', 'ค', 'ฅ', 'ฆ', 'ง',
'จ', 'ฉ', 'ช', 'ซ', 'ฌ', 'ญ', 'ฎ', 'ฏ',
'ฐ', 'ฑ', 'ฒ', 'ณ', 'ด', 'ต', 'ถ', 'ท',
'ธ', 'น', 'บ', 'ป', 'ผ', 'ฝ', 'พ', 'ฟ',
'ภ', 'ม', 'ย', 'ร', 'ฤ', 'ล', 'ฦ', 'ว',
'ศ', 'ษ', 'ส', 'ห', 'ฬ', 'อ', 'ฮ', 'ฯ',
'ะ', 'ั', 'า', 'ำ', 'ิ', 'ี', 'ึ', 'ื',
'ุ', 'ู', 'ฺ', ' ', ' ', ' ', ' ', '฿',
'เ', 'แ', 'โ', 'ใ', 'ไ', 'ๅ', 'ๆ', '็',
'่', '้', '๊', '๋', '์', 'ํ', '๎', '๏',
'๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗',
'๘', '๙', '๚', '๛', ' ', ' ', ' ', ' '];
immutable dchar[] ISO_8859_13 = [
' ', '”', '¢', '£', '¤', '„', '¦', '§',
'Ø', '©', 'Ŗ', '«', '¬', '', '®', 'Æ',
'°', '±', '²', '³', '“', 'µ', '¶', '·',
'ø', '¹', 'ŗ', '»', '¼', '½', '¾', 'æ',
'Ą', 'Į', 'Ā', 'Ć', 'Ä', 'Å', 'Ę', 'Ē',
'Č', 'É', 'Ź', 'Ė', 'Ģ', 'Ķ', 'Ī', 'Ļ',
'Š', 'Ń', 'Ņ', 'Ó', 'Ō', 'Ő', 'Ö', '×',
'Ų', 'Ł', 'Ś', 'Ū', 'Ü', 'Ż', 'Ž', 'ß',
'ą', 'į', 'ā', 'ć', 'ä', 'å', 'ę', 'ē',
'č', 'é', 'ź', 'ė', 'ģ', 'ķ', 'ī', 'ļ',
'š', 'ń', 'ņ', 'ó', 'ō', 'ő', 'ö', '÷',
'ų', 'ł', 'ś', 'ū', 'ü', 'ż', 'ž', '’'];
immutable dchar[] ISO_8859_14 = [
' ', 'Ḃ', 'ḃ', '£', 'Ċ', 'ċ', 'Ḋ', '§',
'Ẁ', '©', 'Ẃ', 'ḋ', 'Ỳ', '', '®', 'Ÿ',
'Ḟ', 'ḟ', 'Ġ', 'ġ', 'Ṁ', 'ṁ', '¶', 'Ṗ',
'ẁ', 'ṗ', 'ẃ', 'Ṡ', 'ỳ', 'Ẅ', 'ẅ', 'ṡ',
'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
'Ŵ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', 'Ṫ',
'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Ŷ', 'ß',
'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
'ŵ', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', 'ṫ',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'ŷ', 'ÿ'];
immutable dchar[] ISO_8859_15 = [
' ', '¡', '¢', '£', '€', '¥', 'Š', '§',
'š', '©', 'ª', '«', '¬', '', '®', '¯',
'°', '±', '²', '³', 'Ž', 'µ', '¶', '·',
'ž', '¹', 'º', '»', 'Œ', 'œ', 'Ÿ', '¿',
'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
'Ð', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', '×',
'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß',
'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
'ð', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ'];
immutable dchar[] ISO_8859_16 = [
' ', 'Ą', 'ą', 'Ł', '€', '„', 'Š', '§',
'š', '©', 'Ș', '«', 'Ź', '', 'ź', 'Ż',
'°', '±', 'Č', 'ł', 'Ž', '”', '¶', '·',
'ž', 'č', 'ș', '»', 'Œ', 'œ', 'Ÿ', 'ż',
'À', 'Á', 'Â', 'Ă', 'Ä', 'Ć', 'Æ', 'Ç',
'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï',
'Ð', 'Ń', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', 'Ś',
'Ű', 'Ù', 'Ú', 'Û', 'Ü', 'Ę', 'Ț', 'ß',
'à', 'á', 'â', 'ă', 'ä', 'ć', 'æ', 'ç',
'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï',
'đ', 'ń', 'ò', 'ó', 'ô', 'ő', 'ö', 'ś',
'ű', 'ù', 'ú', 'û', 'ü', 'ę', 'ț', 'ÿ'];
immutable dchar[] KOI8_R_Lower = [
'─', '│', '┌', '┐', '└', '┘', '├', '┤',
'┬', '┴', '┼', '▀', '▄', '█', '▌', '▐',
'░', '▒', '▓', '⌠', '■', '∙', '√', '≈',
'≤', '≥', '\u00a0', '⌡', '°', '²', '·', '÷'];
immutable dchar[] KOI8_R = [
'═', '║', '╒', 'ё', '╓', '╔', '╕', '╖',
'╗', '╘', '╙', '╚', '╛', '╜', '╝', '╞',
'╟', '╠', '╡', 'ё', '╢', '╣', '╤', '╥',
'╦', '╧', '╨', '╩', '╪', '╫', '╬', '©',
'ю', 'а', 'б', 'ц', 'д', 'е', 'ф', 'г',
'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о',
'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в',
'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ',
'ю', 'а', 'б', 'ц', 'д', 'е', 'ф', 'г',
'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о',
'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в',
'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ'];
immutable dchar[] Windows_1251_Lower = [
'Ђ', 'Ѓ', '‚', 'ѓ', '„', '…', '†', '‡',
'€', '‰', 'Љ', '‹', 'Њ', 'Ќ', 'Ћ', 'Џ',
'ђ', '‘', '’', '“', '”', '•', '–', '—',
' ', '™', 'љ', '›', 'њ', 'ќ', 'ћ', 'џ'];
immutable dchar[] Windows_1251 = [
' ', 'Ў', 'ў', 'Ј', '¤', 'Ґ', '¦', '§',
'Ё', '©', 'Є', '«', '¬', '', '®', 'Ї',
'°', '±', 'І', 'і', 'ґ', 'µ', '¶', '·',
'ё', '№', 'є', '»', 'ј', 'Ѕ', 'ѕ', 'ї',
'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З',
'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П',
'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч',
'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я',
'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з',
'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п',
'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч',
'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я'];