forks
/
arsd
mirror of https://github.com/adamdruppe/arsd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

guessing function for utf

This commit is contained in:
Adam D. Ruppe 2011-11-20 11:50:47 -05:00
parent a7f86d26e0
commit f7d4f5e481
1 changed files with 83 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
characterencodings.d
View File

@ -57,14 +57,17 @@ string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) {
// since the input is immutable, these are ok too. // since the input is immutable, these are ok too.
// just want to cover all the bases with one runtime function. // just want to cover all the bases with one runtime function.
case "utf16": case "utf16":
case "utf16le":
return to!string(cast(wstring) data); return to!string(cast(wstring) data);
case "utf32": case "utf32":
case "utf32le":
return to!string(cast(dstring) data); return to!string(cast(dstring) data);
// FIXME: does the big endian to little endian conversion work?
case "utf8": case "utf8":
return cast(string) data; return cast(string) data;
// and now the various 8 bit encodings we support. // and now the various 8 bit encodings we support.
case "windows1252": case "windows1252":
return decodeImpl(data, ISO_8869_1, Windows_1252); return decodeImpl(data, ISO_8859_1, Windows_1252);
case "latin1": case "latin1":
case "iso88591": case "iso88591":
// Why am I putting Windows_1252 here? A lot of // Why am I putting Windows_1252 here? A lot of
@ -72,43 +75,85 @@ string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) {
// do some good in the Just Works department. // do some good in the Just Works department.
// Regardless, I don't handle the // Regardless, I don't handle the
// control char set in that zone anyway right now. // control char set in that zone anyway right now.
return decodeImpl(data, ISO_8869_1, Windows_1252); return decodeImpl(data, ISO_8859_1, Windows_1252);
case "iso88592": case "iso88592":
return decodeImpl(data, ISO_8869_2); return decodeImpl(data, ISO_8859_2);
case "iso88593": case "iso88593":
return decodeImpl(data, ISO_8869_3); return decodeImpl(data, ISO_8859_3);
case "iso88594": case "iso88594":
return decodeImpl(data, ISO_8869_4); return decodeImpl(data, ISO_8859_4);
case "iso88595": case "iso88595":
return decodeImpl(data, ISO_8869_5); return decodeImpl(data, ISO_8859_5);
case "iso88596": case "iso88596":
return decodeImpl(data, ISO_8869_6); return decodeImpl(data, ISO_8859_6);
case "iso88597": case "iso88597":
return decodeImpl(data, ISO_8869_7); return decodeImpl(data, ISO_8859_7);
case "iso88598": case "iso88598":
return decodeImpl(data, ISO_8869_8); return decodeImpl(data, ISO_8859_8);
case "iso88599": case "iso88599":
return decodeImpl(data, ISO_8869_9); return decodeImpl(data, ISO_8859_9);
case "iso885910": case "iso885910":
return decodeImpl(data, ISO_8869_10); return decodeImpl(data, ISO_8859_10);
case "iso885911": case "iso885911":
return decodeImpl(data, ISO_8869_11); return decodeImpl(data, ISO_8859_11);
case "iso885913": case "iso885913":
return decodeImpl(data, ISO_8869_13); return decodeImpl(data, ISO_8859_13);
case "iso885914": case "iso885914":
return decodeImpl(data, ISO_8869_14); return decodeImpl(data, ISO_8859_14);
case "iso885915": case "iso885915":
return decodeImpl(data, ISO_8869_15); return decodeImpl(data, ISO_8859_15);
case "iso885916": case "iso885916":
return decodeImpl(data, ISO_8869_16); return decodeImpl(data, ISO_8859_16);
} }
assert(0); assert(0);
} }
/// Tries to determine the current encoding based on the content.
/// Only really helps with the UTF variants.
/// Returns null if it can't be reasonably sure.
string tryToDetermineEncoding(in ubyte[] rawdata) {
import std.utf;
try {
validate!string(cast(string) rawdata);
// the odds of non stuff validating as utf-8 are pretty low
return "UTF-8";
} catch(UtfException t) {
// it's definitely not UTF-8!
// we'll look at the first few characters. If there's a
// BOM, it's probably UTF-16 or UTF-32
if(rawdata.length > 4) {
// not checking for utf8 bom; if it was that, we
// wouldn't be here.
if(rawdata[0] == 0xff && rawdata[1] == 0xfe)
return "UTF-16 LE";
else if(rawdata[0] == 0xfe && rawdata[1] == 0xff)
return "UTF-16 BE";
else if(rawdata[0] == 0x00 && rawdata[1] == 0x00
&& rawdata[2] == 0xfe && rawdata[3] == 0xff)
return "UTF-32 BE";
else if(rawdata[0] == 0xff && rawdata[1] == 0xfe
&& rawdata[2] == 0x00 && rawdata[3] == 0x00)
return "UTF-32 LE";
else {
// this space is intentionally left blank
}
}
}
// we don't know with enough confidence. The app will have to find another way.
return null;
}
// this function actually does the work, using the translation tables // this function actually does the work, using the translation tables
// below. // below.
string decodeImpl(in ubyte[] data, in dchar[] chars160to255, in dchar[] chars128to159 = null) string decodeImpl(in ubyte[] data, in dchar[] chars160to255, in dchar[] chars128to159 = null, in dchar[] chars0to127 = null)
in {
assert(chars160to255.length == 256 - 160);
assert(chars128to159 is null || chars128to159.length == 160 - 128);
assert(chars0to127 is null || chars0to127.length == 128 - 0);
}
out(ret) { out(ret) {
import std.utf; import std.utf;
validate(ret); validate(ret);
@ -119,9 +164,12 @@ body {
/// I'm sure this could be a lot more efficient, but whatever, it /// I'm sure this could be a lot more efficient, but whatever, it
/// works. /// works.
foreach(octet; data) { foreach(octet; data) {
if(octet < 128) if(octet < 128) {
utf8 ~= cast(char) octet; if(chars0to127 !is null)
else if(octet < 160) { utf8 ~= chars0to127[octet];
else
utf8 ~= cast(char) octet; // ascii is the same
} else if(octet < 160) {
if(chars128to159 !is null) if(chars128to159 !is null)
utf8 ~= chars128to159[octet - 128]; utf8 ~= chars128to159[octet - 128];
else else
@ -149,7 +197,7 @@ immutable dchar[] Windows_1252 = [
// the following tables give the characters from decimal 160 up to 255 // the following tables give the characters from decimal 160 up to 255
// in the given encodings. // in the given encodings.
immutable dchar[] ISO_8869_1 = [ immutable dchar[] ISO_8859_1 = [
' ', '¡', '¢', '£', '¤', '¥', '¦', '§', ' ', '¡', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', 'ª', '«', '¬', '­', '®', '¯', '¨', '©', 'ª', '«', '¬', '­', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·', '°', '±', '²', '³', '´', 'µ', '¶', '·',
@ -163,7 +211,7 @@ immutable dchar[] ISO_8869_1 = [
'ð', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 'ð', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ']; 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ'];
immutable dchar[] ISO_8869_2 = [ immutable dchar[] ISO_8859_2 = [
' ', 'Ą', '˘', 'Ł', '¤', 'Ľ', 'Ś', '§', ' ', 'Ą', '˘', 'Ł', '¤', 'Ľ', 'Ś', '§',
'¨', 'Š', 'Ş', 'Ť', 'Ź', '­', 'Ž', 'Ż', '¨', 'Š', 'Ş', 'Ť', 'Ź', '­', 'Ž', 'Ż',
'°', 'ą', '˛', 'ł', '´', 'ľ', 'ś', 'ˇ', '°', 'ą', '˛', 'ł', '´', 'ľ', 'ś', 'ˇ',
@ -177,7 +225,7 @@ immutable dchar[] ISO_8869_2 = [
'đ', 'ń', 'ň', 'ó', 'ô', 'ő', 'ö', '÷', 'đ', 'ń', 'ň', 'ó', 'ô', 'ő', 'ö', '÷',
'ř', 'ů', 'ú', 'ű', 'ü', 'ý', 'ţ', '˙']; 'ř', 'ů', 'ú', 'ű', 'ü', 'ý', 'ţ', '˙'];
immutable dchar[] ISO_8869_3 = [ immutable dchar[] ISO_8859_3 = [
' ', 'Ħ', '˘', '£', '¤', ' ', 'Ĥ', '§', ' ', 'Ħ', '˘', '£', '¤', ' ', 'Ĥ', '§',
'¨', 'İ', 'Ş', 'Ğ', 'Ĵ', '­', ' ', 'Ż', '¨', 'İ', 'Ş', 'Ğ', 'Ĵ', '­', ' ', 'Ż',
'°', 'ħ', '²', '³', '´', 'µ', 'ĥ', '·', '°', 'ħ', '²', '³', '´', 'µ', 'ĥ', '·',
@ -191,7 +239,7 @@ immutable dchar[] ISO_8869_3 = [
' ', 'ñ', 'ò', 'ó', 'ô', 'ġ', 'ö', '÷', ' ', 'ñ', 'ò', 'ó', 'ô', 'ġ', 'ö', '÷',
'ĝ', 'ù', 'ú', 'û', 'ü', 'ŭ', 'ŝ', '˙']; 'ĝ', 'ù', 'ú', 'û', 'ü', 'ŭ', 'ŝ', '˙'];
immutable dchar[] ISO_8869_4 = [ immutable dchar[] ISO_8859_4 = [
' ', 'Ą', 'ĸ', 'Ŗ', '¤', 'Ĩ', 'Ļ', '§', ' ', 'Ą', 'ĸ', 'Ŗ', '¤', 'Ĩ', 'Ļ', '§',
'¨', 'Š', 'Ē', 'Ģ', 'Ŧ', '­', 'Ž', '¯', '¨', 'Š', 'Ē', 'Ģ', 'Ŧ', '­', 'Ž', '¯',
'°', 'ą', '˛', 'ŗ', '´', 'ĩ', 'ļ', 'ˇ', '°', 'ą', '˛', 'ŗ', '´', 'ĩ', 'ļ', 'ˇ',
@ -205,7 +253,7 @@ immutable dchar[] ISO_8869_4 = [
'đ', 'ņ', 'ō', 'ķ', 'ô', 'õ', 'ö', '÷', 'đ', 'ņ', 'ō', 'ķ', 'ô', 'õ', 'ö', '÷',
'ø', 'ų', 'ú', 'û', 'ü', 'ũ', 'ū', '˙']; 'ø', 'ų', 'ú', 'û', 'ü', 'ũ', 'ū', '˙'];
immutable dchar[] ISO_8869_5 = [ immutable dchar[] ISO_8859_5 = [
' ', 'Ё', 'Ђ', 'Ѓ', 'Є', 'Ѕ', 'І', 'Ї', ' ', 'Ё', 'Ђ', 'Ѓ', 'Є', 'Ѕ', 'І', 'Ї',
'Ј', 'Љ', 'Њ', 'Ћ', 'Ќ', '­', 'Ў', 'Џ', 'Ј', 'Љ', 'Њ', 'Ћ', 'Ќ', '­', 'Ў', 'Џ',
'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З', 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З',
@ -219,7 +267,7 @@ immutable dchar[] ISO_8869_5 = [
'№', 'ё', 'ђ', 'ѓ', 'є', 'ѕ', 'і', 'ї', '№', 'ё', 'ђ', 'ѓ', 'є', 'ѕ', 'і', 'ї',
'ј', 'љ', 'њ', 'ћ', 'ќ', '§', 'ў', 'џ']; 'ј', 'љ', 'њ', 'ћ', 'ќ', '§', 'ў', 'џ'];
immutable dchar[] ISO_8869_6 = [ immutable dchar[] ISO_8859_6 = [
' ', ' ', ' ', ' ', '¤', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '¤', ' ', ' ', ' ',
' ', ' ', ' ', ' ', '،', '­', ' ', ' ', ' ', ' ', ' ', ' ', '،', '­', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ',
@ -233,7 +281,7 @@ immutable dchar[] ISO_8869_6 = [
'ِ', 'ّ', 'ْ', ' ', ' ', ' ', ' ', ' ', 'ِ', 'ّ', 'ْ', ' ', ' ', ' ', ' ', ' ',
' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ']; ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '];
immutable dchar[] ISO_8869_7 = [ immutable dchar[] ISO_8859_7 = [
' ', '‘', '’', '£', '€', '₯', '¦', '§', ' ', '‘', '’', '£', '€', '₯', '¦', '§',
'¨', '©', 'ͺ', '«', '¬', '­', ' ', '―', '¨', '©', 'ͺ', '«', '¬', '­', ' ', '―',
'°', '±', '²', '³', '΄', '΅', 'Ά', '·', '°', '±', '²', '³', '΄', '΅', 'Ά', '·',
@ -247,7 +295,7 @@ immutable dchar[] ISO_8869_7 = [
'π', 'ρ', 'ς', 'σ', 'τ', 'υ', 'φ', 'χ', 'π', 'ρ', 'ς', 'σ', 'τ', 'υ', 'φ', 'χ',
'ψ', 'ω', 'ϊ', 'ϋ', 'ό', 'ύ', 'ώ', ' ']; 'ψ', 'ω', 'ϊ', 'ϋ', 'ό', 'ύ', 'ώ', ' '];
immutable dchar[] ISO_8869_8 = [ immutable dchar[] ISO_8859_8 = [
' ', ' ', '¢', '£', '¤', '¥', '¦', '§', ' ', ' ', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', '×', '«', '¬', '­', '®', '¯', '¨', '©', '×', '«', '¬', '­', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·', '°', '±', '²', '³', '´', 'µ', '¶', '·',
@ -262,7 +310,7 @@ immutable dchar[] ISO_8869_8 = [
// v v those are wrong // v v those are wrong
'ר', 'ש', 'ת', ' ', ' ', ' ', ' ', ' ']; // FIXME: those ones marked wrong are supposed to be left to right and right to left markers, not spaces. 'ר', 'ש', 'ת', ' ', ' ', ' ', ' ', ' ']; // FIXME: those ones marked wrong are supposed to be left to right and right to left markers, not spaces.
immutable dchar[] ISO_8869_9 = [ immutable dchar[] ISO_8859_9 = [
' ', '¡', '¢', '£', '¤', '¥', '¦', '§', ' ', '¡', '¢', '£', '¤', '¥', '¦', '§',
'¨', '©', 'ª', '«', '¬', '­', '®', '¯', '¨', '©', 'ª', '«', '¬', '­', '®', '¯',
'°', '±', '²', '³', '´', 'µ', '¶', '·', '°', '±', '²', '³', '´', 'µ', '¶', '·',
@ -276,7 +324,7 @@ immutable dchar[] ISO_8869_9 = [
'ğ', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 'ğ', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ı', 'ş', 'ÿ']; 'ø', 'ù', 'ú', 'û', 'ü', 'ı', 'ş', 'ÿ'];
immutable dchar[] ISO_8869_10 = [ immutable dchar[] ISO_8859_10 = [
' ', 'Ą', 'Ē', 'Ģ', 'Ī', 'Ĩ', 'Ķ', '§', ' ', 'Ą', 'Ē', 'Ģ', 'Ī', 'Ĩ', 'Ķ', '§',
'Ļ', 'Đ', 'Š', 'Ŧ', 'Ž', '­', 'Ū', 'Ŋ', 'Ļ', 'Đ', 'Š', 'Ŧ', 'Ž', '­', 'Ū', 'Ŋ',
'°', 'ą', 'ē', 'ģ', 'ī', 'ĩ', 'ķ', '·', '°', 'ą', 'ē', 'ģ', 'ī', 'ĩ', 'ķ', '·',
@ -290,7 +338,7 @@ immutable dchar[] ISO_8869_10 = [
'ð', 'ņ', 'ō', 'ó', 'ô', 'õ', 'ö', 'ũ', 'ð', 'ņ', 'ō', 'ó', 'ô', 'õ', 'ö', 'ũ',
'ø', 'ų', 'ú', 'û', 'ü', 'ý', 'þ', 'ĸ']; 'ø', 'ų', 'ú', 'û', 'ü', 'ý', 'þ', 'ĸ'];
immutable dchar[] ISO_8869_11 = [ immutable dchar[] ISO_8859_11 = [
' ', 'ก', 'ข', 'ฃ', 'ค', 'ฅ', 'ฆ', 'ง', ' ', 'ก', 'ข', 'ฃ', 'ค', 'ฅ', 'ฆ', 'ง',
'จ', 'ฉ', 'ช', 'ซ', 'ฌ', 'ญ', 'ฎ', 'ฏ', 'จ', 'ฉ', 'ช', 'ซ', 'ฌ', 'ญ', 'ฎ', 'ฏ',
'ฐ', 'ฑ', 'ฒ', 'ณ', 'ด', 'ต', 'ถ', 'ท', 'ฐ', 'ฑ', 'ฒ', 'ณ', 'ด', 'ต', 'ถ', 'ท',
@ -304,7 +352,7 @@ immutable dchar[] ISO_8869_11 = [
'๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗',
'๘', '๙', '๚', '๛', ' ', ' ', ' ', ' ']; '๘', '๙', '๚', '๛', ' ', ' ', ' ', ' '];
immutable dchar[] ISO_8869_13 = [ immutable dchar[] ISO_8859_13 = [
' ', '”', '¢', '£', '¤', '„', '¦', '§', ' ', '”', '¢', '£', '¤', '„', '¦', '§',
'Ø', '©', 'Ŗ', '«', '¬', '­', '®', 'Æ', 'Ø', '©', 'Ŗ', '«', '¬', '­', '®', 'Æ',
'°', '±', '²', '³', '“', 'µ', '¶', '·', '°', '±', '²', '³', '“', 'µ', '¶', '·',
@ -318,7 +366,7 @@ immutable dchar[] ISO_8869_13 = [
'š', 'ń', 'ņ', 'ó', 'ō', 'ő', 'ö', '÷', 'š', 'ń', 'ņ', 'ó', 'ō', 'ő', 'ö', '÷',
'ų', 'ł', 'ś', 'ū', 'ü', 'ż', 'ž', '’']; 'ų', 'ł', 'ś', 'ū', 'ü', 'ż', 'ž', '’'];
immutable dchar[] ISO_8869_14 = [ immutable dchar[] ISO_8859_14 = [
' ', 'Ḃ', 'ḃ', '£', 'Ċ', 'ċ', 'Ḋ', '§', ' ', 'Ḃ', 'ḃ', '£', 'Ċ', 'ċ', 'Ḋ', '§',
'Ẁ', '©', 'Ẃ', 'ḋ', 'Ỳ', '­', '®', 'Ÿ', 'Ẁ', '©', 'Ẃ', 'ḋ', 'Ỳ', '­', '®', 'Ÿ',
'Ḟ', 'ḟ', 'Ġ', 'ġ', 'Ṁ', 'ṁ', '¶', 'Ṗ', 'Ḟ', 'ḟ', 'Ġ', 'ġ', 'Ṁ', 'ṁ', '¶', 'Ṗ',
@ -332,7 +380,7 @@ immutable dchar[] ISO_8869_14 = [
'ŵ', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', 'ṫ', 'ŵ', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', 'ṫ',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'ŷ', 'ÿ']; 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'ŷ', 'ÿ'];
immutable dchar[] ISO_8869_15 = [ immutable dchar[] ISO_8859_15 = [
' ', '¡', '¢', '£', '€', '¥', 'Š', '§', ' ', '¡', '¢', '£', '€', '¥', 'Š', '§',
'š', '©', 'ª', '«', '¬', '­', '®', '¯', 'š', '©', 'ª', '«', '¬', '­', '®', '¯',
'°', '±', '²', '³', 'Ž', 'µ', '¶', '·', '°', '±', '²', '³', 'Ž', 'µ', '¶', '·',
@ -346,7 +394,7 @@ immutable dchar[] ISO_8869_15 = [
'ð', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', '÷', 'ð', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', '÷',
'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ']; 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ'];
immutable dchar[] ISO_8869_16 = [ immutable dchar[] ISO_8859_16 = [
' ', 'Ą', 'ą', 'Ł', '€', '„', 'Š', '§', ' ', 'Ą', 'ą', 'Ł', '€', '„', 'Š', '§',
'š', '©', 'Ș', '«', 'Ź', '­', 'ź', 'Ż', 'š', '©', 'Ș', '«', 'Ź', '­', 'ź', 'Ż',
'°', '±', 'Č', 'ł', 'Ž', '”', '¶', '·', '°', '±', 'Č', 'ł', 'Ž', '”', '¶', '·',