forks
/
arsd
mirror of https://github.com/adamdruppe/arsd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use replacement char instead of nothing in lossy read

This commit is contained in:
Adam D. Ruppe 2022-10-01 20:38:17 -04:00
parent 949e91bdbe
commit 13c7ef0eb7
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
characterencodings.d
View File

@ -47,6 +47,8 @@ import std.string;
import std.array; import std.array;
import std.conv; import std.conv;
// FIXME: use replacement char here instead
/// Like convertToUtf8, but if the encoding is unknown, it just strips all chars > 127 and calls it done instead of throwing /// Like convertToUtf8, but if the encoding is unknown, it just strips all chars > 127 and calls it done instead of throwing
string convertToUtf8Lossy(immutable(ubyte)[] data, string dataCharacterEncoding) { string convertToUtf8Lossy(immutable(ubyte)[] data, string dataCharacterEncoding) {
try { try {
@ -59,6 +61,8 @@ string convertToUtf8Lossy(immutable(ubyte)[] data, string dataCharacterEncoding)
foreach(b; data) foreach(b; data)
if(b < 128) if(b < 128)
ret ~= b; ret ~= b;
else
ret ~= '\uFFFD';
return ret; return ret;
} }
} }