Fix UTF-8 filename

rpcs3/Crypto/unpkg.cpp

@@ -1014,7 +1014,8 @@ bool package_reader::fill_data(std::map<std::string, install_entry*>& all_instal
 		}
 		default:
 		{
-			// TODO: check for valid utf8 characters
+			// The name reached "path" through "vfs::escape", which turns whatever the host file system cannot take
+			// into characters it can, a byte that is not valid UTF-8 included (macOS refuses a name carrying one)
 			auto map_ptr = &*all_install_entries.try_emplace(path).first;
 
 			m_install_entries.push_back({

rpcs3/Emu/VFS.cpp

@@ -499,6 +499,46 @@ std::string vfs::retrieve(std::string_view path, const vfs_directory* node, std:
 	return result;
 }
 
+// Size of the well formed UTF-8 sequence starting at "pos", or 0 when the bytes there are not one: an invalid
+// leading byte, a truncated sequence, an overlong encoding or half a surrogate pair, which UTF-8 never carries
+static usz utf8_sequence_size(std::string_view name, usz pos)
+{
+	auto byte = [&](usz i) -> uchar { return i < name.size() ? static_cast<uchar>(name[i]) : 0; };
+
+	const uchar b0 = byte(pos);
+
+	if (b0 < 0x80)
+	{
+		return 1;
+	}
+
+	// The second byte is the one carrying the exceptions: it is what tells an overlong encoding (0xe0, 0xf0), a
+	// surrogate half (0xed) and a code point past U+10FFFF (0xf4) from a well formed sequence. A byte read past
+	// the end of the name comes back as 0, which no rule below accepts
+	const uchar b1 = byte(pos + 1);
+	usz size = 0;
+
+	if (b0 >= 0xc2 && b0 <= 0xdf && b1 >= 0x80 && b1 <= 0xbf) size = 2;
+	else if (b0 == 0xe0 && b1 >= 0xa0 && b1 <= 0xbf) size = 3;
+	else if (b0 == 0xed && b1 >= 0x80 && b1 <= 0x9f) size = 3;
+	else if (b0 >= 0xe1 && b0 <= 0xef && b1 >= 0x80 && b1 <= 0xbf) size = 3;
+	else if (b0 == 0xf0 && b1 >= 0x90 && b1 <= 0xbf) size = 4;
+	else if (b0 == 0xf4 && b1 >= 0x80 && b1 <= 0x8f) size = 4;
+	else if (b0 >= 0xf1 && b0 <= 0xf3 && b1 >= 0x80 && b1 <= 0xbf) size = 4;
+	else return 0;
+
+	// Whatever the leading byte allows, every byte after the second one is a plain continuation
+	for (usz i = 2; i < size; i++)
+	{
+		if (const uchar b = byte(pos + i); b < 0x80 || b > 0xbf)
+		{
+			return 0;
+		}
+	}
+
+	return size;
+}
+
 std::string vfs::escape(std::string_view name, bool escape_slash)
 {
 	std::string result;
@@ -562,9 +602,41 @@ std::string vfs::escape(std::string_view name, bool escape_slash)
 
 	result.reserve(result.size() + name.size());
 
+	// Bytes left of the UTF-8 sequence being copied through: a name is only valid UTF-8 as a whole, so whether
+	// a byte belongs to a sequence or is trash of its own cannot be told by looking at that byte alone
+	usz utf8_left = 0;
+
 	for (usz i = 0, s = name.size(); i < s; i++)
 	{
-		switch (char2 c = name[i])
+		const uchar b = name[i];
+
+		// A name is ASCII nearly every time, and every byte of a sequence is outside it, so this one test is all
+		// such a name pays: "utf8_left" can only be standing on a byte of 0x80 and above
+		if (b >= 0x80)
+		{
+			if (utf8_left)
+			{
+				utf8_left--;
+			}
+			else if (const usz size = utf8_sequence_size(name, i))
+			{
+				utf8_left = size - 1;
+			}
+			else
+			{
+				// A byte no sequence can account for, written as "％" and its two hex digits: a host refusing a name
+				// that is not valid UTF-8 (macOS does) would turn the file away outright, and one writing it as it
+				// comes leaves a name that cannot be told from the escape characters this very function produces
+				constexpr char hex[] = "0123456789ABCDEF";
+
+				result += reinterpret_cast<const char*>(u8"％");
+				result += hex[b >> 4];
+				result += hex[b & 0xf];
+				continue;
+			}
+		}
+
+		switch (char2 c = static_cast<char2>(b))
 		{
 		case 0:
 		case 1:
@@ -769,7 +841,7 @@ std::string vfs::unescape(std::string_view name)
 					{
 						result += static_cast<char>(c3);
 						result.back() -= u8"０"[2];
-						continue;
+						break;
 					}
 					case char2{u8"Ａ"[2]}:
 					case char2{u8"Ｂ"[2]}:
@@ -797,7 +869,7 @@ std::string vfs::unescape(std::string_view name)
 						result += static_cast<char>(c3);
 						result.back() -= u8"Ａ"[2];
 						result.back() += 10;
-						continue;
+						break;
 					}
 					case char2{u8"！"[2]}:
 					{
@@ -814,6 +886,30 @@ std::string vfs::unescape(std::string_view name)
 						i += 3;
 						continue;
 					}
+					case char2{u8"％"[2]}:
+					{
+						// A byte no UTF-8 sequence could account for, written as its two hex digits (see "vfs::escape")
+						auto digit = [](char2 h) -> int
+						{
+							if (h >= '0' && h <= '9') return h - '0';
+							if (h >= 'A' && h <= 'F') return h - 'A' + 10;
+							return -1;
+						};
+
+						const int hi = digit(get_char(i + 3));
+						const int lo = digit(get_char(i + 4));
+
+						if (hi >= 0 && lo >= 0)
+						{
+							result += static_cast<char>(hi * 16 + lo);
+
+							i += 4;
+							continue;
+						}
+
+						// Not what this function wrote: dropped like any other escape character it does not know
+						break;
+					}
 					case char2{u8"＿"[2]}:
 					{
 						result += ' ';