diff --git a/rpcs3/Crypto/unpkg.cpp b/rpcs3/Crypto/unpkg.cpp index ae30973620..e56b28f0cc 100644 --- a/rpcs3/Crypto/unpkg.cpp +++ b/rpcs3/Crypto/unpkg.cpp @@ -1014,7 +1014,8 @@ bool package_reader::fill_data(std::map& all_instal } default: { - // TODO: check for valid utf8 characters + // The name reached "path" through "vfs::escape", which turns whatever the host file system cannot take + // into characters it can, a byte that is not valid UTF-8 included (macOS refuses a name carrying one) auto map_ptr = &*all_install_entries.try_emplace(path).first; m_install_entries.push_back({ diff --git a/rpcs3/Emu/VFS.cpp b/rpcs3/Emu/VFS.cpp index 46a417ee6e..94129713cb 100644 --- a/rpcs3/Emu/VFS.cpp +++ b/rpcs3/Emu/VFS.cpp @@ -499,6 +499,46 @@ std::string vfs::retrieve(std::string_view path, const vfs_directory* node, std: return result; } +// Size of the well formed UTF-8 sequence starting at "pos", or 0 when the bytes there are not one: an invalid +// leading byte, a truncated sequence, an overlong encoding or half a surrogate pair, which UTF-8 never carries +static usz utf8_sequence_size(std::string_view name, usz pos) +{ + auto byte = [&](usz i) -> uchar { return i < name.size() ? static_cast(name[i]) : 0; }; + + const uchar b0 = byte(pos); + + if (b0 < 0x80) + { + return 1; + } + + // The second byte is the one carrying the exceptions: it is what tells an overlong encoding (0xe0, 0xf0), a + // surrogate half (0xed) and a code point past U+10FFFF (0xf4) from a well formed sequence. A byte read past + // the end of the name comes back as 0, which no rule below accepts + const uchar b1 = byte(pos + 1); + usz size = 0; + + if (b0 >= 0xc2 && b0 <= 0xdf && b1 >= 0x80 && b1 <= 0xbf) size = 2; + else if (b0 == 0xe0 && b1 >= 0xa0 && b1 <= 0xbf) size = 3; + else if (b0 == 0xed && b1 >= 0x80 && b1 <= 0x9f) size = 3; + else if (b0 >= 0xe1 && b0 <= 0xef && b1 >= 0x80 && b1 <= 0xbf) size = 3; + else if (b0 == 0xf0 && b1 >= 0x90 && b1 <= 0xbf) size = 4; + else if (b0 == 0xf4 && b1 >= 0x80 && b1 <= 0x8f) size = 4; + else if (b0 >= 0xf1 && b0 <= 0xf3 && b1 >= 0x80 && b1 <= 0xbf) size = 4; + else return 0; + + // Whatever the leading byte allows, every byte after the second one is a plain continuation + for (usz i = 2; i < size; i++) + { + if (const uchar b = byte(pos + i); b < 0x80 || b > 0xbf) + { + return 0; + } + } + + return size; +} + std::string vfs::escape(std::string_view name, bool escape_slash) { std::string result; @@ -562,9 +602,41 @@ std::string vfs::escape(std::string_view name, bool escape_slash) result.reserve(result.size() + name.size()); + // Bytes left of the UTF-8 sequence being copied through: a name is only valid UTF-8 as a whole, so whether + // a byte belongs to a sequence or is trash of its own cannot be told by looking at that byte alone + usz utf8_left = 0; + for (usz i = 0, s = name.size(); i < s; i++) { - switch (char2 c = name[i]) + const uchar b = name[i]; + + // A name is ASCII nearly every time, and every byte of a sequence is outside it, so this one test is all + // such a name pays: "utf8_left" can only be standing on a byte of 0x80 and above + if (b >= 0x80) + { + if (utf8_left) + { + utf8_left--; + } + else if (const usz size = utf8_sequence_size(name, i)) + { + utf8_left = size - 1; + } + else + { + // A byte no sequence can account for, written as "%" and its two hex digits: a host refusing a name + // that is not valid UTF-8 (macOS does) would turn the file away outright, and one writing it as it + // comes leaves a name that cannot be told from the escape characters this very function produces + constexpr char hex[] = "0123456789ABCDEF"; + + result += reinterpret_cast(u8"%"); + result += hex[b >> 4]; + result += hex[b & 0xf]; + continue; + } + } + + switch (char2 c = static_cast(b)) { case 0: case 1: @@ -769,7 +841,7 @@ std::string vfs::unescape(std::string_view name) { result += static_cast(c3); result.back() -= u8"0"[2]; - continue; + break; } case char2{u8"A"[2]}: case char2{u8"B"[2]}: @@ -797,7 +869,7 @@ std::string vfs::unescape(std::string_view name) result += static_cast(c3); result.back() -= u8"A"[2]; result.back() += 10; - continue; + break; } case char2{u8"!"[2]}: { @@ -814,6 +886,30 @@ std::string vfs::unescape(std::string_view name) i += 3; continue; } + case char2{u8"%"[2]}: + { + // A byte no UTF-8 sequence could account for, written as its two hex digits (see "vfs::escape") + auto digit = [](char2 h) -> int + { + if (h >= '0' && h <= '9') return h - '0'; + if (h >= 'A' && h <= 'F') return h - 'A' + 10; + return -1; + }; + + const int hi = digit(get_char(i + 3)); + const int lo = digit(get_char(i + 4)); + + if (hi >= 0 && lo >= 0) + { + result += static_cast(hi * 16 + lo); + + i += 4; + continue; + } + + // Not what this function wrote: dropped like any other escape character it does not know + break; + } case char2{u8"_"[2]}: { result += ' ';