diff --git a/.github/workflows/gamedb-lint.yml b/.github/workflows/gamedb-lint.yml index 342a6971f..2dd4095f7 100644 --- a/.github/workflows/gamedb-lint.yml +++ b/.github/workflows/gamedb-lint.yml @@ -3,13 +3,15 @@ name: GameDB Lint on: pull_request: paths: - - 'data/resources/gamedb.json' + - 'data/resources/gamedb.yaml' + - 'data/resources/discdb.yaml' push: branches: - master - dev paths: - - 'data/resources/gamedb.json' + - 'data/resources/gamedb.yaml' + - 'data/resources/discdb.yaml' workflow_dispatch: jobs: @@ -25,9 +27,12 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get -y install python3-demjson + sudo apt-get -y install yamllint - name: Check GameDB shell: bash - run: | - jsonlint -s data/resources/gamedb.json + run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/gamedb.yaml + + - name: Check DiscDB + shell: bash + run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/discdb.yaml diff --git a/extras/yamllint-config.yaml b/extras/yamllint-config.yaml new file mode 100644 index 000000000..def770861 --- /dev/null +++ b/extras/yamllint-config.yaml @@ -0,0 +1,16 @@ +extends: default + +rules: + line-length: + max: 200 + indentation: + spaces: 2 + indent-sequences: true + document-start: + present: false + document-end: + present: false + comments: + require-starting-space: true + min-spaces-from-content: 1 + diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index e685cfaf6..11e1ded95 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -131,7 +131,7 @@ target_precompile_headers(core PRIVATE "pch.h") target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..") target_link_libraries(core PUBLIC Threads::Threads common util ZLIB::ZLIB) -target_link_libraries(core PRIVATE stb xxhash imgui rapidjson rcheevos) +target_link_libraries(core PRIVATE stb xxhash imgui rapidyaml rcheevos) if(CPU_ARCH_X64) target_compile_definitions(core PUBLIC "ENABLE_RECOMPILER=1" "ENABLE_NEWREC=1" "ENABLE_MMAP_FASTMEM=1") diff --git a/src/core/core.props b/src/core/core.props index 3d34ca16e..7fbaca20b 100644 --- a/src/core/core.props +++ b/src/core/core.props @@ -10,7 +10,11 @@ ENABLE_MMAP_FASTMEM=1;%(PreprocessorDefinitions) ENABLE_NEWREC=1;%(PreprocessorDefinitions) - %(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\rapidjson\include;$(SolutionDir)dep\discord-rpc\include + %(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\discord-rpc\include + + %(PreprocessorDefinitions);C4_NO_DEBUG_BREAK=1 + %(AdditionalIncludeDirectories);$(SolutionDir)dep\rapidyaml\include;$(SolutionDir)dep\rapidjson\include + %(AdditionalIncludeDirectories);$(SolutionDir)dep\rainterface %(AdditionalIncludeDirectories);$(SolutionDir)dep\xbyak\xbyak diff --git a/src/core/core.vcxproj b/src/core/core.vcxproj index 345a0aa94..31428f491 100644 --- a/src/core/core.vcxproj +++ b/src/core/core.vcxproj @@ -172,6 +172,9 @@ {e4357877-d459-45c7-b8f6-dcbb587bb528} + + {1ad23a8a-4c20-434c-ae6b-0e07759eeb1e} + {4ba0a6d4-3ae1-42b2-9347-096fd023ff64} diff --git a/src/core/game_database.cpp b/src/core/game_database.cpp index 08b2fecf8..054416a01 100644 --- a/src/core/game_database.cpp +++ b/src/core/game_database.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) #include "game_database.h" @@ -16,22 +16,18 @@ #include "common/string_util.h" #include "common/timer.h" -#include "rapidjson/document.h" -#include "rapidjson/error/en.h" +#include "ryml.hpp" #include #include #include #include +#include #include "IconsFontAwesome5.h" Log_SetChannel(GameDatabase); -#ifdef _WIN32 -#include "common/windows_headers.h" -#endif - namespace GameDatabase { enum : u32 @@ -46,35 +42,51 @@ static const Entry* GetEntryForId(const std::string_view& code); static bool LoadFromCache(); static bool SaveToCache(); -static bool LoadGameDBJson(); -static bool ParseJsonEntry(Entry* entry, const rapidjson::Value& value); -static bool ParseJsonCodes(u32 index, const rapidjson::Value& value); +static void SetRymlCallbacks(); +static bool LoadGameDBYaml(); +static bool ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value); +static bool ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial); static bool LoadTrackHashes(); -static std::array(GameDatabase::Trait::Count)> s_trait_names = {{ - "ForceInterpreter", - "ForceSoftwareRenderer", - "ForceSoftwareRendererForReadbacks", - "ForceInterlacing", - "DisableTrueColor", - "DisableUpscaling", - "DisableTextureFiltering", - "DisableScaledDithering", - "DisableForceNTSCTimings", - "DisableWidescreen", - "DisablePGXP", - "DisablePGXPCulling", - "DisablePGXPTextureCorrection", - "DisablePGXPColorCorrection", - "DisablePGXPDepthBuffer", - "ForcePGXPVertexCache", - "ForcePGXPCPUMode", - "ForceRecompilerMemoryExceptions", - "ForceRecompilerICache", - "ForceRecompilerLUTFastmem", - "IsLibCryptProtected", +static constexpr const std::array(CompatibilityRating::Count)> + s_compatibility_rating_names = { + {"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}}; + +static constexpr const std::array(CompatibilityRating::Count)> + s_compatibility_rating_display_names = {{TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"), + TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"), + TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"), + TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"), + TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"), + TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}}; + +static constexpr const std::array(GameDatabase::Trait::Count)> s_trait_names = {{ + "forceInterpreter", + "forceSoftwareRenderer", + "forceSoftwareRendererForReadbacks", + "forceInterlacing", + "disableTrueColor", + "disableUpscaling", + "disableTextureFiltering", + "disableScaledDithering", + "disableForceNTSCTimings", + "disableWidescreen", + "disablePGXP", + "disablePGXPCulling", + "disablePGXPTextureCorrection", + "disablePGXPColorCorrection", + "disablePGXPDepthBuffer", + "forcePGXPVertexCache", + "forcePGXPCPUMode", + "forceRecompilerMemoryExceptions", + "forceRecompilerICache", + "forceRecompilerLUTFastmem", + "isLibCryptProtected", }}; +static constexpr const char* GAMEDB_YAML_FILENAME = "gamedb.yaml"; +static constexpr const char* DISCDB_YAML_FILENAME = "discdb.yaml"; + static bool s_loaded = false; static bool s_track_hashes_loaded = false; @@ -84,6 +96,94 @@ static PreferUnorderedStringMap s_code_lookup; static TrackHashesMap s_track_hashes_map; } // namespace GameDatabase +// RapidYAML utility routines. + +ALWAYS_INLINE std::string_view to_stringview(const c4::csubstr& s) +{ + return std::string_view(s.data(), s.size()); +} + +ALWAYS_INLINE std::string_view to_stringview(const c4::substr& s) +{ + return std::string_view(s.data(), s.size()); +} + +ALWAYS_INLINE c4::csubstr to_csubstr(const std::string_view& sv) +{ + return c4::csubstr(sv.data(), sv.length()); +} + +static bool GetStringFromObject(const ryml::ConstNodeRef& object, std::string_view key, std::string* dest) +{ + dest->clear(); + + const ryml::ConstNodeRef member = object.find_child(to_csubstr(key)); + if (!member.valid()) + return false; + + const c4::csubstr val = member.val(); + if (!val.empty()) + dest->assign(val.data(), val.size()); + + return true; +} + +template +static bool GetUIntFromObject(const ryml::ConstNodeRef& object, std::string_view key, T* dest) +{ + *dest = 0; + + const ryml::ConstNodeRef member = object.find_child(to_csubstr(key)); + if (!member.valid()) + return false; + + const c4::csubstr val = member.val(); + if (val.empty()) + { + Log_ErrorFmt("Unexpected empty value in {}", key); + return false; + } + + const std::optional opt_value = StringUtil::FromChars(to_stringview(val)); + if (!opt_value.has_value()) + { + Log_ErrorFmt("Unexpected non-uint value in {}", key); + return false; + } + + *dest = opt_value.value(); + return true; +} + +template +static std::optional GetOptionalTFromObject(const ryml::ConstNodeRef& object, std::string_view key) +{ + std::optional ret; + + const ryml::ConstNodeRef member = object.find_child(to_csubstr(key)); + if (member.valid()) + { + const c4::csubstr val = member.val(); + if (!val.empty()) + { + ret = StringUtil::FromChars(to_stringview(val)); + if (!ret.has_value()) + { + if constexpr (std::is_floating_point_v) + Log_ErrorFmt("Unexpected non-float value in {}", key); + else if constexpr (std::is_integral_v) + Log_ErrorFmt("Unexpected non-int value in {}", key); + } + } + else + { + Log_ErrorFmt("Unexpected empty value in {}", key); + } + } + + return ret; +} + void GameDatabase::EnsureLoaded() { if (s_loaded) @@ -98,11 +198,11 @@ void GameDatabase::EnsureLoaded() s_entries = {}; s_code_lookup = {}; - LoadGameDBJson(); + LoadGameDBYaml(); SaveToCache(); } - Log_InfoPrintf("Database load took %.2f ms", timer.GetTimeMilliseconds()); + Log_InfoFmt("Database load took {:.0f}ms", timer.GetTimeMilliseconds()); } void GameDatabase::Unload() @@ -198,30 +298,16 @@ GameDatabase::Entry* GameDatabase::GetMutableEntry(const std::string_view& seria return nullptr; } -const char* GameDatabase::GetTraitName(Trait trait) -{ - DebugAssert(trait < Trait::Count); - return s_trait_names[static_cast(trait)]; -} - const char* GameDatabase::GetCompatibilityRatingName(CompatibilityRating rating) { - static std::array(CompatibilityRating::Count)> names = { - {"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}}; - return names[static_cast(rating)]; + return s_compatibility_rating_names[static_cast(rating)]; } const char* GameDatabase::GetCompatibilityRatingDisplayName(CompatibilityRating rating) { - static constexpr std::array(CompatibilityRating::Count)> names = { - {TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"), - TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"), - TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"), - TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"), - TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"), - TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}}; return (rating >= CompatibilityRating::Unknown && rating < CompatibilityRating::Count) ? - Host::TranslateToCString("GameListCompatibilityRating", names[static_cast(rating)]) : + Host::TranslateToCString("GameListCompatibilityRating", + s_compatibility_rating_display_names[static_cast(rating)]) : ""; } @@ -580,7 +666,7 @@ bool GameDatabase::LoadFromCache() return false; } - const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0); + const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0); u32 signature, version, num_entries, num_codes; u64 file_gamedb_ts; @@ -674,7 +760,7 @@ bool GameDatabase::LoadFromCache() bool GameDatabase::SaveToCache() { - const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0); + const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0); std::unique_ptr stream( ByteStream::OpenFile(GetCacheFile().c_str(), BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_WRITE | @@ -742,329 +828,258 @@ bool GameDatabase::SaveToCache() return true; } -////////////////////////////////////////////////////////////////////////// -// JSON Parsing -////////////////////////////////////////////////////////////////////////// - -static bool GetStringFromObject(const rapidjson::Value& object, const char* key, std::string* dest) +void GameDatabase::SetRymlCallbacks() { - dest->clear(); - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsString()) - return false; - - dest->assign(member->value.GetString(), member->value.GetStringLength()); - return true; + ryml::Callbacks callbacks = ryml::get_callbacks(); + callbacks.m_error = [](const char* msg, size_t msg_len, ryml::Location loc, void* userdata) { + Log_ErrorFmt("Parse error at {}:{} (bufpos={}): {}", loc.line, loc.col, loc.offset, std::string_view(msg, msg_len)); + }; + ryml::set_callbacks(callbacks); + c4::set_error_callback( + [](const char* msg, size_t msg_size) { Log_ErrorFmt("C4 error: {}", std::string_view(msg, msg_size)); }); } -static bool GetBoolFromObject(const rapidjson::Value& object, const char* key, bool* dest) +bool GameDatabase::LoadGameDBYaml() { - *dest = false; + Common::Timer timer; - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsBool()) - return false; - - *dest = member->value.GetBool(); - return true; -} - -template -static bool GetUIntFromObject(const rapidjson::Value& object, const char* key, T* dest) -{ - *dest = 0; - - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsUint()) - return false; - - *dest = static_cast(member->value.GetUint()); - return true; -} - -static bool GetArrayOfStringsFromObject(const rapidjson::Value& object, const char* key, std::vector* dest) -{ - dest->clear(); - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsArray()) - return false; - - for (const rapidjson::Value& str : member->value.GetArray()) - { - if (str.IsString()) - { - dest->emplace_back(str.GetString(), str.GetStringLength()); - } - } - return true; -} - -template -static std::optional GetOptionalIntFromObject(const rapidjson::Value& object, const char* key) -{ - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsInt()) - return std::nullopt; - - return static_cast(member->value.GetInt()); -} - -template -static std::optional GetOptionalUIntFromObject(const rapidjson::Value& object, const char* key) -{ - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsUint()) - return std::nullopt; - - return static_cast(member->value.GetUint()); -} - -static std::optional GetOptionalFloatFromObject(const rapidjson::Value& object, const char* key) -{ - auto member = object.FindMember(key); - if (member == object.MemberEnd() || !member->value.IsFloat()) - return std::nullopt; - - return member->value.GetFloat(); -} - -bool GameDatabase::LoadGameDBJson() -{ - std::optional gamedb_data(Host::ReadResourceFileToString("gamedb.json", false)); + const std::optional gamedb_data = Host::ReadResourceFileToString(GAMEDB_YAML_FILENAME, false); if (!gamedb_data.has_value()) { - Log_ErrorPrintf("Failed to read game database"); + Log_ErrorPrint("Failed to read game database"); return false; } - // TODO: Parse in-place, avoid string allocations. - std::unique_ptr json = std::make_unique(); - json->Parse(gamedb_data->c_str(), gamedb_data->size()); - if (json->HasParseError()) - { - Log_ErrorPrintf("Failed to parse game database: %s at offset %zu", - rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset()); - return false; - } + SetRymlCallbacks(); - if (!json->IsArray()) - { - Log_ErrorPrintf("Document is not an array"); - return false; - } + const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(GAMEDB_YAML_FILENAME), to_csubstr(gamedb_data.value())); + const ryml::ConstNodeRef root = tree.rootref(); + s_entries.reserve(root.num_children()); - const auto& jarray = json->GetArray(); - s_entries.reserve(jarray.Size()); - - for (const rapidjson::Value& current : json->GetArray()) + for (const ryml::ConstNodeRef& current : root.children()) { // TODO: binary sort const u32 index = static_cast(s_entries.size()); Entry& entry = s_entries.emplace_back(); - if (!ParseJsonEntry(&entry, current)) + if (!ParseYamlEntry(&entry, current)) { s_entries.pop_back(); continue; } - ParseJsonCodes(index, current); + ParseYamlCodes(index, current, entry.serial); } - Log_InfoPrintf("Loaded %zu entries and %zu codes from database", s_entries.size(), s_code_lookup.size()); - return true; + ryml::reset_callbacks(); + + Log_InfoFmt("Loaded {} entries and {} codes from database in {:.0f}ms.", s_entries.size(), s_code_lookup.size(), + timer.GetTimeMilliseconds()); + return !s_entries.empty(); } -bool GameDatabase::ParseJsonEntry(Entry* entry, const rapidjson::Value& value) +bool GameDatabase::ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value) { - if (!value.IsObject()) + entry->serial = to_stringview(value.key()); + if (entry->serial.empty()) { - Log_WarningPrintf("entry is not an object"); + Log_ErrorPrint("Missing serial for entry."); return false; } - if (!GetStringFromObject(value, "serial", &entry->serial) || !GetStringFromObject(value, "name", &entry->title) || - entry->serial.empty()) + GetStringFromObject(value, "name", &entry->title); + + if (const ryml::ConstNodeRef metadata = value.find_child(to_csubstr("metadata")); metadata.valid()) { - Log_ErrorPrintf("Missing serial or title for entry"); - return false; - } + GetStringFromObject(metadata, "genre", &entry->genre); + GetStringFromObject(metadata, "developer", &entry->developer); + GetStringFromObject(metadata, "publisher", &entry->publisher); - GetStringFromObject(value, "genre", &entry->genre); - GetStringFromObject(value, "developer", &entry->developer); - GetStringFromObject(value, "publisher", &entry->publisher); + GetUIntFromObject(metadata, "minPlayers", &entry->min_players); + GetUIntFromObject(metadata, "maxPlayers", &entry->max_players); + GetUIntFromObject(metadata, "minBlocks", &entry->min_blocks); + GetUIntFromObject(metadata, "maxBlocks", &entry->max_blocks); - GetUIntFromObject(value, "minPlayers", &entry->min_players); - GetUIntFromObject(value, "maxPlayers", &entry->max_players); - GetUIntFromObject(value, "minBlocks", &entry->min_blocks); - GetUIntFromObject(value, "maxBlocks", &entry->max_blocks); - - entry->release_date = 0; - { - std::string release_date; - if (GetStringFromObject(value, "releaseDate", &release_date)) + entry->release_date = 0; { - std::istringstream iss(release_date); - struct tm parsed_time = {}; - iss >> std::get_time(&parsed_time, "%Y-%m-%d"); - if (!iss.fail()) + std::string release_date; + if (GetStringFromObject(metadata, "releaseDate", &release_date)) { - parsed_time.tm_isdst = 0; + std::istringstream iss(release_date); + struct tm parsed_time = {}; + iss >> std::get_time(&parsed_time, "%Y-%m-%d"); + if (!iss.fail()) + { + parsed_time.tm_isdst = 0; #ifdef _WIN32 - entry->release_date = _mkgmtime(&parsed_time); + entry->release_date = _mkgmtime(&parsed_time); #else - entry->release_date = timegm(&parsed_time); + entry->release_date = timegm(&parsed_time); #endif + } } } } entry->supported_controllers = static_cast(~0u); - const auto controllers = value.FindMember("controllers"); - if (controllers != value.MemberEnd()) - { - if (controllers->value.IsArray()) - { - bool first = true; - for (const rapidjson::Value& controller : controllers->value.GetArray()) - { - if (!controller.IsString()) - { - Log_WarningPrintf("controller is not a string"); - return false; - } - std::optional ctype = Settings::ParseControllerTypeName(controller.GetString()); - if (!ctype.has_value()) + if (const ryml::ConstNodeRef controllers = value.find_child(to_csubstr("controllers")); + controllers.valid() && controllers.has_children()) + { + bool first = true; + for (const ryml::ConstNodeRef& controller : controllers.children()) + { + const std::string_view controller_str = to_stringview(controller.val()); + if (controller_str.empty()) + { + Log_WarningFmt("controller is not a string in {}", entry->serial); + return false; + } + + std::optional ctype = Settings::ParseControllerTypeName(controller_str); + if (!ctype.has_value()) + { + Log_WarningFmt("Invalid controller type {} in {}", controller_str, entry->serial); + continue; + } + + if (first) + { + entry->supported_controllers = 0; + first = false; + } + + entry->supported_controllers |= (1u << static_cast(ctype.value())); + } + } + + if (const ryml::ConstNodeRef compatibility = value.find_child(to_csubstr("compatibility")); + compatibility.valid() && compatibility.has_children()) + { + const ryml::ConstNodeRef rating = compatibility.find_child(to_csubstr("rating")); + if (rating.valid()) + { + const std::string_view rating_str = to_stringview(rating.val()); + + const auto iter = std::find(s_compatibility_rating_names.begin(), s_compatibility_rating_names.end(), rating_str); + if (iter != s_compatibility_rating_names.end()) + { + const size_t rating_idx = static_cast(std::distance(s_compatibility_rating_names.begin(), iter)); + DebugAssert(rating_idx < static_cast(CompatibilityRating::Count)); + entry->compatibility = static_cast(rating_idx); + } + else + { + Log_WarningFmt("Unknown compatibility rating {} in {}", rating_str, entry->serial); + } + } + } + + if (const ryml::ConstNodeRef traits = value.find_child(to_csubstr("traits")); traits.valid() && traits.has_children()) + { + for (const ryml::ConstNodeRef& trait : traits.children()) + { + const std::string_view trait_str = to_stringview(trait.val()); + if (trait_str.empty()) + { + Log_WarningFmt("Empty trait in {}", entry->serial); + continue; + } + + const auto iter = std::find(s_trait_names.begin(), s_trait_names.end(), trait_str); + if (iter == s_trait_names.end()) + { + Log_WarningFmt("Unknown trait {} in {}", trait_str, entry->serial); + continue; + } + + const size_t trait_idx = static_cast(std::distance(s_trait_names.begin(), iter)); + DebugAssert(trait_idx < static_cast(Trait::Count)); + entry->traits[trait_idx] = true; + } + } + + if (const ryml::ConstNodeRef settings = value.find_child(to_csubstr("settings")); + settings.valid() && settings.has_children()) + { + entry->display_active_start_offset = GetOptionalTFromObject(settings, "displayActiveStartOffset"); + entry->display_active_end_offset = GetOptionalTFromObject(settings, "displayActiveEndOffset"); + entry->display_line_start_offset = GetOptionalTFromObject(settings, "displayLineStartOffset"); + entry->display_line_end_offset = GetOptionalTFromObject(settings, "displayLineEndOffset"); + entry->dma_max_slice_ticks = GetOptionalTFromObject(settings, "dmaMaxSliceTicks"); + entry->dma_halt_ticks = GetOptionalTFromObject(settings, "dmaHaltTicks"); + entry->gpu_fifo_size = GetOptionalTFromObject(settings, "gpuFIFOSize"); + entry->gpu_max_run_ahead = GetOptionalTFromObject(settings, "gpuMaxRunAhead"); + entry->gpu_pgxp_tolerance = GetOptionalTFromObject(settings, "gpuPGXPTolerance"); + entry->gpu_pgxp_depth_threshold = GetOptionalTFromObject(settings, "gpuPGXPDepthThreshold"); + } + + if (const ryml::ConstNodeRef disc_set = value.find_child("discSet"); disc_set.valid() && disc_set.has_children()) + { + GetStringFromObject(disc_set, "name", &entry->disc_set_name); + + if (const ryml::ConstNodeRef set_serials = disc_set.find_child("serials"); + set_serials.valid() && set_serials.has_children()) + { + entry->disc_set_serials.reserve(set_serials.num_children()); + for (const ryml::ConstNodeRef& serial : set_serials) + { + const std::string_view serial_str = to_stringview(serial.val()); + if (serial_str.empty()) { - Log_WarningPrintf("Invalid controller type '%s'", controller.GetString()); + Log_WarningFmt("Empty disc set serial in {}", entry->serial); continue; } - if (first) + if (std::find(entry->disc_set_serials.begin(), entry->disc_set_serials.end(), serial_str) != + entry->disc_set_serials.end()) { - entry->supported_controllers = 0; - first = false; + Log_WarningFmt("Duplicate serial {} in disc set serials for {}", serial_str, entry->serial); + continue; } - entry->supported_controllers |= (1u << static_cast(ctype.value())); + entry->disc_set_serials.emplace_back(serial_str); } } - else - { - Log_WarningPrintf("controllers is not an array"); - } - } - - const auto compatibility = value.FindMember("compatibility"); - if (compatibility != value.MemberEnd()) - { - if (compatibility->value.IsObject()) - { - u32 rating; - if (GetUIntFromObject(compatibility->value, "rating", &rating) && - rating < static_cast(CompatibilityRating::Count)) - { - entry->compatibility = static_cast(rating); - } - } - else - { - Log_WarningPrintf("compatibility is not an object"); - } - } - - const auto traits = value.FindMember("traits"); - if (traits != value.MemberEnd()) - { - if (traits->value.IsObject()) - { - const auto& traitsobj = traits->value; - for (u32 trait = 0; trait < static_cast(Trait::Count); trait++) - { - bool bvalue; - if (GetBoolFromObject(traitsobj, s_trait_names[trait], &bvalue) && bvalue) - entry->traits[trait] = bvalue; - } - - entry->display_active_start_offset = GetOptionalIntFromObject(traitsobj, "DisplayActiveStartOffset"); - entry->display_active_end_offset = GetOptionalIntFromObject(traitsobj, "DisplayActiveEndOffset"); - entry->display_line_start_offset = GetOptionalIntFromObject(traitsobj, "DisplayLineStartOffset"); - entry->display_line_end_offset = GetOptionalIntFromObject(traitsobj, "DisplayLineEndOffset"); - entry->dma_max_slice_ticks = GetOptionalUIntFromObject(traitsobj, "DMAMaxSliceTicks"); - entry->dma_halt_ticks = GetOptionalUIntFromObject(traitsobj, "DMAHaltTicks"); - entry->gpu_fifo_size = GetOptionalUIntFromObject(traitsobj, "GPUFIFOSize"); - entry->gpu_max_run_ahead = GetOptionalUIntFromObject(traitsobj, "GPUMaxRunAhead"); - entry->gpu_pgxp_tolerance = GetOptionalFloatFromObject(traitsobj, "GPUPGXPTolerance"); - entry->gpu_pgxp_depth_threshold = GetOptionalFloatFromObject(traitsobj, "GPUPGXPDepthThreshold"); - } - else - { - Log_WarningPrintf("traits is not an object"); - } - } - - GetStringFromObject(value, "discSetName", &entry->disc_set_name); - const auto disc_set_serials = value.FindMember("discSetSerials"); - if (disc_set_serials != value.MemberEnd()) - { - if (disc_set_serials->value.IsArray()) - { - const auto disc_set_serials_array = disc_set_serials->value.GetArray(); - entry->disc_set_serials.reserve(disc_set_serials_array.Size()); - for (const rapidjson::Value& serial : disc_set_serials_array) - { - if (serial.IsString()) - { - entry->disc_set_serials.emplace_back(serial.GetString(), serial.GetStringLength()); - } - else - { - Log_WarningPrintf("discSetSerial is not a string"); - } - } - } - else - { - Log_WarningPrintf("discSetSerials is not an array"); - } } return true; } -bool GameDatabase::ParseJsonCodes(u32 index, const rapidjson::Value& value) +bool GameDatabase::ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial) { - auto member = value.FindMember("codes"); - if (member == value.MemberEnd()) + const ryml::ConstNodeRef& codes = value.find_child(to_csubstr("codes")); + if (!codes.valid() || !codes.has_children()) { - Log_WarningPrintf("codes member is missing"); - return false; - } + // use serial instead + auto iter = s_code_lookup.find(serial); + if (iter != s_code_lookup.end()) + { + Log_WarningFmt("Duplicate code '{}'", serial); + return false; + } - if (!member->value.IsArray()) - { - Log_WarningPrintf("codes is not an array"); - return false; + s_code_lookup.emplace(serial, index); + return true; } u32 added = 0; - for (const rapidjson::Value& current_code : member->value.GetArray()) + for (const ryml::ConstNodeRef& current_code : codes) { - if (!current_code.IsString()) + const std::string_view current_code_str = to_stringview(current_code.val()); + if (current_code_str.empty()) { - Log_WarningPrintf("code is not a string"); + Log_WarningFmt("code is not a string in {}", serial); continue; } - const std::string_view code(current_code.GetString(), current_code.GetStringLength()); - auto iter = s_code_lookup.find(code); + auto iter = s_code_lookup.find(current_code_str); if (iter != s_code_lookup.end()) { - Log_WarningPrintf("Duplicate code '%.*s'", static_cast(code.size()), code.data()); + Log_WarningFmt("Duplicate code '{}' in {}", current_code_str, serial); continue; } - s_code_lookup.emplace(code, index); + s_code_lookup.emplace(current_code_str, index); added++; } @@ -1082,105 +1097,84 @@ void GameDatabase::EnsureTrackHashesMapLoaded() bool GameDatabase::LoadTrackHashes() { - std::optional gamedb_data(Host::ReadResourceFileToString("gamedb.json", false)); + Common::Timer load_timer; + + std::optional gamedb_data(Host::ReadResourceFileToString(DISCDB_YAML_FILENAME, false)); if (!gamedb_data.has_value()) { - Log_ErrorPrintf("Failed to read game database"); + Log_ErrorPrint("Failed to read game database"); return false; } + SetRymlCallbacks(); + // TODO: Parse in-place, avoid string allocations. - std::unique_ptr json = std::make_unique(); - json->Parse(gamedb_data->c_str(), gamedb_data->size()); - if (json->HasParseError()) - { - Log_ErrorPrintf("Failed to parse game database: %s at offset %zu", - rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset()); - return false; - } - - if (!json->IsArray()) - { - Log_ErrorPrintf("Document is not an array"); - return false; - } + const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(DISCDB_YAML_FILENAME), to_csubstr(gamedb_data.value())); + const ryml::ConstNodeRef root = tree.rootref(); s_track_hashes_map = {}; - for (const rapidjson::Value& current : json->GetArray()) + size_t serials = 0; + for (const ryml::ConstNodeRef& current : root.children()) { - if (!current.IsObject()) + const std::string_view serial = to_stringview(current.key()); + if (serial.empty() || !current.has_children()) { - Log_WarningPrintf("entry is not an object"); + Log_WarningPrint("entry is not an object"); continue; } - std::vector codes; - if (!GetArrayOfStringsFromObject(current, "codes", &codes)) + const ryml::ConstNodeRef track_data = current.find_child(to_csubstr("trackData")); + if (!track_data.valid() || !track_data.has_children()) { - Log_WarningPrintf("codes member is missing"); + Log_WarningFmt("trackData is missing in {}", serial); continue; } - auto track_data = current.FindMember("track_data"); - if (track_data == current.MemberEnd()) + u32 revision = 0; + for (const ryml::ConstNodeRef& track_revisions : track_data.children()) { - Log_WarningPrintf("track_data member is missing"); - continue; - } - - if (!track_data->value.IsArray()) - { - Log_WarningPrintf("track_data is not an array"); - continue; - } - - uint32_t revision = 0; - for (const rapidjson::Value& track_revisions : track_data->value.GetArray()) - { - if (!track_revisions.IsObject()) + const ryml::ConstNodeRef tracks = track_revisions.find_child(to_csubstr("tracks")); + if (!tracks.valid() || !tracks.has_children()) { - Log_WarningPrintf("track_data is not an array of object"); + Log_WarningFmt("tracks member is missing in {}", serial); continue; } - auto tracks = track_revisions.FindMember("tracks"); - if (tracks == track_revisions.MemberEnd()) - { - Log_WarningPrintf("tracks member is missing"); - continue; - } + std::string revision_string; + GetStringFromObject(track_revisions, "version", &revision_string); - if (!tracks->value.IsArray()) + for (const ryml::ConstNodeRef& track : tracks) { - Log_WarningPrintf("tracks is not an array"); - continue; - } - - std::string revisionString; - GetStringFromObject(track_revisions, "version", &revisionString); - - for (const rapidjson::Value& track : tracks->value.GetArray()) - { - auto md5_field = track.FindMember("md5"); - if (md5_field == track.MemberEnd() || !md5_field->value.IsString()) + const ryml::ConstNodeRef md5 = track.find_child("md5"); + std::string_view md5_str; + if (!md5.valid() || (md5_str = to_stringview(md5.val())).empty()) { + Log_WarningFmt("md5 is missing in track in {}", serial); continue; } - auto md5 = CDImageHasher::HashFromString( - std::string_view(md5_field->value.GetString(), md5_field->value.GetStringLength())); - if (md5) + const std::optional md5o = CDImageHasher::HashFromString(md5_str); + if (md5o.has_value()) { - s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5.value()), - std::forward_as_tuple(codes, revisionString, revision)); + s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5o.value()), + std::forward_as_tuple(std::string(serial), revision_string, revision)); + } + else + { + Log_WarningFmt("invalid md5 in {}", serial); } } revision++; } + + serials++; } - return true; + ryml::reset_callbacks(); + Log_InfoFmt("Loaded {} track hashes from {} serials in {:.0f}ms.", s_track_hashes_map.size(), serials, + load_timer.GetTimeMilliseconds()); + return !s_track_hashes_map.empty(); } const GameDatabase::TrackHashesMap& GameDatabase::GetTrackHashesMap() diff --git a/src/core/game_database.h b/src/core/game_database.h index 06e2578d6..81b4ad68c 100644 --- a/src/core/game_database.h +++ b/src/core/game_database.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) #pragma once @@ -98,16 +98,14 @@ const Entry* GetEntryForSerial(const std::string_view& serial); std::string GetSerialForDisc(CDImage* image); std::string GetSerialForPath(const char* path); -const char* GetTraitName(Trait trait); - const char* GetCompatibilityRatingName(CompatibilityRating rating); const char* GetCompatibilityRatingDisplayName(CompatibilityRating rating); /// Map of track hashes for image verification struct TrackData { - TrackData(std::vector codes, std::string revisionString, uint32_t revision) - : codes(std::move(codes)), revisionString(revisionString), revision(revision) + TrackData(std::string serial_, std::string revision_str_, uint32_t revision_) + : serial(std::move(serial_)), revision_str(std::move(revision_str_)), revision(revision_) { } @@ -115,12 +113,12 @@ struct TrackData { // 'revisionString' is deliberately ignored in comparisons as it's redundant with comparing 'revision'! Do not // change! - return left.codes == right.codes && left.revision == right.revision; + return left.serial == right.serial && left.revision == right.revision; } - std::vector codes; - std::string revisionString; - uint32_t revision; + std::string serial; + std::string revision_str; + u32 revision; }; using TrackHashesMap = std::multimap; diff --git a/src/duckstation-qt/gamesummarywidget.cpp b/src/duckstation-qt/gamesummarywidget.cpp index adce80a11..f66baaf44 100644 --- a/src/duckstation-qt/gamesummarywidget.cpp +++ b/src/duckstation-qt/gamesummarywidget.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) #include "gamesummarywidget.h" @@ -13,7 +13,6 @@ #include "fmt/format.h" -#include #include #include @@ -219,13 +218,6 @@ void GameSummaryWidget::onComputeHashClicked() return; } -#ifndef _DEBUGFAST - // Kick off hash preparation asynchronously, as building the map of results may take a while - // This breaks for DebugFast because of the iterator debug level mismatch. - QFuture result = - QtConcurrent::run([]() { return &GameDatabase::GetTrackHashesMap(); }); -#endif - QtModalProgressCallback progress_callback(this); progress_callback.SetProgressRange(image->GetTrackCount()); @@ -259,6 +251,7 @@ void GameSummaryWidget::onComputeHashClicked() if (calculate_hash_success) { std::string found_revision; + std::string found_serial; m_redump_search_keyword = CDImageHasher::HashToString(track_hashes.front()); progress_callback.SetStatusText("Verifying hashes..."); @@ -270,11 +263,7 @@ void GameSummaryWidget::onComputeHashClicked() // 2. For each data track match, try to match all audio tracks // If all match, assume this revision. Else, try other revisions, // and accept the one with the most matches. -#ifndef _DEBUGFAST - const GameDatabase::TrackHashesMap& hashes_map = *result.result(); -#else const GameDatabase::TrackHashesMap& hashes_map = GameDatabase::GetTrackHashesMap(); -#endif auto data_track_matches = hashes_map.equal_range(track_hashes[0]); if (data_track_matches.first != data_track_matches.second) @@ -317,13 +306,30 @@ void GameSummaryWidget::onComputeHashClicked() } } - found_revision = best_data_match->second.revisionString; + found_revision = best_data_match->second.revision_str; + found_serial = best_data_match->second.serial; } + QString text; + if (!found_revision.empty()) + text = tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision)); + + if (found_serial != m_ui.serial->text().toStdString()) { - m_ui.revision->setText( - tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision))); + text = + tr("Serial Mismatch: %1 vs %2%3").arg(QString::fromStdString(found_serial)).arg(m_ui.serial->text()).arg(text); + } + + if (!text.isEmpty()) + { + if (m_ui.verifySpacer) + { + m_ui.verifyLayout->removeItem(m_ui.verifySpacer); + delete m_ui.verifySpacer; + m_ui.verifySpacer = nullptr; + } + m_ui.revision->setText(text); m_ui.revision->setVisible(true); } } diff --git a/src/duckstation-qt/gamesummarywidget.ui b/src/duckstation-qt/gamesummarywidget.ui index 5e71a699b..5c8e047c0 100644 --- a/src/duckstation-qt/gamesummarywidget.ui +++ b/src/duckstation-qt/gamesummarywidget.ui @@ -189,9 +189,9 @@ - + - + Qt::Horizontal @@ -207,7 +207,7 @@ - 100 + 300 0