VolumeVerifier: Check hashes in Wii partitions

Source/Core/DiscIO/Volume.h

@@ -99,7 +99,11 @@ public:
   }
   virtual Platform GetVolumeType() const = 0;
   virtual bool SupportsIntegrityCheck() const { return false; }
-  virtual bool CheckIntegrity(const Partition& partition) const { return false; }
+  virtual bool CheckH3TableIntegrity(const Partition& partition) const { return false; }
+  virtual bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const
+  {
+    return false;
+  }
   virtual Region GetRegion() const = 0;
   virtual Country GetCountry(const Partition& partition = PARTITION_NONE) const = 0;
   virtual BlobType GetBlobType() const = 0;

Source/Core/DiscIO/VolumeVerifier.cpp

@@ -14,6 +14,7 @@
 #include "Common/Align.h"
 #include "Common/Assert.h"
 #include "Common/CommonTypes.h"
+#include "Common/Logging/Log.h"
 #include "Common/MsgHandler.h"
 #include "Common/StringUtil.h"
 #include "Common/Swap.h"
@@ -62,6 +63,9 @@ void VolumeVerifier::Start()
     CheckCorrectlySigned(PARTITION_NONE, GetStringT("This title is not correctly signed."));
   CheckDiscSize();
   CheckMisc();
+
+  std::sort(m_blocks.begin(), m_blocks.end(),
+            [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; });
 }
 
 void VolumeVerifier::CheckPartitions()
@@ -164,18 +168,7 @@ bool VolumeVerifier::CheckPartition(const Partition& partition)
   else if (*type == PARTITION_UPDATE)
     severity = Severity::Low;
 
-  std::string name = NameForPartitionType(*type, false);
-  if (ShouldHaveMasterpiecePartitions() && *type > 0xFF)
-  {
-    // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces
-    // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo
-    // rather than the game mode as a whole, so please use the singular form. Official translations:
-    // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre
-    // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean).
-    // If your language is not one of the languages above, consider leaving the string untranslated
-    // so that people will recognize it as the name of the game mode.
-    name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str());
-  }
+  const std::string name = GetPartitionName(type);
 
   if (partition.offset % VolumeWii::BLOCK_TOTAL_SIZE != 0 ||
       m_volume.PartitionOffsetToRawOffset(0, partition) % VolumeWii::BLOCK_TOTAL_SIZE != 0)
@@ -189,6 +182,13 @@ bool VolumeVerifier::CheckPartition(const Partition& partition)
       partition, StringFromFormat(GetStringT("The %s partition is not correctly signed.").c_str(),
                                   name.c_str()));
 
+  if (m_volume.SupportsIntegrityCheck() && !m_volume.CheckH3TableIntegrity(partition))
+  {
+    const std::string text = StringFromFormat(
+        GetStringT("The H3 hash table for the %s partition is not correct.").c_str(), name.c_str());
+    AddProblem(Severity::Low, text);
+  }
+
   bool invalid_disc_header = false;
   std::vector<u8> disc_header(0x80);
   constexpr u32 WII_MAGIC = 0x5D1C9EA3;
@@ -262,9 +262,43 @@ bool VolumeVerifier::CheckPartition(const Partition& partition)
     }
   }
 
+  // Prepare for hash verification in the Process step
+  if (m_volume.SupportsIntegrityCheck())
+  {
+    u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition);
+    const std::optional<u64> size =
+        m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE);
+    const u64 end_offset = offset + size.value_or(0);
+
+    for (size_t i = 0; offset < end_offset; ++i, offset += VolumeWii::BLOCK_TOTAL_SIZE)
+      m_blocks.emplace_back(BlockToVerify{partition, offset, i});
+
+    m_block_errors.emplace(partition, 0);
+  }
+
   return true;
 }
 
+std::string VolumeVerifier::GetPartitionName(std::optional<u32> type) const
+{
+  if (!type)
+    return "???";
+
+  std::string name = NameForPartitionType(*type, false);
+  if (ShouldHaveMasterpiecePartitions() && *type > 0xFF)
+  {
+    // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces
+    // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo
+    // rather than the game mode as a whole, so please use the singular form. Official translations:
+    // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre
+    // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean).
+    // If your language is not one of the languages above, consider leaving the string untranslated
+    // so that people will recognize it as the name of the game mode.
+    name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str());
+  }
+  return name;
+}
+
 void VolumeVerifier::CheckCorrectlySigned(const Partition& partition, const std::string& error_text)
 {
   IOS::HLE::Kernel ios;
@@ -596,7 +630,30 @@ void VolumeVerifier::Process()
   if (m_progress == m_max_progress)
     return;
 
-  m_progress += std::min(m_max_progress - m_progress, BLOCK_SIZE);
+  u64 bytes_to_read = BLOCK_SIZE;
+  if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress)
+  {
+    bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE;
+  }
+  else if (m_block_index + 1 < m_blocks.size() && m_blocks[m_block_index + 1].offset > m_progress)
+  {
+    bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index + 1].offset - m_progress);
+  }
+  bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress);
+
+  m_progress += bytes_to_read;
+
+  while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress)
+  {
+    if (!m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index,
+                                      m_blocks[m_block_index].partition))
+    {
+      WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64,
+               m_blocks[m_block_index].offset);
+      m_block_errors[m_blocks[m_block_index].partition]++;
+    }
+    m_block_index++;
+  }
 }
 
 u64 VolumeVerifier::GetBytesProcessed() const
@@ -615,6 +672,18 @@ void VolumeVerifier::Finish()
     return;
   m_done = true;
 
+  for (auto pair : m_block_errors)
+  {
+    if (pair.second > 0)
+    {
+      const std::string name = GetPartitionName(m_volume.GetPartitionType(pair.first));
+      AddProblem(Severity::Medium,
+                 StringFromFormat(
+                     GetStringT("Errors were found in %zu blocks in the %s partition.").c_str(),
+                     pair.second, name.c_str()));
+    }
+  }
+
   // Show the most serious problems at the top
   std::stable_sort(m_result.problems.begin(), m_result.problems.end(),
                    [](const Problem& p1, const Problem& p2) { return p1.severity > p2.severity; });

Source/Core/DiscIO/VolumeVerifier.h

@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include <map>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -64,8 +66,16 @@ public:
   const Result& GetResult() const;
 
 private:
+  struct BlockToVerify
+  {
+    Partition partition;
+    u64 offset;
+    u64 block_index;
+  };
+
   void CheckPartitions();
   bool CheckPartition(const Partition& partition);  // Returns false if partition should be ignored
+  std::string GetPartitionName(std::optional<u32> type) const;
   void CheckCorrectlySigned(const Partition& partition, const std::string& error_text);
   bool IsDebugSigned() const;
   bool ShouldHaveChannelPartition() const;
@@ -85,6 +95,10 @@ private:
   bool m_is_datel;
   bool m_is_not_retail;
 
+  std::vector<BlockToVerify> m_blocks;
+  size_t m_block_index = 0;  // Index in m_blocks, not index in a specific partition
+  std::map<Partition, size_t> m_block_errors;
+
   bool m_started;
   bool m_done;
   u64 m_progress;

Source/Core/DiscIO/VolumeWii.cpp

@@ -109,6 +109,19 @@ VolumeWii::VolumeWii(std::unique_ptr<BlobReader> reader)
         return cert_chain;
       };
 
+      auto get_h3_table = [this, partition]() -> std::vector<u8> {
+        if (!m_encrypted)
+          return {};
+        const std::optional<u64> h3_table_offset =
+            ReadSwappedAndShifted(partition.offset + 0x2b4, PARTITION_NONE);
+        if (!h3_table_offset)
+          return {};
+        std::vector<u8> h3_table(H3_TABLE_SIZE);
+        if (!m_reader->Read(partition.offset + *h3_table_offset, H3_TABLE_SIZE, h3_table.data()))
+          return {};
+        return h3_table;
+      };
+
       auto get_key = [this, partition]() -> std::unique_ptr<mbedtls_aes_context> {
         const IOS::ES::TicketReader& ticket = *m_partitions[partition].ticket;
         if (!ticket.IsValid())
@@ -133,6 +146,7 @@ VolumeWii::VolumeWii(std::unique_ptr<BlobReader> reader)
                                       Common::Lazy<IOS::ES::TicketReader>(get_ticket),
                                       Common::Lazy<IOS::ES::TMDReader>(get_tmd),
                                       Common::Lazy<std::vector<u8>>(get_cert_chain),
+                                      Common::Lazy<std::vector<u8>>(get_h3_table),
                                       Common::Lazy<std::unique_ptr<FileSystem>>(get_file_system),
                                       Common::Lazy<u64>(get_data_offset), *partition_type});
     }
@@ -409,82 +423,84 @@ u64 VolumeWii::GetRawSize() const
   return m_reader->GetRawSize();
 }
 
-bool VolumeWii::CheckIntegrity(const Partition& partition) const
+bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const
 {
-  if (!m_encrypted)
-    return false;
-
-  // Get the decryption key for the partition
   auto it = m_partitions.find(partition);
   if (it == m_partitions.end())
     return false;
   const PartitionDetails& partition_details = it->second;
+
+  const std::vector<u8>& h3_table = *partition_details.h3_table;
+  if (h3_table.size() != H3_TABLE_SIZE)
+    return false;
+
+  const IOS::ES::TMDReader& tmd = *partition_details.tmd;
+  if (!tmd.IsValid())
+    return false;
+
+  const std::vector<IOS::ES::Content> contents = tmd.GetContents();
+  if (contents.size() != 1)
+    return false;
+
+  std::array<u8, 20> h3_table_sha1;
+  mbedtls_sha1(h3_table.data(), h3_table.size(), h3_table_sha1.data());
+  return h3_table_sha1 == contents[0].sha1;
+}
+
+bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition) const
+{
+  auto it = m_partitions.find(partition);
+  if (it == m_partitions.end())
+    return false;
+  const PartitionDetails& partition_details = it->second;
+
+  constexpr size_t SHA1_SIZE = 20;
+  if (block_index / 64 * SHA1_SIZE >= partition_details.h3_table->size())
+    return false;
+
   mbedtls_aes_context* aes_context = partition_details.key->get();
   if (!aes_context)
     return false;
 
-  // Get partition data size
-  const auto part_data_size = ReadSwappedAndShifted(partition.offset + 0x2BC, PARTITION_NONE);
-  if (!part_data_size)
+  const u64 cluster_offset =
+      partition.offset + *partition_details.data_offset + block_index * BLOCK_TOTAL_SIZE;
+
+  // Read and decrypt the cluster metadata
+  u8 cluster_metadata_crypted[BLOCK_HEADER_SIZE];
+  u8 cluster_metadata[BLOCK_HEADER_SIZE];
+  u8 iv[16] = {0};
+  if (!m_reader->Read(cluster_offset, BLOCK_HEADER_SIZE, cluster_metadata_crypted))
+    return false;
+  mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_HEADER_SIZE, iv,
+                        cluster_metadata_crypted, cluster_metadata);
+
+  u8 cluster_data[BLOCK_DATA_SIZE];
+  if (!Read(block_index * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE, cluster_data, partition))
     return false;
 
-  const u32 num_clusters = static_cast<u32>(part_data_size.value() / 0x8000);
-  for (u32 cluster_id = 0; cluster_id < num_clusters; ++cluster_id)
+  for (u32 hash_index = 0; hash_index < 31; ++hash_index)
   {
-    const u64 cluster_offset =
-        partition.offset + *partition_details.data_offset + static_cast<u64>(cluster_id) * 0x8000;
-
-    // Read and decrypt the cluster metadata
-    u8 cluster_metadata_crypted[0x400];
-    u8 cluster_metadata[0x400];
-    u8 iv[16] = {0};
-    if (!m_reader->Read(cluster_offset, sizeof(cluster_metadata_crypted), cluster_metadata_crypted))
-    {
-      WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read metadata", cluster_id);
+    u8 h0_hash[SHA1_SIZE];
+    mbedtls_sha1(cluster_data + hash_index * 0x400, 0x400, h0_hash);
+    if (memcmp(h0_hash, cluster_metadata + hash_index * SHA1_SIZE, SHA1_SIZE))
       return false;
-    }
-    mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_metadata), iv,
-                          cluster_metadata_crypted, cluster_metadata);
-
-    // Some clusters have invalid data and metadata because they aren't
-    // meant to be read by the game (for example, holes between files). To
-    // try to avoid reporting errors because of these clusters, we check
-    // the 0x00 paddings in the metadata.
-    //
-    // This may cause some false negatives though: some bad clusters may be
-    // skipped because they are *too* bad and are not even recognized as
-    // valid clusters. To be improved.
-    const u8* pad_begin = cluster_metadata + 0x26C;
-    const u8* pad_end = pad_begin + 0x14;
-    const bool meaningless = std::any_of(pad_begin, pad_end, [](u8 val) { return val != 0; });
-
-    if (meaningless)
-      continue;
-
-    u8 cluster_data[0x7C00];
-    if (!Read(cluster_id * sizeof(cluster_data), sizeof(cluster_data), cluster_data, partition))
-    {
-      WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read data", cluster_id);
-      return false;
-    }
-
-    for (u32 hash_id = 0; hash_id < 31; ++hash_id)
-    {
-      u8 hash[20];
-
-      mbedtls_sha1(cluster_data + hash_id * sizeof(cluster_metadata), sizeof(cluster_metadata),
-                   hash);
-
-      // Note that we do not use strncmp here
-      if (memcmp(hash, cluster_metadata + hash_id * sizeof(hash), sizeof(hash)))
-      {
-        WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: hash %d is invalid", cluster_id,
-                 hash_id);
-        return false;
-      }
-    }
   }
 
+  u8 h1_hash[SHA1_SIZE];
+  mbedtls_sha1(cluster_metadata, SHA1_SIZE * 31, h1_hash);
+  if (memcmp(h1_hash, cluster_metadata + 0x280 + (block_index % 8) * SHA1_SIZE, SHA1_SIZE))
+    return false;
+
+  u8 h2_hash[SHA1_SIZE];
+  mbedtls_sha1(cluster_metadata + 0x280, SHA1_SIZE * 8, h2_hash);
+  if (memcmp(h2_hash, cluster_metadata + 0x340 + (block_index / 8 % 8) * SHA1_SIZE, SHA1_SIZE))
+    return false;
+
+  u8 h3_hash[SHA1_SIZE];
+  mbedtls_sha1(cluster_metadata + 0x340, SHA1_SIZE * 8, h3_hash);
+  if (memcmp(h3_hash, partition_details.h3_table->data() + block_index / 64 * SHA1_SIZE, SHA1_SIZE))
+    return false;
+
   return true;
 }
 

Source/Core/DiscIO/VolumeWii.h

@@ -56,8 +56,9 @@ public:
   std::optional<u8> GetDiscNumber(const Partition& partition = PARTITION_NONE) const override;
 
   Platform GetVolumeType() const override;
-  bool SupportsIntegrityCheck() const override { return true; }
-  bool CheckIntegrity(const Partition& partition) const override;
+  bool SupportsIntegrityCheck() const override { return m_encrypted; }
+  bool CheckH3TableIntegrity(const Partition& partition) const override;
+  bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const override;
 
   Region GetRegion() const override;
   Country GetCountry(const Partition& partition = PARTITION_NONE) const override;
@@ -66,6 +67,8 @@ public:
   bool IsSizeAccurate() const override;
   u64 GetRawSize() const override;
 
+  static constexpr unsigned int H3_TABLE_SIZE = 0x18000;
+
   static constexpr unsigned int BLOCK_HEADER_SIZE = 0x0400;
   static constexpr unsigned int BLOCK_DATA_SIZE = 0x7C00;
   static constexpr unsigned int BLOCK_TOTAL_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE;
@@ -80,6 +83,7 @@ private:
     Common::Lazy<IOS::ES::TicketReader> ticket;
     Common::Lazy<IOS::ES::TMDReader> tmd;
     Common::Lazy<std::vector<u8>> cert_chain;
+    Common::Lazy<std::vector<u8>> h3_table;
     Common::Lazy<std::unique_ptr<FileSystem>> file_system;
     Common::Lazy<u64> data_offset;
     u32 type;

Source/Core/DolphinQt/Config/FilesystemWidget.cpp

@@ -238,12 +238,6 @@ void FilesystemWidget::ShowContextMenu(const QPoint&)
       if (!folder.isEmpty())
         ExtractPartition(partition, folder);
     });
-    if (m_volume->IsEncryptedAndHashed())
-    {
-      menu->addSeparator();
-      menu->addAction(tr("Check Partition Integrity"), this,
-                      [this, partition] { CheckIntegrity(partition); });
-    }
     break;
   case EntryType::File:
     menu->addAction(tr("Extract File..."), this, [this, partition, path] {
@@ -327,35 +321,3 @@ void FilesystemWidget::ExtractFile(const DiscIO::Partition& partition, const QSt
   else
     ModalMessageBox::critical(this, tr("Error"), tr("Failed to extract file."));
 }
-
-void FilesystemWidget::CheckIntegrity(const DiscIO::Partition& partition)
-{
-  QProgressDialog* dialog = new QProgressDialog(this);
-  std::future<bool> is_valid = std::async(
-      std::launch::async, [this, partition] { return m_volume->CheckIntegrity(partition); });
-
-  dialog->setLabelText(tr("Verifying integrity of partition..."));
-  dialog->setWindowFlags(dialog->windowFlags() & ~Qt::WindowContextHelpButtonHint);
-  dialog->setWindowTitle(tr("Verifying partition"));
-
-  dialog->setMinimum(0);
-  dialog->setMaximum(0);
-  dialog->show();
-
-  while (is_valid.wait_for(std::chrono::milliseconds(50)) != std::future_status::ready)
-    QCoreApplication::processEvents();
-
-  dialog->close();
-
-  if (is_valid.get())
-  {
-    ModalMessageBox::information(this, tr("Success"),
-                                 tr("Integrity check completed. No errors have been found."));
-  }
-  else
-  {
-    ModalMessageBox::critical(this, tr("Error"),
-                              tr("Integrity check for partition failed. The disc image is most "
-                                 "likely corrupted or has been patched incorrectly."));
-  }
-}

Source/Core/DolphinQt/Config/FilesystemWidget.h

@@ -43,7 +43,6 @@ private:
                         const QString& out);
   void ExtractFile(const DiscIO::Partition& partition, const QString& path, const QString& out);
   bool ExtractSystemData(const DiscIO::Partition& partition, const QString& out);
-  void CheckIntegrity(const DiscIO::Partition& partition);
 
   DiscIO::Partition GetPartitionFromID(int id);