From a7b47e7a3448220acbab8902de0b32d0db6f31cd Mon Sep 17 00:00:00 2001
From: Mike Nelson <mjn.other@gmail.com>
Date: Thu, 3 Apr 2025 14:56:28 -0500
Subject: [PATCH] Hash check retail strings and weapons tables (#6662)

* hash check retail strings and weapons tables

* const
---
 code/parse/md5_hash.cpp  | 18 ++++++++++++++++++
 code/parse/md5_hash.h    |  9 +++++++++
 code/parse/parselo.cpp   | 39 ++++++++++++++++++++++++++++++++++-----
 code/source_groups.cmake |  2 ++
 4 files changed, 63 insertions(+), 5 deletions(-)
 create mode 100644 code/parse/md5_hash.cpp
 create mode 100644 code/parse/md5_hash.h

diff --git a/code/parse/md5_hash.cpp b/code/parse/md5_hash.cpp
new file mode 100644
index 0000000000..dccc8f643e
--- /dev/null
+++ b/code/parse/md5_hash.cpp
@@ -0,0 +1,18 @@
+#include "md5_hash.h"
+
+#include "globalincs/vmallocator.h"
+
+#include "md5.h"
+
+SCP_string md5_hash(const char* text, size_t length)
+{
+	MD5 md5;
+	md5.update(text, static_cast<MD5::size_type>(length));
+	md5.finalize();
+	return md5.hexdigest();
+}
+
+SCP_string md5_hash(const SCP_string& text)
+{
+	return md5_hash(text.c_str(), text.size());
+}
diff --git a/code/parse/md5_hash.h b/code/parse/md5_hash.h
new file mode 100644
index 0000000000..0ea59c5f54
--- /dev/null
+++ b/code/parse/md5_hash.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "globalincs/vmallocator.h"
+
+// Hashes a block of text as raw bytes
+SCP_string md5_hash(const char* text, size_t length);
+
+// Hashes an SCP_string
+SCP_string md5_hash(const SCP_string& text);
diff --git a/code/parse/parselo.cpp b/code/parse/parselo.cpp
index 314d50d0c9..39d859ccc4 100644
--- a/code/parse/parselo.cpp
+++ b/code/parse/parselo.cpp
@@ -20,6 +20,7 @@
 #include "localization/localize.h"
 #include "mission/missionparse.h"
 #include "parse/encrypt.h"
+#include "parse/md5_hash.h"
 #include "parse/parselo.h"
 #include "parse/sexp.h"
 #include "ship/ship.h"
@@ -60,6 +61,11 @@ SCP_vector<Bookmark> Bookmarks;	// Stack of all our previously paused parsing
 void allocate_parse_text(size_t size);
 static size_t Parse_text_size = 0;
 
+static const SCP_unordered_map<SCP_string, SCP_string> retail_hashes = {
+	{"strings.tbl", "84ab6e5392d7c54752a61161aac9f9fd"},
+	{"weapons.tbl", "ca2c7f305b1f36988c2bb8c371ab2027"}
+};
+
 
 //	Return true if this character is white space, else false.
 int is_white_space(char ch)
@@ -2418,12 +2424,35 @@ void read_raw_file_text(const char *filename, int mode, char *raw_text)
 
 			// We do the additional can_reallocate check here since we need control over raw_text to reencode the file
 			if (isLatin1 && can_reallocate) {
-				// Latin1 is the encoding of retail data and for legacy reasons we convert that to UTF-8.
-				// We still output a warning though...
-				Warning(LOCATION, "Found Latin-1 encoded file %s. This file will be automatically converted to UTF-8 but "
-						"it may cause parsing issues with retail FS2 files since those contained invalid data.\n"
-						"To silence this warning you must convert the files to UTF-8, e.g. by using a program like iconv.",
+
+				// Some retail files are known to be safe to convert to utf-8 so validate that here and only warn otherwise
+				bool downgrade_warning = false;
+
+				// Compare filename by lowercase string
+				SCP_string key = filename;
+				std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+
+				// Check for a hash
+				auto it = retail_hashes.find(key);
+				if (it != retail_hashes.end()) {
+					const auto hash = md5_hash(raw_text, strlen(raw_text));
+					if (it->second == hash) {
+						downgrade_warning = true;
+					} else {
+						mprintf(("Found Latin-1 encoded retail file %s with non-matching hash '%s'\n", filename, hash.c_str()));
+					}
+				}
+
+				// Log if a retail file was converted. Warn otherwise.
+				if (downgrade_warning) {
+					mprintf(("Found Latin-1 encoded retail file %s. The file will be automatically converted to UTF-8.\n", filename));
+				} else {
+					Warning(LOCATION, "Found Latin-1 encoded file %s. This file will be automatically converted to UTF-8 but "
+						"it may cause parsing issues with some files if they contained invalid data.\n"
+						"To silence this warning you must convert the files to UTF-8, e.g. by using a program like "
+						"iconv.",
 						filename);
+				}
 
 				// SDL2 has iconv functionality so we use that to convert from Latin1 to UTF-8
 
diff --git a/code/source_groups.cmake b/code/source_groups.cmake
index 07e05c744b..ee08e71603 100644
--- a/code/source_groups.cmake
+++ b/code/source_groups.cmake
@@ -1076,6 +1076,8 @@ add_file_folder("Parse"
 	parse/encrypt.h
 	parse/generic_log.cpp
 	parse/generic_log.h
+	parse/md5_hash.cpp
+	parse/md5_hash.h
 	parse/parsehi.cpp
 	parse/parsehi.h
 	parse/parselo.cpp