add supported languages info to product code info embed

Clients/CompatApiClient/POCOs/CompatResult.cs

@@ -36,6 +36,7 @@ public class TitleInfo
     public int? Network;
     public string Update;
     public bool? UsingLocalCache;
+    public IReadOnlyCollection<string> Languages;
 }
     
 #nullable restore

Clients/IrdLibraryClient/IrdClient.cs

@@ -1,12 +1,16 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml;
+using System.Xml.Linq;
+using System.Xml.Serialization;
 using CompatApiClient;
 using CompatApiClient.Compression;
 using IrdLibraryClient.IrdFormat;
@@ -18,6 +22,7 @@ namespace IrdLibraryClient
     public class IrdClient
     {
         private static readonly Uri BaseDownloadUri = new("https://github.com/FlexBy420/playstation_3_ird_database/raw/main/");
+        private static readonly Uri RedumpDatDownloadUri = new("http://redump.org/datfile/ps3/serial,version");
         private static readonly HttpClient Client = HttpClientFactory.Create(new CompressionMessageHandler());
         private static readonly JsonSerializerOptions JsonOptions= new()
         {
@@ -127,6 +132,98 @@ namespace IrdLibraryClient
             }
         }
 
+        public static async ValueTask<XDocument?> GetRedumpDatfileAsync(string localCachePath, CancellationToken cancellationToken)
+        {
+            try
+            {
+                string? localZipFilePath;
+                if (Directory.Exists(localCachePath)
+                    && Directory.EnumerateFiles(
+                        localCachePath,
+                        "*Datfile*.zip",
+                        new EnumerationOptions { IgnoreInaccessible = true, RecurseSubdirectories = false }
+                    ).OrderBy(n => n).LastOrDefault() is string localFilePath)
+                {
+                    if (new FileInfo(localFilePath).CreationTimeUtc.AddDays(7) > DateTime.UtcNow)
+                        localZipFilePath = localFilePath;
+                    else
+                        localZipFilePath = await DownloadLatestRedumpDatfileAsync(localCachePath, cancellationToken).ConfigureAwait(false)
+                                           ?? localFilePath;
+                }
+                else
+                    localZipFilePath = await DownloadLatestRedumpDatfileAsync(localCachePath, cancellationToken).ConfigureAwait(false);
+                if (localZipFilePath is not { Length: > 0 })
+                    return null;
+
+                await using var zipStream = File.Open(localZipFilePath, new FileStreamOptions
+                {
+                    Mode = FileMode.Open,
+                    Access = FileAccess.Read,
+                    Share = FileShare.Read,
+                    Options = FileOptions.Asynchronous | FileOptions.SequentialScan,
+                });
+                using var zipFile = new ZipArchive(zipStream, ZipArchiveMode.Read, true);
+                var fileEntry = zipFile.Entries.FirstOrDefault(e => e.Name.EndsWith(".dat", StringComparison.OrdinalIgnoreCase));
+                if (fileEntry is null)
+                {
+                    ApiConfig.Log.Warn($"No datfile inside {localZipFilePath}");
+                    return null;
+                }
+
+                await using var xmlStream = fileEntry.Open();
+                using var xmlReader = XmlReader.Create(xmlStream, new(){ Async = true, DtdProcessing = DtdProcessing.Ignore });
+                if (await XDocument.LoadAsync(xmlReader, LoadOptions.None, cancellationToken).ConfigureAwait(false) is not XDocument doc)
+                {
+                    ApiConfig.Log.Warn($"Failed to deserialize {fileEntry.Name} from {localZipFilePath}");
+                    return null;
+                }
+
+                return doc;
+            }
+            catch (Exception e)
+            {
+                ApiConfig.Log.Warn(e, "Failed to get redump datfile content.");
+            }
+            return null;
+        }
+
+        private static async ValueTask<string?> DownloadLatestRedumpDatfileAsync(string localCachePath, CancellationToken cancellationToken)
+        {
+            string? localFilePath = null;
+            using var request = new HttpRequestMessage(HttpMethod.Get, RedumpDatDownloadUri);
+            var response = await Client.SendAsync(request, cancellationToken).ConfigureAwait(false);
+            if (response.IsSuccessStatusCode &&
+                response.Content.Headers.ContentDisposition?.FileName is string filename)
+            {
+                if (filename.StartsWith('"') && filename.EndsWith('"'))
+                    filename = filename[1..^1];
+                ApiConfig.Log.Info($"Latest redump datfile snapshot: {filename}");
+                var localCacheFilename = Path.Combine(localCachePath, filename);
+                if (File.Exists(localCacheFilename))
+                {
+                    ApiConfig.Log.Info("Using local copy of redump datfile snapshot");
+                    localFilePath = localCacheFilename;
+                }
+                else
+                {
+                    var resultBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
+                    try
+                    {
+                        if (!Directory.Exists(localCachePath))
+                            Directory.CreateDirectory(localCachePath);
+                        ApiConfig.Log.Info($"Saving latest redump datfile snapshot in local cache: {filename}...");
+                        await File.WriteAllBytesAsync(localCacheFilename, resultBytes, cancellationToken);
+                        localFilePath = localCacheFilename;
+                    }
+                    catch (Exception ex)
+                    {
+                        ApiConfig.Log.Warn(ex, $"Failed to write {filename} to local cache: {ex.Message}");
+                    }
+                }
+            }
+            return localFilePath;
+        }
+        
         private static string EscapeSegments(string relativePath)
         {
             var segments = relativePath.Split('/');

CompatBot/Config.cs

@@ -87,6 +87,7 @@ internal static class Config
     public static string PreferredFontFamily => config.GetValue(nameof(PreferredFontFamily), "");
     public static string LogPath => config.GetValue(nameof(LogPath), "./logs/"); // paths are relative to the working directory
     public static string IrdCachePath => config.GetValue(nameof(IrdCachePath), "./ird/");
+    public static string RedumpDatfileCachePath => config.GetValue(nameof(RedumpDatfileCachePath), "./datfile/");
     public static string RenameNameSuffix => config.GetValue(nameof(RenameNameSuffix), " (Rule 7)");
 
     public static double GameTitleMatchThreshold => config.GetValue(nameof(GameTitleMatchThreshold), 0.57);

CompatBot/Database/Providers/DiscLanguageProvider.cs

@@ -0,0 +1,109 @@
+using System.Data;
+using System.IO;
+using System.Text.RegularExpressions;
+using CompatBot.EventHandlers;
+using IrdLibraryClient;
+
+namespace CompatBot.Database.Providers;
+
+public static partial class DiscLanguageProvider
+{
+    [GeneratedRegex(@"^(?<title>.+?) \((?<region>\w+(, \w+)*)\)( \((?<lang>\w{2}(,\w{2})*)\))?", RegexOptions.ExplicitCapture | RegexOptions.Singleline)]
+    private static partial Regex RedumpName();
+    private static readonly Dictionary<string, List<string>> ProductCodeToVersionAndLangList = new();
+    
+    public static async Task RefreshAsync(CancellationToken cancellationToken)
+    {
+        var datXml = await IrdClient.GetRedumpDatfileAsync(Config.RedumpDatfileCachePath, cancellationToken).ConfigureAwait(false);
+        if (datXml?.Root?.Descendants("game").ToList() is not { Count: > 0 } gameList)
+            return;
+
+        foreach (var gameInfo in gameList)
+        {
+            var name = (string?)gameInfo.Attribute("name");
+            var version = (string?)gameInfo.Element("version") ?? "01.00";
+            var serialList = ((string?)gameInfo.Element("serial"))?
+                .Replace(" ", "")
+                .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                .SelectMany(ProductCodeLookup.GetProductIds)
+                .Distinct()
+                .ToList();
+#if DEBUG
+            var desc = (string?)gameInfo.Element("description");
+            if (name != desc)
+                throw new InvalidDataException("Unexpected datfile format discrepancy");
+#endif
+            if (serialList is not { Count: > 0 } || name is not {Length: >0})
+                continue;
+            var langs = ParseLangList(name);
+            foreach (var serial in serialList)
+            {
+                if (!ProductCodeToVersionAndLangList.TryGetValue(serial, out var listOfLangs))
+                    ProductCodeToVersionAndLangList[serial] = listOfLangs = [];
+                if (listOfLangs.Any(l => l.Equals(langs, StringComparison.OrdinalIgnoreCase)))
+                    continue;
+                listOfLangs.Add(langs);
+            }
+        }
+    }
+    
+    public static async ValueTask<IReadOnlyList<string>> GetLanguageListAsync(string productCode)
+    {
+        if (!ProductCodeToVersionAndLangList.TryGetValue(productCode, out var listOfLangs))
+            return [];
+        return listOfLangs.AsReadOnly();
+    }
+
+    private static string ParseLangList(string name)
+    {
+        if (RedumpName().Match(name) is not { Success: true } match)
+            return "";
+
+        if (match.Groups["lang"].Value is { Length: > 0 } lang)
+        {
+            var langs = lang.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                .Distinct()
+                .OrderBy(l => l, StringComparer.OrdinalIgnoreCase)
+                .ToList();
+            return string.Join(",", langs);
+        }
+
+        if (match.Groups["region"].Value is not { Length: > 0 } region)
+            return "";
+
+        var langList = region.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+            .Select(MapRegionToLang)
+            .Distinct()
+            .Where(l => l is { Length: > 0 })
+            .OrderBy(l => l, StringComparer.OrdinalIgnoreCase)
+            .ToList();
+        return string.Join(",", langList);
+    }
+
+    private static string MapRegionToLang(string region)
+        => region switch
+        {
+            "Japan" => "Ja",
+            "Asia" => "Ja",
+            "USA" => "En",
+            "Europe" => "En",
+            "UK" => "En",
+            "Australia" => "En",
+            "Canada" => "En",
+            "India" => "En",
+            "Korea" => "Ko",
+            "Brazil" => "Es",
+            "Spain" => "Es",
+            "Mexico" => "Es",
+            "Poland" => "Pl",
+            "Germany" => "De",
+            "Austria" => "De",
+            "Switzerland" => "De",
+            "Italy" => "It",
+            "France" => "Fr",
+            "Greece" => "El",
+            "Russia" => "Ru",
+            "Turkey" => "Tr",
+            _ => throw new InvalidDataException($"No mapping from region {region} to language")
+        };
+}

CompatBot/Program.cs

@@ -113,6 +113,7 @@ internal static class Program
                 ThumbScrapper.GameTdbScraper.RunAsync(Config.Cts.Token),
                 //TitleUpdateInfoProvider.RefreshGameUpdateInfoAsync(Config.Cts.Token),
 #endif
+                DiscLanguageProvider.RefreshAsync(Config.Cts.Token),
                 StatsStorage.BackgroundSaveAsync(),
                 CompatList.ImportCompatListAsync(),
                 Config.GetAzureDevOpsClient().GetPipelineDurationAsync(Config.Cts.Token),

CompatBot/Utils/ResultFormatters/TitleInfoFormatter.cs

@@ -59,7 +59,7 @@ internal static class TitleInfoFormatter
         string? titleId,
         string? gameTitle = null,
         bool forLog = false,
-        string? thumbnailUrl = null
+        string thumbnailUrl = ""
     )
     {
         if (string.IsNullOrWhiteSpace(gameTitle))
@@ -81,6 +81,8 @@ internal static class TitleInfoFormatter
                 };
             }
         }
+        if (titleId is {Length: 9})
+            info.Languages = await DiscLanguageProvider.GetLanguageListAsync(titleId).ConfigureAwait(false);
         if (info.Status is string status && StatusColors.TryGetValue(status, out var color))
         {
             // apparently there's no formatting in the footer, but you need to escape everything in description; ugh
@@ -106,7 +108,8 @@ internal static class TitleInfoFormatter
                 Url = info.Thread > 0 ? $"https://forums.rpcs3.net/thread-{info.Thread}.html" : null,
                 Description = desc,
                 Color = color,
-            }.WithThumbnail(thumbnailUrl);
+            }.WithThumbnail(thumbnailUrl)
+            .WithLanguages(info.Languages);
         }
         else
         {
@@ -119,20 +122,20 @@ internal static class TitleInfoFormatter
             else
             {
                 embedColor = Config.Colors.CompatStatusUnknown;
-                if (!string.IsNullOrEmpty(titleId))
+                if (titleId is {Length: >0})
                     desc = $"Product code {titleId} was not found in compatibility database";
             }
             var result = new DiscordEmbedBuilder
             {
                 Description = desc,
                 Color = embedColor,
-            }.WithThumbnail(thumbnailUrl);
-            if (gameTitle == null
-                && !string.IsNullOrEmpty(titleId)
-                && ThumbnailProvider.GetTitleNameAsync(titleId, Config.Cts.Token).ConfigureAwait(false).GetAwaiter().GetResult() is string titleName
-                && !string.IsNullOrEmpty(titleName))
+            }.WithThumbnail(thumbnailUrl)
+            .WithLanguages(info.Languages);
+            if (gameTitle is null
+                && titleId is {Length: >0}
+                && await ThumbnailProvider.GetTitleNameAsync(titleId, Config.Cts.Token).ConfigureAwait(false) is {Length: >0} titleName)
                 gameTitle = titleName;
-            if (!string.IsNullOrEmpty(gameTitle))
+            if (gameTitle is {Length: >0})
             {
                 StatsStorage.IncGameStat(gameTitle);
                 result.Title = $"{productCodePart}{gameTitle.Sanitize().Trim(200)}";
@@ -141,12 +144,17 @@ internal static class TitleInfoFormatter
         }
     }
 
+    public static DiscordEmbedBuilder WithLanguages(this DiscordEmbedBuilder embedBuilder, IReadOnlyCollection<string> languages)
+    {
+        if (languages is not {Count: >0})
+            return embedBuilder;
+
+        return embedBuilder.AddField(
+            "Supported Languages",
+            string.Join('\n', languages)
+        );
+    }
+
     public static string AsString(this (string code, TitleInfo info, double score) resultInfo)
         => resultInfo.info.AsString(resultInfo.code);
-
-    public static string AsString(this KeyValuePair<string, TitleInfo> resultInfo)
-        => resultInfo.Value.AsString(resultInfo.Key);
-
-    public static ValueTask<DiscordEmbedBuilder> AsEmbedAsync(this KeyValuePair<string, TitleInfo> resultInfo)
-        => resultInfo.Value.AsEmbedAsync(resultInfo.Key);
 }