implement persistent psn title update api response cache

Clients/PsnClient/POCOs/TitlePatch.cs

@@ -13,6 +13,8 @@ namespace PsnClient.POCOs
         public string Status { get; set; }
         [XmlElement("tag")]
         public TitlePatchTag Tag { get; set; }
+        [XmlIgnore]
+        public bool OfflineCache { get; set; }
     }
 
     public class TitlePatchTag

Clients/PsnClient/PsnClient.cs

@@ -233,25 +233,26 @@ namespace PsnClient
             }
         }
 
-        public async Task<TitlePatch?> GetTitleUpdatesAsync(string? productId, CancellationToken cancellationToken)
+        public async Task<(TitlePatch? patch, string? responseXml)> GetTitleUpdatesAsync(string? productId, CancellationToken cancellationToken)
         {
             if (string.IsNullOrEmpty(productId))
-                return null;
+                return default;
             
             if (ResponseCache.TryGetValue(productId, out TitlePatch patchInfo))
-                return patchInfo;
+                return (patchInfo, default);
 
             using var message = new HttpRequestMessage(HttpMethod.Get, $"https://a0.ww.np.dl.playstation.net/tpl/np/{productId}/{productId}-ver.xml");
             using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
             try
             {
                 if (response.StatusCode == HttpStatusCode.NotFound)
-                    return null;
+                    return default;
 
                 await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
+                var xml = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
                 patchInfo = await response.Content.ReadAsAsync<TitlePatch>(xmlFormatters, cancellationToken).ConfigureAwait(false);
                 ResponseCache.Set(productId, patchInfo, ResponseCacheDuration);
-                return patchInfo;
+                return (patchInfo, xml);
             }
             catch (Exception e)
             {

CompatBot/Commands/Fortune.cs

@@ -31,12 +31,8 @@ namespace CompatBot.Commands
         
         public static async Task ShowFortune(DiscordMessage message, DiscordUser user)
         {
-            var prefix = DateTime.UtcNow.ToString("yyyyMMdd");
+            var prefix = DateTime.UtcNow.ToString("yyyyMMdd")+ user.Id.ToString("x16");
-            using var sha256 = System.Security.Cryptography.SHA256.Create();
+            var rng = new Random(prefix.GetStableHash());
-            var data = Encoding.UTF8.GetBytes(prefix + user.Id.ToString("x16"));
-            var hash = sha256.ComputeHash(data);
-            var seed = BitConverter.ToInt32(hash, 0);
-            var rng = new Random(seed);
             await using var db = new ThumbnailDb();
             Database.Fortune fortune;
             do

CompatBot/Commands/Psn.Check.cs

@@ -5,6 +5,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using CompatApiClient.Utils;
 using CompatBot.Database;
+using CompatBot.Database.Providers;
 using CompatBot.EventHandlers;
 using CompatBot.ThumbScrapper;
 using CompatBot.Utils;
@@ -56,13 +57,13 @@ namespace CompatBot.Commands
                 List<DiscordEmbedBuilder> embeds;
                 try
                 {
-                    var updateInfo = await Client.GetTitleUpdatesAsync(id, Config.Cts.Token).ConfigureAwait(false);
+                    var updateInfo = await TitleUpdateInfoProvider.GetAsync(id, Config.Cts.Token).ConfigureAwait(false);
                     embeds = await updateInfo.AsEmbedAsync(ctx.Client, id).ConfigureAwait(false);
                 }
                 catch (Exception e)
                 {
                     Config.Log.Warn(e, "Failed to get title update info");
-                    embeds = new List<DiscordEmbedBuilder>
+                    embeds = new()
                     {
                         new()
                         {
@@ -153,7 +154,7 @@ namespace CompatBot.Commands
                     await announcementChannel.SendMessageAsync(embed: embed).ConfigureAwait(false);
                     latestFwVersion = newVersion;
                     if (fwVersionState == null)
-                        await db.BotState.AddAsync(new BotState {Key = "Latest-Firmware-Version", Value = latestFwVersion}).ConfigureAwait(false);
+                        await db.BotState.AddAsync(new() {Key = "Latest-Firmware-Version", Value = latestFwVersion}).ConfigureAwait(false);
                     else
                         fwVersionState.Value = latestFwVersion;
                     await db.SaveChangesAsync().ConfigureAwait(false);

CompatBot/Database/Migrations/ThumbnailDb/20210414183007_AddGameUpdateInfo.Designer.cs

@@ -0,0 +1,297 @@
+// <auto-generated />
+using System;
+using CompatBot.Database;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+namespace CompatBot.Migrations
+{
+    [DbContext(typeof(ThumbnailDb))]
+    [Migration("20210414183007_AddGameUpdateInfo")]
+    partial class AddGameUpdateInfo
+    {
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "5.0.5");
+
+            modelBuilder.Entity("CompatBot.Database.Fortune", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Content")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("content");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.ToTable("fortune");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.GameUpdateInfo", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<int>("MetaHash")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("meta_hash");
+
+                    b.Property<string>("MetaXml")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("meta_xml");
+
+                    b.Property<string>("ProductCode")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("product_code");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.HasIndex("ProductCode")
+                        .IsUnique()
+                        .HasDatabaseName("game_update_info_product_code");
+
+                    b.ToTable("game_update_info");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.Metacritic", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<byte?>("CriticScore")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("critic_score");
+
+                    b.Property<string>("Notes")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("notes");
+
+                    b.Property<string>("Title")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("title");
+
+                    b.Property<byte?>("UserScore")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("user_score");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.ToTable("metacritic");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.NamePool", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("name");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.ToTable("name_pool");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.State", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Locale")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("locale");
+
+                    b.Property<long>("Timestamp")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("timestamp");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.HasIndex("Locale")
+                        .IsUnique()
+                        .HasDatabaseName("state_locale");
+
+                    b.HasIndex("Timestamp")
+                        .HasDatabaseName("state_timestamp");
+
+                    b.ToTable("state");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<string>("Function")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("function");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.HasIndex("Function")
+                        .HasDatabaseName("syscall_info_function");
+
+                    b.ToTable("syscall_info");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+                {
+                    b.Property<int>("ProductId")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("product_id");
+
+                    b.Property<int>("SyscallInfoId")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("syscall_info_id");
+
+                    b.HasKey("ProductId", "SyscallInfoId")
+                        .HasName("id");
+
+                    b.HasIndex("SyscallInfoId")
+                        .HasDatabaseName("ix_syscall_to_product_map_syscall_info_id");
+
+                    b.ToTable("syscall_to_product_map");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<long?>("CompatibilityChangeDate")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("compatibility_change_date");
+
+                    b.Property<byte?>("CompatibilityStatus")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("compatibility_status");
+
+                    b.Property<string>("ContentId")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("content_id");
+
+                    b.Property<int?>("EmbedColor")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("embed_color");
+
+                    b.Property<string>("EmbeddableUrl")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("embeddable_url");
+
+                    b.Property<int?>("MetacriticId")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("metacritic_id");
+
+                    b.Property<string>("Name")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("name");
+
+                    b.Property<string>("ProductCode")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("product_code");
+
+                    b.Property<long>("Timestamp")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("timestamp");
+
+                    b.Property<string>("Url")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("url");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.HasIndex("ContentId")
+                        .IsUnique()
+                        .HasDatabaseName("thumbnail_content_id");
+
+                    b.HasIndex("MetacriticId")
+                        .HasDatabaseName("ix_thumbnail_metacritic_id");
+
+                    b.HasIndex("ProductCode")
+                        .IsUnique()
+                        .HasDatabaseName("thumbnail_product_code");
+
+                    b.HasIndex("Timestamp")
+                        .HasDatabaseName("thumbnail_timestamp");
+
+                    b.ToTable("thumbnail");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+                {
+                    b.HasOne("CompatBot.Database.Thumbnail", "Product")
+                        .WithMany("SyscallToProductMap")
+                        .HasForeignKey("ProductId")
+                        .HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
+                        .WithMany("SyscallToProductMap")
+                        .HasForeignKey("SyscallInfoId")
+                        .HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("Product");
+
+                    b.Navigation("SyscallInfo");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
+                {
+                    b.HasOne("CompatBot.Database.Metacritic", "Metacritic")
+                        .WithMany()
+                        .HasForeignKey("MetacriticId")
+                        .HasConstraintName("fk_thumbnail_metacritic_metacritic_id");
+
+                    b.Navigation("Metacritic");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
+                {
+                    b.Navigation("SyscallToProductMap");
+                });
+
+            modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
+                {
+                    b.Navigation("SyscallToProductMap");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

CompatBot/Database/Migrations/ThumbnailDb/20210414183007_AddGameUpdateInfo.cs

@@ -0,0 +1,37 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace CompatBot.Migrations
+{
+    public partial class AddGameUpdateInfo : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "game_update_info",
+                columns: table => new
+                {
+                    id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    product_code = table.Column<string>(type: "TEXT", nullable: false),
+                    meta_hash = table.Column<int>(type: "INTEGER", nullable: false),
+                    meta_xml = table.Column<string>(type: "TEXT", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("id", x => x.id);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "game_update_info_product_code",
+                table: "game_update_info",
+                column: "product_code",
+                unique: true);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "game_update_info");
+        }
+    }
+}

CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs

@@ -14,7 +14,7 @@ namespace CompatBot.Migrations
         {
 #pragma warning disable 612, 618
             modelBuilder
-                .HasAnnotation("ProductVersion", "5.0.3");
+                .HasAnnotation("ProductVersion", "5.0.5");
 
             modelBuilder.Entity("CompatBot.Database.Fortune", b =>
                 {
@@ -34,6 +34,37 @@ namespace CompatBot.Migrations
                     b.ToTable("fortune");
                 });
 
+            modelBuilder.Entity("CompatBot.Database.GameUpdateInfo", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("id");
+
+                    b.Property<int>("MetaHash")
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("meta_hash");
+
+                    b.Property<string>("MetaXml")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("meta_xml");
+
+                    b.Property<string>("ProductCode")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("product_code");
+
+                    b.HasKey("Id")
+                        .HasName("id");
+
+                    b.HasIndex("ProductCode")
+                        .IsUnique()
+                        .HasDatabaseName("game_update_info_product_code");
+
+                    b.ToTable("game_update_info");
+                });
+
             modelBuilder.Entity("CompatBot.Database.Metacritic", b =>
                 {
                     b.Property<int>("Id")

CompatBot/Database/Providers/TitleUpdateInfoProvider.cs

@@ -0,0 +1,51 @@
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Xml.Serialization;
+using CompatBot.Utils;
+using PsnClient.POCOs;
+
+namespace CompatBot.Database.Providers
+{
+    public static class TitleUpdateInfoProvider
+    {
+        private static readonly PsnClient.Client Client = new();
+
+        public static async Task<TitlePatch?> GetAsync(string? productId, CancellationToken cancellationToken)
+        {
+            if (string.IsNullOrEmpty(productId))
+                return default;
+
+            productId = productId.ToUpper();
+            var (update, xml) = await Client.GetTitleUpdatesAsync(productId, cancellationToken).ConfigureAwait(false);
+            if (xml is string {Length: > 10})
+            {
+                var xmlChecksum = xml.GetStableHash();
+                await using var db = new ThumbnailDb();
+                var updateInfo = db.GameUpdateInfo.FirstOrDefault(ui => ui.ProductCode == productId);
+                if (updateInfo is null)
+                    db.GameUpdateInfo.Add(new() {ProductCode = productId, MetaHash = xmlChecksum, MetaXml = xml});
+                else if (updateInfo.MetaHash != xmlChecksum && update?.Tag?.Packages is {Length: >0})
+                {
+                    updateInfo.MetaHash = xmlChecksum;
+                    updateInfo.MetaXml = xml;
+                }
+                await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
+            }
+            if ((update?.Tag?.Packages?.Length ?? 0) == 0)
+            {
+                await using var db = new ThumbnailDb();
+                var updateInfo = db.GameUpdateInfo.FirstOrDefault(ui => ui.ProductCode == productId);
+                if (updateInfo is null)
+                    return update;
+
+                await using var memStream = Config.MemoryStreamManager.GetStream(Encoding.UTF8.GetBytes(updateInfo.MetaXml));
+                var xmlSerializer = new XmlSerializer(typeof(TitlePatch));
+                update = (TitlePatch?)xmlSerializer.Deserialize(memStream);
+            }
+            
+            return update;
+        }
+    }
+}

CompatBot/Database/ThumbnailDb.cs

@@ -10,6 +10,7 @@ namespace CompatBot.Database
     {
         public DbSet<State> State { get; set; } = null!;
         public DbSet<Thumbnail> Thumbnail { get; set; } = null!;
+        public DbSet<GameUpdateInfo> GameUpdateInfo { get; set; } = null!;
         public DbSet<SyscallInfo> SyscallInfo { get; set; } = null!;
         public DbSet<SyscallToProductMap> SyscallToProductMap { get; set; } = null!;
         public DbSet<Metacritic> Metacritic { get; set; } = null!;
@@ -33,6 +34,7 @@ namespace CompatBot.Database
             modelBuilder.Entity<Thumbnail>().HasIndex(m => m.ProductCode).IsUnique().HasDatabaseName("thumbnail_product_code");
             modelBuilder.Entity<Thumbnail>().HasIndex(m => m.ContentId).IsUnique().HasDatabaseName("thumbnail_content_id");
             modelBuilder.Entity<Thumbnail>().HasIndex(m => m.Timestamp).HasDatabaseName("thumbnail_timestamp");
+            modelBuilder.Entity<GameUpdateInfo>().HasIndex(ui => ui.ProductCode).IsUnique().HasDatabaseName("game_update_info_product_code");
             modelBuilder.Entity<SyscallInfo>().HasIndex(sci => sci.Function).HasDatabaseName("syscall_info_function");
             modelBuilder.Entity<SyscallToProductMap>().HasKey(m => new {m.ProductId, m.SyscallInfoId});
             modelBuilder.Entity<Fortune>();
@@ -73,6 +75,16 @@ namespace CompatBot.Database
         public List<SyscallToProductMap> SyscallToProductMap { get; set; } = null!;
     }
 
+    internal class GameUpdateInfo
+    {
+        public int Id { get; set; }
+        [Required]
+        public string ProductCode { get; set; } = null!;
+        public int MetaHash { get; set; }
+        [Required]
+        public string MetaXml { get; set; } = null!;
+    }
+
     public enum CompatStatus : byte
     {
         Unknown = 0,

CompatBot/Utils/Extensions/StringUtils.cs

@@ -443,6 +443,14 @@ namespace CompatBot.Utils
             return string.Compare(a, b, CultureInfo.InvariantCulture, CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreWidth | CompareOptions.IgnoreCase) == 0;
         }
 
+        internal static int GetStableHash(this string str)
+        {
+            using var sha256 = System.Security.Cryptography.SHA256.Create();
+            var data = Encoding.UTF8.GetBytes(str);
+            var hash = sha256.ComputeHash(data);
+            return BitConverter.ToInt32(hash, 0);
+        }
+
         private static double GetScoreWithAcronym(this string strA, string strB)
         {
             var fullMatch = strA.DiceIshCoefficientIsh(strB);

CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs

@@ -251,7 +251,7 @@ namespace CompatBot.Utils.ResultFormatters
                     collection["game_category"] = "HG";
                 }
                 serial = collection["serial"] ?? "";
-                var titleUpdateInfoTask = PsnClient.GetTitleUpdatesAsync(serial, Config.Cts.Token);
+                var titleUpdateInfoTask = TitleUpdateInfoProvider.GetAsync(serial, Config.Cts.Token);
                 var titleMetaTask = PsnClient.GetTitleMetaAsync(serial, Config.Cts.Token);
                 var gameInfo = await client.LookupGameInfoWithEmbedAsync(serial, collection["game_title"], true, category: collection["game_category"]).ConfigureAwait(false);
                 try