RPCS3/discord-bot
1
0
Fork 0
mirror of https://github.com/RPCS3/discord-bot.git synced 2024-11-23 18:29:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

implement persistent psn title update api response cache

Browse Source
This commit is contained in:
13xforever 2021-04-15 00:01:48 +05:00
parent 6fd9af844b
commit bf440604e2
11 changed files with 452 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Clients/PsnClient/POCOs/TitlePatch.cs
View File

@ -13,6 +13,8 @@ namespace PsnClient.POCOs
public string Status { get; set; }
[XmlElement("tag")]
public TitlePatchTag Tag { get; set; }
[XmlIgnore]
public bool OfflineCache { get; set; }
}
public class TitlePatchTag

11
Clients/PsnClient/PsnClient.cs
View File

@ -233,25 +233,26 @@ namespace PsnClient
}
}
public async Task<TitlePatch?> GetTitleUpdatesAsync(string? productId, CancellationToken cancellationToken)
public async Task<(TitlePatch? patch, string? responseXml)> GetTitleUpdatesAsync(string? productId, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(productId))
return null;
return default;
if (ResponseCache.TryGetValue(productId, out TitlePatch patchInfo))
return patchInfo;
return (patchInfo, default);
using var message = new HttpRequestMessage(HttpMethod.Get, $"https://a0.ww.np.dl.playstation.net/tpl/np/{productId}/{productId}-ver.xml");
using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
try
{
if (response.StatusCode == HttpStatusCode.NotFound)
return null;
return default;
await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
var xml = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
patchInfo = await response.Content.ReadAsAsync<TitlePatch>(xmlFormatters, cancellationToken).ConfigureAwait(false);
ResponseCache.Set(productId, patchInfo, ResponseCacheDuration);
return patchInfo;
return (patchInfo, xml);
}
catch (Exception e)
{

8
CompatBot/Commands/Fortune.cs
View File

@ -31,12 +31,8 @@ namespace CompatBot.Commands
public static async Task ShowFortune(DiscordMessage message, DiscordUser user)
{
var prefix = DateTime.UtcNow.ToString("yyyyMMdd");
using var sha256 = System.Security.Cryptography.SHA256.Create();
var data = Encoding.UTF8.GetBytes(prefix + user.Id.ToString("x16"));
var hash = sha256.ComputeHash(data);
var seed = BitConverter.ToInt32(hash, 0);
var rng = new Random(seed);
var prefix = DateTime.UtcNow.ToString("yyyyMMdd")+ user.Id.ToString("x16");
var rng = new Random(prefix.GetStableHash());
await using var db = new ThumbnailDb();
Database.Fortune fortune;
do

7
CompatBot/Commands/Psn.Check.cs
View File

@ -5,6 +5,7 @@ using System.Threading;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers;
using CompatBot.ThumbScrapper;
using CompatBot.Utils;
@ -56,13 +57,13 @@ namespace CompatBot.Commands
List<DiscordEmbedBuilder> embeds;
try
{
var updateInfo = await Client.GetTitleUpdatesAsync(id, Config.Cts.Token).ConfigureAwait(false);
var updateInfo = await TitleUpdateInfoProvider.GetAsync(id, Config.Cts.Token).ConfigureAwait(false);
embeds = await updateInfo.AsEmbedAsync(ctx.Client, id).ConfigureAwait(false);
}
catch (Exception e)
{
Config.Log.Warn(e, "Failed to get title update info");
embeds = new List<DiscordEmbedBuilder>
embeds = new()
{
new()
{
@ -153,7 +154,7 @@ namespace CompatBot.Commands
await announcementChannel.SendMessageAsync(embed: embed).ConfigureAwait(false);
latestFwVersion = newVersion;
if (fwVersionState == null)
await db.BotState.AddAsync(new BotState {Key = "Latest-Firmware-Version", Value = latestFwVersion}).ConfigureAwait(false);
await db.BotState.AddAsync(new() {Key = "Latest-Firmware-Version", Value = latestFwVersion}).ConfigureAwait(false);
else
fwVersionState.Value = latestFwVersion;
await db.SaveChangesAsync().ConfigureAwait(false);

297
CompatBot/Database/Migrations/ThumbnailDb/20210414183007_AddGameUpdateInfo.Designer.cs generated Normal file
View File

@ -0,0 +1,297 @@
// <auto-generated />
using System;
using CompatBot.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace CompatBot.Migrations
{
[DbContext(typeof(ThumbnailDb))]
[Migration("20210414183007_AddGameUpdateInfo")]
partial class AddGameUpdateInfo
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "5.0.5");
modelBuilder.Entity("CompatBot.Database.Fortune", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Content")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("content");
b.HasKey("Id")
.HasName("id");
b.ToTable("fortune");
});
modelBuilder.Entity("CompatBot.Database.GameUpdateInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<int>("MetaHash")
.HasColumnType("INTEGER")
.HasColumnName("meta_hash");
b.Property<string>("MetaXml")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("meta_xml");
b.Property<string>("ProductCode")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("product_code");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ProductCode")
.IsUnique()
.HasDatabaseName("game_update_info_product_code");
b.ToTable("game_update_info");
});
modelBuilder.Entity("CompatBot.Database.Metacritic", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<byte?>("CriticScore")
.HasColumnType("INTEGER")
.HasColumnName("critic_score");
b.Property<string>("Notes")
.HasColumnType("TEXT")
.HasColumnName("notes");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("title");
b.Property<byte?>("UserScore")
.HasColumnType("INTEGER")
.HasColumnName("user_score");
b.HasKey("Id")
.HasName("id");
b.ToTable("metacritic");
});
modelBuilder.Entity("CompatBot.Database.NamePool", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("name");
b.HasKey("Id")
.HasName("id");
b.ToTable("name_pool");
});
modelBuilder.Entity("CompatBot.Database.State", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Locale")
.HasColumnType("TEXT")
.HasColumnName("locale");
b.Property<long>("Timestamp")
.HasColumnType("INTEGER")
.HasColumnName("timestamp");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Locale")
.IsUnique()
.HasDatabaseName("state_locale");
b.HasIndex("Timestamp")
.HasDatabaseName("state_timestamp");
b.ToTable("state");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Function")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("function");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Function")
.HasDatabaseName("syscall_info_function");
b.ToTable("syscall_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.Property<int>("ProductId")
.HasColumnType("INTEGER")
.HasColumnName("product_id");
b.Property<int>("SyscallInfoId")
.HasColumnType("INTEGER")
.HasColumnName("syscall_info_id");
b.HasKey("ProductId", "SyscallInfoId")
.HasName("id");
b.HasIndex("SyscallInfoId")
.HasDatabaseName("ix_syscall_to_product_map_syscall_info_id");
b.ToTable("syscall_to_product_map");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<long?>("CompatibilityChangeDate")
.HasColumnType("INTEGER")
.HasColumnName("compatibility_change_date");
b.Property<byte?>("CompatibilityStatus")
.HasColumnType("INTEGER")
.HasColumnName("compatibility_status");
b.Property<string>("ContentId")
.HasColumnType("TEXT")
.HasColumnName("content_id");
b.Property<int?>("EmbedColor")
.HasColumnType("INTEGER")
.HasColumnName("embed_color");
b.Property<string>("EmbeddableUrl")
.HasColumnType("TEXT")
.HasColumnName("embeddable_url");
b.Property<int?>("MetacriticId")
.HasColumnType("INTEGER")
.HasColumnName("metacritic_id");
b.Property<string>("Name")
.HasColumnType("TEXT")
.HasColumnName("name");
b.Property<string>("ProductCode")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("product_code");
b.Property<long>("Timestamp")
.HasColumnType("INTEGER")
.HasColumnName("timestamp");
b.Property<string>("Url")
.HasColumnType("TEXT")
.HasColumnName("url");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ContentId")
.IsUnique()
.HasDatabaseName("thumbnail_content_id");
b.HasIndex("MetacriticId")
.HasDatabaseName("ix_thumbnail_metacritic_id");
b.HasIndex("ProductCode")
.IsUnique()
.HasDatabaseName("thumbnail_product_code");
b.HasIndex("Timestamp")
.HasDatabaseName("thumbnail_timestamp");
b.ToTable("thumbnail");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.HasOne("CompatBot.Database.Thumbnail", "Product")
.WithMany("SyscallToProductMap")
.HasForeignKey("ProductId")
.HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
.WithMany("SyscallToProductMap")
.HasForeignKey("SyscallInfoId")
.HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Product");
b.Navigation("SyscallInfo");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.HasOne("CompatBot.Database.Metacritic", "Metacritic")
.WithMany()
.HasForeignKey("MetacriticId")
.HasConstraintName("fk_thumbnail_metacritic_metacritic_id");
b.Navigation("Metacritic");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Navigation("SyscallToProductMap");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Navigation("SyscallToProductMap");
});
#pragma warning restore 612, 618
}
}
}

37
CompatBot/Database/Migrations/ThumbnailDb/20210414183007_AddGameUpdateInfo.cs Normal file
View File

@ -0,0 +1,37 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace CompatBot.Migrations
{
public partial class AddGameUpdateInfo : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "game_update_info",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
product_code = table.Column<string>(type: "TEXT", nullable: false),
meta_hash = table.Column<int>(type: "INTEGER", nullable: false),
meta_xml = table.Column<string>(type: "TEXT", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("id", x => x.id);
});
migrationBuilder.CreateIndex(
name: "game_update_info_product_code",
table: "game_update_info",
column: "product_code",
unique: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "game_update_info");
}
}
}

33
CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
View File

@ -14,7 +14,7 @@ namespace CompatBot.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "5.0.3");
.HasAnnotation("ProductVersion", "5.0.5");
modelBuilder.Entity("CompatBot.Database.Fortune", b =>
{
@ -34,6 +34,37 @@ namespace CompatBot.Migrations
b.ToTable("fortune");
});
modelBuilder.Entity("CompatBot.Database.GameUpdateInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<int>("MetaHash")
.HasColumnType("INTEGER")
.HasColumnName("meta_hash");
b.Property<string>("MetaXml")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("meta_xml");
b.Property<string>("ProductCode")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("product_code");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ProductCode")
.IsUnique()
.HasDatabaseName("game_update_info_product_code");
b.ToTable("game_update_info");
});
modelBuilder.Entity("CompatBot.Database.Metacritic", b =>
{
b.Property<int>("Id")

51
CompatBot/Database/Providers/TitleUpdateInfoProvider.cs Normal file
View File

@ -0,0 +1,51 @@
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Serialization;
using CompatBot.Utils;
using PsnClient.POCOs;
namespace CompatBot.Database.Providers
{
public static class TitleUpdateInfoProvider
{
private static readonly PsnClient.Client Client = new();
public static async Task<TitlePatch?> GetAsync(string? productId, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(productId))
return default;
productId = productId.ToUpper();
var (update, xml) = await Client.GetTitleUpdatesAsync(productId, cancellationToken).ConfigureAwait(false);
if (xml is string {Length: > 10})
{
var xmlChecksum = xml.GetStableHash();
await using var db = new ThumbnailDb();
var updateInfo = db.GameUpdateInfo.FirstOrDefault(ui => ui.ProductCode == productId);
if (updateInfo is null)
db.GameUpdateInfo.Add(new() {ProductCode = productId, MetaHash = xmlChecksum, MetaXml = xml});
else if (updateInfo.MetaHash != xmlChecksum && update?.Tag?.Packages is {Length: >0})
{
updateInfo.MetaHash = xmlChecksum;
updateInfo.MetaXml = xml;
}
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
if ((update?.Tag?.Packages?.Length ?? 0) == 0)
{
await using var db = new ThumbnailDb();
var updateInfo = db.GameUpdateInfo.FirstOrDefault(ui => ui.ProductCode == productId);
if (updateInfo is null)
return update;
await using var memStream = Config.MemoryStreamManager.GetStream(Encoding.UTF8.GetBytes(updateInfo.MetaXml));
var xmlSerializer = new XmlSerializer(typeof(TitlePatch));
update = (TitlePatch?)xmlSerializer.Deserialize(memStream);
}
return update;
}
}
}

12
CompatBot/Database/ThumbnailDb.cs
View File

@ -10,6 +10,7 @@ namespace CompatBot.Database
{
public DbSet<State> State { get; set; } = null!;
public DbSet<Thumbnail> Thumbnail { get; set; } = null!;
public DbSet<GameUpdateInfo> GameUpdateInfo { get; set; } = null!;
public DbSet<SyscallInfo> SyscallInfo { get; set; } = null!;
public DbSet<SyscallToProductMap> SyscallToProductMap { get; set; } = null!;
public DbSet<Metacritic> Metacritic { get; set; } = null!;
@ -33,6 +34,7 @@ namespace CompatBot.Database
modelBuilder.Entity<Thumbnail>().HasIndex(m => m.ProductCode).IsUnique().HasDatabaseName("thumbnail_product_code");
modelBuilder.Entity<Thumbnail>().HasIndex(m => m.ContentId).IsUnique().HasDatabaseName("thumbnail_content_id");
modelBuilder.Entity<Thumbnail>().HasIndex(m => m.Timestamp).HasDatabaseName("thumbnail_timestamp");
modelBuilder.Entity<GameUpdateInfo>().HasIndex(ui => ui.ProductCode).IsUnique().HasDatabaseName("game_update_info_product_code");
modelBuilder.Entity<SyscallInfo>().HasIndex(sci => sci.Function).HasDatabaseName("syscall_info_function");
modelBuilder.Entity<SyscallToProductMap>().HasKey(m => new {m.ProductId, m.SyscallInfoId});
modelBuilder.Entity<Fortune>();
@ -73,6 +75,16 @@ namespace CompatBot.Database
public List<SyscallToProductMap> SyscallToProductMap { get; set; } = null!;
}
internal class GameUpdateInfo
{
public int Id { get; set; }
[Required]
public string ProductCode { get; set; } = null!;
public int MetaHash { get; set; }
[Required]
public string MetaXml { get; set; } = null!;
}
public enum CompatStatus : byte
{
Unknown = 0,

8
CompatBot/Utils/Extensions/StringUtils.cs
View File

@ -443,6 +443,14 @@ namespace CompatBot.Utils
return string.Compare(a, b, CultureInfo.InvariantCulture, CompareOptions.IgnoreNonSpace | CompareOptions.IgnoreWidth | CompareOptions.IgnoreCase) == 0;
}
internal static int GetStableHash(this string str)
{
using var sha256 = System.Security.Cryptography.SHA256.Create();
var data = Encoding.UTF8.GetBytes(str);
var hash = sha256.ComputeHash(data);
return BitConverter.ToInt32(hash, 0);
}
private static double GetScoreWithAcronym(this string strA, string strB)
{
var fullMatch = strA.DiceIshCoefficientIsh(strB);

2
CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs
View File

@ -251,7 +251,7 @@ namespace CompatBot.Utils.ResultFormatters
collection["game_category"] = "HG";
}
serial = collection["serial"] ?? "";
var titleUpdateInfoTask = PsnClient.GetTitleUpdatesAsync(serial, Config.Cts.Token);
var titleUpdateInfoTask = TitleUpdateInfoProvider.GetAsync(serial, Config.Cts.Token);
var titleMetaTask = PsnClient.GetTitleMetaAsync(serial, Config.Cts.Token);
var gameInfo = await client.LookupGameInfoWithEmbedAsync(serial, collection["game_title"], true, category: collection["game_category"]).ConfigureAwait(false);
try