From fda07e446f0e749ce96ed70c8de9df94b5a937c7 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Wed, 7 Aug 2019 20:00:00 +0500 Subject: [PATCH] persist syscall information --- .../20190807141221_SyscallInfo.Designer.cs | 183 ++++++++++++++++++ .../ThumbnailDb/20190807141221_SyscallInfo.cs | 72 +++++++ .../ThumbnailDb/ThumbnailDbModelSnapshot.cs | 60 +++++- .../Database/Providers/SyscallInfoProvider.cs | 47 +++++ CompatBot/Database/ThumbnailDb.cs | 28 +++ CompatBot/EventHandlers/LogParsingHandler.cs | 51 +++-- 6 files changed, 421 insertions(+), 20 deletions(-) create mode 100644 CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs create mode 100644 CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs create mode 100644 CompatBot/Database/Providers/SyscallInfoProvider.cs diff --git a/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs new file mode 100644 index 00000000..a65ec09e --- /dev/null +++ b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs @@ -0,0 +1,183 @@ +// +using System; +using CompatBot.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace CompatBot.Migrations +{ + [DbContext(typeof(ThumbnailDb))] + [Migration("20190807141221_SyscallInfo")] + partial class SyscallInfo + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "2.2.6-servicing-10079"); + + modelBuilder.Entity("CompatBot.Database.State", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("id"); + + b.Property("Locale") + .HasColumnName("locale"); + + b.Property("Timestamp") + .HasColumnName("timestamp"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Locale") + .IsUnique() + .HasName("state_locale"); + + b.HasIndex("Timestamp") + .HasName("state_timestamp"); + + b.ToTable("state"); + }); + + modelBuilder.Entity("CompatBot.Database.SyscallInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("id"); + + b.Property("Function") + .IsRequired() + .HasColumnName("function"); + + b.Property("Module") + .IsRequired() + .HasColumnName("module"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Function") + .HasName("syscall_info_function"); + + b.HasIndex("Module") + .HasName("syscall_info_module"); + + b.ToTable("syscall_info"); + }); + + modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b => + { + b.Property("ProductId") + .HasColumnName("product_id"); + + b.Property("SyscallInfoId") + .HasColumnName("syscall_info_id"); + + b.HasKey("ProductId", "SyscallInfoId") + .HasName("id"); + + b.HasIndex("SyscallInfoId") + .HasName("ix_syscall_to_product_map_syscall_info_id"); + + b.ToTable("syscall_to_product_map"); + }); + + modelBuilder.Entity("CompatBot.Database.Thumbnail", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("id"); + + b.Property("ContentId") + .HasColumnName("content_id"); + + b.Property("EmbeddableUrl") + .HasColumnName("embeddable_url"); + + b.Property("Name") + .HasColumnName("name"); + + b.Property("ProductCode") + .IsRequired() + .HasColumnName("product_code"); + + b.Property("Timestamp") + .HasColumnName("timestamp"); + + b.Property("Url") + .HasColumnName("url"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("ContentId") + .IsUnique() + .HasName("thumbnail_content_id"); + + b.HasIndex("ProductCode") + .IsUnique() + .HasName("thumbnail_product_code"); + + b.HasIndex("Timestamp") + .HasName("thumbnail_timestamp"); + + b.ToTable("thumbnail"); + }); + + modelBuilder.Entity("CompatBot.Database.TitleInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("id"); + + b.Property("ContentId") + .IsRequired() + .HasColumnName("content_id"); + + b.Property("EmbedColor") + .HasColumnName("embed_color"); + + b.Property("ThumbnailEmbeddableUrl") + .HasColumnName("thumbnail_embeddable_url"); + + b.Property("ThumbnailUrl") + .HasColumnName("thumbnail_url"); + + b.Property("Timestamp") + .HasColumnName("timestamp"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("ContentId") + .IsUnique() + .HasName("title_info_content_id"); + + b.HasIndex("Timestamp") + .HasName("title_info_timestamp"); + + b.ToTable("title_info"); + }); + + modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b => + { + b.HasOne("CompatBot.Database.Thumbnail", "Product") + .WithMany("SyscallToProductMap") + .HasForeignKey("ProductId") + .HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo") + .WithMany("SyscallToProductMap") + .HasForeignKey("SyscallInfoId") + .HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id") + .OnDelete(DeleteBehavior.Cascade); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs new file mode 100644 index 00000000..a64f2d2e --- /dev/null +++ b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs @@ -0,0 +1,72 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace CompatBot.Migrations +{ + public partial class SyscallInfo : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "syscall_info", + columns: table => new + { + id = table.Column(nullable: false) + .Annotation("Sqlite:Autoincrement", true), + module = table.Column(nullable: false), + function = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("id", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "syscall_to_product_map", + columns: table => new + { + product_id = table.Column(nullable: false), + syscall_info_id = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("id", x => new { x.product_id, x.syscall_info_id }); + table.ForeignKey( + name: "fk_syscall_to_product_map__thumbnail_product_id", + column: x => x.product_id, + principalTable: "thumbnail", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "fk_syscall_to_product_map_syscall_info_syscall_info_id", + column: x => x.syscall_info_id, + principalTable: "syscall_info", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "syscall_info_function", + table: "syscall_info", + column: "function"); + + migrationBuilder.CreateIndex( + name: "syscall_info_module", + table: "syscall_info", + column: "module"); + + migrationBuilder.CreateIndex( + name: "ix_syscall_to_product_map_syscall_info_id", + table: "syscall_to_product_map", + column: "syscall_info_id"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "syscall_to_product_map"); + + migrationBuilder.DropTable( + name: "syscall_info"); + } + } +} diff --git a/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs b/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs index a1bfba00..1b254e33 100644 --- a/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs +++ b/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs @@ -14,7 +14,7 @@ namespace CompatBot.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "2.2.2-servicing-10034"); + .HasAnnotation("ProductVersion", "2.2.6-servicing-10079"); modelBuilder.Entity("CompatBot.Database.State", b => { @@ -41,6 +41,49 @@ namespace CompatBot.Migrations b.ToTable("state"); }); + modelBuilder.Entity("CompatBot.Database.SyscallInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnName("id"); + + b.Property("Function") + .IsRequired() + .HasColumnName("function"); + + b.Property("Module") + .IsRequired() + .HasColumnName("module"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Function") + .HasName("syscall_info_function"); + + b.HasIndex("Module") + .HasName("syscall_info_module"); + + b.ToTable("syscall_info"); + }); + + modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b => + { + b.Property("ProductId") + .HasColumnName("product_id"); + + b.Property("SyscallInfoId") + .HasColumnName("syscall_info_id"); + + b.HasKey("ProductId", "SyscallInfoId") + .HasName("id"); + + b.HasIndex("SyscallInfoId") + .HasName("ix_syscall_to_product_map_syscall_info_id"); + + b.ToTable("syscall_to_product_map"); + }); + modelBuilder.Entity("CompatBot.Database.Thumbnail", b => { b.Property("Id") @@ -117,6 +160,21 @@ namespace CompatBot.Migrations b.ToTable("title_info"); }); + + modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b => + { + b.HasOne("CompatBot.Database.Thumbnail", "Product") + .WithMany("SyscallToProductMap") + .HasForeignKey("ProductId") + .HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo") + .WithMany("SyscallToProductMap") + .HasForeignKey("SyscallInfoId") + .HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id") + .OnDelete(DeleteBehavior.Cascade); + }); #pragma warning restore 612, 618 } } diff --git a/CompatBot/Database/Providers/SyscallInfoProvider.cs b/CompatBot/Database/Providers/SyscallInfoProvider.cs new file mode 100644 index 00000000..c4463746 --- /dev/null +++ b/CompatBot/Database/Providers/SyscallInfoProvider.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace CompatBot.Database.Providers +{ + internal static class SyscallInfoProvider + { + private static readonly SemaphoreSlim Limiter = new SemaphoreSlim(1, 1); + + public static async Task SaveAsync(Dictionary>> syscallInfo) + { + if (syscallInfo == null || syscallInfo.Count == 0) + return; + + if (await Limiter.WaitAsync(1000, Config.Cts.Token)) + { + try + { + using (var db = new ThumbnailDb()) + { + foreach (var productCodeMap in syscallInfo) + { + var product = db.Thumbnail.AsNoTracking().FirstOrDefault(t => t.ProductCode == productCodeMap.Key) + ?? db.Thumbnail.Add(new Thumbnail {ProductCode = productCodeMap.Key}).Entity; + foreach (var moduleMap in productCodeMap.Value) + foreach (var func in moduleMap.Value) + { + var syscall = db.SyscallInfo.AsNoTracking().FirstOrDefault(sci => sci.Module == moduleMap.Key && sci.Function == func) + ?? db.SyscallInfo.Add(new SyscallInfo {Module = moduleMap.Key, Function = func}).Entity; + if (!db.SyscallToProductMap.Any(m => m.ProductId == product.Id && m.SyscallInfoId == syscall.Id)) + db.SyscallToProductMap.Add(new SyscallToProductMap {ProductId = product.Id, SyscallInfoId = syscall.Id}); + } + } + await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false); + } + } + finally + { + Limiter.Release(); + } + } + } + } +} diff --git a/CompatBot/Database/ThumbnailDb.cs b/CompatBot/Database/ThumbnailDb.cs index c2f0d2d7..b41a8b47 100644 --- a/CompatBot/Database/ThumbnailDb.cs +++ b/CompatBot/Database/ThumbnailDb.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using CompatApiClient; using Microsoft.EntityFrameworkCore; @@ -10,6 +11,8 @@ namespace CompatBot.Database public DbSet State { get; set; } public DbSet Thumbnail { get; set; } public DbSet TitleInfo { get; set; } + public DbSet SyscallInfo { get; set; } + public DbSet SyscallToProductMap { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { @@ -30,6 +33,9 @@ namespace CompatBot.Database modelBuilder.Entity().HasIndex(m => m.Timestamp).HasName("thumbnail_timestamp"); modelBuilder.Entity().HasIndex(ti => ti.ContentId).IsUnique().HasName("title_info_content_id"); modelBuilder.Entity().HasIndex(ti => ti.Timestamp).HasName("title_info_timestamp"); + modelBuilder.Entity().HasIndex(sci => sci.Module).HasName("syscall_info_module"); + modelBuilder.Entity().HasIndex(sci => sci.Function).HasName("syscall_info_function"); + modelBuilder.Entity().HasKey(m => new {m.ProductId, m.SyscallInfoId}); //configure default policy of Id being the primary key modelBuilder.ConfigureDefaultPkConvention(); @@ -56,6 +62,8 @@ namespace CompatBot.Database public string Url { get; set; } public string EmbeddableUrl { get; set; } public long Timestamp { get; set; } + + public List SyscallToProductMap { get; set; } } internal class TitleInfo @@ -68,4 +76,24 @@ namespace CompatBot.Database public int? EmbedColor { get; set; } public long Timestamp { get; set; } } + + internal class SyscallInfo + { + public int Id { get; set; } + [Required] + public string Module { get; set; } + [Required] + public string Function { get; set; } + + public List SyscallToProductMap { get; set; } + } + + internal class SyscallToProductMap + { + public int ProductId { get; set; } + public Thumbnail Product { get; set; } + + public int SyscallInfoId { get; set; } + public SyscallInfo SyscallInfo { get; set; } + } } diff --git a/CompatBot/EventHandlers/LogParsingHandler.cs b/CompatBot/EventHandlers/LogParsingHandler.cs index 9ce3acc9..225c6d84 100644 --- a/CompatBot/EventHandlers/LogParsingHandler.cs +++ b/CompatBot/EventHandlers/LogParsingHandler.cs @@ -94,19 +94,29 @@ namespace CompatBot.EventHandlers botMsg = await channel.SendMessageAsync(embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false); parsedLog = true; - var result = await ParseLogAsync(source, async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false)).ConfigureAwait(false); - + var timeout = new CancellationTokenSource(Config.LogParsingTimeout); + var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token); + var tries = 0; + LogParseState result = null; + do + { + result = await ParseLogAsync( + source, + async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false), + combinedTokenSource.Token + ).ConfigureAwait(false); + tries++; + } while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3); if (result == null) { - botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, - embed: new DiscordEmbedBuilder - { - Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" + - "Please run the game again and re-upload a new copy.", - Color = Config.Colors.LogResultFailed, - } - .AddAuthor(client, message, source) - .Build() + botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: new DiscordEmbedBuilder + { + Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" + + "Please run the game again and re-upload a new copy.", + Color = Config.Colors.LogResultFailed, + } + .AddAuthor(client, message, source) + .Build() ).ConfigureAwait(false); } else @@ -222,23 +232,20 @@ namespace CompatBot.EventHandlers } } - public static async Task ParseLogAsync(ISource source, Func onProgressAsync) + public static async Task ParseLogAsync(ISource source, Func onProgressAsync, CancellationToken cancellationToken) { LogParseState result = null; try { - var timeout = new CancellationTokenSource(Config.LogParsingTimeout); - var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token); - var pipe = new Pipe(); - var fillPipeTask = source.FillPipeAsync(pipe.Writer, combinedTokenSource.Token); - var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, combinedTokenSource.Token); + var fillPipeTask = source.FillPipeAsync(pipe.Writer, cancellationToken); + var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, cancellationToken); do { - await Task.WhenAny(readPipeTask, Task.Delay(5000, combinedTokenSource.Token)).ConfigureAwait(false); + await Task.WhenAny(readPipeTask, Task.Delay(5000, cancellationToken)).ConfigureAwait(false); if (!readPipeTask.IsCompleted) await onProgressAsync().ConfigureAwait(false); - } while (!readPipeTask.IsCompleted && !combinedTokenSource.IsCancellationRequested); + } while (!readPipeTask.IsCompleted && !cancellationToken.IsCancellationRequested); result = await readPipeTask.ConfigureAwait(false); await fillPipeTask.ConfigureAwait(false); result.TotalBytes = source.LogFileSize; @@ -280,6 +287,12 @@ namespace CompatBot.EventHandlers Config.Log.Debug("Product keys: " + serialCount); Config.Log.Debug("Modules: " + moduleCount); Config.Log.Debug("Functions: " + functionCount); + Config.Log.Debug("Saving syscall information..."); + var sw = Stopwatch.StartNew(); +#endif + await SyscallInfoProvider.SaveAsync(result.Syscalls).ConfigureAwait(false); +#if DEBUG + Config.Log.Debug("Saving syscall information took " + sw.Elapsed); #endif } catch (Exception e)