diff --git a/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs
new file mode 100644
index 00000000..a65ec09e
--- /dev/null
+++ b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs
@@ -0,0 +1,183 @@
+//
+using System;
+using CompatBot.Database;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+
+namespace CompatBot.Migrations
+{
+ [DbContext(typeof(ThumbnailDb))]
+ [Migration("20190807141221_SyscallInfo")]
+ partial class SyscallInfo
+ {
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "2.2.6-servicing-10079");
+
+ modelBuilder.Entity("CompatBot.Database.State", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnName("id");
+
+ b.Property("Locale")
+ .HasColumnName("locale");
+
+ b.Property("Timestamp")
+ .HasColumnName("timestamp");
+
+ b.HasKey("Id")
+ .HasName("id");
+
+ b.HasIndex("Locale")
+ .IsUnique()
+ .HasName("state_locale");
+
+ b.HasIndex("Timestamp")
+ .HasName("state_timestamp");
+
+ b.ToTable("state");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnName("id");
+
+ b.Property("Function")
+ .IsRequired()
+ .HasColumnName("function");
+
+ b.Property("Module")
+ .IsRequired()
+ .HasColumnName("module");
+
+ b.HasKey("Id")
+ .HasName("id");
+
+ b.HasIndex("Function")
+ .HasName("syscall_info_function");
+
+ b.HasIndex("Module")
+ .HasName("syscall_info_module");
+
+ b.ToTable("syscall_info");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+ {
+ b.Property("ProductId")
+ .HasColumnName("product_id");
+
+ b.Property("SyscallInfoId")
+ .HasColumnName("syscall_info_id");
+
+ b.HasKey("ProductId", "SyscallInfoId")
+ .HasName("id");
+
+ b.HasIndex("SyscallInfoId")
+ .HasName("ix_syscall_to_product_map_syscall_info_id");
+
+ b.ToTable("syscall_to_product_map");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnName("id");
+
+ b.Property("ContentId")
+ .HasColumnName("content_id");
+
+ b.Property("EmbeddableUrl")
+ .HasColumnName("embeddable_url");
+
+ b.Property("Name")
+ .HasColumnName("name");
+
+ b.Property("ProductCode")
+ .IsRequired()
+ .HasColumnName("product_code");
+
+ b.Property("Timestamp")
+ .HasColumnName("timestamp");
+
+ b.Property("Url")
+ .HasColumnName("url");
+
+ b.HasKey("Id")
+ .HasName("id");
+
+ b.HasIndex("ContentId")
+ .IsUnique()
+ .HasName("thumbnail_content_id");
+
+ b.HasIndex("ProductCode")
+ .IsUnique()
+ .HasName("thumbnail_product_code");
+
+ b.HasIndex("Timestamp")
+ .HasName("thumbnail_timestamp");
+
+ b.ToTable("thumbnail");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.TitleInfo", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnName("id");
+
+ b.Property("ContentId")
+ .IsRequired()
+ .HasColumnName("content_id");
+
+ b.Property("EmbedColor")
+ .HasColumnName("embed_color");
+
+ b.Property("ThumbnailEmbeddableUrl")
+ .HasColumnName("thumbnail_embeddable_url");
+
+ b.Property("ThumbnailUrl")
+ .HasColumnName("thumbnail_url");
+
+ b.Property("Timestamp")
+ .HasColumnName("timestamp");
+
+ b.HasKey("Id")
+ .HasName("id");
+
+ b.HasIndex("ContentId")
+ .IsUnique()
+ .HasName("title_info_content_id");
+
+ b.HasIndex("Timestamp")
+ .HasName("title_info_timestamp");
+
+ b.ToTable("title_info");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+ {
+ b.HasOne("CompatBot.Database.Thumbnail", "Product")
+ .WithMany("SyscallToProductMap")
+ .HasForeignKey("ProductId")
+ .HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
+ .WithMany("SyscallToProductMap")
+ .HasForeignKey("SyscallInfoId")
+ .HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
+#pragma warning restore 612, 618
+ }
+ }
+}
diff --git a/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs
new file mode 100644
index 00000000..a64f2d2e
--- /dev/null
+++ b/CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs
@@ -0,0 +1,72 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace CompatBot.Migrations
+{
+ public partial class SyscallInfo : Migration
+ {
+ protected override void Up(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.CreateTable(
+ name: "syscall_info",
+ columns: table => new
+ {
+ id = table.Column(nullable: false)
+ .Annotation("Sqlite:Autoincrement", true),
+ module = table.Column(nullable: false),
+ function = table.Column(nullable: false)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("id", x => x.id);
+ });
+
+ migrationBuilder.CreateTable(
+ name: "syscall_to_product_map",
+ columns: table => new
+ {
+ product_id = table.Column(nullable: false),
+ syscall_info_id = table.Column(nullable: false)
+ },
+ constraints: table =>
+ {
+ table.PrimaryKey("id", x => new { x.product_id, x.syscall_info_id });
+ table.ForeignKey(
+ name: "fk_syscall_to_product_map__thumbnail_product_id",
+ column: x => x.product_id,
+ principalTable: "thumbnail",
+ principalColumn: "id",
+ onDelete: ReferentialAction.Cascade);
+ table.ForeignKey(
+ name: "fk_syscall_to_product_map_syscall_info_syscall_info_id",
+ column: x => x.syscall_info_id,
+ principalTable: "syscall_info",
+ principalColumn: "id",
+ onDelete: ReferentialAction.Cascade);
+ });
+
+ migrationBuilder.CreateIndex(
+ name: "syscall_info_function",
+ table: "syscall_info",
+ column: "function");
+
+ migrationBuilder.CreateIndex(
+ name: "syscall_info_module",
+ table: "syscall_info",
+ column: "module");
+
+ migrationBuilder.CreateIndex(
+ name: "ix_syscall_to_product_map_syscall_info_id",
+ table: "syscall_to_product_map",
+ column: "syscall_info_id");
+ }
+
+ protected override void Down(MigrationBuilder migrationBuilder)
+ {
+ migrationBuilder.DropTable(
+ name: "syscall_to_product_map");
+
+ migrationBuilder.DropTable(
+ name: "syscall_info");
+ }
+ }
+}
diff --git a/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs b/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
index a1bfba00..1b254e33 100644
--- a/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
+++ b/CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
@@ -14,7 +14,7 @@ namespace CompatBot.Migrations
{
#pragma warning disable 612, 618
modelBuilder
- .HasAnnotation("ProductVersion", "2.2.2-servicing-10034");
+ .HasAnnotation("ProductVersion", "2.2.6-servicing-10079");
modelBuilder.Entity("CompatBot.Database.State", b =>
{
@@ -41,6 +41,49 @@ namespace CompatBot.Migrations
b.ToTable("state");
});
+ modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnName("id");
+
+ b.Property("Function")
+ .IsRequired()
+ .HasColumnName("function");
+
+ b.Property("Module")
+ .IsRequired()
+ .HasColumnName("module");
+
+ b.HasKey("Id")
+ .HasName("id");
+
+ b.HasIndex("Function")
+ .HasName("syscall_info_function");
+
+ b.HasIndex("Module")
+ .HasName("syscall_info_module");
+
+ b.ToTable("syscall_info");
+ });
+
+ modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+ {
+ b.Property("ProductId")
+ .HasColumnName("product_id");
+
+ b.Property("SyscallInfoId")
+ .HasColumnName("syscall_info_id");
+
+ b.HasKey("ProductId", "SyscallInfoId")
+ .HasName("id");
+
+ b.HasIndex("SyscallInfoId")
+ .HasName("ix_syscall_to_product_map_syscall_info_id");
+
+ b.ToTable("syscall_to_product_map");
+ });
+
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Property("Id")
@@ -117,6 +160,21 @@ namespace CompatBot.Migrations
b.ToTable("title_info");
});
+
+ modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
+ {
+ b.HasOne("CompatBot.Database.Thumbnail", "Product")
+ .WithMany("SyscallToProductMap")
+ .HasForeignKey("ProductId")
+ .HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
+ .OnDelete(DeleteBehavior.Cascade);
+
+ b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
+ .WithMany("SyscallToProductMap")
+ .HasForeignKey("SyscallInfoId")
+ .HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
+ .OnDelete(DeleteBehavior.Cascade);
+ });
#pragma warning restore 612, 618
}
}
diff --git a/CompatBot/Database/Providers/SyscallInfoProvider.cs b/CompatBot/Database/Providers/SyscallInfoProvider.cs
new file mode 100644
index 00000000..c4463746
--- /dev/null
+++ b/CompatBot/Database/Providers/SyscallInfoProvider.cs
@@ -0,0 +1,47 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+namespace CompatBot.Database.Providers
+{
+ internal static class SyscallInfoProvider
+ {
+ private static readonly SemaphoreSlim Limiter = new SemaphoreSlim(1, 1);
+
+ public static async Task SaveAsync(Dictionary>> syscallInfo)
+ {
+ if (syscallInfo == null || syscallInfo.Count == 0)
+ return;
+
+ if (await Limiter.WaitAsync(1000, Config.Cts.Token))
+ {
+ try
+ {
+ using (var db = new ThumbnailDb())
+ {
+ foreach (var productCodeMap in syscallInfo)
+ {
+ var product = db.Thumbnail.AsNoTracking().FirstOrDefault(t => t.ProductCode == productCodeMap.Key)
+ ?? db.Thumbnail.Add(new Thumbnail {ProductCode = productCodeMap.Key}).Entity;
+ foreach (var moduleMap in productCodeMap.Value)
+ foreach (var func in moduleMap.Value)
+ {
+ var syscall = db.SyscallInfo.AsNoTracking().FirstOrDefault(sci => sci.Module == moduleMap.Key && sci.Function == func)
+ ?? db.SyscallInfo.Add(new SyscallInfo {Module = moduleMap.Key, Function = func}).Entity;
+ if (!db.SyscallToProductMap.Any(m => m.ProductId == product.Id && m.SyscallInfoId == syscall.Id))
+ db.SyscallToProductMap.Add(new SyscallToProductMap {ProductId = product.Id, SyscallInfoId = syscall.Id});
+ }
+ }
+ await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false);
+ }
+ }
+ finally
+ {
+ Limiter.Release();
+ }
+ }
+ }
+ }
+}
diff --git a/CompatBot/Database/ThumbnailDb.cs b/CompatBot/Database/ThumbnailDb.cs
index c2f0d2d7..b41a8b47 100644
--- a/CompatBot/Database/ThumbnailDb.cs
+++ b/CompatBot/Database/ThumbnailDb.cs
@@ -1,4 +1,5 @@
using System;
+using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using CompatApiClient;
using Microsoft.EntityFrameworkCore;
@@ -10,6 +11,8 @@ namespace CompatBot.Database
public DbSet State { get; set; }
public DbSet Thumbnail { get; set; }
public DbSet TitleInfo { get; set; }
+ public DbSet SyscallInfo { get; set; }
+ public DbSet SyscallToProductMap { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -30,6 +33,9 @@ namespace CompatBot.Database
modelBuilder.Entity().HasIndex(m => m.Timestamp).HasName("thumbnail_timestamp");
modelBuilder.Entity().HasIndex(ti => ti.ContentId).IsUnique().HasName("title_info_content_id");
modelBuilder.Entity().HasIndex(ti => ti.Timestamp).HasName("title_info_timestamp");
+ modelBuilder.Entity().HasIndex(sci => sci.Module).HasName("syscall_info_module");
+ modelBuilder.Entity().HasIndex(sci => sci.Function).HasName("syscall_info_function");
+ modelBuilder.Entity().HasKey(m => new {m.ProductId, m.SyscallInfoId});
//configure default policy of Id being the primary key
modelBuilder.ConfigureDefaultPkConvention();
@@ -56,6 +62,8 @@ namespace CompatBot.Database
public string Url { get; set; }
public string EmbeddableUrl { get; set; }
public long Timestamp { get; set; }
+
+ public List SyscallToProductMap { get; set; }
}
internal class TitleInfo
@@ -68,4 +76,24 @@ namespace CompatBot.Database
public int? EmbedColor { get; set; }
public long Timestamp { get; set; }
}
+
+ internal class SyscallInfo
+ {
+ public int Id { get; set; }
+ [Required]
+ public string Module { get; set; }
+ [Required]
+ public string Function { get; set; }
+
+ public List SyscallToProductMap { get; set; }
+ }
+
+ internal class SyscallToProductMap
+ {
+ public int ProductId { get; set; }
+ public Thumbnail Product { get; set; }
+
+ public int SyscallInfoId { get; set; }
+ public SyscallInfo SyscallInfo { get; set; }
+ }
}
diff --git a/CompatBot/EventHandlers/LogParsingHandler.cs b/CompatBot/EventHandlers/LogParsingHandler.cs
index 9ce3acc9..225c6d84 100644
--- a/CompatBot/EventHandlers/LogParsingHandler.cs
+++ b/CompatBot/EventHandlers/LogParsingHandler.cs
@@ -94,19 +94,29 @@ namespace CompatBot.EventHandlers
botMsg = await channel.SendMessageAsync(embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false);
parsedLog = true;
- var result = await ParseLogAsync(source, async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false)).ConfigureAwait(false);
-
+ var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
+ var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
+ var tries = 0;
+ LogParseState result = null;
+ do
+ {
+ result = await ParseLogAsync(
+ source,
+ async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false),
+ combinedTokenSource.Token
+ ).ConfigureAwait(false);
+ tries++;
+ } while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3);
if (result == null)
{
- botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
- embed: new DiscordEmbedBuilder
- {
- Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
- "Please run the game again and re-upload a new copy.",
- Color = Config.Colors.LogResultFailed,
- }
- .AddAuthor(client, message, source)
- .Build()
+ botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: new DiscordEmbedBuilder
+ {
+ Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
+ "Please run the game again and re-upload a new copy.",
+ Color = Config.Colors.LogResultFailed,
+ }
+ .AddAuthor(client, message, source)
+ .Build()
).ConfigureAwait(false);
}
else
@@ -222,23 +232,20 @@ namespace CompatBot.EventHandlers
}
}
- public static async Task ParseLogAsync(ISource source, Func onProgressAsync)
+ public static async Task ParseLogAsync(ISource source, Func onProgressAsync, CancellationToken cancellationToken)
{
LogParseState result = null;
try
{
- var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
- var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
-
var pipe = new Pipe();
- var fillPipeTask = source.FillPipeAsync(pipe.Writer, combinedTokenSource.Token);
- var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, combinedTokenSource.Token);
+ var fillPipeTask = source.FillPipeAsync(pipe.Writer, cancellationToken);
+ var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, cancellationToken);
do
{
- await Task.WhenAny(readPipeTask, Task.Delay(5000, combinedTokenSource.Token)).ConfigureAwait(false);
+ await Task.WhenAny(readPipeTask, Task.Delay(5000, cancellationToken)).ConfigureAwait(false);
if (!readPipeTask.IsCompleted)
await onProgressAsync().ConfigureAwait(false);
- } while (!readPipeTask.IsCompleted && !combinedTokenSource.IsCancellationRequested);
+ } while (!readPipeTask.IsCompleted && !cancellationToken.IsCancellationRequested);
result = await readPipeTask.ConfigureAwait(false);
await fillPipeTask.ConfigureAwait(false);
result.TotalBytes = source.LogFileSize;
@@ -280,6 +287,12 @@ namespace CompatBot.EventHandlers
Config.Log.Debug("Product keys: " + serialCount);
Config.Log.Debug("Modules: " + moduleCount);
Config.Log.Debug("Functions: " + functionCount);
+ Config.Log.Debug("Saving syscall information...");
+ var sw = Stopwatch.StartNew();
+#endif
+ await SyscallInfoProvider.SaveAsync(result.Syscalls).ConfigureAwait(false);
+#if DEBUG
+ Config.Log.Debug("Saving syscall information took " + sw.Elapsed);
#endif
}
catch (Exception e)