RPCS3/archived-discord-bot
1
0
Fork 0
mirror of https://github.com/RPCS3/discord-bot.git synced 2026-01-31 01:25:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

persist syscall information

Browse Source
This commit is contained in:
13xforever
2019-08-07 20:00:00 +05:00
parent bc14931d39
commit fda07e446f
6 changed files with 421 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.Designer.cs generated Normal file
View File

@@ -0,0 +1,183 @@
// <auto-generated />
using System;
using CompatBot.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace CompatBot.Migrations
{
[DbContext(typeof(ThumbnailDb))]
[Migration("20190807141221_SyscallInfo")]
partial class SyscallInfo
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "2.2.6-servicing-10079");
modelBuilder.Entity("CompatBot.Database.State", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnName("id");
b.Property<string>("Locale")
.HasColumnName("locale");
b.Property<long>("Timestamp")
.HasColumnName("timestamp");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Locale")
.IsUnique()
.HasName("state_locale");
b.HasIndex("Timestamp")
.HasName("state_timestamp");
b.ToTable("state");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnName("id");
b.Property<string>("Function")
.IsRequired()
.HasColumnName("function");
b.Property<string>("Module")
.IsRequired()
.HasColumnName("module");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Function")
.HasName("syscall_info_function");
b.HasIndex("Module")
.HasName("syscall_info_module");
b.ToTable("syscall_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.Property<int>("ProductId")
.HasColumnName("product_id");
b.Property<int>("SyscallInfoId")
.HasColumnName("syscall_info_id");
b.HasKey("ProductId", "SyscallInfoId")
.HasName("id");
b.HasIndex("SyscallInfoId")
.HasName("ix_syscall_to_product_map_syscall_info_id");
b.ToTable("syscall_to_product_map");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnName("id");
b.Property<string>("ContentId")
.HasColumnName("content_id");
b.Property<string>("EmbeddableUrl")
.HasColumnName("embeddable_url");
b.Property<string>("Name")
.HasColumnName("name");
b.Property<string>("ProductCode")
.IsRequired()
.HasColumnName("product_code");
b.Property<long>("Timestamp")
.HasColumnName("timestamp");
b.Property<string>("Url")
.HasColumnName("url");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ContentId")
.IsUnique()
.HasName("thumbnail_content_id");
b.HasIndex("ProductCode")
.IsUnique()
.HasName("thumbnail_product_code");
b.HasIndex("Timestamp")
.HasName("thumbnail_timestamp");
b.ToTable("thumbnail");
});
modelBuilder.Entity("CompatBot.Database.TitleInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnName("id");
b.Property<string>("ContentId")
.IsRequired()
.HasColumnName("content_id");
b.Property<int?>("EmbedColor")
.HasColumnName("embed_color");
b.Property<string>("ThumbnailEmbeddableUrl")
.HasColumnName("thumbnail_embeddable_url");
b.Property<string>("ThumbnailUrl")
.HasColumnName("thumbnail_url");
b.Property<long>("Timestamp")
.HasColumnName("timestamp");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ContentId")
.IsUnique()
.HasName("title_info_content_id");
b.HasIndex("Timestamp")
.HasName("title_info_timestamp");
b.ToTable("title_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.HasOne("CompatBot.Database.Thumbnail", "Product")
.WithMany("SyscallToProductMap")
.HasForeignKey("ProductId")
.HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
.WithMany("SyscallToProductMap")
.HasForeignKey("SyscallInfoId")
.HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
.OnDelete(DeleteBehavior.Cascade);
});
#pragma warning restore 612, 618
}
}
}

72
CompatBot/Database/Migrations/ThumbnailDb/20190807141221_SyscallInfo.cs Normal file
View File

@@ -0,0 +1,72 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace CompatBot.Migrations
{
public partial class SyscallInfo : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "syscall_info",
columns: table => new
{
id = table.Column<int>(nullable: false)
.Annotation("Sqlite:Autoincrement", true),
module = table.Column<string>(nullable: false),
function = table.Column<string>(nullable: false)
},
constraints: table =>
{
table.PrimaryKey("id", x => x.id);
});
migrationBuilder.CreateTable(
name: "syscall_to_product_map",
columns: table => new
{
product_id = table.Column<int>(nullable: false),
syscall_info_id = table.Column<int>(nullable: false)
},
constraints: table =>
{
table.PrimaryKey("id", x => new { x.product_id, x.syscall_info_id });
table.ForeignKey(
name: "fk_syscall_to_product_map__thumbnail_product_id",
column: x => x.product_id,
principalTable: "thumbnail",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "fk_syscall_to_product_map_syscall_info_syscall_info_id",
column: x => x.syscall_info_id,
principalTable: "syscall_info",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "syscall_info_function",
table: "syscall_info",
column: "function");
migrationBuilder.CreateIndex(
name: "syscall_info_module",
table: "syscall_info",
column: "module");
migrationBuilder.CreateIndex(
name: "ix_syscall_to_product_map_syscall_info_id",
table: "syscall_to_product_map",
column: "syscall_info_id");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "syscall_to_product_map");
migrationBuilder.DropTable(
name: "syscall_info");
}
}
}

60
CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
View File

@@ -14,7 +14,7 @@ namespace CompatBot.Migrations
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "2.2.2-servicing-10034");
.HasAnnotation("ProductVersion", "2.2.6-servicing-10079");
modelBuilder.Entity("CompatBot.Database.State", b =>
{
@@ -41,6 +41,49 @@ namespace CompatBot.Migrations
b.ToTable("state");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnName("id");
b.Property<string>("Function")
.IsRequired()
.HasColumnName("function");
b.Property<string>("Module")
.IsRequired()
.HasColumnName("module");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Function")
.HasName("syscall_info_function");
b.HasIndex("Module")
.HasName("syscall_info_module");
b.ToTable("syscall_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.Property<int>("ProductId")
.HasColumnName("product_id");
b.Property<int>("SyscallInfoId")
.HasColumnName("syscall_info_id");
b.HasKey("ProductId", "SyscallInfoId")
.HasName("id");
b.HasIndex("SyscallInfoId")
.HasName("ix_syscall_to_product_map_syscall_info_id");
b.ToTable("syscall_to_product_map");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Property<int>("Id")
@@ -117,6 +160,21 @@ namespace CompatBot.Migrations
b.ToTable("title_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.HasOne("CompatBot.Database.Thumbnail", "Product")
.WithMany("SyscallToProductMap")
.HasForeignKey("ProductId")
.HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
.OnDelete(DeleteBehavior.Cascade);
b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
.WithMany("SyscallToProductMap")
.HasForeignKey("SyscallInfoId")
.HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
.OnDelete(DeleteBehavior.Cascade);
});
#pragma warning restore 612, 618
}
}

47
CompatBot/Database/Providers/SyscallInfoProvider.cs Normal file
View File

@@ -0,0 +1,47 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
namespace CompatBot.Database.Providers
{
internal static class SyscallInfoProvider
{
private static readonly SemaphoreSlim Limiter = new SemaphoreSlim(1, 1);
public static async Task SaveAsync(Dictionary<string, Dictionary<string, HashSet<string>>> syscallInfo)
{
if (syscallInfo == null || syscallInfo.Count == 0)
return;
if (await Limiter.WaitAsync(1000, Config.Cts.Token))
{
try
{
using (var db = new ThumbnailDb())
{
foreach (var productCodeMap in syscallInfo)
{
var product = db.Thumbnail.AsNoTracking().FirstOrDefault(t => t.ProductCode == productCodeMap.Key)
?? db.Thumbnail.Add(new Thumbnail {ProductCode = productCodeMap.Key}).Entity;
foreach (var moduleMap in productCodeMap.Value)
foreach (var func in moduleMap.Value)
{
var syscall = db.SyscallInfo.AsNoTracking().FirstOrDefault(sci => sci.Module == moduleMap.Key && sci.Function == func)
?? db.SyscallInfo.Add(new SyscallInfo {Module = moduleMap.Key, Function = func}).Entity;
if (!db.SyscallToProductMap.Any(m => m.ProductId == product.Id && m.SyscallInfoId == syscall.Id))
db.SyscallToProductMap.Add(new SyscallToProductMap {ProductId = product.Id, SyscallInfoId = syscall.Id});
}
}
await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false);
}
}
finally
{
Limiter.Release();
}
}
}
}
}

28
CompatBot/Database/ThumbnailDb.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using CompatApiClient;
using Microsoft.EntityFrameworkCore;
@@ -10,6 +11,8 @@ namespace CompatBot.Database
public DbSet<State> State { get; set; }
public DbSet<Thumbnail> Thumbnail { get; set; }
public DbSet<TitleInfo> TitleInfo { get; set; }
public DbSet<SyscallInfo> SyscallInfo { get; set; }
public DbSet<SyscallToProductMap> SyscallToProductMap { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -30,6 +33,9 @@ namespace CompatBot.Database
modelBuilder.Entity<Thumbnail>().HasIndex(m => m.Timestamp).HasName("thumbnail_timestamp");
modelBuilder.Entity<TitleInfo>().HasIndex(ti => ti.ContentId).IsUnique().HasName("title_info_content_id");
modelBuilder.Entity<TitleInfo>().HasIndex(ti => ti.Timestamp).HasName("title_info_timestamp");
modelBuilder.Entity<SyscallInfo>().HasIndex(sci => sci.Module).HasName("syscall_info_module");
modelBuilder.Entity<SyscallInfo>().HasIndex(sci => sci.Function).HasName("syscall_info_function");
modelBuilder.Entity<SyscallToProductMap>().HasKey(m => new {m.ProductId, m.SyscallInfoId});
//configure default policy of Id being the primary key
modelBuilder.ConfigureDefaultPkConvention();
@@ -56,6 +62,8 @@ namespace CompatBot.Database
public string Url { get; set; }
public string EmbeddableUrl { get; set; }
public long Timestamp { get; set; }
public List<SyscallToProductMap> SyscallToProductMap { get; set; }
}
internal class TitleInfo
@@ -68,4 +76,24 @@ namespace CompatBot.Database
public int? EmbedColor { get; set; }
public long Timestamp { get; set; }
}
internal class SyscallInfo
{
public int Id { get; set; }
[Required]
public string Module { get; set; }
[Required]
public string Function { get; set; }
public List<SyscallToProductMap> SyscallToProductMap { get; set; }
}
internal class SyscallToProductMap
{
public int ProductId { get; set; }
public Thumbnail Product { get; set; }
public int SyscallInfoId { get; set; }
public SyscallInfo SyscallInfo { get; set; }
}
}

51
CompatBot/EventHandlers/LogParsingHandler.cs
View File

@@ -94,19 +94,29 @@ namespace CompatBot.EventHandlers
botMsg = await channel.SendMessageAsync(embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false);
parsedLog = true;
var result = await ParseLogAsync(source, async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false)).ConfigureAwait(false);
var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
var tries = 0;
LogParseState result = null;
do
{
result = await ParseLogAsync(
source,
async () => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false),
combinedTokenSource.Token
).ConfigureAwait(false);
tries++;
} while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3);
if (result == null)
{
botMsg = await botMsg.UpdateOrCreateMessageAsync(channel,
embed: new DiscordEmbedBuilder
{
Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
"Please run the game again and re-upload a new copy.",
Color = Config.Colors.LogResultFailed,
}
.AddAuthor(client, message, source)
.Build()
botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed: new DiscordEmbedBuilder
{
Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" +
"Please run the game again and re-upload a new copy.",
Color = Config.Colors.LogResultFailed,
}
.AddAuthor(client, message, source)
.Build()
).ConfigureAwait(false);
}
else
@@ -222,23 +232,20 @@ namespace CompatBot.EventHandlers
}
}
public static async Task<LogParseState> ParseLogAsync(ISource source, Func<Task> onProgressAsync)
public static async Task<LogParseState> ParseLogAsync(ISource source, Func<Task> onProgressAsync, CancellationToken cancellationToken)
{
LogParseState result = null;
try
{
var timeout = new CancellationTokenSource(Config.LogParsingTimeout);
var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token);
var pipe = new Pipe();
var fillPipeTask = source.FillPipeAsync(pipe.Writer, combinedTokenSource.Token);
var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, combinedTokenSource.Token);
var fillPipeTask = source.FillPipeAsync(pipe.Writer, cancellationToken);
var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, cancellationToken);
do
{
await Task.WhenAny(readPipeTask, Task.Delay(5000, combinedTokenSource.Token)).ConfigureAwait(false);
await Task.WhenAny(readPipeTask, Task.Delay(5000, cancellationToken)).ConfigureAwait(false);
if (!readPipeTask.IsCompleted)
await onProgressAsync().ConfigureAwait(false);
} while (!readPipeTask.IsCompleted && !combinedTokenSource.IsCancellationRequested);
} while (!readPipeTask.IsCompleted && !cancellationToken.IsCancellationRequested);
result = await readPipeTask.ConfigureAwait(false);
await fillPipeTask.ConfigureAwait(false);
result.TotalBytes = source.LogFileSize;
@@ -280,6 +287,12 @@ namespace CompatBot.EventHandlers
Config.Log.Debug("Product keys: " + serialCount);
Config.Log.Debug("Modules: " + moduleCount);
Config.Log.Debug("Functions: " + functionCount);
Config.Log.Debug("Saving syscall information...");
var sw = Stopwatch.StartNew();
#endif
await SyscallInfoProvider.SaveAsync(result.Syscalls).ConfigureAwait(false);
#if DEBUG
Config.Log.Debug("Saving syscall information took " + sw.Elapsed);
#endif
}
catch (Exception e)