RPCS3/archived-discord-bot
1
0
Fork 0
mirror of https://github.com/RPCS3/discord-bot.git synced 2026-01-31 01:25:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

redesign the name pool to reduce compilation time resource usage

Browse Source
This commit is contained in:
13xforever
2021-03-10 02:31:38 +05:00
parent 4ecbd62eb0
commit 222eaa2a35
11 changed files with 433 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CompatBot/CompatBot.csproj
View File

@@ -43,9 +43,7 @@
<ItemGroup>
<None Remove="..\win32_error_codes.txt" />
<None Remove="..\names_*.txt" />
<AdditionalFiles Include="..\win32_error_codes.txt" />
<AdditionalFiles Include="..\names_*.txt" />
</ItemGroup>
<ItemGroup>

1
CompatBot/Config.cs
View File

@@ -85,6 +85,7 @@ namespace CompatBot
public static string IrdCachePath => config.GetValue(nameof(IrdCachePath), "./ird/");
public static double GameTitleMatchThreshold => config.GetValue(nameof(GameTitleMatchThreshold), 0.57);
public static byte[] CryptoSalt => Convert.FromBase64String(config.GetValue(nameof(CryptoSalt), ""));
public static string RenameNameSuffix => config.GetValue(nameof(RenameNameSuffix), " (Rule 7)");
internal static class AllowedMentions
{

99
CompatBot/Database/DbImporter.cs
View File

@@ -1,5 +1,9 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using CompatBot.Database.Migrations;
@@ -7,6 +11,7 @@ using CompatBot.Utils;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Migrations.Internal;
using PsnClient.Utils;
namespace CompatBot.Database
{
@@ -146,5 +151,99 @@ namespace CompatBot.Database
}
return dbPath;
}
public static async Task<bool> ImportNamesPool(ThumbnailDb db, CancellationToken cancellationToken)
{
Config.Log.Debug("Importing name pool...");
var rootDir = Environment.CurrentDirectory;
while (rootDir is not null && !Directory.EnumerateFiles(rootDir, "names_*.txt", SearchOption.TopDirectoryOnly).Any())
rootDir = Path.GetDirectoryName(rootDir);
if (rootDir is null)
{
Config.Log.Error("Couldn't find any name sources");
return db.NamePool.Any();
}
var resources = Directory.GetFiles(rootDir, "names_*.txt", SearchOption.TopDirectoryOnly)
.OrderBy(f => f)
.ToList();
if (resources.Count == 0)
{
Config.Log.Error("Couldn't find any name sources (???)");
return db.NamePool.Any();
}
var timestamp = -1L;
using (var sha256 = System.Security.Cryptography.SHA256.Create())
{
byte[] buf;
foreach (var path in resources)
{
var fileInfo = new FileInfo(path);
buf = BitConverter.GetBytes(fileInfo.Length);
sha256.TransformBlock(buf, 0, buf.Length, null, 0);
}
buf = Encoding.UTF8.GetBytes(Config.RenameNameSuffix);
buf = sha256.TransformFinalBlock(buf, 0, buf.Length);
timestamp = BitConverter.ToInt64(buf, 0);
}
const string renameStateKey = "rename-name-pool";
var stateEntry = db.State.FirstOrDefault(n => n.Locale == renameStateKey);
if (stateEntry?.Timestamp == timestamp)
{
Config.Log.Info("Name pool is up-to-date");
return true;
}
Config.Log.Info("Updating name pool...");
try
{
var names = new HashSet<string>();
foreach (var resourcePath in resources)
{
await using var stream = File.Open(resourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new StreamReader(stream);
while (await reader.ReadLineAsync().ConfigureAwait(false) is string line)
{
if (line.Length < 2 || line.StartsWith("#"))
continue;
var commentPos = line.IndexOf(" (");
if (commentPos > 1)
line = line.Substring(0, commentPos);
line = line.Trim()
.Replace(" ", " ")
.Replace('`', '\'') // consider ’
.Replace("\"", "\\\"");
if (line.Length + Config.RenameNameSuffix.Length > 32)
continue;
if (line.Contains('@')
|| line.Contains('#')
|| line.Contains(':'))
continue;
names.Add(line);
}
}
await using var tx = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
db.NamePool.RemoveRange(db.NamePool);
foreach (var name in names)
await db.NamePool.AddAsync(new() {Name = name}, cancellationToken).ConfigureAwait(false);
if (stateEntry is null)
await db.State.AddAsync(new() {Locale = renameStateKey, Timestamp = timestamp}, cancellationToken).ConfigureAwait(false);
else
stateEntry.Timestamp = timestamp;
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
return names.Count > 0;
}
catch (Exception e)
{
Config.Log.Error(e);
return false;
}
}
}
}

266
CompatBot/Database/Migrations/ThumbnailDb/20210309212939_AddUserNamePool.Designer.cs generated Normal file
View File

@@ -0,0 +1,266 @@
// <auto-generated />
using System;
using CompatBot.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace CompatBot.Migrations
{
[DbContext(typeof(ThumbnailDb))]
[Migration("20210309212939_AddUserNamePool")]
partial class AddUserNamePool
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "5.0.3");
modelBuilder.Entity("CompatBot.Database.Fortune", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Content")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("content");
b.HasKey("Id")
.HasName("id");
b.ToTable("fortune");
});
modelBuilder.Entity("CompatBot.Database.Metacritic", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<byte?>("CriticScore")
.HasColumnType("INTEGER")
.HasColumnName("critic_score");
b.Property<string>("Notes")
.HasColumnType("TEXT")
.HasColumnName("notes");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("title");
b.Property<byte?>("UserScore")
.HasColumnType("INTEGER")
.HasColumnName("user_score");
b.HasKey("Id")
.HasName("id");
b.ToTable("metacritic");
});
modelBuilder.Entity("CompatBot.Database.NamePool", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("name");
b.HasKey("Id")
.HasName("id");
b.ToTable("name_pool");
});
modelBuilder.Entity("CompatBot.Database.State", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Locale")
.HasColumnType("TEXT")
.HasColumnName("locale");
b.Property<long>("Timestamp")
.HasColumnType("INTEGER")
.HasColumnName("timestamp");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Locale")
.IsUnique()
.HasDatabaseName("state_locale");
b.HasIndex("Timestamp")
.HasDatabaseName("state_timestamp");
b.ToTable("state");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Function")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("function");
b.HasKey("Id")
.HasName("id");
b.HasIndex("Function")
.HasDatabaseName("syscall_info_function");
b.ToTable("syscall_info");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.Property<int>("ProductId")
.HasColumnType("INTEGER")
.HasColumnName("product_id");
b.Property<int>("SyscallInfoId")
.HasColumnType("INTEGER")
.HasColumnName("syscall_info_id");
b.HasKey("ProductId", "SyscallInfoId")
.HasName("id");
b.HasIndex("SyscallInfoId")
.HasDatabaseName("ix_syscall_to_product_map_syscall_info_id");
b.ToTable("syscall_to_product_map");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<long?>("CompatibilityChangeDate")
.HasColumnType("INTEGER")
.HasColumnName("compatibility_change_date");
b.Property<byte?>("CompatibilityStatus")
.HasColumnType("INTEGER")
.HasColumnName("compatibility_status");
b.Property<string>("ContentId")
.HasColumnType("TEXT")
.HasColumnName("content_id");
b.Property<int?>("EmbedColor")
.HasColumnType("INTEGER")
.HasColumnName("embed_color");
b.Property<string>("EmbeddableUrl")
.HasColumnType("TEXT")
.HasColumnName("embeddable_url");
b.Property<int?>("MetacriticId")
.HasColumnType("INTEGER")
.HasColumnName("metacritic_id");
b.Property<string>("Name")
.HasColumnType("TEXT")
.HasColumnName("name");
b.Property<string>("ProductCode")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("product_code");
b.Property<long>("Timestamp")
.HasColumnType("INTEGER")
.HasColumnName("timestamp");
b.Property<string>("Url")
.HasColumnType("TEXT")
.HasColumnName("url");
b.HasKey("Id")
.HasName("id");
b.HasIndex("ContentId")
.IsUnique()
.HasDatabaseName("thumbnail_content_id");
b.HasIndex("MetacriticId")
.HasDatabaseName("ix_thumbnail_metacritic_id");
b.HasIndex("ProductCode")
.IsUnique()
.HasDatabaseName("thumbnail_product_code");
b.HasIndex("Timestamp")
.HasDatabaseName("thumbnail_timestamp");
b.ToTable("thumbnail");
});
modelBuilder.Entity("CompatBot.Database.SyscallToProductMap", b =>
{
b.HasOne("CompatBot.Database.Thumbnail", "Product")
.WithMany("SyscallToProductMap")
.HasForeignKey("ProductId")
.HasConstraintName("fk_syscall_to_product_map__thumbnail_product_id")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("CompatBot.Database.SyscallInfo", "SyscallInfo")
.WithMany("SyscallToProductMap")
.HasForeignKey("SyscallInfoId")
.HasConstraintName("fk_syscall_to_product_map_syscall_info_syscall_info_id")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Product");
b.Navigation("SyscallInfo");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.HasOne("CompatBot.Database.Metacritic", "Metacritic")
.WithMany()
.HasForeignKey("MetacriticId")
.HasConstraintName("fk_thumbnail_metacritic_metacritic_id");
b.Navigation("Metacritic");
});
modelBuilder.Entity("CompatBot.Database.SyscallInfo", b =>
{
b.Navigation("SyscallToProductMap");
});
modelBuilder.Entity("CompatBot.Database.Thumbnail", b =>
{
b.Navigation("SyscallToProductMap");
});
#pragma warning restore 612, 618
}
}
}

29
CompatBot/Database/Migrations/ThumbnailDb/20210309212939_AddUserNamePool.cs Normal file
View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
namespace CompatBot.Migrations
{
public partial class AddUserNamePool : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "name_pool",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
name = table.Column<string>(type: "TEXT", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("id", x => x.id);
});
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "name_pool");
}
}
}

18
CompatBot/Database/Migrations/ThumbnailDb/ThumbnailDbModelSnapshot.cs
View File

@@ -64,6 +64,24 @@ namespace CompatBot.Migrations
b.ToTable("metacritic");
});
modelBuilder.Entity("CompatBot.Database.NamePool", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("name");
b.HasKey("Id")
.HasName("id");
b.ToTable("name_pool");
});
modelBuilder.Entity("CompatBot.Database.State", b =>
{
b.Property<int>("Id")

9
CompatBot/Database/ThumbnailDb.cs
View File

@@ -14,6 +14,7 @@ namespace CompatBot.Database
public DbSet<SyscallToProductMap> SyscallToProductMap { get; set; } = null!;
public DbSet<Metacritic> Metacritic { get; set; } = null!;
public DbSet<Fortune> Fortune { get; set; } = null!;
public DbSet<NamePool> NamePool { get; set; } = null!;
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -35,6 +36,7 @@ namespace CompatBot.Database
modelBuilder.Entity<SyscallInfo>().HasIndex(sci => sci.Function).HasDatabaseName("syscall_info_function");
modelBuilder.Entity<SyscallToProductMap>().HasKey(m => new {m.ProductId, m.SyscallInfoId});
modelBuilder.Entity<Fortune>();
modelBuilder.Entity<NamePool>();
//configure default policy of Id being the primary key
modelBuilder.ConfigureDefaultPkConvention();
@@ -126,4 +128,11 @@ namespace CompatBot.Database
[Required]
public string Content { get; set; } = null!;
}
internal class NamePool
{
public int Id { get; set; }
[Required]
public string Name { get; set; } = null!;
}
}

8
CompatBot/EventHandlers/UsernameZalgoMonitor.cs
View File

@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CompatApiClient.Utils;
using CompatBot.Database;
using CompatBot.Utils;
using DSharpPlus;
using DSharpPlus.Entities;
@@ -166,8 +168,10 @@ namespace CompatBot.EventHandlers
{
var hash = userId.GetHashCode();
var rng = new Random(hash);
var name = NamesPool.List[rng.Next(NamesPool.NameCount)];
return name + NamesPool.NameSuffix;
using var db = new ThumbnailDb();
var count = db.NamePool.Count();
var name = db.NamePool.Skip(rng.Next(count)).First().Name;
return name + Config.RenameNameSuffix;
}
}
}

5
CompatBot/Program.cs
View File

@@ -101,9 +101,14 @@ namespace CompatBot
return;
await using (var db = new ThumbnailDb())
{
if (!await DbImporter.UpgradeAsync(db, Config.Cts.Token))
return;
if (!await DbImporter.ImportNamesPool(db, Config.Cts.Token))
return;
}
await SqlConfiguration.RestoreAsync().ConfigureAwait(false);
Config.Log.Debug("Restored configuration variables from persistent storage");

1
README.md
View File

@@ -65,5 +65,4 @@ External resources that need manual updates
-------------------------------------------
* [Unicode Confusables](http://www.unicode.org/Public/security/latest/confusables.txt), for Homoglyph checks
* [Windows Error Codes](https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-erref/), for error decoding on non-Windows host
* Optionally [Redump disc key database](http://redump.org/downloads/) in text format (requires membership)
* Optionally pool of names (one name per line), files named as `names_<category>.txt`

91
SourceGenerators/NamesSourceGenerator.cs
View File

@@ -1,91 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Text;
namespace SourceGenerators
{
[Generator]
public class NamesSourceGenerator : ISourceGenerator
{
private const string Indent = " ";
private const string NameSuffix = " (Rule 7)";
//private const int DiscordUsernameLengthLimit = 32-10; //" #12345678"
private const int DiscordUsernameLengthLimit = 32;
public void Initialize(GeneratorInitializationContext context)
{
}
public void Execute(GeneratorExecutionContext context)
{
var resources = context.AdditionalFiles
.Where(f => Path.GetFileName(f.Path).ToLower().StartsWith("names_") && f.Path.ToLower().EndsWith(".txt"))
.OrderBy(f => f.Path)
.ToList();
if (resources.Count == 0)
return;
var names = new HashSet<string>();
foreach (var resource in resources)
{
using var stream = File.Open(resource.Path, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new StreamReader(stream);
while (reader.ReadLine() is string line)
{
if (line.Length < 2 || line.StartsWith("#"))
continue;
var commentPos = line.IndexOf(" (");
if (commentPos > 1)
line = line.Substring(0, commentPos);
line = line.Trim()
.Replace(" ", " ")
.Replace('`', '\'') // consider ’
.Replace("\"", "\\\"");
//if (line.Length + NameSuffix.Length > DiscordUsernameLengthLimit)
// line = line.Split(' ')[0];
if (line.Length + NameSuffix.Length > DiscordUsernameLengthLimit)
continue;
if (line.Contains('@')
|| line.Contains('#')
|| line.Contains(':'))
continue;
names.Add(line);
//if (line.Contains(' '))
// names.Add(line.Split(' ')[0]);
}
}
if (!context.AnalyzerConfigOptions.GlobalOptions.TryGetValue("build_property.RootNamespace", out var ns))
ns = context.Compilation.AssemblyName;
var cn = "NamesPool";
var result = new StringBuilder()
.AppendLine("using System.Collections.Generic;")
.AppendLine()
.AppendLine($"namespace {ns}")
.AppendLine("{")
.AppendLine($"{Indent}public static class {cn}")
.AppendLine($"{Indent}{{")
.AppendLine($"{Indent}{Indent}public const string NameSuffix = \"{NameSuffix}\";")
.AppendLine()
.AppendLine($"{Indent}{Indent}public const int NameCount = {names.Count};")
.AppendLine()
.AppendLine($"{Indent}{Indent}public static readonly List<string> List = new()")
.AppendLine($"{Indent}{Indent}{{");
foreach (var name in names.OrderBy(n => n))
result.AppendLine($"{Indent}{Indent}{Indent}\"{name}\",");
result.AppendLine($"{Indent}{Indent}}};")
.AppendLine($"{Indent}}}")
.AppendLine("}");
context.AddSource($"{cn}.Generated.cs", SourceText.From(result.ToString(), Encoding.UTF8));
}
}
}