From 8518b7824de960ce00de5fd2914584e000d16328 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 4 Jul 2022 21:48:32 +0500 Subject: [PATCH 1/3] update handling of bot usage stats * wrap everything inside the provider * do hour-long buckets to workaround the long-standing issue of sticky data bias * remove stale data on stats restore instead of nuking-n-paving on every stat save --- CompatBot/Commands/BaseCommandModuleCustom.cs | 3 +- CompatBot/Commands/BotStats.cs | 21 +- CompatBot/Commands/CompatList.cs | 5 +- CompatBot/Commands/Explain.cs | 3 +- CompatBot/Database/BotDb.cs | 3 +- ...704163631_AddStatsBucketColumn.Designer.cs | 433 ++++++++++++++++++ .../20220704163631_AddStatsBucketColumn.cs | 45 ++ .../Migrations/BotDb/BotDbModelSnapshot.cs | 13 +- CompatBot/Database/Providers/StatsStorage.cs | 96 +++- .../EventHandlers/IsTheGamePlayableHandler.cs | 5 +- .../ResultFormatters/TitleInfoFormatter.cs | 8 +- 11 files changed, 581 insertions(+), 54 deletions(-) create mode 100644 CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.Designer.cs create mode 100644 CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.cs diff --git a/CompatBot/Commands/BaseCommandModuleCustom.cs b/CompatBot/Commands/BaseCommandModuleCustom.cs index a1d11c80..4bf88ec7 100644 --- a/CompatBot/Commands/BaseCommandModuleCustom.cs +++ b/CompatBot/Commands/BaseCommandModuleCustom.cs @@ -63,8 +63,7 @@ internal class BaseCommandModuleCustom : BaseCommandModule { if (ctx.Command?.QualifiedName is string qualifiedName) { - StatsStorage.CmdStatCache.TryGetValue(qualifiedName, out int counter); - StatsStorage.CmdStatCache.Set(qualifiedName, ++counter, StatsStorage.CacheTime); + StatsStorage.IncCmdStat(qualifiedName); Config.TelemetryClient?.TrackRequest(qualifiedName, executionStart, DateTimeOffset.UtcNow - executionStart, HttpStatusCode.OK.ToString(), true); } diff --git a/CompatBot/Commands/BotStats.cs b/CompatBot/Commands/BotStats.cs index da10be7a..c5395515 100644 --- a/CompatBot/Commands/BotStats.cs +++ b/CompatBot/Commands/BotStats.cs @@ -171,12 +171,7 @@ internal sealed class BotStats: BaseCommandModuleCustom private static void AppendCmdStats(DiscordEmbedBuilder embed) { - var commandStats = StatsStorage.CmdStatCache.GetCacheKeys(); - var sortedCommandStats = commandStats - .Select(c => (name: c, stat: StatsStorage.CmdStatCache.Get(c) as int?)) - .Where(c => c.stat.HasValue) - .OrderByDescending(c => c.stat) - .ToList(); + var sortedCommandStats = StatsStorage.GetCmdStats(); var totalCalls = sortedCommandStats.Sum(c => c.stat); var top = sortedCommandStats.Take(5).ToList(); if (top.Count == 0) @@ -192,12 +187,7 @@ internal sealed class BotStats: BaseCommandModuleCustom private static void AppendExplainStats(DiscordEmbedBuilder embed) { - var terms = StatsStorage.ExplainStatCache.GetCacheKeys(); - var sortedTerms = terms - .Select(t => (term: t, stat: StatsStorage.ExplainStatCache.Get(t) as int?)) - .Where(t => t.stat.HasValue) - .OrderByDescending(t => t.stat) - .ToList(); + var sortedTerms = StatsStorage.GetExplainStats(); var totalExplains = sortedTerms.Sum(t => t.stat); var top = sortedTerms.Take(5).ToList(); if (top.Count == 0) @@ -213,12 +203,7 @@ internal sealed class BotStats: BaseCommandModuleCustom private static void AppendGameLookupStats(DiscordEmbedBuilder embed) { - var gameTitles = StatsStorage.GameStatCache.GetCacheKeys(); - var sortedTitles = gameTitles - .Select(t => (title: t, stat: StatsStorage.GameStatCache.Get(t) as int?)) - .Where(t => t.stat.HasValue) - .OrderByDescending(t => t.stat) - .ToList(); + var sortedTitles = StatsStorage.GetGameStats(); var totalLookups = sortedTitles.Sum(t => t.stat); var top = sortedTitles.Take(5).ToList(); if (top.Count == 0) diff --git a/CompatBot/Commands/CompatList.cs b/CompatBot/Commands/CompatList.cs index 4f04799a..4ee32c4f 100644 --- a/CompatBot/Commands/CompatList.cs +++ b/CompatBot/Commands/CompatList.cs @@ -508,10 +508,7 @@ internal sealed class CompatList : BaseCommandModuleCustom || (t.info.Title?.StartsWith(searchTerm, StringComparison.InvariantCultureIgnoreCase) ?? false) || (t.info.AlternativeTitle?.StartsWith(searchTerm, StringComparison.InvariantCultureIgnoreCase) ?? false)); foreach (var title in searchHits.Select(t => t.info.Title).Distinct()) - { - StatsStorage.GameStatCache.TryGetValue(title, out int stat); - StatsStorage.GameStatCache.Set(title, ++stat, StatsStorage.CacheTime); - } + StatsStorage.IncGameStat(title); foreach (var resultInfo in sortedList.Take(request.AmountRequested)) { var info = resultInfo.AsString(); diff --git a/CompatBot/Commands/Explain.cs b/CompatBot/Commands/Explain.cs index 32fe30d6..3683a2d4 100644 --- a/CompatBot/Commands/Explain.cs +++ b/CompatBot/Commands/Explain.cs @@ -397,8 +397,7 @@ internal sealed class Explain: BaseCommandModuleCustom } var explain = termLookupResult.explanation; - StatsStorage.ExplainStatCache.TryGetValue(explain.Keyword, out int stat); - StatsStorage.ExplainStatCache.Set(explain.Keyword, ++stat, StatsStorage.CacheTime); + StatsStorage.IncExplainStat(explain.Keyword); msgBuilder = new DiscordMessageBuilder().WithContent(explain.Text); if (!usedReply && useReply) msgBuilder.WithReply(sourceMessage.Id); diff --git a/CompatBot/Database/BotDb.cs b/CompatBot/Database/BotDb.cs index f3e3dc5a..907e2488 100644 --- a/CompatBot/Database/BotDb.cs +++ b/CompatBot/Database/BotDb.cs @@ -44,7 +44,7 @@ internal class BotDb: DbContext modelBuilder.Entity().HasIndex(c => c.Command).IsUnique().HasDatabaseName("disabled_command_command"); modelBuilder.Entity().HasIndex(i => i.GuildId).IsUnique().HasDatabaseName("whitelisted_invite_guild_id"); modelBuilder.Entity().HasIndex(e => new {e.Year, e.EventName}).HasDatabaseName("event_schedule_year_event_name"); - modelBuilder.Entity().HasIndex(s => new { s.Category, s.Key }).IsUnique().HasDatabaseName("stats_category_key"); + modelBuilder.Entity().HasIndex(s => new { s.Category, s.Bucket, s.Key }).IsUnique().HasDatabaseName("stats_category_bucket_key"); modelBuilder.Entity().HasIndex(k => k.UserId).IsUnique().HasDatabaseName("kot_user_id"); modelBuilder.Entity().HasIndex(d => d.UserId).IsUnique().HasDatabaseName("doggo_user_id"); modelBuilder.Entity().HasIndex(d => new { d.GuildId, d.UserId }).IsUnique().HasDatabaseName("forced_nickname_guild_id_user_id"); @@ -169,6 +169,7 @@ internal class Stats public int Id { get; set; } [Required] public string Category { get; set; } = null!; + public string? Bucket { get; set; } [Required] public string Key { get; set; } = null!; public int Value { get; set; } diff --git a/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.Designer.cs b/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.Designer.cs new file mode 100644 index 00000000..5ea2fa14 --- /dev/null +++ b/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.Designer.cs @@ -0,0 +1,433 @@ +// +using System; +using CompatBot.Database; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace CompatBot.Database.Migrations +{ + [DbContext(typeof(BotDb))] + [Migration("20220704163631_AddStatsBucketColumn")] + partial class AddStatsBucketColumn + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "6.0.6"); + + modelBuilder.Entity("CompatBot.Database.BotState", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("Key") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("key"); + + b.Property("Value") + .HasColumnType("TEXT") + .HasColumnName("value"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("bot_state_key"); + + b.ToTable("bot_state"); + }); + + modelBuilder.Entity("CompatBot.Database.DisabledCommand", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("Command") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("command"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Command") + .IsUnique() + .HasDatabaseName("disabled_command_command"); + + b.ToTable("disabled_commands"); + }); + + modelBuilder.Entity("CompatBot.Database.Doggo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("UserId") + .HasColumnType("INTEGER") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("UserId") + .IsUnique() + .HasDatabaseName("doggo_user_id"); + + b.ToTable("doggo"); + }); + + modelBuilder.Entity("CompatBot.Database.EventSchedule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("End") + .HasColumnType("INTEGER") + .HasColumnName("end"); + + b.Property("EventName") + .HasColumnType("TEXT") + .HasColumnName("event_name"); + + b.Property("Name") + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.Property("Start") + .HasColumnType("INTEGER") + .HasColumnName("start"); + + b.Property("Year") + .HasColumnType("INTEGER") + .HasColumnName("year"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Year", "EventName") + .HasDatabaseName("event_schedule_year_event_name"); + + b.ToTable("event_schedule"); + }); + + modelBuilder.Entity("CompatBot.Database.Explanation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("Attachment") + .HasMaxLength(7340032) + .HasColumnType("BLOB") + .HasColumnName("attachment"); + + b.Property("AttachmentFilename") + .HasColumnType("TEXT") + .HasColumnName("attachment_filename"); + + b.Property("Keyword") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("keyword"); + + b.Property("Text") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("text"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Keyword") + .IsUnique() + .HasDatabaseName("explanation_keyword"); + + b.ToTable("explanation"); + }); + + modelBuilder.Entity("CompatBot.Database.ForcedNickname", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("GuildId") + .HasColumnType("INTEGER") + .HasColumnName("guild_id"); + + b.Property("Nickname") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("nickname"); + + b.Property("UserId") + .HasColumnType("INTEGER") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("GuildId", "UserId") + .IsUnique() + .HasDatabaseName("forced_nickname_guild_id_user_id"); + + b.ToTable("forced_nicknames"); + }); + + modelBuilder.Entity("CompatBot.Database.Kot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("UserId") + .HasColumnType("INTEGER") + .HasColumnName("user_id"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("UserId") + .IsUnique() + .HasDatabaseName("kot_user_id"); + + b.ToTable("kot"); + }); + + modelBuilder.Entity("CompatBot.Database.Moderator", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("DiscordId") + .HasColumnType("INTEGER") + .HasColumnName("discord_id"); + + b.Property("Sudoer") + .HasColumnType("INTEGER") + .HasColumnName("sudoer"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("DiscordId") + .IsUnique() + .HasDatabaseName("moderator_discord_id"); + + b.ToTable("moderator"); + }); + + modelBuilder.Entity("CompatBot.Database.Piracystring", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("Actions") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasDefaultValue(11) + .HasColumnName("actions"); + + b.Property("Context") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasDefaultValue((byte)3) + .HasColumnName("context"); + + b.Property("CustomMessage") + .HasColumnType("TEXT") + .HasColumnName("custom_message"); + + b.Property("Disabled") + .HasColumnType("INTEGER") + .HasColumnName("disabled"); + + b.Property("ExplainTerm") + .HasColumnType("TEXT") + .HasColumnName("explain_term"); + + b.Property("String") + .IsRequired() + .HasColumnType("varchar(255)") + .HasColumnName("string"); + + b.Property("ValidatingRegex") + .HasColumnType("TEXT") + .HasColumnName("validating_regex"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("String") + .HasDatabaseName("piracystring_string"); + + b.ToTable("piracystring"); + }); + + modelBuilder.Entity("CompatBot.Database.Stats", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("Bucket") + .HasColumnType("TEXT") + .HasColumnName("bucket"); + + b.Property("Category") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("category"); + + b.Property("ExpirationTimestamp") + .HasColumnType("INTEGER") + .HasColumnName("expiration_timestamp"); + + b.Property("Key") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("key"); + + b.Property("Value") + .HasColumnType("INTEGER") + .HasColumnName("value"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("Category", "Bucket", "Key") + .IsUnique() + .HasDatabaseName("stats_category_bucket_key"); + + b.ToTable("stats"); + }); + + modelBuilder.Entity("CompatBot.Database.SuspiciousString", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("String") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("string"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("String") + .HasDatabaseName("suspicious_string_string"); + + b.ToTable("suspicious_string"); + }); + + modelBuilder.Entity("CompatBot.Database.Warning", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("DiscordId") + .HasColumnType("INTEGER") + .HasColumnName("discord_id"); + + b.Property("FullReason") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("full_reason"); + + b.Property("IssuerId") + .HasColumnType("INTEGER") + .HasColumnName("issuer_id"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("reason"); + + b.Property("Retracted") + .HasColumnType("INTEGER") + .HasColumnName("retracted"); + + b.Property("RetractedBy") + .HasColumnType("INTEGER") + .HasColumnName("retracted_by"); + + b.Property("RetractionReason") + .HasColumnType("TEXT") + .HasColumnName("retraction_reason"); + + b.Property("RetractionTimestamp") + .HasColumnType("INTEGER") + .HasColumnName("retraction_timestamp"); + + b.Property("Timestamp") + .HasColumnType("INTEGER") + .HasColumnName("timestamp"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("DiscordId") + .HasDatabaseName("warning_discord_id"); + + b.ToTable("warning"); + }); + + modelBuilder.Entity("CompatBot.Database.WhitelistedInvite", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id"); + + b.Property("GuildId") + .HasColumnType("INTEGER") + .HasColumnName("guild_id"); + + b.Property("InviteCode") + .HasColumnType("TEXT") + .HasColumnName("invite_code"); + + b.Property("Name") + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("id"); + + b.HasIndex("GuildId") + .IsUnique() + .HasDatabaseName("whitelisted_invite_guild_id"); + + b.ToTable("whitelisted_invites"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.cs b/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.cs new file mode 100644 index 00000000..65d4e330 --- /dev/null +++ b/CompatBot/Database/Migrations/BotDb/20220704163631_AddStatsBucketColumn.cs @@ -0,0 +1,45 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace CompatBot.Database.Migrations +{ + public partial class AddStatsBucketColumn : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "stats_category_key", + table: "stats"); + + migrationBuilder.AddColumn( + name: "bucket", + table: "stats", + type: "TEXT", + nullable: true); + + migrationBuilder.CreateIndex( + name: "stats_category_bucket_key", + table: "stats", + columns: new[] { "category", "bucket", "key" }, + unique: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "stats_category_bucket_key", + table: "stats"); + + migrationBuilder.DropColumn( + name: "bucket", + table: "stats"); + + migrationBuilder.CreateIndex( + name: "stats_category_key", + table: "stats", + columns: new[] { "category", "key" }, + unique: true); + } + } +} diff --git a/CompatBot/Database/Migrations/BotDb/BotDbModelSnapshot.cs b/CompatBot/Database/Migrations/BotDb/BotDbModelSnapshot.cs index b2bc6e81..87fa11a1 100644 --- a/CompatBot/Database/Migrations/BotDb/BotDbModelSnapshot.cs +++ b/CompatBot/Database/Migrations/BotDb/BotDbModelSnapshot.cs @@ -5,6 +5,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +#nullable disable + namespace CompatBot.Database.Migrations { [DbContext(typeof(BotDb))] @@ -13,8 +15,7 @@ namespace CompatBot.Database.Migrations protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "5.0.5"); + modelBuilder.HasAnnotation("ProductVersion", "6.0.6"); modelBuilder.Entity("CompatBot.Database.BotState", b => { @@ -289,6 +290,10 @@ namespace CompatBot.Database.Migrations .HasColumnType("INTEGER") .HasColumnName("id"); + b.Property("Bucket") + .HasColumnType("TEXT") + .HasColumnName("bucket"); + b.Property("Category") .IsRequired() .HasColumnType("TEXT") @@ -310,9 +315,9 @@ namespace CompatBot.Database.Migrations b.HasKey("Id") .HasName("id"); - b.HasIndex("Category", "Key") + b.HasIndex("Category", "Bucket", "Key") .IsUnique() - .HasDatabaseName("stats_category_key"); + .HasDatabaseName("stats_category_bucket_key"); b.ToTable("stats"); }); diff --git a/CompatBot/Database/Providers/StatsStorage.cs b/CompatBot/Database/Providers/StatsStorage.cs index 69a917e3..9766b09f 100644 --- a/CompatBot/Database/Providers/StatsStorage.cs +++ b/CompatBot/Database/Providers/StatsStorage.cs @@ -11,12 +11,14 @@ namespace CompatBot.Database.Providers; internal static class StatsStorage { - internal static readonly TimeSpan CacheTime = TimeSpan.FromDays(1); - internal static readonly MemoryCache CmdStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); - internal static readonly MemoryCache ExplainStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); - internal static readonly MemoryCache GameStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); + private static readonly TimeSpan CacheTime = TimeSpan.FromDays(1); + private static readonly MemoryCache CmdStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); + private static readonly MemoryCache ExplainStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); + private static readonly MemoryCache GameStatCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromDays(1) }); + private const char PrefixSeparator = '\0'; private static readonly SemaphoreSlim Barrier = new(1, 1); + private static readonly SemaphoreSlim BucketLock = new(1, 1); private static readonly (string name, MemoryCache cache)[] AllCaches = { (nameof(CmdStatCache), CmdStatCache), @@ -24,6 +26,50 @@ internal static class StatsStorage (nameof(GameStatCache), GameStatCache), }; + private static ((int y, int m, int d, int h) Key, string Value) bucketPrefix = ((0, 0, 0, 0), ""); + + private static string Prefix + { + get + { + var ts = DateTime.UtcNow; + var key = (ts.Year, ts.Month, ts.Day, ts.Hour); + if (bucketPrefix.Key == key) + return bucketPrefix.Value; + + if (!BucketLock.Wait(0)) + return bucketPrefix.Value; + + bucketPrefix = (key, ts.ToString("yyyyMMddHH") + PrefixSeparator); + BucketLock.Release(); + return bucketPrefix.Value; + } + } + + public static void IncCmdStat(string qualifiedName) => IncStat(qualifiedName, CmdStatCache); + public static void IncExplainStat(string term) => IncStat(term, ExplainStatCache); + public static void IncGameStat(string title) => IncStat(title, GameStatCache); + private static void IncStat(string key, MemoryCache cache) + { + var bucketKey = Prefix + key; + cache.TryGetValue(bucketKey, out int stat); + cache.Set(bucketKey, ++stat, CacheTime); + } + + public static List<(string name, int stat)> GetCmdStats() => GetStats(CmdStatCache); + public static List<(string name, int stat)> GetExplainStats() => GetStats(ExplainStatCache); + public static List<(string name, int stat)> GetGameStats() => GetStats(GameStatCache); + private static List<(string name, int stat)> GetStats(MemoryCache cache) + { + return cache.GetCacheKeys() + .Select(c => (name: c.Split(PrefixSeparator, 2)[^1], stat: cache.Get(c) as int?)) + .Where(s => s.stat.HasValue) + .GroupBy(s => s.name) + .Select(g => (name: g.Key, stat: (int)g.Sum(s => s.stat)!)) + .OrderByDescending(s => s.stat) + .ToList(); + } + public static async Task SaveAsync(bool wait = false) { if (await Barrier.WaitAsync(0).ConfigureAwait(false)) @@ -32,21 +78,35 @@ internal static class StatsStorage { Config.Log.Debug("Got stats saving lock"); await using var db = new BotDb(); - db.Stats.RemoveRange(db.Stats); - await db.SaveChangesAsync().ConfigureAwait(false); foreach (var (category, cache) in AllCaches) { var entries = cache.GetCacheEntries(); var savedKeys = new HashSet(); foreach (var (key, value) in entries) if (savedKeys.Add(key)) - await db.Stats.AddAsync(new Stats + { + var keyParts = key.Split(PrefixSeparator, 2); + var bucket = keyParts.Length == 2 ? keyParts[0] : null; + var statKey = keyParts[^1]; + var statValue = (int?)value?.Value ?? 0; + var ts = value?.AbsoluteExpiration?.ToUniversalTime().Ticks ?? 0; + + var currentEntry = db.Stats.FirstOrDefault(e => e.Category == category && e.Bucket == bucket && e.Key == statKey); + if (currentEntry is null) + await db.Stats.AddAsync(new() + { + Category = category, + Bucket = bucket, + Key = statKey, + Value = statValue, + ExpirationTimestamp = ts + }).ConfigureAwait(false); + else { - Category = category, - Key = key, - Value = (int?)value?.Value ?? 0, - ExpirationTimestamp = value?.AbsoluteExpiration?.ToUniversalTime().Ticks ?? 0 - }).ConfigureAwait(false); + currentEntry.Value = statValue; + currentEntry.ExpirationTimestamp = ts; + } + } else Config.Log.Warn($"Somehow there's another '{key}' in the {category} cache"); } @@ -80,9 +140,19 @@ internal static class StatsStorage { var time = entry.ExpirationTimestamp.AsUtc(); if (time > now) - cache.Set(entry.Key, entry.Value, time); + { + var key = entry.Key; + if (entry.Bucket is { Length: > 0 } bucket) + key = bucket + PrefixSeparator + key; + cache.Set(key, entry.Value, time); + } + else + { + db.Stats.Remove(entry); + } } } + await db.SaveChangesAsync(Config.Cts.Token).ConfigureAwait(false); } public static async Task BackgroundSaveAsync() diff --git a/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs b/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs index 953d7b7f..0b0146b7 100644 --- a/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs +++ b/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs @@ -125,10 +125,7 @@ internal static class IsTheGamePlayableHandler return (null, null); if (!string.IsNullOrEmpty(info.Title)) - { - StatsStorage.GameStatCache.TryGetValue(info.Title, out int stat); - StatsStorage.GameStatCache.Set(info.Title, ++stat, StatsStorage.CacheTime); - } + StatsStorage.IncGameStat(info.Title); return (code, info); } catch (Exception e) diff --git a/CompatBot/Utils/ResultFormatters/TitleInfoFormatter.cs b/CompatBot/Utils/ResultFormatters/TitleInfoFormatter.cs index 46338ca9..680d1881 100644 --- a/CompatBot/Utils/ResultFormatters/TitleInfoFormatter.cs +++ b/CompatBot/Utils/ResultFormatters/TitleInfoFormatter.cs @@ -102,10 +102,7 @@ internal static class TitleInfoFormatter desc += " (cached)"; var cacheTitle = info.Title ?? gameTitle; if (!string.IsNullOrEmpty(cacheTitle)) - { - StatsStorage.GameStatCache.TryGetValue(cacheTitle, out int stat); - StatsStorage.GameStatCache.Set(cacheTitle, ++stat, StatsStorage.CacheTime); - } + StatsStorage.IncGameStat(cacheTitle); var title = $"{productCodePart}{cacheTitle?.Trim(200)}{onlineOnlyPart}"; if (string.IsNullOrEmpty(title)) desc = ""; @@ -143,8 +140,7 @@ internal static class TitleInfoFormatter gameTitle = titleName; if (!string.IsNullOrEmpty(gameTitle)) { - StatsStorage.GameStatCache.TryGetValue(gameTitle, out int stat); - StatsStorage.GameStatCache.Set(gameTitle, ++stat, StatsStorage.CacheTime); + StatsStorage.IncGameStat(gameTitle); result.Title = $"{productCodePart}{gameTitle.Sanitize().Trim(200)}"; } return result; From 47b7f2fd4e58576b8911a2fd33e236a77f7775ae Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 4 Jul 2022 22:12:20 +0500 Subject: [PATCH 2/3] try to fix another exception I've seen in logs --- CompatBot/EventHandlers/IsTheGamePlayableHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs b/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs index 0b0146b7..5bd23932 100644 --- a/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs +++ b/CompatBot/EventHandlers/IsTheGamePlayableHandler.cs @@ -32,7 +32,7 @@ internal static class IsTheGamePlayableHandler public static async Task OnMessageCreated(DiscordClient c, MessageCreateEventArgs args) { - if (DefaultHandlerFilter.IsFluff(args.Message)) + if (DefaultHandlerFilter.IsFluff(args.Message) || args.Channel is null) return; #if !DEBUG From 659c569aa7e1db5fb23b4e94811696e38bc95f53 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 4 Jul 2022 22:26:17 +0500 Subject: [PATCH 3/3] downgrade dsharpplus packages until they fix PopulateMentions() bug --- CompatBot/CompatBot.csproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CompatBot/CompatBot.csproj b/CompatBot/CompatBot.csproj index 8c6e7326..265a2920 100644 --- a/CompatBot/CompatBot.csproj +++ b/CompatBot/CompatBot.csproj @@ -39,10 +39,10 @@ - - - - + + + +