From 49a6e202dd3c6a2bf9920842d110692f211187f7 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 5 Nov 2018 16:01:31 +0500 Subject: [PATCH] IRD downloads with local caching --- CompatBot/Commands/CompatList.cs | 10 +-- CompatBot/Config.cs | 1 + CompatBot/Program.cs | 12 +++- IrdLibraryClient/IrdClient.cs | 101 +++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/CompatBot/Commands/CompatList.cs b/CompatBot/Commands/CompatList.cs index 9b90d9b8..f5675c62 100644 --- a/CompatBot/Commands/CompatList.cs +++ b/CompatBot/Commands/CompatList.cs @@ -33,7 +33,7 @@ namespace CompatBot.Commands } [Command("compat"), Aliases("c")] - [Description("Searches the compatibility database, USE: !compat searchterm")] + [Description("Searches the compatibility database, USE: !compat search term")] public async Task Compat(CommandContext ctx, [RemainingText, Description("Game title to look up")] string title) { try @@ -205,7 +205,7 @@ Example usage: result.AppendFormat(returnCode.info, compatResult.SearchTerm); yield return result.ToString(); result.Clear(); - var footer = $"Retrieved from: *<{request.Build(false).ToString().Replace(' ', '+')}>* in {compatResult.RequestDuration.TotalMilliseconds:0} milliseconds!"; + //var footer = $"Retrieved from: *<{request.Build(false).ToString().Replace(' ', '+')}>* in {compatResult.RequestDuration.TotalMilliseconds:0} milliseconds!"; if (returnCode.displayResults) { @@ -217,10 +217,10 @@ Example usage: } result.Append("```"); yield return result.ToString(); - yield return footer; + //yield return footer; } - else if (returnCode.displayFooter) - yield return footer; + //else if (returnCode.displayFooter) + // yield return footer; } } } diff --git a/CompatBot/Config.cs b/CompatBot/Config.cs index ebf163cd..e45c52cf 100644 --- a/CompatBot/Config.cs +++ b/CompatBot/Config.cs @@ -29,6 +29,7 @@ namespace CompatBot public static readonly string Token; public static readonly string LogPath = "../../../logs/bot.log"; // paths are relative to the assembly, so this will put it in the project's root + public static readonly string IrdCachePath = "./ird/"; internal static readonly ILogger Log; diff --git a/CompatBot/Program.cs b/CompatBot/Program.cs index ae97a109..401b69e1 100644 --- a/CompatBot/Program.cs +++ b/CompatBot/Program.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using CompatBot.Commands; @@ -81,6 +82,16 @@ namespace CompatBot var gameTdbScrapingTask = GameTdbScraper.RunAsync(Config.Cts.Token); await amdDriverRefreshTask.ConfigureAwait(false); + try + { + if (!Directory.Exists(Config.IrdCachePath)) + Directory.CreateDirectory(Config.IrdCachePath); + } + catch (Exception e) + { + Config.Log.Warn(e, $"Failed to create new folder {Config.IrdCachePath}: {e.Message}"); + } + var config = new DiscordConfiguration { Token = Config.Token, @@ -88,7 +99,6 @@ namespace CompatBot //UseInternalLogHandler = true, //LogLevel = LogLevel.Debug, }; - using (var client = new DiscordClient(config)) { var commands = client.UseCommandsNext(new CommandsNextConfiguration diff --git a/IrdLibraryClient/IrdClient.cs b/IrdLibraryClient/IrdClient.cs index 5d435af8..1430c509 100644 --- a/IrdLibraryClient/IrdClient.cs +++ b/IrdLibraryClient/IrdClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; @@ -101,6 +102,106 @@ namespace IrdLibraryClient } } + public async Task> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken) + { + var result = new List(); + try + { + // first we search local cache and try to load whatever data we can + var localCacheItems = new List(); + try + { + var tmpCacheItemList = Directory.GetFiles(localCachePath, productCode + "*.ird", SearchOption.TopDirectoryOnly).Select(Path.GetFileName).ToList(); + foreach (var item in tmpCacheItemList) + { + try + { + result.Add(File.ReadAllBytes(Path.Combine(localCachePath, item))); + localCacheItems.Add(item); + } + catch (Exception ex) + { + ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message); + } + } + } + catch (Exception e) + { + ApiConfig.Log.Warn(e, "Error accessing local IRD cache: " + e.Message); + } + ApiConfig.Log.Debug($"Found {localCacheItems.Count} cached items for {productCode}"); + SearchResult searchResult = null; + + // then try to do IRD Library search + try + { + searchResult = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + } + var tmpFilesToGet = searchResult?.Data.Select(i => i.Filename).Except(localCacheItems, StringComparer.InvariantCultureIgnoreCase).ToList(); + if ((tmpFilesToGet?.Count ?? 0) == 0) + return result; + + // as IRD Library could return more data than we found, try to check for all the items locally + var filesToDownload = new List(); + foreach (var item in tmpFilesToGet) + { + try + { + var localItemPath = Path.Combine(localCachePath, item); + if (File.Exists(localItemPath)) + { + result.Add(File.ReadAllBytes(localItemPath)); + localCacheItems.Add(item); + } + else + filesToDownload.Add(item); + } + catch (Exception ex) + { + ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message); + filesToDownload.Add(item); + } + } + ApiConfig.Log.Debug($"Found {tmpFilesToGet.Count} total matches for {productCode}, {result.Count} already cached"); + if (filesToDownload.Count == 0) + return result; + + // download the remaining .ird files + foreach (var item in filesToDownload) + { + try + { + var resultBytes = await client.GetByteArrayAsync(GetDownloadLink(item)).ConfigureAwait(false); + result.Add(resultBytes); + try + { + //todo: check if it's a valid .ird file before caching + File.WriteAllBytes(Path.Combine(localCachePath, item), resultBytes); + } + catch (Exception ex) + { + ApiConfig.Log.Warn(ex, $"Failed to write {item} to local cache: {ex.Message}"); + } + } + catch (Exception e) + { + ApiConfig.Log.Warn(e, $"Failed to download {item}: {e.Message}"); + } + } + ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}"); + return result; + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + return result; + } + } + private static string GetIrdFilename(string html) { if (string.IsNullOrEmpty(html))