RPCS3/discord-bot
1
0
Fork 0
mirror of https://github.com/RPCS3/discord-bot.git synced 2024-11-23 10:19:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

game title scraping from gametdb

Browse Source
This commit is contained in:
13xforever 2018-08-19 02:21:50 +05:00 committed by Roberto Anić Banić
parent 176d5ab919
commit 0956410d69
7 changed files with 165 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CompatApiClient/Utils/ConsoleLogger.cs
View File

@ -5,7 +5,6 @@ namespace CompatApiClient.Utils
{
public static class ConsoleLogger
{
public static void PrintError(Exception e, HttpResponseMessage response, ConsoleColor color = ConsoleColor.Red)
{
Console.ForegroundColor = ConsoleColor.Red;

2
CompatBot/Commands/Psn.Check.cs
View File

@ -55,7 +55,7 @@ namespace CompatBot.Commands
}
foreach (var id in itemsToCheck)
PsnScraper.CheckContentId(id, Config.Cts.Token);
PsnScraper.CheckContentIdAsync(ctx, id, Config.Cts.Token);
await ctx.ReactWithAsync(Config.Reactions.Success, $"Added {itemsToCheck.Count} ID{StringUtils.GetSuffix(itemsToCheck.Count)} to the scraping queue").ConfigureAwait(false);
}

13
CompatBot/Database/Providers/ScrapeStateProvider.cs
View File

@ -31,6 +31,19 @@ namespace CompatBot.Database.Providers
return false;
}
public static bool IsFresh(string locale, DateTime dataTimestamp)
{
using (var db = new ThumbnailDb())
{
var timestamp = string.IsNullOrEmpty(locale) ? db.State.OrderBy(s => s.Timestamp).FirstOrDefault() : db.State.FirstOrDefault(s => s.Locale == locale);
if (timestamp?.Timestamp is long checkDate && checkDate > 0)
return new DateTime(checkDate, DateTimeKind.Utc) > dataTimestamp;
}
return false;
}
public static async Task SetLastRunTimestampAsync(string locale, string containerId = null)
{
if (string.IsNullOrEmpty(locale))

5
CompatBot/Database/Providers/ThumbnailProvider.cs
View File

@ -10,6 +10,8 @@ namespace CompatBot.Database.Providers
{
internal static class ThumbnailProvider
{
private static readonly HttpClient HttpClient = HttpClientFactory.Create();
public static async Task<string> GetThumbnailUrlAsync(this DiscordClient client, string productCode)
{
using (var db = new ThumbnailDb())
@ -30,8 +32,7 @@ namespace CompatBot.Database.Providers
try
{
using (var httpClient = new HttpClient())
using (var imgStream = await httpClient.GetStreamAsync(url).ConfigureAwait(false))
using (var imgStream = await HttpClient.GetStreamAsync(url).ConfigureAwait(false))
using (var memStream = new MemoryStream())
{
await imgStream.CopyToAsync(memStream).ConfigureAwait(false);

9
CompatBot/Program.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CompatBot.Commands;
@ -59,7 +58,8 @@ namespace CompatBot
if (!await DbImporter.UpgradeAsync(db, Config.Cts.Token))
return;
var psnScrappingTask = new PsnScraper().Run(Config.Cts.Token);
var psnScrappingTask = new PsnScraper().RunAsync(Config.Cts.Token);
var gameTdbScrapingTask = GameTdbScraper.RunAsync(Config.Cts.Token);
var config = new DiscordConfiguration
{
@ -153,7 +153,10 @@ namespace CompatBot
await Task.Delay(TimeSpan.FromMinutes(1), Config.Cts.Token).ContinueWith(dt => {/* in case it was cancelled */}).ConfigureAwait(false);
}
}
await psnScrappingTask.ConfigureAwait(false);
await Task.WhenAll(
psnScrappingTask,
gameTdbScrapingTask
).ConfigureAwait(false);
}
catch (TaskCanceledException)
{

136
CompatBot/ThumbScrapper/GameTdbScraper.cs Normal file
View File

@ -0,0 +1,136 @@
using System;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.IO.Pipelines;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using CompatApiClient.Compression;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers;
using Microsoft.EntityFrameworkCore;
namespace CompatBot.ThumbScrapper
{
internal static class GameTdbScraper
{
private static readonly HttpClient HttpClient = HttpClientFactory.Create(new CompressionMessageHandler());
private static readonly Uri TitleDownloadLink = new Uri("https://www.gametdb.com/ps3tdb.zip?LANG=EN");
public static async Task RunAsync(CancellationToken cancellationToken)
{
do
{
if (cancellationToken.IsCancellationRequested)
break;
try
{
await UpdateGameTitlesAsync(cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
PrintError(e);
}
await Task.Delay(TimeSpan.FromDays(30), cancellationToken).ConfigureAwait(false);
} while (!cancellationToken.IsCancellationRequested);
}
public static async Task UpdateGameTitlesAsync(CancellationToken cancellationToken)
{
var container = Path.GetFileName(TitleDownloadLink.AbsolutePath);
try
{
if (ScrapeStateProvider.IsFresh(container))
return;
Console.WriteLine("Scraping GameTDB for game titles...");
using (var fileStream = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.Read, 16384, FileOptions.Asynchronous | FileOptions.RandomAccess | FileOptions.DeleteOnClose))
{
using (var downloadStream = await HttpClient.GetStreamAsync(TitleDownloadLink).ConfigureAwait(false))
await downloadStream.CopyToAsync(fileStream, 16384, cancellationToken).ConfigureAwait(false);
fileStream.Seek(0, SeekOrigin.Begin);
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read))
{
var logEntry = zipArchive.Entries.FirstOrDefault(e => e.Name.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase));
if (logEntry == null)
throw new InvalidOperationException("No zip entries that match the .xml criteria");
using (var zipStream = logEntry.Open())
using (var xmlReader = XmlReader.Create(zipStream))
{
xmlReader.ReadToFollowing("PS3TDB");
var version = xmlReader.GetAttribute("version");
if (!DateTime.TryParseExact(version, "yyyyMMddHHmmss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var timestamp))
return;
if (ScrapeStateProvider.IsFresh("PS3TDB", timestamp))
{
await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false);
return;
}
while (!cancellationToken.IsCancellationRequested && xmlReader.ReadToFollowing("game"))
{
if (xmlReader.ReadToFollowing("id"))
{
var productId = xmlReader.ReadElementContentAsString().ToUpperInvariant();
if (!ProductCodeLookup.ProductCode.IsMatch(productId))
continue;
string title = null;
if (xmlReader.ReadToFollowing("locale") && xmlReader.ReadToFollowing("title"))
title = xmlReader.ReadElementContentAsString();
if (!string.IsNullOrEmpty(title))
{
using (var db = new ThumbnailDb())
{
var item = await db.Thumbnail.FirstOrDefaultAsync(t => t.ProductCode == productId, cancellationToken).ConfigureAwait(false);
if (item == null)
{
await db.Thumbnail.AddAsync(new Thumbnail {ProductCode = productId, Name = title}, cancellationToken).ConfigureAwait(false);
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
else
{
if (item.Name != title && item.Timestamp == 0)
{
item.Name = title;
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
}
}
}
}
}
await ScrapeStateProvider.SetLastRunTimestampAsync("PS3TDB").ConfigureAwait(false);
}
}
}
await ScrapeStateProvider.SetLastRunTimestampAsync(container).ConfigureAwait(false);
}
catch (Exception e)
{
PrintError(e);
}
finally
{
Console.WriteLine("Finished scraping GameTDB for game titles");
}
}
private static void PrintError(Exception e)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error scraping titles from GameTDB: " + e);
Console.ResetColor();
}
}
}

11
CompatBot/ThumbScrapper/PsnScraper.cs
View File

@ -8,12 +8,13 @@ using System.Threading.Tasks;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers;
using DSharpPlus.CommandsNext;
using PsnClient.POCOs;
using PsnClient.Utils;
namespace CompatBot.ThumbScrapper
{
internal class PsnScraper
internal sealed class PsnScraper
{
private static readonly PsnClient.Client Client = new PsnClient.Client();
public static readonly Regex ContentIdMatcher = new Regex(@"(?<content_id>(?<service_id>(?<service_letters>\w\w)(?<service_number>\d{4}))-(?<product_id>(?<product_letters>\w{4})(?<product_number>\d{5}))_(?<part>\d\d)-(?<label>\w{16}))", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.ExplicitCapture);
@ -22,7 +23,7 @@ namespace CompatBot.ThumbScrapper
private static DateTime StoreRefreshTimestamp = DateTime.MinValue;
private static readonly SemaphoreSlim QueueLimiter = new SemaphoreSlim(32, 32);
public async Task Run(CancellationToken cancellationToken)
public async Task RunAsync(CancellationToken cancellationToken)
{
do
{
@ -43,7 +44,7 @@ namespace CompatBot.ThumbScrapper
} while (!cancellationToken.IsCancellationRequested);
}
public static async void CheckContentId(string contentId, CancellationToken cancellationToken)
public static async void CheckContentIdAsync(CommandContext ctx, string contentId, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(contentId))
return;
@ -75,11 +76,11 @@ namespace CompatBot.ThumbScrapper
if (relatedContainer == null)
continue;
Console.WriteLine($"\tFound {contentId} in {locale} store");
await ctx.RespondAsync($"Found {contentId} in {locale} store").ConfigureAwait(false);
await ProcessIncludedGamesAsync(locale, relatedContainer, cancellationToken, false).ConfigureAwait(false);
return;
}
Console.WriteLine($"\tDidn't find {contentId} in any PSN store");
await ctx.RespondAsync($"Didn't find {contentId} in any PSN store").ConfigureAwait(false);
}
finally
{