From b98bd3daee5dace3cf6364e835a49cddddc12830 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sat, 27 Dec 2025 05:31:07 +0500 Subject: [PATCH] Add User-Agent to GameTDB scraper --- Clients/CompatApiClient/Client.cs | 2 +- Clients/CompatApiClient/Utils/Utils.cs | 7 ++++++ Clients/GithubClient/Client.cs | 3 +-- Clients/MediafireClient/Client.cs | 22 ++++++------------ Clients/OneDriveClient/Client.cs | 23 ++++++------------- Clients/YandexDiskClient/Client.cs | 19 +++++---------- .../EventHandlers/DiscordInviteFilter.cs | 7 +++--- CompatBot/ThumbScrapper/GameTdbScraper.cs | 3 ++- 8 files changed, 34 insertions(+), 52 deletions(-) diff --git a/Clients/CompatApiClient/Client.cs b/Clients/CompatApiClient/Client.cs index 99f3a12a..663b8e9c 100644 --- a/Clients/CompatApiClient/Client.cs +++ b/Clients/CompatApiClient/Client.cs @@ -19,7 +19,7 @@ public class Client: IDisposable private static readonly MemoryCache ResponseCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromHours(1) }); private static readonly string[] BuildArchList = [ArchType.X64, ArchType.Arm]; - private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()); + private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent(); private readonly JsonSerializerOptions jsonOptions = new() { PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, diff --git a/Clients/CompatApiClient/Utils/Utils.cs b/Clients/CompatApiClient/Utils/Utils.cs index 21dd1336..f9365caf 100644 --- a/Clients/CompatApiClient/Utils/Utils.cs +++ b/Clients/CompatApiClient/Utils/Utils.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; namespace CompatApiClient.Utils; @@ -57,4 +58,10 @@ public static class Utils < UnderGB => $"{bytes / (1024.0 * 1024):0.##} MB", _ => $"{bytes / (1024.0 * 1024 * 1024):0.##} GB" }; + + public static HttpClient WithUserAgent(this HttpClient client) + { + client.DefaultRequestHeaders.UserAgent.Add(ApiConfig.ProductInfoHeader); + return client; + } } \ No newline at end of file diff --git a/Clients/GithubClient/Client.cs b/Clients/GithubClient/Client.cs index 996172f7..f90bcfd8 100644 --- a/Clients/GithubClient/Client.cs +++ b/Clients/GithubClient/Client.cs @@ -13,7 +13,7 @@ namespace GithubClient; public partial class Client { - private readonly GitHubClient client; + private readonly GitHubClient client = new(new ProductHeaderValue(ApiConfig.ProductName, ApiConfig.ProductVersion)); private const string OwnerId = "RPCS3"; private const string RepoId = "rpcs3"; @@ -28,7 +28,6 @@ public partial class Client public Client(string? githubToken) { - client = new(new ProductHeaderValue(ApiConfig.ProductName, ApiConfig.ProductVersion)); if (githubToken is {Length: >0}) client.Credentials = new(githubToken); } diff --git a/Clients/MediafireClient/Client.cs b/Clients/MediafireClient/Client.cs index 84025c5d..881fb386 100644 --- a/Clients/MediafireClient/Client.cs +++ b/Clients/MediafireClient/Client.cs @@ -17,8 +17,13 @@ namespace MediafireClient; public sealed partial class Client { - private readonly HttpClient client; - private readonly JsonSerializerOptions jsonOptions; + private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent(); + private readonly JsonSerializerOptions jsonOptions = new() + { + PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + IncludeFields = true, + }; //var optSecurityToken = "1605819132.376f3d84695f46daa7b69ee67fbc5edb0a00843a8b2d5ac7d3d1b1ad8a4212b0"; //private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?.+)""", RegexOptions.ExplicitCapture); @@ -26,24 +31,12 @@ public sealed partial class Client [GeneratedRegex(@"(var\s+optDirectURL|href)\s*=\s*""(?https?://download\d+\.mediafire\.com/.+)""")] private static partial Regex DirectUrlRegex(); - public Client() - { - client = HttpClientFactory.Create(new CompressionMessageHandler()); - jsonOptions = new() - { - PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - IncludeFields = true, - }; - } - public async Task GetWebLinkAsync(string quickKey, CancellationToken cancellationToken) { try { var uri = new Uri($"https://www.mediafire.com/api/1.5/file/get_links.php?quick_key={quickKey}&response_format=json"); using var message = new HttpRequestMessage(HttpMethod.Get, uri); - message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false); try { @@ -67,7 +60,6 @@ public sealed partial class Client try { using var message = new HttpRequestMessage(HttpMethod.Get, webLink); - message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false); if (response.StatusCode is HttpStatusCode.Redirect or HttpStatusCode.TemporaryRedirect) { diff --git a/Clients/OneDriveClient/Client.cs b/Clients/OneDriveClient/Client.cs index 8d15e0de..74a37d7e 100644 --- a/Clients/OneDriveClient/Client.cs +++ b/Clients/OneDriveClient/Client.cs @@ -14,28 +14,20 @@ namespace OneDriveClient; public class Client { - private readonly HttpClient client; - private readonly HttpClient noRedirectsClient; - private readonly JsonSerializerOptions jsonOptions; - - public Client() + private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent(); + private readonly HttpClient noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler { AllowAutoRedirect = false }).WithUserAgent(); + private readonly JsonSerializerOptions jsonOptions = new() { - client = HttpClientFactory.Create(new CompressionMessageHandler()); - noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler {AllowAutoRedirect = false}); - jsonOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - IncludeFields = true, - }; - } + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + IncludeFields = true, + }; private async Task ResolveShortLink(Uri shortLink, CancellationToken cancellationToken) { try { using var message = new HttpRequestMessage(HttpMethod.Head, shortLink); - message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await noRedirectsClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); return response.Headers.Location; } @@ -82,7 +74,6 @@ public class Client ("select", "id,@content.downloadUrl,name,size") ); using var message = new HttpRequestMessage(HttpMethod.Get, resourceMetaUri); - message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { diff --git a/Clients/YandexDiskClient/Client.cs b/Clients/YandexDiskClient/Client.cs index 3828ad86..6bbc6cfe 100644 --- a/Clients/YandexDiskClient/Client.cs +++ b/Clients/YandexDiskClient/Client.cs @@ -15,19 +15,13 @@ namespace YandexDiskClient; public sealed class Client { - private readonly HttpClient client; - private readonly JsonSerializerOptions jsonOptions; - - public Client() + private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()); + private readonly JsonSerializerOptions jsonOptions = new() { - client = HttpClientFactory.Create(new CompressionMessageHandler()); - jsonOptions = new() - { - PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - IncludeFields = true, - }; - } + PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + IncludeFields = true, + }; public Task GetResourceInfoAsync(string shareKey, CancellationToken cancellationToken) => GetResourceInfoAsync(new Uri($"https://yadi.sk/d/{shareKey}"), cancellationToken); @@ -41,7 +35,6 @@ public sealed class Client ("fields", "size,name,file") ); using var message = new HttpRequestMessage(HttpMethod.Get, uri); - message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false); try { diff --git a/CompatBot/EventHandlers/DiscordInviteFilter.cs b/CompatBot/EventHandlers/DiscordInviteFilter.cs index f749b419..fd1d75ad 100644 --- a/CompatBot/EventHandlers/DiscordInviteFilter.cs +++ b/CompatBot/EventHandlers/DiscordInviteFilter.cs @@ -2,7 +2,9 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text.RegularExpressions; +using CompatApiClient; using CompatApiClient.Compression; +using CompatApiClient.Utils; using CompatBot.Commands; using CompatBot.Database; using CompatBot.Database.Providers; @@ -232,11 +234,10 @@ internal static partial class DiscordInviteFilter try { using var handler = new HttpClientHandler {AllowAutoRedirect = false}; // needed to store cloudflare session cookies - using var httpClient = HttpClientFactory.Create(handler, new CompressionMessageHandler()); + using var httpClient = HttpClientFactory.Create(handler, new CompressionMessageHandler()).WithUserAgent(); using var request = new HttpRequestMessage(HttpMethod.Get, "https://discord.me/" + meLink); request.Headers.Accept.Add(new("text/html")); request.Headers.CacheControl = CacheControlHeaderValue.Parse("no-cache"); - request.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0")); using var response = await httpClient.SendAsync(request).ConfigureAwait(false); var html = await response.Content.ReadAsStringAsync().ConfigureAwait(false); if (response.IsSuccessStatusCode) @@ -258,7 +259,6 @@ internal static partial class DiscordInviteFilter }!), }; postRequest.Headers.Accept.Add(new("text/html")); - postRequest.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0")); using var postResponse = await httpClient.SendAsync(postRequest).ConfigureAwait(false); if (postResponse.StatusCode == HttpStatusCode.Redirect) { @@ -267,7 +267,6 @@ internal static partial class DiscordInviteFilter { using var getDiscordRequest = new HttpRequestMessage(HttpMethod.Get, "https://discord.me/server/join/redirect/" + redirectId); getDiscordRequest.Headers.CacheControl = CacheControlHeaderValue.Parse("no-cache"); - getDiscordRequest.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0")); using var discordRedirect = await httpClient.SendAsync(getDiscordRequest).ConfigureAwait(false); if (discordRedirect.StatusCode == HttpStatusCode.Redirect) { diff --git a/CompatBot/ThumbScrapper/GameTdbScraper.cs b/CompatBot/ThumbScrapper/GameTdbScraper.cs index dd4aa689..df3ca4b6 100644 --- a/CompatBot/ThumbScrapper/GameTdbScraper.cs +++ b/CompatBot/ThumbScrapper/GameTdbScraper.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Text.RegularExpressions; using System.Xml; using CompatApiClient.Compression; +using CompatApiClient.Utils; using CompatBot.Database; using CompatBot.Database.Providers; using CompatBot.EventHandlers; @@ -14,7 +15,7 @@ namespace CompatBot.ThumbScrapper; internal static partial class GameTdbScraper { - private static readonly HttpClient HttpClient = HttpClientFactory.Create(new CompressionMessageHandler()); + private static readonly HttpClient HttpClient = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent(); private static readonly Uri TitleDownloadLink = new("https://www.gametdb.com/ps3tdb.zip?LANG=EN"); [GeneratedRegex( @"(?https?://art\.gametdb\.com/ps3/cover(?!full)[/\w\d]+\.jpg(\?\d+)?)",