RPCS3/archived-discord-bot
1
0
Fork 0
mirror of https://github.com/RPCS3/discord-bot.git synced 2026-01-31 01:25:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add User-Agent to GameTDB scraper

Browse Source
This commit is contained in:
13xforever
2025-12-27 05:31:07 +05:00
parent f32c141a67
commit b98bd3daee
8 changed files with 34 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Clients/CompatApiClient/Client.cs
View File

@@ -19,7 +19,7 @@ public class Client: IDisposable
private static readonly MemoryCache ResponseCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromHours(1) });
private static readonly string[] BuildArchList = [ArchType.X64, ArchType.Arm];
private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler());
private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent();
private readonly JsonSerializerOptions jsonOptions = new()
{
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,

7
Clients/CompatApiClient/Utils/Utils.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Net.Http;
namespace CompatApiClient.Utils;
@@ -57,4 +58,10 @@ public static class Utils
< UnderGB => $"{bytes / (1024.0 * 1024):0.##} MB",
_ => $"{bytes / (1024.0 * 1024 * 1024):0.##} GB"
};
public static HttpClient WithUserAgent(this HttpClient client)
{
client.DefaultRequestHeaders.UserAgent.Add(ApiConfig.ProductInfoHeader);
return client;
}
}

3
Clients/GithubClient/Client.cs
View File

@@ -13,7 +13,7 @@ namespace GithubClient;
public partial class Client
{
private readonly GitHubClient client;
private readonly GitHubClient client = new(new ProductHeaderValue(ApiConfig.ProductName, ApiConfig.ProductVersion));
private const string OwnerId = "RPCS3";
private const string RepoId = "rpcs3";
@@ -28,7 +28,6 @@ public partial class Client
public Client(string? githubToken)
{
client = new(new ProductHeaderValue(ApiConfig.ProductName, ApiConfig.ProductVersion));
if (githubToken is {Length: >0})
client.Credentials = new(githubToken);
}

22
Clients/MediafireClient/Client.cs
View File

@@ -17,8 +17,13 @@ namespace MediafireClient;
public sealed partial class Client
{
private readonly HttpClient client;
private readonly JsonSerializerOptions jsonOptions;
private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent();
private readonly JsonSerializerOptions jsonOptions = new()
{
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
//var optSecurityToken = "1605819132.376f3d84695f46daa7b69ee67fbc5edb0a00843a8b2d5ac7d3d1b1ad8a4212b0";
//private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?<security_token>.+)""", RegexOptions.ExplicitCapture);
@@ -26,24 +31,12 @@ public sealed partial class Client
[GeneratedRegex(@"(var\s+optDirectURL|href)\s*=\s*""(?<direct_link>https?://download\d+\.mediafire\.com/.+)""")]
private static partial Regex DirectUrlRegex();
public Client()
{
client = HttpClientFactory.Create(new CompressionMessageHandler());
jsonOptions = new()
{
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
}
public async Task<LinksResult?> GetWebLinkAsync(string quickKey, CancellationToken cancellationToken)
{
try
{
var uri = new Uri($"https://www.mediafire.com/api/1.5/file/get_links.php?quick_key={quickKey}&response_format=json");
using var message = new HttpRequestMessage(HttpMethod.Get, uri);
message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
try
{
@@ -67,7 +60,6 @@ public sealed partial class Client
try
{
using var message = new HttpRequestMessage(HttpMethod.Get, webLink);
message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
if (response.StatusCode is HttpStatusCode.Redirect or HttpStatusCode.TemporaryRedirect)
{

23
Clients/OneDriveClient/Client.cs
View File

@@ -14,28 +14,20 @@ namespace OneDriveClient;
public class Client
{
private readonly HttpClient client;
private readonly HttpClient noRedirectsClient;
private readonly JsonSerializerOptions jsonOptions;
public Client()
private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent();
private readonly HttpClient noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler { AllowAutoRedirect = false }).WithUserAgent();
private readonly JsonSerializerOptions jsonOptions = new()
{
client = HttpClientFactory.Create(new CompressionMessageHandler());
noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler {AllowAutoRedirect = false});
jsonOptions = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
}
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
private async Task<Uri?> ResolveShortLink(Uri shortLink, CancellationToken cancellationToken)
{
try
{
using var message = new HttpRequestMessage(HttpMethod.Head, shortLink);
message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
using var response = await noRedirectsClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
return response.Headers.Location;
}
@@ -82,7 +74,6 @@ public class Client
("select", "id,@content.downloadUrl,name,size")
);
using var message = new HttpRequestMessage(HttpMethod.Get, resourceMetaUri);
message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false);
try
{

19
Clients/YandexDiskClient/Client.cs
View File

@@ -15,19 +15,13 @@ namespace YandexDiskClient;
public sealed class Client
{
private readonly HttpClient client;
private readonly JsonSerializerOptions jsonOptions;
public Client()
private readonly HttpClient client = HttpClientFactory.Create(new CompressionMessageHandler());
private readonly JsonSerializerOptions jsonOptions = new()
{
client = HttpClientFactory.Create(new CompressionMessageHandler());
jsonOptions = new()
{
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
}
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
public Task<ResourceInfo?> GetResourceInfoAsync(string shareKey, CancellationToken cancellationToken)
=> GetResourceInfoAsync(new Uri($"https://yadi.sk/d/{shareKey}"), cancellationToken);
@@ -41,7 +35,6 @@ public sealed class Client
("fields", "size,name,file")
);
using var message = new HttpRequestMessage(HttpMethod.Get, uri);
message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
try
{

7
CompatBot/EventHandlers/DiscordInviteFilter.cs
View File

@@ -2,7 +2,9 @@
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using CompatApiClient;
using CompatApiClient.Compression;
using CompatApiClient.Utils;
using CompatBot.Commands;
using CompatBot.Database;
using CompatBot.Database.Providers;
@@ -232,11 +234,10 @@ internal static partial class DiscordInviteFilter
try
{
using var handler = new HttpClientHandler {AllowAutoRedirect = false}; // needed to store cloudflare session cookies
using var httpClient = HttpClientFactory.Create(handler, new CompressionMessageHandler());
using var httpClient = HttpClientFactory.Create(handler, new CompressionMessageHandler()).WithUserAgent();
using var request = new HttpRequestMessage(HttpMethod.Get, "https://discord.me/" + meLink);
request.Headers.Accept.Add(new("text/html"));
request.Headers.CacheControl = CacheControlHeaderValue.Parse("no-cache");
request.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0"));
using var response = await httpClient.SendAsync(request).ConfigureAwait(false);
var html = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
if (response.IsSuccessStatusCode)
@@ -258,7 +259,6 @@ internal static partial class DiscordInviteFilter
}!),
};
postRequest.Headers.Accept.Add(new("text/html"));
postRequest.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0"));
using var postResponse = await httpClient.SendAsync(postRequest).ConfigureAwait(false);
if (postResponse.StatusCode == HttpStatusCode.Redirect)
{
@@ -267,7 +267,6 @@ internal static partial class DiscordInviteFilter
{
using var getDiscordRequest = new HttpRequestMessage(HttpMethod.Get, "https://discord.me/server/join/redirect/" + redirectId);
getDiscordRequest.Headers.CacheControl = CacheControlHeaderValue.Parse("no-cache");
getDiscordRequest.Headers.UserAgent.Add(new("RPCS3CompatibilityBot", "2.0"));
using var discordRedirect = await httpClient.SendAsync(getDiscordRequest).ConfigureAwait(false);
if (discordRedirect.StatusCode == HttpStatusCode.Redirect)
{

3
CompatBot/ThumbScrapper/GameTdbScraper.cs
View File

@@ -5,6 +5,7 @@ using System.Net.Http;
using System.Text.RegularExpressions;
using System.Xml;
using CompatApiClient.Compression;
using CompatApiClient.Utils;
using CompatBot.Database;
using CompatBot.Database.Providers;
using CompatBot.EventHandlers;
@@ -14,7 +15,7 @@ namespace CompatBot.ThumbScrapper;
internal static partial class GameTdbScraper
{
private static readonly HttpClient HttpClient = HttpClientFactory.Create(new CompressionMessageHandler());
private static readonly HttpClient HttpClient = HttpClientFactory.Create(new CompressionMessageHandler()).WithUserAgent();
private static readonly Uri TitleDownloadLink = new("https://www.gametdb.com/ps3tdb.zip?LANG=EN");
[GeneratedRegex(
@"(?<cover_link>https?://art\.gametdb\.com/ps3/cover(?!full)[/\w\d]+\.jpg(\?\d+)?)",