From 8ac2c40c1a2a86bbffb68c632ee42933c0e52af3 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Fri, 20 Nov 2020 13:48:48 +0500 Subject: [PATCH] add support for ya.disk log links --- Clients/MediafireClient/Client.cs | 6 +- Clients/YandexDiskClient/Client.cs | 63 +++++++++++ .../YandexDiskClient/POCOs/ResourceInfo.cs | 20 ++++ .../YandexDiskClient/YandexDiskClient.csproj | 12 ++ CompatBot/CompatBot.csproj | 1 + .../SourceHandlers/MediafireHandler.cs | 2 - .../SourceHandlers/YandexDiskHandler.cs | 107 ++++++++++++++++++ CompatBot/EventHandlers/LogParsingHandler.cs | 1 + discord-bot-net.sln | 7 ++ 9 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 Clients/YandexDiskClient/Client.cs create mode 100644 Clients/YandexDiskClient/POCOs/ResourceInfo.cs create mode 100644 Clients/YandexDiskClient/YandexDiskClient.csproj create mode 100644 CompatBot/EventHandlers/LogParsing/SourceHandlers/YandexDiskHandler.cs diff --git a/Clients/MediafireClient/Client.cs b/Clients/MediafireClient/Client.cs index dd45ce1d..737106d6 100644 --- a/Clients/MediafireClient/Client.cs +++ b/Clients/MediafireClient/Client.cs @@ -13,21 +13,19 @@ using MediafireClient.POCOs; namespace MediafireClient { - public class Client + public sealed class Client { private readonly HttpClient client; - private readonly HttpClient noRedirectsClient; private readonly JsonSerializerOptions jsonOptions; //var optSecurityToken = "1605819132.376f3d84695f46daa7b69ee67fbc5edb0a00843a8b2d5ac7d3d1b1ad8a4212b0"; - private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?.+)""", RegexOptions.ExplicitCapture); + //private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?.+)""", RegexOptions.ExplicitCapture); //var optDirectURL = "https://download1499.mediafire.com/12zqzob7gbfg/tmybrjpmtrpcejl/DemonsSouls_CrashLog_Nov.19th.zip"; private static readonly Regex DirectUrlRegex = new(@"(var\s+optDirectURL|href)\s*=\s*""(?https?://download\d+\.mediafire\.com/.+)"""); public Client() { client = HttpClientFactory.Create(new CompressionMessageHandler()); - noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler {AllowAutoRedirect = false}); jsonOptions = new JsonSerializerOptions { PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, diff --git a/Clients/YandexDiskClient/Client.cs b/Clients/YandexDiskClient/Client.cs new file mode 100644 index 00000000..087d0f12 --- /dev/null +++ b/Clients/YandexDiskClient/Client.cs @@ -0,0 +1,63 @@ +using System; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using CompatApiClient; +using CompatApiClient.Compression; +using CompatApiClient.Utils; +using System.Text.Json; +using CompatApiClient.Formatters; +using YandexDiskClient.POCOs; + +namespace YandexDiskClient +{ + public sealed class Client + { + private readonly HttpClient client; + private readonly JsonSerializerOptions jsonOptions; + + public Client() + { + client = HttpClientFactory.Create(new CompressionMessageHandler()); + jsonOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, + IgnoreNullValues = true, + IncludeFields = true, + }; + } + + public Task GetResourceInfoAsync(string shareKey, CancellationToken cancellationToken) + => GetResourceInfoAsync(new Uri($"https://yadi.sk/d/{shareKey}"), cancellationToken); + + public async Task GetResourceInfoAsync(Uri publicUri, CancellationToken cancellationToken) + { + try + { + var uri = new Uri($"https://cloud-api.yandex.net/v1/disk/public/resources").SetQueryParameters( + ("public_key", publicUri.ToString()), + ("fields", "size,name,file") + ); + using var message = new HttpRequestMessage(HttpMethod.Get, uri); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); + using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false); + try + { + await response.Content.LoadIntoBufferAsync().ConfigureAwait(false); + return await response.Content.ReadFromJsonAsync(jsonOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + ConsoleLogger.PrintError(e, response); + } + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + } + return null; + } + + } +} \ No newline at end of file diff --git a/Clients/YandexDiskClient/POCOs/ResourceInfo.cs b/Clients/YandexDiskClient/POCOs/ResourceInfo.cs new file mode 100644 index 00000000..9a52edc1 --- /dev/null +++ b/Clients/YandexDiskClient/POCOs/ResourceInfo.cs @@ -0,0 +1,20 @@ +namespace YandexDiskClient.POCOs +{ + #nullable disable + + public sealed class ResourceInfo + { + public int? Size; + public string Name; //RPCS3.log.gz + public string PublicKey; + public string Type; //file + public string MimeType; //application/x-gzip + public string File; // + public string MediaType; //compressed + public string Md5; + public string Sha256; + public long? Revision; + } + + #nullable restore +} \ No newline at end of file diff --git a/Clients/YandexDiskClient/YandexDiskClient.csproj b/Clients/YandexDiskClient/YandexDiskClient.csproj new file mode 100644 index 00000000..b16a631e --- /dev/null +++ b/Clients/YandexDiskClient/YandexDiskClient.csproj @@ -0,0 +1,12 @@ + + + + net5.0 + enable + + + + + + + diff --git a/CompatBot/CompatBot.csproj b/CompatBot/CompatBot.csproj index 003d9317..a8f43dbd 100644 --- a/CompatBot/CompatBot.csproj +++ b/CompatBot/CompatBot.csproj @@ -74,6 +74,7 @@ + diff --git a/CompatBot/EventHandlers/LogParsing/SourceHandlers/MediafireHandler.cs b/CompatBot/EventHandlers/LogParsing/SourceHandlers/MediafireHandler.cs index 841b8765..07fa195c 100644 --- a/CompatBot/EventHandlers/LogParsing/SourceHandlers/MediafireHandler.cs +++ b/CompatBot/EventHandlers/LogParsing/SourceHandlers/MediafireHandler.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Text.RegularExpressions; using System.Threading.Tasks; using CompatBot.EventHandlers.LogParsing.ArchiveHandlers; @@ -9,7 +8,6 @@ using CompatBot.Utils; using System.IO.Pipelines; using System.Net.Http; using System.Threading; -using CompatApiClient.Utils; using MediafireClient; namespace CompatBot.EventHandlers.LogParsing.SourceHandlers diff --git a/CompatBot/EventHandlers/LogParsing/SourceHandlers/YandexDiskHandler.cs b/CompatBot/EventHandlers/LogParsing/SourceHandlers/YandexDiskHandler.cs new file mode 100644 index 00000000..f608fbd9 --- /dev/null +++ b/CompatBot/EventHandlers/LogParsing/SourceHandlers/YandexDiskHandler.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using CompatBot.EventHandlers.LogParsing.ArchiveHandlers; +using DSharpPlus.Entities; +using CompatBot.Utils; +using System.IO.Pipelines; +using System.Net.Http; +using System.Threading; +using YandexDiskClient; + +namespace CompatBot.EventHandlers.LogParsing.SourceHandlers +{ + internal sealed class YandexDiskHandler: BaseSourceHandler + { + private static readonly Regex ExternalLink = new(@"(?(https?://)?(www\.)?yadi\.sk/d/(?[^/>\s]+))\b", DefaultOptions); + private static readonly Client Client = new(); + + public override async Task<(ISource? source, string? failReason)> FindHandlerAsync(DiscordMessage message, ICollection handlers) + { + if (string.IsNullOrEmpty(message.Content)) + return (null, null); + + var matches = ExternalLink.Matches(message.Content); + if (matches.Count == 0) + return (null, null); + + using var client = HttpClientFactory.Create(); + foreach (Match m in matches) + { + if (m.Groups["yadisk_link"].Value is string lnk + && !string.IsNullOrEmpty(lnk) + && Uri.TryCreate(lnk, UriKind.Absolute, out var webLink)) + { + try + { + var filename = ""; + var filesize = -1; + + var resourceInfo = await Client.GetResourceInfoAsync(webLink, Config.Cts.Token).ConfigureAwait(false); + if (string.IsNullOrEmpty(resourceInfo?.File)) + return (null, null); + + if (resourceInfo.Size.HasValue) + filesize = resourceInfo.Size.Value; + if (!string.IsNullOrEmpty(resourceInfo.Name)) + filename = resourceInfo.Name; + + await using var stream = await client.GetStreamAsync(resourceInfo.File).ConfigureAwait(false); + var buf = BufferPool.Rent(SnoopBufferSize); + try + { + var read = await stream.ReadBytesAsync(buf).ConfigureAwait(false); + foreach (var handler in handlers) + { + var (canHandle, reason) = handler.CanHandle(filename, filesize, buf.AsSpan(0, read)); + if (canHandle) + return (new YaDiskSource(resourceInfo.File, handler, filename, filesize), null); + else if (!string.IsNullOrEmpty(reason)) + return (null, reason); + } + } + finally + { + BufferPool.Return(buf); + } + } + + catch (Exception e) + { + Config.Log.Warn(e, $"Error sniffing {m.Groups["yadisk_link"].Value}"); + } + } + } + return (null, null); + } + + private sealed class YaDiskSource : ISource + { + private readonly Uri uri; + private readonly IArchiveHandler handler; + + public string SourceType => "Ya.Disk"; + public string FileName { get; } + public long SourceFileSize { get; } + public long SourceFilePosition => handler.SourcePosition; + public long LogFileSize => handler.LogSize; + + internal YaDiskSource(string uri, IArchiveHandler handler, string fileName, int fileSize) + { + this.uri = new Uri(uri); + this.handler = handler; + FileName = fileName; + SourceFileSize = fileSize; + } + + public async Task FillPipeAsync(PipeWriter writer, CancellationToken cancellationToken) + { + using var client = HttpClientFactory.Create(); + await using var stream = await client.GetStreamAsync(uri, cancellationToken).ConfigureAwait(false); + await handler.FillPipeAsync(stream, writer, cancellationToken).ConfigureAwait(false); + } + } + + } +} \ No newline at end of file diff --git a/CompatBot/EventHandlers/LogParsingHandler.cs b/CompatBot/EventHandlers/LogParsingHandler.cs index 79efac6d..bf58040e 100644 --- a/CompatBot/EventHandlers/LogParsingHandler.cs +++ b/CompatBot/EventHandlers/LogParsingHandler.cs @@ -35,6 +35,7 @@ namespace CompatBot.EventHandlers new DropboxHandler(), new MegaHandler(), new OneDriveSourceHandler(), + new YandexDiskHandler(), new MediafireHandler(), new GenericLinkHandler(), new PastebinHandler(), diff --git a/discord-bot-net.sln b/discord-bot-net.sln index 405b7536..faeab2c1 100644 --- a/discord-bot-net.sln +++ b/discord-bot-net.sln @@ -36,6 +36,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneDriveClient", "Clients\O EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MediafireClient", "Clients\MediafireClient\MediafireClient.csproj", "{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YandexDiskClient", "Clients\YandexDiskClient\YandexDiskClient.csproj", "{CABC3E5E-2153-443B-A5A8-DA3E389359EC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -77,6 +79,10 @@ Global {1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Debug|Any CPU.Build.0 = Debug|Any CPU {1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Release|Any CPU.ActiveCfg = Release|Any CPU {1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Release|Any CPU.Build.0 = Release|Any CPU + {CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -88,6 +94,7 @@ Global {AF8FDA29-864E-4A1C-9568-99DECB7E4B36} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} {1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} + {CABC3E5E-2153-443B-A5A8-DA3E389359EC} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D7696F56-AEAC-4D83-9BD8-BE0C122A5DCE}