add support for ya.disk log links

Clients/MediafireClient/Client.cs

@@ -13,21 +13,19 @@ using MediafireClient.POCOs;
 
 namespace MediafireClient
 {
-    public class Client
+    public sealed class Client
     {
         private readonly HttpClient client;
-        private readonly HttpClient noRedirectsClient;
         private readonly JsonSerializerOptions jsonOptions;
         
         //var optSecurityToken = "1605819132.376f3d84695f46daa7b69ee67fbc5edb0a00843a8b2d5ac7d3d1b1ad8a4212b0";
-        private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?<security_token>.+)""", RegexOptions.ExplicitCapture);
+        //private static readonly Regex SecurityTokenRegex = new(@"(var\s+optSecurityToken|name=""security"" value)\s*=\s*""(?<security_token>.+)""", RegexOptions.ExplicitCapture);
         //var optDirectURL = "https://download1499.mediafire.com/12zqzob7gbfg/tmybrjpmtrpcejl/DemonsSouls_CrashLog_Nov.19th.zip";
         private static readonly Regex DirectUrlRegex = new(@"(var\s+optDirectURL|href)\s*=\s*""(?<direct_link>https?://download\d+\.mediafire\.com/.+)""");
 
         public Client()
         {
             client = HttpClientFactory.Create(new CompressionMessageHandler());
-            noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler {AllowAutoRedirect = false});
             jsonOptions = new JsonSerializerOptions
             {
                 PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,

Clients/YandexDiskClient/Client.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Net.Http;
+using System.Net.Http.Json;
+using System.Threading;
+using System.Threading.Tasks;
+using CompatApiClient;
+using CompatApiClient.Compression;
+using CompatApiClient.Utils;
+using System.Text.Json;
+using CompatApiClient.Formatters;
+using YandexDiskClient.POCOs;
+
+namespace YandexDiskClient
+{
+    public sealed class Client
+    {
+        private readonly HttpClient client;
+        private readonly JsonSerializerOptions jsonOptions;
+
+        public Client()
+        {
+            client = HttpClientFactory.Create(new CompressionMessageHandler());
+            jsonOptions = new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
+                IgnoreNullValues = true,
+                IncludeFields = true,
+            };
+        }
+
+        public Task<ResourceInfo?> GetResourceInfoAsync(string shareKey, CancellationToken cancellationToken)
+            => GetResourceInfoAsync(new Uri($"https://yadi.sk/d/{shareKey}"), cancellationToken);
+        
+        public async Task<ResourceInfo?> GetResourceInfoAsync(Uri publicUri, CancellationToken cancellationToken)
+        {
+            try
+            {
+                var uri = new Uri($"https://cloud-api.yandex.net/v1/disk/public/resources").SetQueryParameters(
+                    ("public_key", publicUri.ToString()),
+                    ("fields", "size,name,file")
+                );
+                using var message = new HttpRequestMessage(HttpMethod.Get, uri);
+                message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader);
+                using var response = await client.SendAsync(message, cancellationToken).ConfigureAwait(false);
+                try
+                {
+                    await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
+                    return await response.Content.ReadFromJsonAsync<ResourceInfo>(jsonOptions, cancellationToken).ConfigureAwait(false);
+                }
+                catch (Exception e)
+                {
+                    ConsoleLogger.PrintError(e, response);
+                }
+            }
+            catch (Exception e)
+            {
+                ApiConfig.Log.Error(e);
+            }
+            return null;
+        }
+
+    }
+}

Clients/YandexDiskClient/POCOs/ResourceInfo.cs

@@ -0,0 +1,20 @@
+namespace YandexDiskClient.POCOs
+{
+    #nullable disable
+    
+    public sealed class ResourceInfo
+    {
+        public int? Size;
+        public string Name; //RPCS3.log.gz
+        public string PublicKey;
+        public string Type; //file
+        public string MimeType; //application/x-gzip
+        public string File; //<direct download url>
+        public string MediaType; //compressed
+        public string Md5;
+        public string Sha256;
+        public long? Revision;
+    }
+    
+    #nullable restore
+}

Clients/YandexDiskClient/YandexDiskClient.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net5.0</TargetFramework>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\CompatApiClient\CompatApiClient.csproj" />
+    </ItemGroup>
+
+</Project>

CompatBot/CompatBot.csproj

@@ -74,6 +74,7 @@
     <ProjectReference Include="..\Clients\GithubClient\GithubClient.csproj" />
     <ProjectReference Include="..\Clients\MediafireClient\MediafireClient.csproj" />
     <ProjectReference Include="..\Clients\OneDriveClient\OneDriveClient.csproj" />
+    <ProjectReference Include="..\Clients\YandexDiskClient\YandexDiskClient.csproj" />
     <ProjectReference Include="..\HomoglyphConverter\HomoglyphConverter.csproj" />
     <ProjectReference Include="..\Clients\IrdLibraryClient\IrdLibraryClient.csproj" />
     <ProjectReference Include="..\Clients\PsnClient\PsnClient.csproj" />

CompatBot/EventHandlers/LogParsing/SourceHandlers/MediafireHandler.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using CompatBot.EventHandlers.LogParsing.ArchiveHandlers;
@@ -9,7 +8,6 @@ using CompatBot.Utils;
 using System.IO.Pipelines;
 using System.Net.Http;
 using System.Threading;
-using CompatApiClient.Utils;
 using MediafireClient;
 
 namespace CompatBot.EventHandlers.LogParsing.SourceHandlers

CompatBot/EventHandlers/LogParsing/SourceHandlers/YandexDiskHandler.cs

@@ -0,0 +1,107 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using CompatBot.EventHandlers.LogParsing.ArchiveHandlers;
+using DSharpPlus.Entities;
+using CompatBot.Utils;
+using System.IO.Pipelines;
+using System.Net.Http;
+using System.Threading;
+using YandexDiskClient;
+
+namespace CompatBot.EventHandlers.LogParsing.SourceHandlers
+{
+    internal sealed class YandexDiskHandler: BaseSourceHandler
+    {
+        private static readonly Regex ExternalLink = new(@"(?<yadisk_link>(https?://)?(www\.)?yadi\.sk/d/(?<share_key>[^/>\s]+))\b", DefaultOptions);
+        private static readonly Client Client = new();
+
+        public override async Task<(ISource? source, string? failReason)> FindHandlerAsync(DiscordMessage message, ICollection<IArchiveHandler> handlers)
+        {
+            if (string.IsNullOrEmpty(message.Content))
+                return (null, null);
+
+            var matches = ExternalLink.Matches(message.Content);
+            if (matches.Count == 0)
+                return (null, null);
+
+            using var client = HttpClientFactory.Create();
+            foreach (Match m in matches)
+            {
+                if (m.Groups["yadisk_link"].Value is string lnk
+                    && !string.IsNullOrEmpty(lnk)
+                    && Uri.TryCreate(lnk, UriKind.Absolute, out var webLink))
+                {
+                    try
+                    {
+                        var filename = "";
+                        var filesize = -1;
+
+                        var resourceInfo = await Client.GetResourceInfoAsync(webLink, Config.Cts.Token).ConfigureAwait(false);
+                        if (string.IsNullOrEmpty(resourceInfo?.File))
+                            return (null, null);
+
+                        if (resourceInfo.Size.HasValue)
+                            filesize = resourceInfo.Size.Value;
+                        if (!string.IsNullOrEmpty(resourceInfo.Name))
+                            filename = resourceInfo.Name;
+
+                        await using var stream = await client.GetStreamAsync(resourceInfo.File).ConfigureAwait(false);
+                        var buf = BufferPool.Rent(SnoopBufferSize);
+                        try
+                        {
+                            var read = await stream.ReadBytesAsync(buf).ConfigureAwait(false);
+                            foreach (var handler in handlers)
+                            {
+                                var (canHandle, reason) = handler.CanHandle(filename, filesize, buf.AsSpan(0, read));
+                                if (canHandle)
+                                    return (new YaDiskSource(resourceInfo.File, handler, filename, filesize), null);
+                                else if (!string.IsNullOrEmpty(reason))
+                                    return (null, reason);
+                            }
+                        }
+                        finally
+                        {
+                            BufferPool.Return(buf);
+                        }
+                    }
+
+                    catch (Exception e)
+                    {
+                        Config.Log.Warn(e, $"Error sniffing {m.Groups["yadisk_link"].Value}");
+                    }
+                }
+            }
+            return (null, null);
+        }
+
+        private sealed class YaDiskSource : ISource
+        {
+            private readonly Uri uri;
+            private readonly IArchiveHandler handler;
+
+            public string SourceType => "Ya.Disk";
+            public string FileName { get; }
+            public long SourceFileSize { get; }
+            public long SourceFilePosition => handler.SourcePosition;
+            public long LogFileSize => handler.LogSize;
+
+            internal YaDiskSource(string uri, IArchiveHandler handler, string fileName, int fileSize)
+            {
+                this.uri = new Uri(uri);
+                this.handler = handler;
+                FileName = fileName;
+                SourceFileSize = fileSize;
+            }
+
+            public async Task FillPipeAsync(PipeWriter writer, CancellationToken cancellationToken)
+            {
+                using var client = HttpClientFactory.Create();
+                await using var stream = await client.GetStreamAsync(uri, cancellationToken).ConfigureAwait(false);
+                await handler.FillPipeAsync(stream, writer, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+    }
+}

CompatBot/EventHandlers/LogParsingHandler.cs

@@ -35,6 +35,7 @@ namespace CompatBot.EventHandlers
             new DropboxHandler(),
             new MegaHandler(),
             new OneDriveSourceHandler(),
+            new YandexDiskHandler(),
             new MediafireHandler(),
             new GenericLinkHandler(),
             new PastebinHandler(),

discord-bot-net.sln

@@ -36,6 +36,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneDriveClient", "Clients\O
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MediafireClient", "Clients\MediafireClient\MediafireClient.csproj", "{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "YandexDiskClient", "Clients\YandexDiskClient\YandexDiskClient.csproj", "{CABC3E5E-2153-443B-A5A8-DA3E389359EC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -77,6 +79,10 @@ Global
 		{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CABC3E5E-2153-443B-A5A8-DA3E389359EC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -88,6 +94,7 @@ Global
 		{AF8FDA29-864E-4A1C-9568-99DECB7E4B36} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2}
 		{5C4BCF33-2EC6-455F-B026-8A0001B7B7AD} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2}
 		{1F743D3D-4A87-47EF-B88D-A0DCEE1C5FB7} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2}
+		{CABC3E5E-2153-443B-A5A8-DA3E389359EC} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D7696F56-AEAC-4D83-9BD8-BE0C122A5DCE}