From d7b018a3dd6a02813e46f1d2e1ba9fd226e2f908 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 9 Mar 2020 22:16:19 +0500 Subject: [PATCH] support for OneDrive log links --- Clients/AppveyorClient/Client.cs | 9 +- Clients/CompatApiClient/ApiConfig.cs | 2 + .../CompatApiClient/Utils/UriExtensions.cs | 8 ++ Clients/GithubClient/Client.cs | 9 +- Clients/OneDriveClient/Client.cs | 109 +++++++++++++++++ Clients/OneDriveClient/OneDriveClient.csproj | 15 +++ Clients/OneDriveClient/POCOs/DriveItemMeta.cs | 15 +++ CompatBot/CompatBot.csproj | 7 +- .../SourceHandlers/GenericLinkHandler.cs | 1 - .../SourceHandlers/GoogleDriveHandler.cs | 1 - .../SourceHandlers/OneDriveSourceHandler.cs | 110 ++++++++++++++++++ CompatBot/EventHandlers/LogParsingHandler.cs | 1 + discord-bot-net.sln | 7 ++ 13 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 Clients/OneDriveClient/Client.cs create mode 100644 Clients/OneDriveClient/OneDriveClient.csproj create mode 100644 Clients/OneDriveClient/POCOs/DriveItemMeta.cs create mode 100644 CompatBot/EventHandlers/LogParsing/SourceHandlers/OneDriveSourceHandler.cs diff --git a/Clients/AppveyorClient/Client.cs b/Clients/AppveyorClient/Client.cs index d3922db3..0e01a025 100644 --- a/Clients/AppveyorClient/Client.cs +++ b/Clients/AppveyorClient/Client.cs @@ -21,7 +21,6 @@ namespace AppveyorClient private readonly HttpClient client; private readonly MediaTypeFormatterCollection formatters; - private static readonly ProductInfoHeaderValue ProductInfoHeader = new ProductInfoHeaderValue("RPCS3CompatibilityBot", "2.0"); private static readonly TimeSpan CacheTime = TimeSpan.FromDays(1); private static readonly TimeSpan PrToArtifactCacheTime = TimeSpan.FromMinutes(1); private static readonly TimeSpan JobToBuildCacheTime = TimeSpan.FromDays(30); @@ -140,7 +139,7 @@ namespace AppveyorClient try { using var message = new HttpRequestMessage(HttpMethod.Get, buildUrl); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -203,7 +202,7 @@ namespace AppveyorClient try { using var message = new HttpRequestMessage(HttpMethod.Get, requestUri); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -272,7 +271,7 @@ namespace AppveyorClient do { using var message = new HttpRequestMessage(HttpMethod.Get, historyUrl); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -314,7 +313,7 @@ namespace AppveyorClient do { using var message = new HttpRequestMessage(HttpMethod.Get, historyUrl); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { diff --git a/Clients/CompatApiClient/ApiConfig.cs b/Clients/CompatApiClient/ApiConfig.cs index 0634df04..300d9096 100644 --- a/Clients/CompatApiClient/ApiConfig.cs +++ b/Clients/CompatApiClient/ApiConfig.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http.Headers; using Microsoft.IO; using NLog; @@ -10,6 +11,7 @@ namespace CompatApiClient public static class ApiConfig { + public static readonly ProductInfoHeaderValue ProductInfoHeader = new ProductInfoHeaderValue("RPCS3CompatibilityBot", "2.0"); public static int Version { get; } = 1; public static Uri BaseUrl { get; } = new Uri("https://rpcs3.net/compatibility"); public static string DateInputFormat { get; } = "yyyy-M-d"; diff --git a/Clients/CompatApiClient/Utils/UriExtensions.cs b/Clients/CompatApiClient/Utils/UriExtensions.cs index 359674ac..3cbbc6d1 100644 --- a/Clients/CompatApiClient/Utils/UriExtensions.cs +++ b/Clients/CompatApiClient/Utils/UriExtensions.cs @@ -59,6 +59,14 @@ namespace CompatApiClient return SetQueryValue(uri, FormatUriParams(parameters)); } + public static Uri SetQueryParameters(this Uri uri, IEnumerable<(string name, string value)> items) + { + var parameters = ParseQueryString(uri); + foreach (var item in items) + parameters[item.name] = item.value; + return SetQueryValue(uri, FormatUriParams(parameters)); + } + public static string FormatUriParams(NameValueCollection parameters) { if (parameters == null || parameters.Count == 0) diff --git a/Clients/GithubClient/Client.cs b/Clients/GithubClient/Client.cs index 4d1fcf7a..1ee42b3c 100644 --- a/Clients/GithubClient/Client.cs +++ b/Clients/GithubClient/Client.cs @@ -21,7 +21,6 @@ namespace GithubClient private readonly HttpClient client; private readonly MediaTypeFormatterCollection formatters; - private static readonly ProductInfoHeaderValue ProductInfoHeader = new ProductInfoHeaderValue("RPCS3CompatibilityBot", "2.0"); private static readonly TimeSpan PrStatusCacheTime = TimeSpan.FromMinutes(3); private static readonly TimeSpan IssueStatusCacheTime = TimeSpan.FromMinutes(30); private static readonly MemoryCache StatusesCache = new MemoryCache(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromMinutes(1) }); @@ -53,7 +52,7 @@ namespace GithubClient try { using var message = new HttpRequestMessage(HttpMethod.Get, "https://api.github.com/repos/RPCS3/rpcs3/pulls/" + pr); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -92,7 +91,7 @@ namespace GithubClient try { using var message = new HttpRequestMessage(HttpMethod.Get, "https://api.github.com/repos/RPCS3/rpcs3/issues/" + issue); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -135,7 +134,7 @@ namespace GithubClient try { using var message = new HttpRequestMessage(HttpMethod.Get, requestUri); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { @@ -173,7 +172,7 @@ namespace GithubClient try { using var message = new HttpRequestMessage(HttpMethod.Get, statusesUrl); - message.Headers.UserAgent.Add(ProductInfoHeader); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); try { diff --git a/Clients/OneDriveClient/Client.cs b/Clients/OneDriveClient/Client.cs new file mode 100644 index 00000000..2ffc6856 --- /dev/null +++ b/Clients/OneDriveClient/Client.cs @@ -0,0 +1,109 @@ +using System; +using System.Net.Http; +using System.Net.Http.Formatting; +using System.Threading; +using System.Threading.Tasks; +using CompatApiClient; +using CompatApiClient.Compression; +using CompatApiClient.Utils; +using Newtonsoft.Json; +using OneDriveClient.POCOs; +using JsonContractResolver = CompatApiClient.JsonContractResolver; + +namespace OneDriveClient +{ + public class Client + { + private readonly HttpClient client; + private readonly HttpClient noRedirectsClient; + private readonly MediaTypeFormatterCollection formatters; + + public Client() + { + client = HttpClientFactory.Create(new CompressionMessageHandler()); + noRedirectsClient = HttpClientFactory.Create(new HttpClientHandler {AllowAutoRedirect = false}); + var settings = new JsonSerializerSettings + { + ContractResolver = new JsonContractResolver(NamingStyles.CamelCase), + NullValueHandling = NullValueHandling.Ignore + }; + formatters = new MediaTypeFormatterCollection(new[] { new JsonMediaTypeFormatter { SerializerSettings = settings } }); + } + + private async Task ResolveShortLink(Uri shortLink, CancellationToken cancellationToken) + { + try + { + using var message = new HttpRequestMessage(HttpMethod.Head, shortLink); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); + using var response = await noRedirectsClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + return response.Headers.Location; + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + } + return null; + } + + // https://1drv.ms/u/s!AruI8iDXabVJ1ShAMIqxgU2tiHZ3 redirects to https://onedrive.live.com/redir?resid=49B569D720F288BB!10920&authkey=!AEAwirGBTa2Idnc + // https://onedrive.live.com/?authkey=!AEAwirGBTa2Idnc&cid=49B569D720F288BB&id=49B569D720F288BB!10920&parId=49B569D720F288BB!4371&o=OneUp + public async Task ResolveContentLinkAsync(Uri shareLink, CancellationToken cancellationToken) + { + if (shareLink?.Host == "1drv.ms") + shareLink = await ResolveShortLink(shareLink, cancellationToken).ConfigureAwait(false); + if (shareLink == null) + return null; + + var queryParams = shareLink.ParseQueryString(); + string resourceId, authKey; + if (queryParams["resid"] is string resId && queryParams["authkey"] is string akey) + { + resourceId = resId; + authKey = akey; + } + else if (queryParams["id"] is string rid && queryParams["authkey"] is string aukey) + { + resourceId = rid; + authKey = aukey; + } + else + { + ApiConfig.Log.Warn("Unknown or invalid OneDrive resource link: " + shareLink); + return null; + } + + var itemId = resourceId.Split('!')[0]; + try + { + var resourceMetaUri = new Uri($"https://api.onedrive.com/v1.0/drives/{itemId}/items/{resourceId}") + .SetQueryParameters(new (string name, string value)[] + { + ("authkey", authKey), + ("select", "id,@content.downloadUrl,name,size"), + }); + using var message = new HttpRequestMessage(HttpMethod.Get, resourceMetaUri); + message.Headers.UserAgent.Add(ApiConfig.ProductInfoHeader); + using var response = await client.SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + try + { + await response.Content.LoadIntoBufferAsync().ConfigureAwait(false); + var meta = await response.Content.ReadAsAsync(formatters, cancellationToken).ConfigureAwait(false); + if (meta.ContentDownloadUrl == null) + throw new InvalidOperationException("Failed to properly deserialize response body"); + + return meta; + } + catch (Exception e) + { + ConsoleLogger.PrintError(e, response); + } + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + } + return null; + } + } +} diff --git a/Clients/OneDriveClient/OneDriveClient.csproj b/Clients/OneDriveClient/OneDriveClient.csproj new file mode 100644 index 00000000..ad7e3654 --- /dev/null +++ b/Clients/OneDriveClient/OneDriveClient.csproj @@ -0,0 +1,15 @@ + + + + netstandard2.1 + + + + + + + + + + + diff --git a/Clients/OneDriveClient/POCOs/DriveItemMeta.cs b/Clients/OneDriveClient/POCOs/DriveItemMeta.cs new file mode 100644 index 00000000..6e7a8a78 --- /dev/null +++ b/Clients/OneDriveClient/POCOs/DriveItemMeta.cs @@ -0,0 +1,15 @@ +using Newtonsoft.Json; + +namespace OneDriveClient.POCOs +{ + public class DriveItemMeta + { + public string Id; + public string Name; + public int Size; + [JsonProperty(PropertyName = "@odata.context")] + public string OdataContext; + [JsonProperty(PropertyName = "@content.downloadUrl")] + public string ContentDownloadUrl; + } +} diff --git a/CompatBot/CompatBot.csproj b/CompatBot/CompatBot.csproj index d12fffff..d67e95ec 100644 --- a/CompatBot/CompatBot.csproj +++ b/CompatBot/CompatBot.csproj @@ -24,9 +24,9 @@ - - - + + + @@ -62,6 +62,7 @@ + diff --git a/CompatBot/EventHandlers/LogParsing/SourceHandlers/GenericLinkHandler.cs b/CompatBot/EventHandlers/LogParsing/SourceHandlers/GenericLinkHandler.cs index 5ea4bc24..4db807af 100644 --- a/CompatBot/EventHandlers/LogParsing/SourceHandlers/GenericLinkHandler.cs +++ b/CompatBot/EventHandlers/LogParsing/SourceHandlers/GenericLinkHandler.cs @@ -68,7 +68,6 @@ namespace CompatBot.EventHandlers.LogParsing.SourceHandlers bufferPool.Return(buf); } } - catch (Exception e) { Config.Log.Warn(e, $"Error sniffing {m.Groups["link"].Value}"); diff --git a/CompatBot/EventHandlers/LogParsing/SourceHandlers/GoogleDriveHandler.cs b/CompatBot/EventHandlers/LogParsing/SourceHandlers/GoogleDriveHandler.cs index cc25691c..0b73215e 100644 --- a/CompatBot/EventHandlers/LogParsing/SourceHandlers/GoogleDriveHandler.cs +++ b/CompatBot/EventHandlers/LogParsing/SourceHandlers/GoogleDriveHandler.cs @@ -12,7 +12,6 @@ using Google.Apis.Auth.OAuth2; using Google.Apis.Download; using Google.Apis.Drive.v3; using Google.Apis.Services; -using Nerdbank.Streams; using FileMeta = Google.Apis.Drive.v3.Data.File; namespace CompatBot.EventHandlers.LogParsing.SourceHandlers diff --git a/CompatBot/EventHandlers/LogParsing/SourceHandlers/OneDriveSourceHandler.cs b/CompatBot/EventHandlers/LogParsing/SourceHandlers/OneDriveSourceHandler.cs new file mode 100644 index 00000000..3b8393d0 --- /dev/null +++ b/CompatBot/EventHandlers/LogParsing/SourceHandlers/OneDriveSourceHandler.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Pipelines; +using System.Net.Http; +using System.Runtime.InteropServices.ComTypes; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using CG.Web.MegaApiClient; +using CompatBot.EventHandlers.LogParsing.ArchiveHandlers; +using CompatBot.Utils; +using DSharpPlus.Entities; +using OneDriveClient; +using OneDriveClient.POCOs; + +namespace CompatBot.EventHandlers.LogParsing.SourceHandlers +{ + internal sealed class OneDriveSourceHandler : BaseSourceHandler + { + private static readonly Regex ExternalLink = new Regex(@"(?(https?://)?(1drv\.ms|onedrive\.live\.com)/[^>\s]+)", DefaultOptions); + private static readonly Client client = new Client(); + + public async override Task<(ISource source, string failReason)> FindHandlerAsync(DiscordMessage message, ICollection handlers) + { + if (string.IsNullOrEmpty(message.Content)) + return (null, null); + + var matches = ExternalLink.Matches(message.Content); + if (matches.Count == 0) + return (null, null); + + using var httpClient = HttpClientFactory.Create(); + foreach (Match m in matches) + { + try + { + if (m.Groups["onedrive_link"].Value is string lnk + && !string.IsNullOrEmpty(lnk) + && Uri.TryCreate(lnk, UriKind.Absolute, out var uri) + && await client.ResolveContentLinkAsync(uri, Config.Cts.Token).ConfigureAwait(false) is DriveItemMeta itemMeta) + { + try + { + var filename = itemMeta.Name; + var filesize = itemMeta.Size; + uri = new Uri(itemMeta.ContentDownloadUrl); + + using var stream = await httpClient.GetStreamAsync(uri).ConfigureAwait(false); + var buf = bufferPool.Rent(1024); + try + { + var read = await stream.ReadBytesAsync(buf).ConfigureAwait(false); + foreach (var handler in handlers) + { + var (canHandle, reason) = handler.CanHandle(filename, filesize, buf.AsSpan(0, read)); + if (canHandle) + return (new OneDriveSource(uri, handler, filename, filesize), null); + else if (!string.IsNullOrEmpty(reason)) + return (null, reason); + } + } + finally + { + bufferPool.Return(buf); + } + } + catch (Exception e) + { + Config.Log.Warn(e, $"Error sniffing {m.Groups["link"].Value}"); + } + } + } + catch (Exception e) + { + Config.Log.Warn(e, $"Error sniffing {m.Groups["mega_link"].Value}"); + } + } + return (null, null); + } + + + private sealed class OneDriveSource : ISource + { + private readonly Uri uri; + private readonly IArchiveHandler handler; + + public string SourceType => "OneDrive"; + public string FileName { get; } + public long SourceFileSize { get; } + public long SourceFilePosition => handler.SourcePosition; + public long LogFileSize => handler.LogSize; + + internal OneDriveSource(Uri uri, IArchiveHandler handler, string fileName, int fileSize) + { + this.uri = uri; + this.handler = handler; + FileName = fileName; + SourceFileSize = fileSize; + } + + public async Task FillPipeAsync(PipeWriter writer, CancellationToken cancellationToken) + { + using var client = HttpClientFactory.Create(); + using var stream = await client.GetStreamAsync(uri).ConfigureAwait(false); + await handler.FillPipeAsync(stream, writer, cancellationToken).ConfigureAwait(false); + } + } + } +} \ No newline at end of file diff --git a/CompatBot/EventHandlers/LogParsingHandler.cs b/CompatBot/EventHandlers/LogParsingHandler.cs index 37ba7fb2..63824c94 100644 --- a/CompatBot/EventHandlers/LogParsingHandler.cs +++ b/CompatBot/EventHandlers/LogParsingHandler.cs @@ -32,6 +32,7 @@ namespace CompatBot.EventHandlers new GoogleDriveHandler(), new DropboxHandler(), new MegaHandler(), + new OneDriveSourceHandler(), new GenericLinkHandler(), new PastebinHandler(), }; diff --git a/discord-bot-net.sln b/discord-bot-net.sln index 485d623e..a12f599e 100644 --- a/discord-bot-net.sln +++ b/discord-bot-net.sln @@ -31,6 +31,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution SECURITY.md = SECURITY.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneDriveClient", "Clients\OneDriveClient\OneDriveClient.csproj", "{5C4BCF33-2EC6-455F-B026-8A0001B7B7AD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -68,6 +70,10 @@ Global {595ED201-1456-49F9-AD60-54B08499A5C1}.Debug|Any CPU.Build.0 = Debug|Any CPU {595ED201-1456-49F9-AD60-54B08499A5C1}.Release|Any CPU.ActiveCfg = Release|Any CPU {595ED201-1456-49F9-AD60-54B08499A5C1}.Release|Any CPU.Build.0 = Release|Any CPU + {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -78,6 +84,7 @@ Global {AA2A333B-CD30-41A5-A680-CC9BCB2D726B} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} {AF8FDA29-864E-4A1C-9568-99DECB7E4B36} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} {595ED201-1456-49F9-AD60-54B08499A5C1} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} + {5C4BCF33-2EC6-455F-B026-8A0001B7B7AD} = {E7FE0ADD-CBA6-4321-8A1C-0A3B5C3F54C2} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D7696F56-AEAC-4D83-9BD8-BE0C122A5DCE}