diff --git a/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs b/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs index 00ecbb8d..db43c041 100644 --- a/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs @@ -11,6 +11,7 @@ internal interface IModDatasetRepository *********/ /// Fetch the latest mod dataset, if it changed or hasn't been fetched yet. /// A callback which should receive progress messages for logging. + /// The downloaded archive isn't a valid mod dataset. Task UpdateAsync(Action? log = null); /// Get the full path to a file in the mod dataset. diff --git a/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs b/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs index 09215b6e..612da40d 100644 --- a/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs @@ -1,6 +1,10 @@ using System; -using System.Diagnostics; using System.IO; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; using System.Threading.Tasks; namespace StardewModdingAPI.Web.Framework.Caching.ModDataset; @@ -11,75 +15,171 @@ internal class ModDatasetRepository : IModDatasetRepository /********* ** Fields *********/ - /// The HTTPS URL of the mod dataset Git repo. - private readonly string RepoUrl; + /// The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL). + private readonly string DownloadZipUrl; - /// The full path to the mod dataset repo. - private readonly string LocalRepoPath; + /// The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted. + private readonly string LocalRootPath; + + /// The HTTP client with which to fetch the mod dataset archive. + private readonly HttpClient HttpClient; + + /// The name of the file within the in which to cache data about the last fetched dataset. + private const string CacheFileName = ".cache.json"; + + /// The path to the actual 'dataset' folder. + private string? DatasetPath; /********* ** Public methods *********/ /// Construct an instance. - /// - /// The path to the mod dataset repo. - public ModDatasetRepository(string repoUrl, string localRepoPath) + /// + /// + /// The user agent to use when fetching the archive URL. + public ModDatasetRepository(string downloadZipUrl, string localPath, string userAgent) { - this.RepoUrl = repoUrl; - this.LocalRepoPath = Path.GetFullPath( - Environment.ExpandEnvironmentVariables(localRepoPath) - ); + this.DownloadZipUrl = downloadZipUrl; + this.LocalRootPath = Path.GetFullPath(this.ExpandEnvironmentVariables(localPath)); + + this.HttpClient = new HttpClient(); + this.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); } /// public async Task UpdateAsync(Action? log = null) { - if (!Directory.Exists(Path.Combine(this.LocalRepoPath, ".git"))) + // check for new archive + log?.Invoke(" Checking for newer dataset..."); + LastDownload? cached = await this.GetCacheInfoAsync(); + using HttpRequestMessage request = this.BuildDownloadRequest(cached?.ETag); + using HttpResponseMessage response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + + // download new archive + if (cached is null || response.StatusCode is not HttpStatusCode.NotModified) { - log?.Invoke("Cloning mod dataset repo..."); - Directory.CreateDirectory(Path.GetDirectoryName(this.LocalRepoPath)!); - await this.RunGitAsync("clone", "--depth", "1", this.RepoUrl, this.LocalRepoPath); + // read response data + if (response.StatusCode is not HttpStatusCode.NotModified) + response.EnsureSuccessStatusCode(); + string? newEtag = response.Headers.ETag?.ToString(); + + // download & unzip dataset + log?.Invoke(" Downloading dataset..."); + string newFolderName = Guid.NewGuid().ToString("N"); + string newRootPath = Path.Combine(this.LocalRootPath, newFolderName); + Directory.CreateDirectory(newRootPath); + await using (Stream downloadStream = await response.Content.ReadAsStreamAsync()) + await using (ZipArchive archive = new(downloadStream, ZipArchiveMode.Read)) + await archive.ExtractToDirectoryAsync(newRootPath); + + // locate 'dataset' folder + log?.Invoke(" Locating 'dataset' folder..."); + string? newDatasetPath = null; + foreach (DirectoryInfo entry in new DirectoryInfo(newRootPath).EnumerateDirectories("*", SearchOption.AllDirectories)) + { + if (entry.Name == "dataset") + { + newDatasetPath = Path.GetRelativePath(newRootPath, entry.FullName); + break; + } + } + if (newDatasetPath is null) + throw new InvalidOperationException("No 'dataset' folder found in the downloaded archive."); + + // save cache info + cached = new LastDownload(newFolderName, newDatasetPath, newEtag); + await this.SaveCacheInfoAsync(cached); + log?.Invoke($" Dataset saved to {newRootPath} with {(newEtag != null ? $"ETag header {newEtag}" : "no ETag header")}."); } - else + + // track dataset path + this.DatasetPath = Path.Combine(this.LocalRootPath, cached.FolderName, cached.RelativePathToDataset); + + // clear previous datasets if possible + foreach (DirectoryInfo directory in new DirectoryInfo(this.LocalRootPath).EnumerateDirectories()) { - log?.Invoke("Fetching latest changes from mod dataset repo..."); - await this.RunGitAsync("-C", this.LocalRepoPath, "fetch", "--depth", "1", "origin"); - await this.RunGitAsync("-C", this.LocalRepoPath, "reset", "--hard", "FETCH_HEAD"); + if (directory.Name == cached.FolderName) + continue; + + log?.Invoke($" Deleting previous dataset download at {directory.FullName}..."); + try + { + directory.Delete(recursive: true); + } + catch (Exception ex) + { + log?.Invoke($" Deletion failed: {ex}"); + } } } /// public string GetFilePath(string relativePath) { - return Path.Combine(this.LocalRepoPath, "dataset", relativePath); + return this.DatasetPath is not null + ? Path.Combine(this.DatasetPath, relativePath) + : throw new InvalidOperationException($"Must call '{nameof(this.UpdateAsync)}' before '{nameof(this.GetFilePath)}'."); } /********* ** Private methods *********/ - /// Run a Git command and wait for it to complete. - /// The git arguments. - /// The Git command exited with a non-zero exit code. - private async Task RunGitAsync(params string[] args) + /// Build the HTTP request to download the archive from the server if it changed. + /// The ETag value for the last successful dataset download, if any. + private HttpRequestMessage BuildDownloadRequest(string? etag) { - ProcessStartInfo startInfo = new("git", args) + HttpRequestMessage request = new(HttpMethod.Get, this.DownloadZipUrl); + try { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; + if (etag != null) + request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag)); + } + catch + { + request.Dispose(); + throw; + } - using Process process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start Git process."); - - Task readOutput = process.StandardOutput.ReadToEndAsync(); - Task readError = process.StandardError.ReadToEndAsync(); - await process.WaitForExitAsync(); - await Task.WhenAll(readOutput, readError); - - if (process.ExitCode != 0) - throw new InvalidOperationException($"Git command failed with exit code {process.ExitCode}: {await readError}."); + return request; } + + /// Get the cached info about the last dataset download, if it was previously downloaded. + private async Task GetCacheInfoAsync() + { + string path = Path.Combine(this.LocalRootPath, CacheFileName); + if (!File.Exists(path)) + return null; + + await using FileStream stream = File.OpenRead(path); + return await JsonSerializer.DeserializeAsync(stream); + } + + /// Save the cache info about a dataset download. + /// The cache info to save. + private async Task SaveCacheInfoAsync(LastDownload cacheInfo) + { + string path = Path.Combine(this.LocalRootPath, CacheFileName); + + await using FileStream stream = File.Create(path); + await JsonSerializer.SerializeAsync(stream, cacheInfo); + } + + /// Expand environment variables in a path. + /// The path to parse. + /// Returns the path with any environment variables replaced. + private string ExpandEnvironmentVariables(string path) + { + if (OperatingSystem.IsLinux()) + path = path.Replace("%TEMP%", Path.GetTempPath(), StringComparison.OrdinalIgnoreCase); + + return Environment.ExpandEnvironmentVariables(path); + } + + /// The cached metadata about the last dataset download. + /// The folder name within the root folder. + /// The relative path to the 'dataset' folder within the . + /// The ETag value for the downloaded archive, if available. + private record LastDownload(string FolderName, string RelativePathToDataset, string? ETag); } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs index 658626a2..b9cc760a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs @@ -6,9 +6,9 @@ internal class ModDatasetConfig /********* ** Accessors *********/ - /// The HTTPS URL of the mod dataset Git repo. - public string RepoUrl { get; set; } = null!; + /// The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL). + public string DownloadZipUrl { get; set; } = null!; - /// The local path into which to clone the mod dataset repo. - public string LocalPath { get; set; } = null!; + /// The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted. + public string LocalRootPath { get; set; } = null!; } diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 95f9cf81..68edd551 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -83,6 +83,15 @@ internal class Startup /// The service injection container. public void ConfigureServices(IServiceCollection services) { + // load config + ApiClientsConfig apiConfig = this.Configuration.GetRequiredSection("ApiClients").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); + BackgroundServicesConfig backgroundServicesConfig = this.Configuration.GetRequiredSection("BackgroundServices").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'BackgroundServices' config section couldn't be loaded."); + ModDatasetConfig datasetConfig = this.Configuration.GetRequiredSection("ModDataset").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded."); + + // init user agent + string userAgentVersion = this.GetType().Assembly.GetName().Version!.ToString(3); + string userAgent = string.Format(apiConfig.UserAgent, userAgentVersion); + // init basic services services .Configure(this.Configuration.GetRequiredSection("ApiClients")) @@ -110,10 +119,7 @@ internal class Startup services.AddSingleton(new NexusExportCacheMemoryRepository()); // init mod dataset - { - ModDatasetConfig config = this.Configuration.GetRequiredSection("ModDataset").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded."); - services.AddSingleton(new ModDatasetRepository(config.RepoUrl, config.LocalPath)); - } + services.AddSingleton(new ModDatasetRepository(datasetConfig.DownloadZipUrl, datasetConfig.LocalRootPath, userAgent)); // init Hangfire services @@ -128,29 +134,22 @@ internal class Startup }); // init background service - { - BackgroundServicesConfig config = this.Configuration.GetRequiredSection("BackgroundServices").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); - if (config.Enabled) - services.AddHostedService(); - } + if (backgroundServicesConfig.Enabled) + services.AddHostedService(); // init API clients { - ApiClientsConfig api = this.Configuration.GetRequiredSection("ApiClients").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); - string version = this.GetType().Assembly.GetName().Version!.ToString(3); - string userAgent = string.Format(api.UserAgent, version); - services.AddSingleton(new ChucklefishClient( userAgent: userAgent, - baseUrl: api.ChucklefishBaseUrl, - modPageUrlFormat: api.ChucklefishModPageUrlFormat + baseUrl: apiConfig.ChucklefishBaseUrl, + modPageUrlFormat: apiConfig.ChucklefishModPageUrlFormat )); - if (!string.IsNullOrWhiteSpace(api.CurseForgeExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.CurseForgeExportUrl)) { services.AddSingleton(new CurseForgeExportApiClient( userAgent: userAgent, - baseUrl: api.CurseForgeExportUrl + baseUrl: apiConfig.CurseForgeExportUrl )); } else @@ -159,26 +158,26 @@ internal class Startup services.AddSingleton( provider => new CurseForgeClient( userAgent: userAgent, - apiUrl: api.CurseForgeBaseUrl, - apiKey: api.CurseForgeApiKey, - webModUrl: api.CurseForgeWebPageUrl, + apiUrl: apiConfig.CurseForgeBaseUrl, + apiKey: apiConfig.CurseForgeApiKey, + webModUrl: apiConfig.CurseForgeWebPageUrl, exportCache: provider.GetRequiredService() ) ); services.AddSingleton(new GitHubClient( - baseUrl: api.GitHubBaseUrl, + baseUrl: apiConfig.GitHubBaseUrl, userAgent: userAgent, - acceptHeader: api.GitHubAcceptHeader, - username: api.GitHubUsername, - password: api.GitHubPassword + acceptHeader: apiConfig.GitHubAcceptHeader, + username: apiConfig.GitHubUsername, + password: apiConfig.GitHubPassword )); - if (!string.IsNullOrWhiteSpace(api.ModDropExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.ModDropExportUrl)) { services.AddSingleton(new ModDropExportApiClient( userAgent: userAgent, - baseUrl: api.ModDropExportUrl + baseUrl: apiConfig.ModDropExportUrl )); } else @@ -187,34 +186,34 @@ internal class Startup services.AddSingleton( provider => new ModDropClient( userAgent: userAgent, - apiUrl: api.ModDropApiUrl, - modUrlFormat: api.ModDropModPageUrl, + apiUrl: apiConfig.ModDropApiUrl, + modUrlFormat: apiConfig.ModDropModPageUrl, exportCache: provider.GetRequiredService() ) ); - if (!string.IsNullOrWhiteSpace(api.NexusExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.NexusExportUrl)) { services.AddSingleton( new NexusExportApiClient( userAgent: userAgent, - baseUrl: api.NexusExportUrl + baseUrl: apiConfig.NexusExportUrl ) ); } else services.AddSingleton(new DisabledNexusExportApiClient()); - if (!string.IsNullOrWhiteSpace(api.NexusApiKey)) + if (!string.IsNullOrWhiteSpace(apiConfig.NexusApiKey)) { services.AddSingleton( provider => new NexusClient( webUserAgent: userAgent, - webBaseUrl: api.NexusBaseUrl, - webModUrlFormat: api.NexusModUrlFormat, - webModScrapeUrlFormat: api.NexusModScrapeUrlFormat, - apiAppVersion: version, - apiKey: api.NexusApiKey, + webBaseUrl: apiConfig.NexusBaseUrl, + webModUrlFormat: apiConfig.NexusModUrlFormat, + webModScrapeUrlFormat: apiConfig.NexusModScrapeUrlFormat, + apiAppVersion: userAgentVersion, + apiKey: apiConfig.NexusApiKey, exportCache: provider.GetRequiredService() )); } @@ -222,7 +221,7 @@ internal class Startup services.AddSingleton(new DisabledNexusClient()); services.AddSingleton(new PastebinClient( - baseUrl: api.PastebinBaseUrl, + baseUrl: apiConfig.PastebinBaseUrl, userAgent: userAgent )); diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index ed47694c..3f1ffe55 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -71,8 +71,8 @@ }, "ModDataset": { - "RepoUrl": "https://github.com/Pathoschild/StardewModDataset.git", - "LocalPath": "%TEMP%/StardewModDataset" + "DownloadZipUrl": "https://github.com/Pathoschild/StardewModDataset/archive/refs/heads/main.zip", + "LocalRootPath": "%TEMP%/StardewModDataset" }, "ModUpdateCheck": {