From 5689c8d6aeecf54f670559ffaaed6684a5febc25 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Thu, 25 Jun 2026 01:07:48 -0400 Subject: [PATCH] switch to zip download for fetching mod dataset Azure App Services doesn't have Git installed on the servers by default, and alternatives like LibGit2Sharp are slower than just redownloading the latest branch when it changes. --- .../ModDataset/IModDatasetRepository.cs | 1 + .../ModDataset/ModDatasetRepository.cs | 180 ++++++++++++++---- .../ConfigModels/ModDatasetConfig.cs | 8 +- src/SMAPI.Web/Startup.cs | 73 ++++--- src/SMAPI.Web/appsettings.json | 4 +- 5 files changed, 183 insertions(+), 83 deletions(-) diff --git a/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs b/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs index 00ecbb8d..db43c041 100644 --- a/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/ModDataset/IModDatasetRepository.cs @@ -11,6 +11,7 @@ internal interface IModDatasetRepository *********/ /// Fetch the latest mod dataset, if it changed or hasn't been fetched yet. /// A callback which should receive progress messages for logging. + /// The downloaded archive isn't a valid mod dataset. Task UpdateAsync(Action? log = null); /// Get the full path to a file in the mod dataset. diff --git a/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs b/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs index 09215b6e..612da40d 100644 --- a/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/ModDataset/ModDatasetRepository.cs @@ -1,6 +1,10 @@ using System; -using System.Diagnostics; using System.IO; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; using System.Threading.Tasks; namespace StardewModdingAPI.Web.Framework.Caching.ModDataset; @@ -11,75 +15,171 @@ internal class ModDatasetRepository : IModDatasetRepository /********* ** Fields *********/ - /// The HTTPS URL of the mod dataset Git repo. - private readonly string RepoUrl; + /// The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL). + private readonly string DownloadZipUrl; - /// The full path to the mod dataset repo. - private readonly string LocalRepoPath; + /// The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted. + private readonly string LocalRootPath; + + /// The HTTP client with which to fetch the mod dataset archive. + private readonly HttpClient HttpClient; + + /// The name of the file within the in which to cache data about the last fetched dataset. + private const string CacheFileName = ".cache.json"; + + /// The path to the actual 'dataset' folder. + private string? DatasetPath; /********* ** Public methods *********/ /// Construct an instance. - /// - /// The path to the mod dataset repo. - public ModDatasetRepository(string repoUrl, string localRepoPath) + /// + /// + /// The user agent to use when fetching the archive URL. + public ModDatasetRepository(string downloadZipUrl, string localPath, string userAgent) { - this.RepoUrl = repoUrl; - this.LocalRepoPath = Path.GetFullPath( - Environment.ExpandEnvironmentVariables(localRepoPath) - ); + this.DownloadZipUrl = downloadZipUrl; + this.LocalRootPath = Path.GetFullPath(this.ExpandEnvironmentVariables(localPath)); + + this.HttpClient = new HttpClient(); + this.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent); } /// public async Task UpdateAsync(Action? log = null) { - if (!Directory.Exists(Path.Combine(this.LocalRepoPath, ".git"))) + // check for new archive + log?.Invoke(" Checking for newer dataset..."); + LastDownload? cached = await this.GetCacheInfoAsync(); + using HttpRequestMessage request = this.BuildDownloadRequest(cached?.ETag); + using HttpResponseMessage response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + + // download new archive + if (cached is null || response.StatusCode is not HttpStatusCode.NotModified) { - log?.Invoke("Cloning mod dataset repo..."); - Directory.CreateDirectory(Path.GetDirectoryName(this.LocalRepoPath)!); - await this.RunGitAsync("clone", "--depth", "1", this.RepoUrl, this.LocalRepoPath); + // read response data + if (response.StatusCode is not HttpStatusCode.NotModified) + response.EnsureSuccessStatusCode(); + string? newEtag = response.Headers.ETag?.ToString(); + + // download & unzip dataset + log?.Invoke(" Downloading dataset..."); + string newFolderName = Guid.NewGuid().ToString("N"); + string newRootPath = Path.Combine(this.LocalRootPath, newFolderName); + Directory.CreateDirectory(newRootPath); + await using (Stream downloadStream = await response.Content.ReadAsStreamAsync()) + await using (ZipArchive archive = new(downloadStream, ZipArchiveMode.Read)) + await archive.ExtractToDirectoryAsync(newRootPath); + + // locate 'dataset' folder + log?.Invoke(" Locating 'dataset' folder..."); + string? newDatasetPath = null; + foreach (DirectoryInfo entry in new DirectoryInfo(newRootPath).EnumerateDirectories("*", SearchOption.AllDirectories)) + { + if (entry.Name == "dataset") + { + newDatasetPath = Path.GetRelativePath(newRootPath, entry.FullName); + break; + } + } + if (newDatasetPath is null) + throw new InvalidOperationException("No 'dataset' folder found in the downloaded archive."); + + // save cache info + cached = new LastDownload(newFolderName, newDatasetPath, newEtag); + await this.SaveCacheInfoAsync(cached); + log?.Invoke($" Dataset saved to {newRootPath} with {(newEtag != null ? $"ETag header {newEtag}" : "no ETag header")}."); } - else + + // track dataset path + this.DatasetPath = Path.Combine(this.LocalRootPath, cached.FolderName, cached.RelativePathToDataset); + + // clear previous datasets if possible + foreach (DirectoryInfo directory in new DirectoryInfo(this.LocalRootPath).EnumerateDirectories()) { - log?.Invoke("Fetching latest changes from mod dataset repo..."); - await this.RunGitAsync("-C", this.LocalRepoPath, "fetch", "--depth", "1", "origin"); - await this.RunGitAsync("-C", this.LocalRepoPath, "reset", "--hard", "FETCH_HEAD"); + if (directory.Name == cached.FolderName) + continue; + + log?.Invoke($" Deleting previous dataset download at {directory.FullName}..."); + try + { + directory.Delete(recursive: true); + } + catch (Exception ex) + { + log?.Invoke($" Deletion failed: {ex}"); + } } } /// public string GetFilePath(string relativePath) { - return Path.Combine(this.LocalRepoPath, "dataset", relativePath); + return this.DatasetPath is not null + ? Path.Combine(this.DatasetPath, relativePath) + : throw new InvalidOperationException($"Must call '{nameof(this.UpdateAsync)}' before '{nameof(this.GetFilePath)}'."); } /********* ** Private methods *********/ - /// Run a Git command and wait for it to complete. - /// The git arguments. - /// The Git command exited with a non-zero exit code. - private async Task RunGitAsync(params string[] args) + /// Build the HTTP request to download the archive from the server if it changed. + /// The ETag value for the last successful dataset download, if any. + private HttpRequestMessage BuildDownloadRequest(string? etag) { - ProcessStartInfo startInfo = new("git", args) + HttpRequestMessage request = new(HttpMethod.Get, this.DownloadZipUrl); + try { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; + if (etag != null) + request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag)); + } + catch + { + request.Dispose(); + throw; + } - using Process process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start Git process."); - - Task readOutput = process.StandardOutput.ReadToEndAsync(); - Task readError = process.StandardError.ReadToEndAsync(); - await process.WaitForExitAsync(); - await Task.WhenAll(readOutput, readError); - - if (process.ExitCode != 0) - throw new InvalidOperationException($"Git command failed with exit code {process.ExitCode}: {await readError}."); + return request; } + + /// Get the cached info about the last dataset download, if it was previously downloaded. + private async Task GetCacheInfoAsync() + { + string path = Path.Combine(this.LocalRootPath, CacheFileName); + if (!File.Exists(path)) + return null; + + await using FileStream stream = File.OpenRead(path); + return await JsonSerializer.DeserializeAsync(stream); + } + + /// Save the cache info about a dataset download. + /// The cache info to save. + private async Task SaveCacheInfoAsync(LastDownload cacheInfo) + { + string path = Path.Combine(this.LocalRootPath, CacheFileName); + + await using FileStream stream = File.Create(path); + await JsonSerializer.SerializeAsync(stream, cacheInfo); + } + + /// Expand environment variables in a path. + /// The path to parse. + /// Returns the path with any environment variables replaced. + private string ExpandEnvironmentVariables(string path) + { + if (OperatingSystem.IsLinux()) + path = path.Replace("%TEMP%", Path.GetTempPath(), StringComparison.OrdinalIgnoreCase); + + return Environment.ExpandEnvironmentVariables(path); + } + + /// The cached metadata about the last dataset download. + /// The folder name within the root folder. + /// The relative path to the 'dataset' folder within the . + /// The ETag value for the downloaded archive, if available. + private record LastDownload(string FolderName, string RelativePathToDataset, string? ETag); } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs index 658626a2..b9cc760a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModDatasetConfig.cs @@ -6,9 +6,9 @@ internal class ModDatasetConfig /********* ** Accessors *********/ - /// The HTTPS URL of the mod dataset Git repo. - public string RepoUrl { get; set; } = null!; + /// The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL). + public string DownloadZipUrl { get; set; } = null!; - /// The local path into which to clone the mod dataset repo. - public string LocalPath { get; set; } = null!; + /// The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted. + public string LocalRootPath { get; set; } = null!; } diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 95f9cf81..68edd551 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -83,6 +83,15 @@ internal class Startup /// The service injection container. public void ConfigureServices(IServiceCollection services) { + // load config + ApiClientsConfig apiConfig = this.Configuration.GetRequiredSection("ApiClients").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); + BackgroundServicesConfig backgroundServicesConfig = this.Configuration.GetRequiredSection("BackgroundServices").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'BackgroundServices' config section couldn't be loaded."); + ModDatasetConfig datasetConfig = this.Configuration.GetRequiredSection("ModDataset").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded."); + + // init user agent + string userAgentVersion = this.GetType().Assembly.GetName().Version!.ToString(3); + string userAgent = string.Format(apiConfig.UserAgent, userAgentVersion); + // init basic services services .Configure(this.Configuration.GetRequiredSection("ApiClients")) @@ -110,10 +119,7 @@ internal class Startup services.AddSingleton(new NexusExportCacheMemoryRepository()); // init mod dataset - { - ModDatasetConfig config = this.Configuration.GetRequiredSection("ModDataset").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded."); - services.AddSingleton(new ModDatasetRepository(config.RepoUrl, config.LocalPath)); - } + services.AddSingleton(new ModDatasetRepository(datasetConfig.DownloadZipUrl, datasetConfig.LocalRootPath, userAgent)); // init Hangfire services @@ -128,29 +134,22 @@ internal class Startup }); // init background service - { - BackgroundServicesConfig config = this.Configuration.GetRequiredSection("BackgroundServices").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); - if (config.Enabled) - services.AddHostedService(); - } + if (backgroundServicesConfig.Enabled) + services.AddHostedService(); // init API clients { - ApiClientsConfig api = this.Configuration.GetRequiredSection("ApiClients").Get() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded."); - string version = this.GetType().Assembly.GetName().Version!.ToString(3); - string userAgent = string.Format(api.UserAgent, version); - services.AddSingleton(new ChucklefishClient( userAgent: userAgent, - baseUrl: api.ChucklefishBaseUrl, - modPageUrlFormat: api.ChucklefishModPageUrlFormat + baseUrl: apiConfig.ChucklefishBaseUrl, + modPageUrlFormat: apiConfig.ChucklefishModPageUrlFormat )); - if (!string.IsNullOrWhiteSpace(api.CurseForgeExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.CurseForgeExportUrl)) { services.AddSingleton(new CurseForgeExportApiClient( userAgent: userAgent, - baseUrl: api.CurseForgeExportUrl + baseUrl: apiConfig.CurseForgeExportUrl )); } else @@ -159,26 +158,26 @@ internal class Startup services.AddSingleton( provider => new CurseForgeClient( userAgent: userAgent, - apiUrl: api.CurseForgeBaseUrl, - apiKey: api.CurseForgeApiKey, - webModUrl: api.CurseForgeWebPageUrl, + apiUrl: apiConfig.CurseForgeBaseUrl, + apiKey: apiConfig.CurseForgeApiKey, + webModUrl: apiConfig.CurseForgeWebPageUrl, exportCache: provider.GetRequiredService() ) ); services.AddSingleton(new GitHubClient( - baseUrl: api.GitHubBaseUrl, + baseUrl: apiConfig.GitHubBaseUrl, userAgent: userAgent, - acceptHeader: api.GitHubAcceptHeader, - username: api.GitHubUsername, - password: api.GitHubPassword + acceptHeader: apiConfig.GitHubAcceptHeader, + username: apiConfig.GitHubUsername, + password: apiConfig.GitHubPassword )); - if (!string.IsNullOrWhiteSpace(api.ModDropExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.ModDropExportUrl)) { services.AddSingleton(new ModDropExportApiClient( userAgent: userAgent, - baseUrl: api.ModDropExportUrl + baseUrl: apiConfig.ModDropExportUrl )); } else @@ -187,34 +186,34 @@ internal class Startup services.AddSingleton( provider => new ModDropClient( userAgent: userAgent, - apiUrl: api.ModDropApiUrl, - modUrlFormat: api.ModDropModPageUrl, + apiUrl: apiConfig.ModDropApiUrl, + modUrlFormat: apiConfig.ModDropModPageUrl, exportCache: provider.GetRequiredService() ) ); - if (!string.IsNullOrWhiteSpace(api.NexusExportUrl)) + if (!string.IsNullOrWhiteSpace(apiConfig.NexusExportUrl)) { services.AddSingleton( new NexusExportApiClient( userAgent: userAgent, - baseUrl: api.NexusExportUrl + baseUrl: apiConfig.NexusExportUrl ) ); } else services.AddSingleton(new DisabledNexusExportApiClient()); - if (!string.IsNullOrWhiteSpace(api.NexusApiKey)) + if (!string.IsNullOrWhiteSpace(apiConfig.NexusApiKey)) { services.AddSingleton( provider => new NexusClient( webUserAgent: userAgent, - webBaseUrl: api.NexusBaseUrl, - webModUrlFormat: api.NexusModUrlFormat, - webModScrapeUrlFormat: api.NexusModScrapeUrlFormat, - apiAppVersion: version, - apiKey: api.NexusApiKey, + webBaseUrl: apiConfig.NexusBaseUrl, + webModUrlFormat: apiConfig.NexusModUrlFormat, + webModScrapeUrlFormat: apiConfig.NexusModScrapeUrlFormat, + apiAppVersion: userAgentVersion, + apiKey: apiConfig.NexusApiKey, exportCache: provider.GetRequiredService() )); } @@ -222,7 +221,7 @@ internal class Startup services.AddSingleton(new DisabledNexusClient()); services.AddSingleton(new PastebinClient( - baseUrl: api.PastebinBaseUrl, + baseUrl: apiConfig.PastebinBaseUrl, userAgent: userAgent )); diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index ed47694c..3f1ffe55 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -71,8 +71,8 @@ }, "ModDataset": { - "RepoUrl": "https://github.com/Pathoschild/StardewModDataset.git", - "LocalPath": "%TEMP%/StardewModDataset" + "DownloadZipUrl": "https://github.com/Pathoschild/StardewModDataset/archive/refs/heads/main.zip", + "LocalRootPath": "%TEMP%/StardewModDataset" }, "ModUpdateCheck": {