switch to zip download for fetching mod dataset

Azure App Services doesn't have Git installed on the servers by default, and alternatives like LibGit2Sharp are slower than just redownloading the latest branch when it changes.
2026-10-12 02:08:51 +00:00 · 2026-06-25 01:07:48 -04:00
parent a444b2b359
commit 5689c8d6ae
5 changed files with 183 additions and 83 deletions
@@ -11,6 +11,7 @@ internal interface IModDatasetRepository
    *********/
    /// <summary>Fetch the latest mod dataset, if it changed or hasn't been fetched yet.</summary>
    /// <param name="log">A callback which should receive progress messages for logging.</param>
+    /// <exception cref="InvalidOperationException">The downloaded archive isn't a valid mod dataset.</exception>
    Task UpdateAsync(Action<string>? log = null);

    /// <summary>Get the full path to a file in the mod dataset.</summary>
@@ -1,6 +1,10 @@
 using System;
-using System.Diagnostics;
 using System.IO;
+using System.IO.Compression;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text.Json;
 using System.Threading.Tasks;

 namespace StardewModdingAPI.Web.Framework.Caching.ModDataset;
@@ -11,75 +15,171 @@ internal class ModDatasetRepository : IModDatasetRepository
    /*********
    ** Fields
    *********/
-    /// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
-    private readonly string RepoUrl;
+    /// <summary>The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL).</summary>
+    private readonly string DownloadZipUrl;

-    /// <summary>The full path to the mod dataset repo.</summary>
-    private readonly string LocalRepoPath;
+    /// <summary>The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted.</summary>
+    private readonly string LocalRootPath;
+
+    /// <summary>The HTTP client with which to fetch the mod dataset archive.</summary>
+    private readonly HttpClient HttpClient;
+
+    /// <summary>The name of the file within the <see cref="LocalRootPath"/> in which to cache data about the last fetched dataset.</summary>
+    private const string CacheFileName = ".cache.json";
+
+    /// <summary>The path to the actual 'dataset' folder.</summary>
+    private string? DatasetPath;


    /*********
    ** Public methods
    *********/
    /// <summary>Construct an instance.</summary>
-    /// <param name="repoUrl"><inheritdoc cref="RepoUrl" path="/summary" /></param>
-    /// <param name="localRepoPath">The path to the mod dataset repo.</param>
-    public ModDatasetRepository(string repoUrl, string localRepoPath)
+    /// <param name="downloadZipUrl"><inheritdoc cref="DownloadZipUrl" path="/summary" /></param>
+    /// <param name="localPath"><inheritdoc cref="LocalRootPath" path="/summary" /></param>
+    /// <param name="userAgent">The user agent to use when fetching the archive URL.</param>
+    public ModDatasetRepository(string downloadZipUrl, string localPath, string userAgent)
    {
-        this.RepoUrl = repoUrl;
-        this.LocalRepoPath = Path.GetFullPath(
-            Environment.ExpandEnvironmentVariables(localRepoPath)
-        );
+        this.DownloadZipUrl = downloadZipUrl;
+        this.LocalRootPath = Path.GetFullPath(this.ExpandEnvironmentVariables(localPath));
+
+        this.HttpClient = new HttpClient();
+        this.HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent);
    }

    /// <inheritdoc />
    public async Task UpdateAsync(Action<string>? log = null)
    {
-        if (!Directory.Exists(Path.Combine(this.LocalRepoPath, ".git")))
+        // check for new archive
+        log?.Invoke("  Checking for newer dataset...");
+        LastDownload? cached = await this.GetCacheInfoAsync();
+        using HttpRequestMessage request = this.BuildDownloadRequest(cached?.ETag);
+        using HttpResponseMessage response = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+
+        // download new archive
+        if (cached is null || response.StatusCode is not HttpStatusCode.NotModified)
        {
-            log?.Invoke("Cloning mod dataset repo...");
-            Directory.CreateDirectory(Path.GetDirectoryName(this.LocalRepoPath)!);
-            await this.RunGitAsync("clone", "--depth", "1", this.RepoUrl, this.LocalRepoPath);
+            // read response data
+            if (response.StatusCode is not HttpStatusCode.NotModified)
+                response.EnsureSuccessStatusCode();
+            string? newEtag = response.Headers.ETag?.ToString();
+
+            // download & unzip dataset
+            log?.Invoke("  Downloading dataset...");
+            string newFolderName = Guid.NewGuid().ToString("N");
+            string newRootPath = Path.Combine(this.LocalRootPath, newFolderName);
+            Directory.CreateDirectory(newRootPath);
+            await using (Stream downloadStream = await response.Content.ReadAsStreamAsync())
+            await using (ZipArchive archive = new(downloadStream, ZipArchiveMode.Read))
+                await archive.ExtractToDirectoryAsync(newRootPath);
+
+            // locate 'dataset' folder
+            log?.Invoke("  Locating 'dataset' folder...");
+            string? newDatasetPath = null;
+            foreach (DirectoryInfo entry in new DirectoryInfo(newRootPath).EnumerateDirectories("*", SearchOption.AllDirectories))
+            {
+                if (entry.Name == "dataset")
+                {
+                    newDatasetPath = Path.GetRelativePath(newRootPath, entry.FullName);
+                    break;
+                }
+            }
+            if (newDatasetPath is null)
+                throw new InvalidOperationException("No 'dataset' folder found in the downloaded archive.");
+
+            // save cache info
+            cached = new LastDownload(newFolderName, newDatasetPath, newEtag);
+            await this.SaveCacheInfoAsync(cached);
+            log?.Invoke($"  Dataset saved to {newRootPath} with {(newEtag != null ? $"ETag header {newEtag}" : "no ETag header")}.");
        }
-        else
+
+        // track dataset path
+        this.DatasetPath = Path.Combine(this.LocalRootPath, cached.FolderName, cached.RelativePathToDataset);
+
+        // clear previous datasets if possible
+        foreach (DirectoryInfo directory in new DirectoryInfo(this.LocalRootPath).EnumerateDirectories())
        {
-            log?.Invoke("Fetching latest changes from mod dataset repo...");
-            await this.RunGitAsync("-C", this.LocalRepoPath, "fetch", "--depth", "1", "origin");
-            await this.RunGitAsync("-C", this.LocalRepoPath, "reset", "--hard", "FETCH_HEAD");
+            if (directory.Name == cached.FolderName)
+                continue;
+
+            log?.Invoke($"  Deleting previous dataset download at {directory.FullName}...");
+            try
+            {
+                directory.Delete(recursive: true);
+            }
+            catch (Exception ex)
+            {
+                log?.Invoke($"    Deletion failed: {ex}");
+            }
        }
    }

    /// <inheritdoc />
    public string GetFilePath(string relativePath)
    {
-        return Path.Combine(this.LocalRepoPath, "dataset", relativePath);
+        return this.DatasetPath is not null
+            ? Path.Combine(this.DatasetPath, relativePath)
+            : throw new InvalidOperationException($"Must call '{nameof(this.UpdateAsync)}' before '{nameof(this.GetFilePath)}'.");
    }


    /*********
    ** Private methods
    *********/
-    /// <summary>Run a Git command and wait for it to complete.</summary>
-    /// <param name="args">The git arguments.</param>
-    /// <exception cref="InvalidOperationException">The Git command exited with a non-zero exit code.</exception>
-    private async Task RunGitAsync(params string[] args)
+    /// <summary>Build the HTTP request to download the archive from the server if it changed.</summary>
+    /// <param name="etag">The ETag value for the last successful dataset download, if any.</param>
+    private HttpRequestMessage BuildDownloadRequest(string? etag)
    {
-        ProcessStartInfo startInfo = new("git", args)
+        HttpRequestMessage request = new(HttpMethod.Get, this.DownloadZipUrl);
+        try
        {
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-            UseShellExecute = false,
-            CreateNoWindow = true
-        };
+            if (etag != null)
+                request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue(etag));
+        }
+        catch
+        {
+            request.Dispose();
+            throw;
+        }

-        using Process process = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start Git process.");
-
-        Task readOutput = process.StandardOutput.ReadToEndAsync();
-        Task<string> readError = process.StandardError.ReadToEndAsync();
-        await process.WaitForExitAsync();
-        await Task.WhenAll(readOutput, readError);
-
-        if (process.ExitCode != 0)
-            throw new InvalidOperationException($"Git command failed with exit code {process.ExitCode}: {await readError}.");
+        return request;
    }
+
+    /// <summary>Get the cached info about the last dataset download, if it was previously downloaded.</summary>
+    private async Task<LastDownload?> GetCacheInfoAsync()
+    {
+        string path = Path.Combine(this.LocalRootPath, CacheFileName);
+        if (!File.Exists(path))
+            return null;
+
+        await using FileStream stream = File.OpenRead(path);
+        return await JsonSerializer.DeserializeAsync<LastDownload>(stream);
+    }
+
+    /// <summary>Save the cache info about a dataset download.</summary>
+    /// <param name="cacheInfo">The cache info to save.</param>
+    private async Task SaveCacheInfoAsync(LastDownload cacheInfo)
+    {
+        string path = Path.Combine(this.LocalRootPath, CacheFileName);
+
+        await using FileStream stream = File.Create(path);
+        await JsonSerializer.SerializeAsync(stream, cacheInfo);
+    }
+
+    /// <summary>Expand environment variables in a path.</summary>
+    /// <param name="path">The path to parse.</param>
+    /// <returns>Returns the path with any environment variables replaced.</returns>
+    private string ExpandEnvironmentVariables(string path)
+    {
+        if (OperatingSystem.IsLinux())
+            path = path.Replace("%TEMP%", Path.GetTempPath(), StringComparison.OrdinalIgnoreCase);
+
+        return Environment.ExpandEnvironmentVariables(path);
+    }
+
+    /// <summary>The cached metadata about the last dataset download.</summary>
+    /// <param name="FolderName">The folder name within the root folder.</param>
+    /// <param name="RelativePathToDataset">The relative path to the 'dataset' folder within the <see cref="FolderName"/>.</param>
+    /// <param name="ETag">The ETag value for the downloaded archive, if available.</param>
+    private record LastDownload(string FolderName, string RelativePathToDataset, string? ETag);
 }
@@ -6,9 +6,9 @@ internal class ModDatasetConfig
    /*********
    ** Accessors
    *********/
-    /// <summary>The HTTPS URL of the mod dataset Git repo.</summary>
-    public string RepoUrl { get; set; } = null!;
+    /// <summary>The URL of the zip file to download which contains the Stardew mod dataset (e.g. a GitHub branch download URL).</summary>
+    public string DownloadZipUrl { get; set; } = null!;

-    /// <summary>The local path into which to clone the mod dataset repo.</summary>
-    public string LocalPath { get; set; } = null!;
+    /// <summary>The full path to the folder into which to download the mod dataset. This should be a new folder, since anything inside it may be deleted.</summary>
+    public string LocalRootPath { get; set; } = null!;
 }
@@ -83,6 +83,15 @@ internal class Startup
    /// <param name="services">The service injection container.</param>
    public void ConfigureServices(IServiceCollection services)
    {
+        // load config
+        ApiClientsConfig apiConfig = this.Configuration.GetRequiredSection("ApiClients").Get<ApiClientsConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
+        BackgroundServicesConfig backgroundServicesConfig = this.Configuration.GetRequiredSection("BackgroundServices").Get<BackgroundServicesConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'BackgroundServices' config section couldn't be loaded.");
+        ModDatasetConfig datasetConfig = this.Configuration.GetRequiredSection("ModDataset").Get<ModDatasetConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded.");
+
+        // init user agent
+        string userAgentVersion = this.GetType().Assembly.GetName().Version!.ToString(3);
+        string userAgent = string.Format(apiConfig.UserAgent, userAgentVersion);
+
        // init basic services
        services
            .Configure<ApiClientsConfig>(this.Configuration.GetRequiredSection("ApiClients"))
@@ -110,10 +119,7 @@ internal class Startup
        services.AddSingleton<INexusExportCacheRepository>(new NexusExportCacheMemoryRepository());

        // init mod dataset
-        {
-            ModDatasetConfig config = this.Configuration.GetRequiredSection("ModDataset").Get<ModDatasetConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ModDataset' config section couldn't be loaded.");
-            services.AddSingleton<IModDatasetRepository>(new ModDatasetRepository(config.RepoUrl, config.LocalPath));
-        }
+        services.AddSingleton<IModDatasetRepository>(new ModDatasetRepository(datasetConfig.DownloadZipUrl, datasetConfig.LocalRootPath, userAgent));

        // init Hangfire
        services
@@ -128,29 +134,22 @@ internal class Startup
            });

        // init background service
-        {
-            BackgroundServicesConfig config = this.Configuration.GetRequiredSection("BackgroundServices").Get<BackgroundServicesConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
-            if (config.Enabled)
-                services.AddHostedService<BackgroundService>();
-        }
+        if (backgroundServicesConfig.Enabled)
+            services.AddHostedService<BackgroundService>();

        // init API clients
        {
-            ApiClientsConfig api = this.Configuration.GetRequiredSection("ApiClients").Get<ApiClientsConfig>() ?? throw new InvalidOperationException("Can't initialize server: required 'ApiClients' config section couldn't be loaded.");
-            string version = this.GetType().Assembly.GetName().Version!.ToString(3);
-            string userAgent = string.Format(api.UserAgent, version);
-
            services.AddSingleton<IChucklefishClient>(new ChucklefishClient(
                userAgent: userAgent,
-                baseUrl: api.ChucklefishBaseUrl,
-                modPageUrlFormat: api.ChucklefishModPageUrlFormat
+                baseUrl: apiConfig.ChucklefishBaseUrl,
+                modPageUrlFormat: apiConfig.ChucklefishModPageUrlFormat
            ));

-            if (!string.IsNullOrWhiteSpace(api.CurseForgeExportUrl))
+            if (!string.IsNullOrWhiteSpace(apiConfig.CurseForgeExportUrl))
            {
                services.AddSingleton<ICurseForgeExportApiClient>(new CurseForgeExportApiClient(
                    userAgent: userAgent,
-                    baseUrl: api.CurseForgeExportUrl
+                    baseUrl: apiConfig.CurseForgeExportUrl
                ));
            }
            else
@@ -159,26 +158,26 @@ internal class Startup
            services.AddSingleton<ICurseForgeClient>(
                provider => new CurseForgeClient(
                    userAgent: userAgent,
-                    apiUrl: api.CurseForgeBaseUrl,
-                    apiKey: api.CurseForgeApiKey,
-                    webModUrl: api.CurseForgeWebPageUrl,
+                    apiUrl: apiConfig.CurseForgeBaseUrl,
+                    apiKey: apiConfig.CurseForgeApiKey,
+                    webModUrl: apiConfig.CurseForgeWebPageUrl,
                    exportCache: provider.GetRequiredService<ICurseForgeExportCacheRepository>()
                )
            );

            services.AddSingleton<IGitHubClient>(new GitHubClient(
-                baseUrl: api.GitHubBaseUrl,
+                baseUrl: apiConfig.GitHubBaseUrl,
                userAgent: userAgent,
-                acceptHeader: api.GitHubAcceptHeader,
-                username: api.GitHubUsername,
-                password: api.GitHubPassword
+                acceptHeader: apiConfig.GitHubAcceptHeader,
+                username: apiConfig.GitHubUsername,
+                password: apiConfig.GitHubPassword
            ));

-            if (!string.IsNullOrWhiteSpace(api.ModDropExportUrl))
+            if (!string.IsNullOrWhiteSpace(apiConfig.ModDropExportUrl))
            {
                services.AddSingleton<IModDropExportApiClient>(new ModDropExportApiClient(
                    userAgent: userAgent,
-                    baseUrl: api.ModDropExportUrl
+                    baseUrl: apiConfig.ModDropExportUrl
                ));
            }
            else
@@ -187,34 +186,34 @@ internal class Startup
            services.AddSingleton<IModDropClient>(
                provider => new ModDropClient(
                    userAgent: userAgent,
-                    apiUrl: api.ModDropApiUrl,
-                    modUrlFormat: api.ModDropModPageUrl,
+                    apiUrl: apiConfig.ModDropApiUrl,
+                    modUrlFormat: apiConfig.ModDropModPageUrl,
                    exportCache: provider.GetRequiredService<IModDropExportCacheRepository>()
                )
            );

-            if (!string.IsNullOrWhiteSpace(api.NexusExportUrl))
+            if (!string.IsNullOrWhiteSpace(apiConfig.NexusExportUrl))
            {
                services.AddSingleton<INexusExportApiClient>(
                    new NexusExportApiClient(
                        userAgent: userAgent,
-                        baseUrl: api.NexusExportUrl
+                        baseUrl: apiConfig.NexusExportUrl
                    )
                );
            }
            else
                services.AddSingleton<INexusExportApiClient>(new DisabledNexusExportApiClient());

-            if (!string.IsNullOrWhiteSpace(api.NexusApiKey))
+            if (!string.IsNullOrWhiteSpace(apiConfig.NexusApiKey))
            {
                services.AddSingleton<INexusClient>(
                    provider => new NexusClient(
                        webUserAgent: userAgent,
-                        webBaseUrl: api.NexusBaseUrl,
-                        webModUrlFormat: api.NexusModUrlFormat,
-                        webModScrapeUrlFormat: api.NexusModScrapeUrlFormat,
-                        apiAppVersion: version,
-                        apiKey: api.NexusApiKey,
+                        webBaseUrl: apiConfig.NexusBaseUrl,
+                        webModUrlFormat: apiConfig.NexusModUrlFormat,
+                        webModScrapeUrlFormat: apiConfig.NexusModScrapeUrlFormat,
+                        apiAppVersion: userAgentVersion,
+                        apiKey: apiConfig.NexusApiKey,
                        exportCache: provider.GetRequiredService<INexusExportCacheRepository>()
                    ));
            }
@@ -222,7 +221,7 @@ internal class Startup
                services.AddSingleton<INexusClient>(new DisabledNexusClient());

            services.AddSingleton<IPastebinClient>(new PastebinClient(
-                baseUrl: api.PastebinBaseUrl,
+                baseUrl: apiConfig.PastebinBaseUrl,
                userAgent: userAgent
            ));

@@ -71,8 +71,8 @@
    },

    "ModDataset": {
-        "RepoUrl": "https://github.com/Pathoschild/StardewModDataset.git",
-        "LocalPath": "%TEMP%/StardewModDataset"
+        "DownloadZipUrl": "https://github.com/Pathoschild/StardewModDataset/archive/refs/heads/main.zip",
+        "LocalRootPath": "%TEMP%/StardewModDataset"
    },

    "ModUpdateCheck": {