From d8ea53de21e42eb0f61d9847696f94cff77ee6a0 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 21 Apr 2025 14:24:39 -0400 Subject: [PATCH] cache mod compatibility list data & fetch it from the frontend The backend now saves the parsed mod list to storage, and returns a SAS URI which the frontend uses to fetch it. This should significantly reduce bandwidth usage and response times, by avoiding the repeated backend parsing and data transfers. --- docs/release-notes.md | 3 + .../CompatibilityRepo/ModCompatibilityInfo.cs | 11 +++ .../Controllers/LogParserController.cs | 2 +- src/SMAPI.Web/Controllers/ModsController.cs | 68 ++++++++++++++++--- .../Framework/Storage/IStorageProvider.cs | 8 +++ .../Framework/Storage/StorageProvider.cs | 19 +++++- src/SMAPI.Web/ViewModels/ModListModel.cs | 17 +++-- src/SMAPI.Web/ViewModels/ModModel.cs | 20 +++--- src/SMAPI.Web/Views/Mods/Index.cshtml | 38 +++++++---- src/SMAPI.Web/wwwroot/Content/js/mods.js | 56 ++++++++------- 10 files changed, 175 insertions(+), 67 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index cbd64a95..d4d597f6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -18,6 +18,9 @@ * Removed support for very old SMAPI logs. * You can now download a JSON representation of the parsed log (see the download link at the bottom of the log page). * Fixed server error if a JSON file contains nested comment syntax. + * Improved mod compatibility list: + * Reduced response times with a new cache and client-side fetch. + * Fixed sort order for mods with non-Latin characters in the name. * Third-party libraries are now served from `smapi.io` instead of external CDNs. ## 4.2.1 diff --git a/src/SMAPI.Toolkit/Framework/Clients/CompatibilityRepo/ModCompatibilityInfo.cs b/src/SMAPI.Toolkit/Framework/Clients/CompatibilityRepo/ModCompatibilityInfo.cs index f099b991..9d33bd54 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/CompatibilityRepo/ModCompatibilityInfo.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/CompatibilityRepo/ModCompatibilityInfo.cs @@ -44,4 +44,15 @@ public class ModCompatibilityInfo this.UnofficialVersion = unofficialVersion; this.UnofficialUrl = unofficialUrl; } + + /// Get whether this mod is compatible with no notes. + public bool IsDefault() + { + return + this.Status == ModCompatibilityStatus.Ok + && this.Summary is "use latest version." + && this.BrokeIn is null + && this.UnofficialUrl is null + && this.UnofficialVersion is null; + } } diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index b643b7e3..8fef4695 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -135,7 +135,7 @@ internal class LogParserController : Controller // upload log string id = Guid.NewGuid().ToString("N"); - UploadResult uploadResult = await this.Storage.SaveAsync($"parsed-{id}", JsonConvert.SerializeObject(log), "application/json"); + UploadResult uploadResult = await this.Storage.SaveToJsonAsync($"parsed-{id}", log); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); diff --git a/src/SMAPI.Web/Controllers/ModsController.cs b/src/SMAPI.Web/Controllers/ModsController.cs index f15c2c2e..c6873575 100644 --- a/src/SMAPI.Web/Controllers/ModsController.cs +++ b/src/SMAPI.Web/Controllers/ModsController.cs @@ -1,11 +1,15 @@ using System; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework.Caching; using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo; using StardewModdingAPI.Web.Framework.ConfigModels; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels; namespace StardewModdingAPI.Web.Controllers; @@ -19,51 +23,93 @@ internal class ModsController : Controller /// The cache in which to store mod metadata. private readonly ICompatibilityCacheRepository Cache; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; + /// The number of minutes before which compatibility list data should be considered old. private readonly int StaleMinutes; + /// When this server instance last refreshed the mod list cache file. + private DateTimeOffset LastCacheUpdate = DateTimeOffset.MinValue; + /********* ** Public methods *********/ /// Construct an instance. /// The cache in which to store mod metadata. + /// Provides access to raw data storage. /// The config settings for mod update checks. - public ModsController(ICompatibilityCacheRepository cache, IOptions configProvider) + public ModsController(ICompatibilityCacheRepository cache, IStorageProvider storage, IOptions configProvider) { ModCompatibilityListConfig config = configProvider.Value; this.Cache = cache; + this.Storage = storage; this.StaleMinutes = config.StaleMinutes; } /// Display information for all mods. [HttpGet] [Route("mods")] - public ViewResult Index() + public async Task Index() { - return this.View("Index", this.FetchData()); + return this.View("Index", await this.BuildModelAsync()); } /********* ** Private methods *********/ - /// Asynchronously fetch mod metadata from the compatibility list. - public ModListModel FetchData() + /// Build a view model for the current mod compatibility list data. + private async Task BuildModelAsync() { - // fetch cached data + // get metadata if (!this.Cache.TryGetCacheMetadata(out Cached? metadata)) - return new ModListModel(Array.Empty(), lastUpdated: DateTimeOffset.UtcNow, isStale: true); + return new ModListModel(null, [], lastUpdated: DateTimeOffset.UtcNow, isStale: true); + + // fetch data + (string? fetchUri, ModModel[]? overrideData) = await this.FetchDataAsync(metadata); // build model return new ModListModel( - mods: this.Cache - .GetMods() - .Select(mod => new ModModel(mod.Data)) - .OrderBy(p => Regex.Replace((p.Name ?? "").ToLower(), "[^a-z0-9]", "")), // ignore case, spaces, and special characters when sorting + fetchUri: fetchUri, + mods: overrideData, lastUpdated: metadata.LastUpdated, isStale: this.Cache.IsStale(metadata.LastUpdated, this.StaleMinutes) ); } + + /// Get the cached data to send to the frontend, creating it if needed. + /// The mod list cache metadata. + private async Task<(string? FetchUri, ModModel[]? OverrideData)> FetchDataAsync(Cached metadata) + { + const string cacheKey = "mod-list"; + + // fetch file if still fresh + StoredFileInfo? fileInfo = this.LastCacheUpdate >= metadata.LastRequested + ? await this.Storage.GetAsync(cacheKey, false) + : null; + + // recreate cache file if needed + if (fileInfo?.Success != true) + { + var mods = this.Cache + .GetMods() + .Select(mod => new ModModel(mod.Data)) + .OrderBy(p => PathUtilities.CreateSlug(p.Name)?.ToLower() ?? ""); // ignore case, spaces, and special characters when sorting + + await this.Storage.SaveToJsonAsync(cacheKey, mods); + this.LastCacheUpdate = DateTimeOffset.UtcNow; + + fileInfo = await this.Storage.GetAsync(cacheKey, false); + } + + // get info + string? fetchUri = fileInfo.FetchUri; + ModModel[]? overrideData = fileInfo.FetchUri is null && fileInfo.FetchedData != null + ? JsonConvert.DeserializeObject(fileInfo.FetchedData) + : null; + return (fetchUri, overrideData); + } } diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index 6e3fcb68..e03beb35 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -13,6 +13,14 @@ internal interface IStorageProvider /// Returns metadata about the save attempt. Task SaveAsync(string id, string content, string contentType, bool compress = true); + /// Serialize a data model to JSON and save it to storage. + /// The data type to save. + /// The ID with which to save the result. + /// The content to serialize and upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + Task SaveToJsonAsync(string id, T content, bool compress = true); + /// Fetch raw text from storage. /// The storage ID used to upload the file. /// Whether to reset the file expiry. diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index f550378c..50cfff3b 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -8,6 +8,7 @@ using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Sas; using Microsoft.Extensions.Options; +using Newtonsoft.Json; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; using StardewModdingAPI.Web.Framework.ConfigModels; @@ -38,6 +39,12 @@ internal class StorageProvider : IStorageProvider /// private int AutoRenewalDays => this.ClientsConfig.AzureBlobTempExpiryAutoRenewalDays; + /// The JSON serializer settings to use when saving JSON data to storage. + private JsonSerializerSettings DefaultJsonSettings = new() + { + NullValueHandling = NullValueHandling.Ignore + }; + /********* ** Public methods @@ -84,13 +91,19 @@ internal class StorageProvider : IStorageProvider } } + /// + public Task SaveToJsonAsync(string id, T content, bool compress = true) + { + string json = JsonConvert.SerializeObject(content, this.DefaultJsonSettings); + + return this.SaveAsync(id, json, "application/json", compress); + } + /// public async Task GetAsync(string id, bool forceRenew, bool forceDownloadContent = false) { // fetch from blob storage - bool isBlobStorage = id.StartsWith("parsed-") - ? Guid.TryParseExact(id.Substring("parsed-".Length), "N", out _) - : Guid.TryParseExact(id, "N", out _); + bool isBlobStorage = id.StartsWith("parsed-") || id == "mod-list" || Guid.TryParseExact(id, "N", out _); if (isBlobStorage) { // Azure Blob storage diff --git a/src/SMAPI.Web/ViewModels/ModListModel.cs b/src/SMAPI.Web/ViewModels/ModListModel.cs index 9a6a0f89..28082656 100644 --- a/src/SMAPI.Web/ViewModels/ModListModel.cs +++ b/src/SMAPI.Web/ViewModels/ModListModel.cs @@ -10,8 +10,11 @@ public class ModListModel /********* ** Accessors *********/ - /// The mods to display. - public ModModel[] Mods { get; } + /// The URI from which to fetch the mod data. + public string? FetchUri { get; } + + /// The mods to display instead of the , if set. + public ModModel[]? Mods { get; } /// When the data was last updated. public DateTimeOffset LastUpdated { get; } @@ -20,19 +23,21 @@ public class ModListModel public bool IsStale { get; } /// Whether the mod metadata is available. - public bool HasData => this.Mods.Any(); + public bool HasData => this.FetchUri != null || this.Mods?.Length > 0; /********* ** Public methods *********/ /// Construct an instance. - /// The mods to display. + /// The URI from which to fetch the mod data. + /// The mods to display instead of the , if set. /// When the data was last updated. /// Whether the data hasn't been updated in a while. - public ModListModel(IEnumerable mods, DateTimeOffset lastUpdated, bool isStale) + public ModListModel(string? fetchUri, IEnumerable? mods, DateTimeOffset lastUpdated, bool isStale) { - this.Mods = mods.ToArray(); + this.FetchUri = fetchUri; + this.Mods = mods?.ToArray(); this.LastUpdated = lastUpdated; this.IsStale = isStale; } diff --git a/src/SMAPI.Web/ViewModels/ModModel.cs b/src/SMAPI.Web/ViewModels/ModModel.cs index d19dd233..ebcfc107 100644 --- a/src/SMAPI.Web/ViewModels/ModModel.cs +++ b/src/SMAPI.Web/ViewModels/ModModel.cs @@ -19,13 +19,13 @@ public class ModModel public string? Name { get; } /// The mod's alternative names, if any. - public string AlternateNames { get; } + public string? AlternateNames { get; } /// The mod author's name. public string? Author { get; } /// The mod author's alternative names, if any. - public string AlternateAuthors { get; } + public string? AlternateAuthors { get; } /// The GitHub repo, if any. public string? GitHubRepo { get; } @@ -33,14 +33,14 @@ public class ModModel /// The URL to the mod's source code, if any. public string? SourceUrl { get; } - /// The compatibility status for the stable version of the game. - public ModCompatibilityModel Compatibility { get; } + /// The compatibility status for the stable version of the game, or null if it's compatible with no notes. + public ModCompatibilityModel? Compatibility { get; } /// Links to the available mod pages. public ModLinkModel[] ModPages { get; } /// The human-readable warnings for players about this mod. - public string[] Warnings { get; } + public string[]? Warnings { get; } /// Special notes intended for developers who maintain unofficial updates or submit pull requests. public string? DevNote { get; } @@ -66,7 +66,7 @@ public class ModModel /// Special notes intended for developers who maintain unofficial updates or submit pull requests. /// A unique identifier for the mod that can be used in an anchor URL. [JsonConstructor] - public ModModel(string[] id, string? name, string alternateNames, string author, string alternateAuthors, string gitHubRepo, string sourceUrl, ModCompatibilityModel compatibility, ModLinkModel[] modPages, string[] warnings, string devNote, string slug) + public ModModel(string[] id, string? name, string? alternateNames, string? author, string? alternateAuthors, string? gitHubRepo, string? sourceUrl, ModCompatibilityModel? compatibility, ModLinkModel[] modPages, string[]? warnings, string? devNote, string? slug) { this.Id = id; this.Name = name; @@ -89,14 +89,14 @@ public class ModModel // basic info this.Id = entry.ID.ToArray(); this.Name = entry.Name.FirstOrDefault(); - this.AlternateNames = string.Join(", ", entry.Name.Skip(1).ToArray()); + this.AlternateNames = entry.Name.Length > 1 ? string.Join(", ", entry.Name.Skip(1)) : null; this.Author = entry.Author.FirstOrDefault(); - this.AlternateAuthors = string.Join(", ", entry.Author.Skip(1).ToArray()); + this.AlternateAuthors = entry.Author.Length > 1 ? string.Join(", ", entry.Author.Skip(1)) : null; this.GitHubRepo = entry.GitHubRepo; this.SourceUrl = this.GetSourceUrl(entry); - this.Compatibility = new ModCompatibilityModel(entry.Compatibility); + this.Compatibility = !entry.Compatibility.IsDefault() ? new ModCompatibilityModel(entry.Compatibility) : null; this.ModPages = this.GetModPageUrls(entry).ToArray(); - this.Warnings = entry.Warnings; + this.Warnings = entry.Warnings.Length > 1 ? entry.Warnings : null; this.DevNote = entry.DevNote; this.Slug = entry.Anchor; } diff --git a/src/SMAPI.Web/Views/Mods/Index.cshtml b/src/SMAPI.Web/Views/Mods/Index.cshtml index cdbbacc8..dd9d6a8c 100644 --- a/src/SMAPI.Web/Views/Mods/Index.cshtml +++ b/src/SMAPI.Web/Views/Mods/Index.cshtml @@ -8,23 +8,24 @@ TimeSpan staleAge = DateTimeOffset.UtcNow - Model.LastUpdated; } @section Head { - + @* jQuery *@ - + @* jQuery tablesorter *@ - + @* Vue *@ - + @* page CSS/JS *@ - } @@ -35,17 +36,28 @@ } else { + @* non-JavaScript error *@ + + @if (Model.IsStale) {
Showing data from @staleAge.Humanize(maxUnit: TimeUnit.Hour, minUnit: TimeUnit.Minute) ago. (Couldn't fetch newer data; the GitHub API may be offline.)
} -
-
-

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

+
+

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

-

The list is updated regularly (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

-
+

The list is updated regularly (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

+
+ +
+ @* loading indicator *@ +
@@ -97,7 +109,7 @@ else
-
⚠ {{warning}}
+
⚠ {{warning}}
diff --git a/src/SMAPI.Web/wwwroot/Content/js/mods.js b/src/SMAPI.Web/wwwroot/Content/js/mods.js index 900bc798..60a92e70 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/mods.js +++ b/src/SMAPI.Web/wwwroot/Content/js/mods.js @@ -1,8 +1,8 @@ var smapi = smapi || {}; var app; -smapi.modList = function (mods) { +smapi.modList = async function (state) { // init data - var defaultStats = { + const defaultStats = { total: 0, compatible: 0, workaround: 0, @@ -14,8 +14,8 @@ smapi.modList = function (mods) { percentBroken: 0, percentObsolete: 0 }; - var data = { - mods: mods, + const data = { + mods: [], showAdvanced: false, visibleMainStats: $.extend({}, defaultStats), filters: { @@ -51,6 +51,12 @@ smapi.modList = function (mods) { search: "" }; + // fetch data + if (state.fetchUri) + data.mods = await $.getJSON(state.fetchUri); + else + data.mods = state.fetchedData ?? []; + // init filters Object.entries(data.filters).forEach(([groupKey, filterGroup]) => { filterGroup.label = filterGroup.label || groupKey; @@ -61,12 +67,16 @@ smapi.modList = function (mods) { }); // init mods - for (var i = 0; i < data.mods.length; i++) { - var mod = mods[i]; + for (let i = 0; i < data.mods.length; i++) { + const mod = data.mods[i]; // set initial visibility mod.Visible = true; + // set default compatibility + if (!mod.Compatibility) + mod.Compatibility = { "Status": "ok", "Summary": "use latest version." }; + // concatenate searchable text mod.SearchableText = [mod.Name, mod.AlternateNames, mod.Author, mod.AlternateAuthors, mod.Compatibility.Summary, mod.BrokeIn]; if (mod.Compatibility.UnofficialVersion) @@ -75,18 +85,18 @@ smapi.modList = function (mods) { } // init app - var methods = { + const methods = { /** * Update the visibility of all mods based on the current search text and filters. */ applyFilters: function () { // get search terms - var words = data.search.toLowerCase().split(" "); + const words = data.search.toLowerCase().split(" "); // apply criteria - var stats = data.visibleMainStats = $.extend({}, defaultStats); - for (var i = 0; i < data.mods.length; i++) { - var mod = data.mods[i]; + const stats = data.visibleMainStats = $.extend({}, defaultStats); + for (let i = 0; i < data.mods.length; i++) { + const mod = data.mods[i]; mod.Visible = true; // check filters @@ -107,7 +117,7 @@ smapi.modList = function (mods) { * Fix the window position for the current hash. */ fixHashPosition: function () { - var anchor = location.hash.substring(1); + const anchor = location.hash.substring(1); if (!anchor) return; @@ -116,10 +126,10 @@ smapi.modList = function (mods) { return; // else jump to mod by ID - for (var i = 0; i < data.mods.length; i++) { - var mod = data.mods[i]; + for (let i = 0; i < data.mods.length; i++) { + const mod = data.mods[i]; - for (var n = 0; n < mod.Id.length; n++) { + for (let n = 0; n < mod.Id.length; n++) { if (anchor.localeCompare(mod.Id[n], undefined, { sensitivity: 'accent' }) === 0) { if (methods.scrollToRow(mod.Slug)) return; @@ -134,8 +144,8 @@ smapi.modList = function (mods) { * @returns {boolean} Whether the target was found and scrolled into view. */ scrollToRow: function(id) { - var row = $("#" + id); - var target = row.prev().get(0) || row.get(0); // show previous row if possible, to push the actual row below the sticky header + const row = $("#" + id); + const target = row.prev().get(0) || row.get(0); // show previous row if possible, to push the actual row below the sticky header target?.scrollIntoView(); return !!target; }, @@ -147,7 +157,7 @@ smapi.modList = function (mods) { * @returns {boolean} Whether the mod matches the filters. */ matchesFilters: function (mod, searchWords) { - var filters = data.filters; + const filters = data.filters; // check hash if (location.hash === "#" + mod.Slug) @@ -160,12 +170,12 @@ smapi.modList = function (mods) { return false; // check status - var mainStatus = mod.Compatibility.Status; + const mainStatus = mod.Compatibility.Status; if (filters.status.value[mainStatus] && !filters.status.value[mainStatus].value) return false; // check download sites - var ignoreSites = []; + const ignoreSites = []; if (!filters.download.value.chucklefish.value) ignoreSites.push("Chucklefish"); @@ -179,8 +189,8 @@ smapi.modList = function (mods) { ignoreSites.push("custom"); if (ignoreSites.length) { - var anyLeft = false; - for (var i = 0; i < mod.ModPages.length; i++) { + let anyLeft = false; + for (let i = 0; i < mod.ModPages.length; i++) { if (ignoreSites.indexOf(mod.ModPages[i].Text) === -1) { anyLeft = true; break; @@ -192,7 +202,7 @@ smapi.modList = function (mods) { } // check search terms - for (var w = 0; w < searchWords.length; w++) { + for (let w = 0; w < searchWords.length; w++) { if (mod.SearchableText.indexOf(searchWords[w]) === -1) return false; }