mirror of
https://github.com/Pathoschild/SMAPI.git
synced 2026-10-12 02:08:51 +00:00
split up BackgroundService
This splits background jobs into: - a `BackgroundService` which manages the server and retry policy. - one class per job which defines its logic, schedule options, and state.
This commit is contained in:
Jesse Plamondon-Willard
@@ -0,0 +1,50 @@
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Hangfire.Console;
|
||||
using Hangfire.Server;
|
||||
using StardewModdingAPI.Toolkit;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.CompatibilityRepo;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which updates the cached compatibility list data.</summary>
|
||||
internal class CompatibilityListJob : IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The cache in which to store compatibility list data.</summary>
|
||||
private readonly ICompatibilityCacheRepository CompatibilityCache;
|
||||
|
||||
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update compatibility list", Cron.MinuteInterval(10));
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="compatibilityCache"><inheritdoc cref="CompatibilityCache" path="/summary"/></param>
|
||||
public CompatibilityListJob(ICompatibilityCacheRepository compatibilityCache)
|
||||
{
|
||||
this.CompatibilityCache = compatibilityCache;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task RunAsync(PerformContext? context)
|
||||
{
|
||||
context.WriteLine("Fetching data from compatibility repo...");
|
||||
ModCompatibilityEntry[] compatList = await new ModToolkit().GetCompatibilityListAsync();
|
||||
|
||||
context.WriteLine("Saving data...");
|
||||
this.CompatibilityCache.SaveData(compatList);
|
||||
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
|
||||
using StardewModdingAPI.Web.Framework.Clients.CurseForge;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which updates the cached CurseForge mod export.</summary>
|
||||
internal class CurseForgeExportJob : ModSiteExportJob<ICurseForgeExportCacheRepository, ICurseForgeExportApiClient>, IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update CurseForge export", Cron.MinuteInterval(10));
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool IsEnabled => this.Client is not DisabledCurseForgeExportApiClient;
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public CurseForgeExportJob(ICurseForgeExportCacheRepository cache, ICurseForgeExportApiClient client, IOptions<ModUpdateCheckConfig> updateCheckConfig)
|
||||
: base(cache, client, updateCheckConfig) { }
|
||||
|
||||
|
||||
/*********
|
||||
** Protected methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
protected override Task<ApiCacheHeaders> FetchCacheHeadersAsync()
|
||||
{
|
||||
return this.Client.FetchCacheHeadersAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task FetchDataAsync()
|
||||
{
|
||||
this.Cache.SetData(await this.Client.FetchExportAsync());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Hangfire.Console;
|
||||
using Hangfire.Server;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit.Framework.ModBlacklistData;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Clients.GitHub;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which fetches the malware blacklist and converts it into the static file fetched by SMAPI.</summary>
|
||||
internal class MalwareBlacklistJob : IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The HTTP client for fetching data from GitHub.</summary>
|
||||
private readonly IGitHubClient GitHubClient;
|
||||
|
||||
/// <summary>The config settings for syncing the malware blacklist.</summary>
|
||||
private readonly IOptions<MalwareBlacklistConfig> Config;
|
||||
|
||||
/// <summary>The web root path for writing static data files.</summary>
|
||||
private readonly string WebRootPath;
|
||||
|
||||
/// <summary>The HTTP ETag header for the last fetched malware blacklist file, if any.</summary>
|
||||
private string? LastETag;
|
||||
|
||||
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update malware blacklist", Cron.MinuteInterval(10));
|
||||
|
||||
/// <summary>Whether the job should be scheduled.</summary>
|
||||
public bool IsEnabled => !string.IsNullOrWhiteSpace(this.Config.Value.GitHubRepo);
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="gitHubClient"><inheritdoc cref="GitHubClient" path="/summary"/></param>
|
||||
/// <param name="config"><inheritdoc cref="Config" path="/summary"/></param>
|
||||
/// <param name="hostEnvironment">The web host environment metadata.</param>
|
||||
public MalwareBlacklistJob(IGitHubClient gitHubClient, IOptions<MalwareBlacklistConfig> config, IWebHostEnvironment hostEnvironment)
|
||||
{
|
||||
this.GitHubClient = gitHubClient;
|
||||
this.Config = config;
|
||||
this.WebRootPath = hostEnvironment.WebRootPath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task RunAsync(PerformContext? context)
|
||||
{
|
||||
// get config
|
||||
MalwareBlacklistConfig config = this.Config.Value;
|
||||
string repo = config.GitHubRepo ?? throw new InvalidOperationException("Can't update the malware blacklist: no GitHub repository is configured.");
|
||||
string outputPath = Path.Combine(this.WebRootPath, Startup.BlacklistFileName);
|
||||
|
||||
// fetch file
|
||||
GitFile file;
|
||||
{
|
||||
context.WriteLine($"Fetching '{config.FilePath}' from {repo}{(config.GitRef != null ? $"@{config.GitRef}" : "")}...");
|
||||
|
||||
string? previousETag = File.Exists(outputPath)
|
||||
? this.LastETag
|
||||
: null;
|
||||
|
||||
file =
|
||||
await this.GitHubClient.GetFileAsync(repo, config.FilePath, config.GitRef, previousETag)
|
||||
?? throw new InvalidOperationException($"Can't update the malware blacklist: file '{config.FilePath}' not found in GitHub repository '{repo}'. If the repository is private, the configured GitHub credentials may not have read access to it.");
|
||||
}
|
||||
|
||||
// skip if unchanged
|
||||
if (!file.IsModified)
|
||||
{
|
||||
context.WriteLine($"Skipped: file hasn't changed since the last sync (ETag {file.ETag}).");
|
||||
return;
|
||||
}
|
||||
|
||||
// convert to public format
|
||||
context.WriteLine($"Parsing blacklist (ETag {file.ETag})...");
|
||||
MalwareListModel rawBlacklist = MalwareBlacklistConverter.FromInternalFormat(file.Content);
|
||||
ModBlacklistModel blacklist = MalwareBlacklistConverter.ToPublicModel(rawBlacklist);
|
||||
string publicJson = MalwareBlacklistConverter.ToPublicJson(blacklist);
|
||||
context.WriteLine($"Updated malware blacklist ({blacklist.Blacklist.Length} mod entries and {blacklist.LooseFileBlacklist.Length} loose file entries).");
|
||||
|
||||
// save file
|
||||
{
|
||||
string tempPath = $"{outputPath}.tmp";
|
||||
await File.WriteAllTextAsync(tempPath, publicJson, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
File.Move(tempPath, outputPath, overwrite: true);
|
||||
|
||||
context.WriteLine($"Saved to {Startup.BlacklistFileName}.");
|
||||
}
|
||||
|
||||
this.LastETag = file.ETag;
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Hangfire.Console;
|
||||
using Hangfire.Server;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.ModDataset;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which updates the local copy of the Stardew mod dataset for use by the frontend scripts.</summary>
|
||||
internal class ModDatasetJob : IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The mod dataset repository.</summary>
|
||||
private readonly IModDatasetRepository ModDatasetRepo;
|
||||
|
||||
/// <summary>The web root path for writing static data files.</summary>
|
||||
private readonly string WebRootPath;
|
||||
|
||||
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update mod dataset", Cron.Hourly());
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="modDatasetRepo"><inheritdoc cref="ModDatasetRepo" path="/summary"/></param>
|
||||
/// <param name="hostEnvironment">The web host environment metadata.</param>
|
||||
public ModDatasetJob(IModDatasetRepository modDatasetRepo, IWebHostEnvironment hostEnvironment)
|
||||
{
|
||||
this.ModDatasetRepo = modDatasetRepo;
|
||||
this.WebRootPath = hostEnvironment.WebRootPath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task RunAsync(PerformContext? context)
|
||||
{
|
||||
// update repo
|
||||
context.WriteLine("Updating mod dataset repo...");
|
||||
DatasetDownload result = await this.ModDatasetRepo.UpdateAsync(context.WriteLine);
|
||||
|
||||
// copy files
|
||||
context.WriteLine("Copying data files for script use...");
|
||||
this.CopyToData("stats", "mods by type.jsonl");
|
||||
this.CopyToData("stats", "Content Patcher packs by format version.jsonl", "content-packs-by-format.jsonl");
|
||||
this.CopyToData("reference-data", "SMAPI costs.jsonl");
|
||||
this.CopyToData("reference-data", "SMAPI DNS queries.json");
|
||||
|
||||
Program.ModDatasetCacheBustValue = result.ETag ?? $"epoch-{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
|
||||
|
||||
/*********
|
||||
** Private methods
|
||||
*********/
|
||||
/// <summary>Copy a fetched file into the web content data directory.</summary>
|
||||
/// <param name="fromRelativeDir">The relative path to the directory containing the file to copy within the fetched repo.</param>
|
||||
/// <param name="fromFileName">The file name within the fetched repo to copy.</param>
|
||||
/// <param name="toFileName">The file name to save it to within the web content data directory, or <c>null</c> to use a lowercase spaces-to-hyphenated version of <paramref name="fromFileName"/>.</param>
|
||||
private void CopyToData(string fromRelativeDir, string fromFileName, string? toFileName = null)
|
||||
{
|
||||
string fromRelativePath = Path.Combine(fromRelativeDir, fromFileName);
|
||||
|
||||
toFileName ??= fromFileName.Replace(' ', '-').ToLowerInvariant();
|
||||
string toAbsolutePath = Path.Combine(this.WebRootPath, "Content", "data", toFileName);
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(toAbsolutePath)!);
|
||||
|
||||
File.Copy(this.ModDatasetRepo.GetFilePath(fromRelativePath), toAbsolutePath, overwrite: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.ModDropExport;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.ModDropExport;
|
||||
using StardewModdingAPI.Web.Framework.Clients.ModDrop;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which updates the cached ModDrop mod export.</summary>
|
||||
internal class ModDropExportJob : ModSiteExportJob<IModDropExportCacheRepository, IModDropExportApiClient>, IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update ModDrop export", Cron.MinuteInterval(10));
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool IsEnabled => this.Client is not DisabledModDropExportApiClient;
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public ModDropExportJob(IModDropExportCacheRepository cache, IModDropExportApiClient client, IOptions<ModUpdateCheckConfig> updateCheckConfig)
|
||||
: base(cache, client, updateCheckConfig) { }
|
||||
|
||||
|
||||
/*********
|
||||
** Protected methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
protected override Task<ApiCacheHeaders> FetchCacheHeadersAsync()
|
||||
{
|
||||
return this.Client.FetchCacheHeadersAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task FetchDataAsync()
|
||||
{
|
||||
this.Cache.SetData(await this.Client.FetchExportAsync());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.NexusExport;
|
||||
using StardewModdingAPI.Web.Framework.Clients.Nexus;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which updates the cached Nexus mod export.</summary>
|
||||
internal class NexusExportJob : ModSiteExportJob<INexusExportCacheRepository, INexusExportApiClient>, IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("update Nexus export", Cron.MinuteInterval(10));
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool IsEnabled => this.Client is not DisabledNexusExportApiClient;
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public NexusExportJob(INexusExportCacheRepository cache, INexusExportApiClient client, IOptions<ModUpdateCheckConfig> updateCheckConfig)
|
||||
: base(cache, client, updateCheckConfig) { }
|
||||
|
||||
|
||||
/*********
|
||||
** Protected methods
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
protected override Task<ApiCacheHeaders> FetchCacheHeadersAsync()
|
||||
{
|
||||
return this.Client.FetchCacheHeadersAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task FetchDataAsync()
|
||||
{
|
||||
this.Cache.SetData(await this.Client.FetchExportAsync());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire.Server;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.Caching.Mods;
|
||||
|
||||
namespace StardewModdingAPI.Web.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job which removes mods from the cache if they haven't been requested recently.</summary>
|
||||
internal class RemoveStaleModsJob : IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The time for which to keep cached mods after they're last requested.</summary>
|
||||
private readonly TimeSpan CacheTime = TimeSpan.FromHours(48);
|
||||
|
||||
/// <summary>The cache in which to store mod data.</summary>
|
||||
private readonly IModCacheRepository ModCache;
|
||||
|
||||
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <inheritdoc />
|
||||
public static JobOptions Options { get; } = new("remove stale mods", "2/10 * * * *"); // every 10 minutes, but offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc)
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="modCache"><inheritdoc cref="ModCache" path="/summary"/></param>
|
||||
public RemoveStaleModsJob(IModCacheRepository modCache)
|
||||
{
|
||||
this.ModCache = modCache;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task RunAsync(PerformContext? context)
|
||||
{
|
||||
this.ModCache.RemoveStaleMods(this.CacheTime);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -1,469 +1,110 @@
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire;
|
||||
using Hangfire.Console;
|
||||
using Hangfire.Server;
|
||||
using Humanizer;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.CompatibilityRepo;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.ModDropExport;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
|
||||
using StardewModdingAPI.Toolkit.Framework.ModBlacklistData;
|
||||
using StardewModdingAPI.Web.Framework.Caching;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
|
||||
using StardewModdingAPI.Web.Framework.Caching.ModDataset;
|
||||
using StardewModdingAPI.Web.Framework.Caching.ModDropExport;
|
||||
using StardewModdingAPI.Web.Framework.Caching.Mods;
|
||||
using StardewModdingAPI.Web.Framework.Caching.NexusExport;
|
||||
using StardewModdingAPI.Web.Framework.Clients.CurseForge;
|
||||
using StardewModdingAPI.Web.Framework.Clients.GitHub;
|
||||
using StardewModdingAPI.Web.Framework.Clients.ModDrop;
|
||||
using StardewModdingAPI.Web.Framework.Clients.Nexus;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
using StardewModdingAPI.Web.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
|
||||
namespace StardewModdingAPI.Web;
|
||||
|
||||
/// <summary>A hosted service which runs background data updates.</summary>
|
||||
/// <remarks>Task methods need to be static, since otherwise Hangfire will try to serialize the entire instance.</remarks>
|
||||
/// <summary>A hosted service which schedules and runs the background jobs.</summary>
|
||||
internal class BackgroundService : IHostedService, IDisposable
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The background task server.</summary>
|
||||
private static BackgroundJobServer? JobServer;
|
||||
/// <summary>The background job server, if started.</summary>
|
||||
private BackgroundJobServer? JobServer;
|
||||
|
||||
/// <summary>The cache in which to store compatibility list data.</summary>
|
||||
private static ICompatibilityCacheRepository? CompatibilityCache;
|
||||
/// <summary>Whether the <see cref="CurseForgeExportJob"/> job is enabled.</summary>
|
||||
private readonly bool EnableCurseForgeExport;
|
||||
|
||||
/// <summary>The cache in which to store mod data.</summary>
|
||||
private static IModCacheRepository? ModCache;
|
||||
/// <summary>Whether the <see cref="ModDropExportJob"/> job is enabled.</summary>
|
||||
private readonly bool EnableModDropExport;
|
||||
|
||||
/// <summary>The HTTP client for fetching the mod export from the CurseForge export API.</summary>
|
||||
private static ICurseForgeExportApiClient? CurseForgeExportApiClient;
|
||||
/// <summary>Whether the <see cref="NexusExportJob"/> job is enabled.</summary>
|
||||
private readonly bool EnableNexusExport;
|
||||
|
||||
/// <summary>The cache in which to store the mod data from the CurseForge export API.</summary>
|
||||
private static ICurseForgeExportCacheRepository? CurseForgeExportCache;
|
||||
|
||||
/// <summary>The HTTP client for fetching data from GitHub.</summary>
|
||||
private static IGitHubClient? GitHubClient;
|
||||
|
||||
/// <summary>The HTTP client for fetching the mod export from the ModDrop export API.</summary>
|
||||
private static IModDropExportApiClient? ModDropExportApiClient;
|
||||
|
||||
/// <summary>The cache in which to store the mod data from the ModDrop export API.</summary>
|
||||
private static IModDropExportCacheRepository? ModDropExportCache;
|
||||
|
||||
/// <summary>The cache in which to store mod data from the Nexus export API.</summary>
|
||||
private static INexusExportCacheRepository? NexusExportCache;
|
||||
|
||||
/// <summary>The HTTP client for fetching the mod export from the Nexus Mods export API.</summary>
|
||||
private static INexusExportApiClient? NexusExportApiClient;
|
||||
|
||||
/// <summary>The config settings for mod update checks.</summary>
|
||||
private static IOptions<ModUpdateCheckConfig>? UpdateCheckConfig;
|
||||
|
||||
/// <summary>The mod dataset repository.</summary>
|
||||
private static IModDatasetRepository? ModDatasetRepo;
|
||||
|
||||
/// <summary>The config settings for syncing the malware blacklist.</summary>
|
||||
private static IOptions<MalwareBlacklistConfig>? MalwareBlacklistConfig;
|
||||
|
||||
/// <summary>The HTTP ETag header for the last fetched malware blacklist file, if any.</summary>
|
||||
private static string? LastMalwareBlacklistETag;
|
||||
|
||||
/// <summary>The web root path for writing static data files.</summary>
|
||||
private static string? WebRootPath;
|
||||
|
||||
/// <summary>Whether the service has been started.</summary>
|
||||
[MemberNotNullWhen(true,
|
||||
nameof(BackgroundService.JobServer),
|
||||
nameof(BackgroundService.ModCache),
|
||||
nameof(BackgroundService.CompatibilityCache),
|
||||
nameof(BackgroundService.CurseForgeExportApiClient),
|
||||
nameof(BackgroundService.CurseForgeExportCache),
|
||||
nameof(BackgroundService.GitHubClient),
|
||||
nameof(BackgroundService.ModDropExportApiClient),
|
||||
nameof(BackgroundService.ModDropExportCache),
|
||||
nameof(BackgroundService.NexusExportApiClient),
|
||||
nameof(BackgroundService.NexusExportCache),
|
||||
nameof(BackgroundService.ModDatasetRepo),
|
||||
nameof(BackgroundService.MalwareBlacklistConfig),
|
||||
nameof(BackgroundService.UpdateCheckConfig),
|
||||
nameof(BackgroundService.WebRootPath)
|
||||
)]
|
||||
private static bool IsStarted { get; set; }
|
||||
|
||||
/// <summary>The number of minutes a site export should be considered valid based on its last-updated date before it's ignored.</summary>
|
||||
private static int ExportStaleAge => (BackgroundService.UpdateCheckConfig?.Value.SuccessCacheMinutes ?? 0) + 10;
|
||||
/// <summary>Whether the <see cref="MalwareBlacklistJob"/> job is enabled.</summary>
|
||||
private readonly bool EnableMalwareBlacklistSync;
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/****
|
||||
** Hosted service
|
||||
****/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="compatibilityCache"><inheritdoc cref="CompatibilityCache" path="/summary"/></param>
|
||||
/// <param name="modCache"><inheritdoc cref="ModCache" path="/summary"/></param>
|
||||
/// <param name="curseForgeExportCache"><inheritdoc cref="CurseForgeExportCache" path="/summary"/></param>
|
||||
/// <param name="curseForgeExportApiClient"><inheritdoc cref="CurseForgeExportApiClient" path="/summary"/></param>
|
||||
/// <param name="gitHubClient"><inheritdoc cref="GitHubClient" path="/summary"/></param>
|
||||
/// <param name="modDropExportCache"><inheritdoc cref="ModDropExportCache" path="/summary"/></param>
|
||||
/// <param name="modDropExportApiClient"><inheritdoc cref="ModDropExportApiClient" path="/summary"/></param>
|
||||
/// <param name="nexusExportCache"><inheritdoc cref="NexusExportCache" path="/summary"/></param>
|
||||
/// <param name="nexusExportApiClient"><inheritdoc cref="NexusExportApiClient" path="/summary"/></param>
|
||||
/// <param name="modDatasetRepo"><inheritdoc cref="ModDatasetRepo" path="/summary"/></param>
|
||||
/// <param name="curseForgeExportJob"><inheritdoc cref="CurseForgeExportJob" path="/summary"/></param>
|
||||
/// <param name="modDropExportJob"><inheritdoc cref="ModDropExportJob" path="/summary"/></param>
|
||||
/// <param name="nexusExportJob"><inheritdoc cref="NexusExportJob" path="/summary"/></param>
|
||||
/// <param name="malwareBlacklistJob"><inheritdoc cref="MalwareBlacklistJob" path="/summary"/></param>
|
||||
/// <param name="hangfireStorage">The Hangfire storage implementation.</param>
|
||||
/// <param name="malwareBlacklistConfig"><inheritdoc cref="MalwareBlacklistConfig" path="/summary"/></param>
|
||||
/// <param name="updateCheckConfig"><inheritdoc cref="UpdateCheckConfig" path="/summary"/></param>
|
||||
/// <param name="hostEnvironment">The web host environment metadata.</param>
|
||||
[SuppressMessage("ReSharper", "UnusedParameter.Local", Justification = "The Hangfire reference forces it to initialize first, since it's needed by the background service.")]
|
||||
public BackgroundService(
|
||||
ICompatibilityCacheRepository compatibilityCache,
|
||||
IModCacheRepository modCache,
|
||||
ICurseForgeExportCacheRepository curseForgeExportCache,
|
||||
ICurseForgeExportApiClient curseForgeExportApiClient,
|
||||
IGitHubClient gitHubClient,
|
||||
IModDropExportCacheRepository modDropExportCache,
|
||||
IModDropExportApiClient modDropExportApiClient,
|
||||
INexusExportCacheRepository nexusExportCache,
|
||||
INexusExportApiClient nexusExportApiClient,
|
||||
IModDatasetRepository modDatasetRepo,
|
||||
JobStorage hangfireStorage,
|
||||
IOptions<MalwareBlacklistConfig> malwareBlacklistConfig,
|
||||
IOptions<ModUpdateCheckConfig> updateCheckConfig,
|
||||
IWebHostEnvironment hostEnvironment
|
||||
)
|
||||
public BackgroundService(CurseForgeExportJob curseForgeExportJob, ModDropExportJob modDropExportJob, NexusExportJob nexusExportJob, MalwareBlacklistJob malwareBlacklistJob, JobStorage hangfireStorage)
|
||||
{
|
||||
BackgroundService.CompatibilityCache = compatibilityCache;
|
||||
BackgroundService.ModCache = modCache;
|
||||
BackgroundService.CurseForgeExportApiClient = curseForgeExportApiClient;
|
||||
BackgroundService.CurseForgeExportCache = curseForgeExportCache;
|
||||
BackgroundService.GitHubClient = gitHubClient;
|
||||
BackgroundService.ModDropExportApiClient = modDropExportApiClient;
|
||||
BackgroundService.ModDropExportCache = modDropExportCache;
|
||||
BackgroundService.NexusExportCache = nexusExportCache;
|
||||
BackgroundService.NexusExportApiClient = nexusExportApiClient;
|
||||
BackgroundService.ModDatasetRepo = modDatasetRepo;
|
||||
BackgroundService.MalwareBlacklistConfig = malwareBlacklistConfig;
|
||||
BackgroundService.UpdateCheckConfig = updateCheckConfig;
|
||||
BackgroundService.WebRootPath = hostEnvironment.WebRootPath;
|
||||
this.EnableCurseForgeExport = curseForgeExportJob.IsEnabled;
|
||||
this.EnableModDropExport = modDropExportJob.IsEnabled;
|
||||
this.EnableNexusExport = nexusExportJob.IsEnabled;
|
||||
this.EnableMalwareBlacklistSync = malwareBlacklistJob.IsEnabled;
|
||||
|
||||
_ = hangfireStorage; // parameter is only received to initialize it before the background service
|
||||
}
|
||||
|
||||
/// <summary>Start the service.</summary>
|
||||
/// <param name="cancellationToken">Tracks whether the start process has been aborted.</param>
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
this.TryInit();
|
||||
if (this.JobServer != null)
|
||||
throw new InvalidOperationException("The scheduler service is already started.");
|
||||
|
||||
bool enableCurseForgeExport = BackgroundService.CurseForgeExportApiClient is not DisabledCurseForgeExportApiClient;
|
||||
bool enableModDropExport = BackgroundService.ModDropExportApiClient is not DisabledModDropExportApiClient;
|
||||
bool enableNexusExport = BackgroundService.NexusExportApiClient is not DisabledNexusExportApiClient;
|
||||
bool enableMalwareBlacklist = !string.IsNullOrWhiteSpace(BackgroundService.MalwareBlacklistConfig?.Value.GitHubRepo);
|
||||
// set retry policy
|
||||
GlobalJobFilters.Filters.Remove<AutomaticRetryAttribute>();
|
||||
GlobalJobFilters.Filters.Add(new AutomaticRetryAttribute { Attempts = 3, DelaysInSeconds = [30, 60, 120] });
|
||||
|
||||
// set startup tasks
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateCompatibilityListAsync(null));
|
||||
if (enableCurseForgeExport)
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateCurseForgeExportAsync(null));
|
||||
if (enableModDropExport)
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateModDropExportAsync(null));
|
||||
if (enableNexusExport)
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync(null));
|
||||
BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync());
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateModDatasetAsync(null));
|
||||
if (enableMalwareBlacklist)
|
||||
BackgroundJob.Enqueue(() => BackgroundService.UpdateMalwareBlacklistAsync(null));
|
||||
// start Hangfire server
|
||||
this.JobServer = new BackgroundJobServer();
|
||||
|
||||
// set recurring tasks
|
||||
RecurringJob.AddOrUpdate("update compatibility list", () => BackgroundService.UpdateCompatibilityListAsync(null), "*/10 * * * *"); // every 10 minutes
|
||||
if (enableCurseForgeExport)
|
||||
RecurringJob.AddOrUpdate("update CurseForge export", () => BackgroundService.UpdateCurseForgeExportAsync(null), "*/10 * * * *");
|
||||
if (enableModDropExport)
|
||||
RecurringJob.AddOrUpdate("update ModDrop export", () => BackgroundService.UpdateModDropExportAsync(null), "*/10 * * * *");
|
||||
if (enableNexusExport)
|
||||
RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(null), "*/10 * * * *");
|
||||
RecurringJob.AddOrUpdate("remove stale mods", () => BackgroundService.RemoveStaleModsAsync(), "2/10 * * * *"); // offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc)
|
||||
RecurringJob.AddOrUpdate("update mod dataset", () => BackgroundService.UpdateModDatasetAsync(null), "0 * * * *"); // hourly
|
||||
if (enableMalwareBlacklist)
|
||||
RecurringJob.AddOrUpdate("update malware blacklist", () => BackgroundService.UpdateMalwareBlacklistAsync(null), "*/10 * * * *");
|
||||
|
||||
BackgroundService.IsStarted = true;
|
||||
// register jobs
|
||||
this.RegisterJob<CompatibilityListJob>();
|
||||
this.RegisterJob<ModDatasetJob>();
|
||||
this.RegisterJob<RemoveStaleModsJob>();
|
||||
if (this.EnableMalwareBlacklistSync)
|
||||
this.RegisterJob<MalwareBlacklistJob>();
|
||||
if (this.EnableCurseForgeExport)
|
||||
this.RegisterJob<CurseForgeExportJob>();
|
||||
if (this.EnableModDropExport)
|
||||
this.RegisterJob<ModDropExportJob>();
|
||||
if (this.EnableNexusExport)
|
||||
this.RegisterJob<NexusExportJob>();
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Triggered when the application host is performing a graceful shutdown.</summary>
|
||||
/// <param name="cancellationToken">Tracks whether the shutdown process should no longer be graceful.</param>
|
||||
/// <inheritdoc />
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
BackgroundService.IsStarted = false;
|
||||
|
||||
if (BackgroundService.JobServer != null)
|
||||
await BackgroundService.JobServer.WaitForShutdownAsync(cancellationToken);
|
||||
if (this.JobServer != null)
|
||||
await this.JobServer.WaitForShutdownAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.</summary>
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
BackgroundService.IsStarted = false;
|
||||
|
||||
BackgroundService.JobServer?.Dispose();
|
||||
}
|
||||
|
||||
/****
|
||||
** Tasks
|
||||
****/
|
||||
/// <summary>Update the cached compatibility list data.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateCompatibilityListAsync(PerformContext? context)
|
||||
{
|
||||
if (!BackgroundService.IsStarted)
|
||||
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
|
||||
|
||||
context.WriteLine("Fetching data from compatibility repo...");
|
||||
ModCompatibilityEntry[] compatList = await new ModToolkit().GetCompatibilityListAsync();
|
||||
|
||||
context.WriteLine("Saving data...");
|
||||
BackgroundService.CompatibilityCache.SaveData(compatList);
|
||||
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
|
||||
/// <summary>Update the cached CurseForge mod export.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateCurseForgeExportAsync(PerformContext? context)
|
||||
{
|
||||
await UpdateExportAsync(
|
||||
context,
|
||||
BackgroundService.CurseForgeExportCache!,
|
||||
BackgroundService.CurseForgeExportApiClient!,
|
||||
fetchCacheHeadersAsync: client => client.FetchCacheHeadersAsync(),
|
||||
fetchDataAsync: async (cache, client) => cache.SetData(await client.FetchExportAsync())
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Update the cached ModDrop mod export.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateModDropExportAsync(PerformContext? context)
|
||||
{
|
||||
await UpdateExportAsync(
|
||||
context,
|
||||
BackgroundService.ModDropExportCache!,
|
||||
BackgroundService.ModDropExportApiClient!,
|
||||
fetchCacheHeadersAsync: client => client.FetchCacheHeadersAsync(),
|
||||
fetchDataAsync: async (cache, client) => cache.SetData(await client.FetchExportAsync())
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Update the cached Nexus mod export.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateNexusExportAsync(PerformContext? context)
|
||||
{
|
||||
await UpdateExportAsync(
|
||||
context,
|
||||
BackgroundService.NexusExportCache!,
|
||||
BackgroundService.NexusExportApiClient!,
|
||||
fetchCacheHeadersAsync: client => client.FetchCacheHeadersAsync(),
|
||||
fetchDataAsync: async (cache, client) => cache.SetData(await client.FetchExportAsync())
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Clone or pull the Stardew mod dataset repo, then copy the latest data files into the web root for use by frontend scripts.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateModDatasetAsync(PerformContext? context)
|
||||
{
|
||||
if (!BackgroundService.IsStarted)
|
||||
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
|
||||
|
||||
// update repo
|
||||
context.WriteLine("Updating mod dataset repo...");
|
||||
DatasetDownload result = await BackgroundService.ModDatasetRepo.UpdateAsync(context.WriteLine);
|
||||
|
||||
// copy files
|
||||
context.WriteLine("Copying data files for script use...");
|
||||
string copyToPath = Path.Combine(BackgroundService.WebRootPath, "Content", "data", "mods-by-type.jsonl");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(copyToPath)!);
|
||||
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("stats/mods by type.jsonl"), copyToPath, overwrite: true);
|
||||
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("stats/Content Patcher packs by format version.jsonl"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "content-packs-by-format.jsonl"), overwrite: true);
|
||||
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("reference-data/SMAPI costs.jsonl"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "smapi-costs.jsonl"), overwrite: true);
|
||||
File.Copy(BackgroundService.ModDatasetRepo.GetFilePath("reference-data/SMAPI DNS queries.json"), Path.Combine(BackgroundService.WebRootPath, "Content", "data", "smapi-dns-queries.json"), overwrite: true);
|
||||
|
||||
Program.ModDatasetCacheBustValue = result.ETag ?? $"epoch-{DateTimeOffset.UtcNow.ToUnixTimeSeconds()}";
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
|
||||
/// <summary>Fetch the malware blacklist and convert it into the static file fetched by SMAPI.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 60, 120])]
|
||||
public static async Task UpdateMalwareBlacklistAsync(PerformContext? context)
|
||||
{
|
||||
if (!BackgroundService.IsStarted)
|
||||
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
|
||||
|
||||
// get config
|
||||
MalwareBlacklistConfig config = BackgroundService.MalwareBlacklistConfig.Value;
|
||||
string repo = config.GitHubRepo ?? throw new InvalidOperationException("Can't update the malware blacklist: no GitHub repository is configured.");
|
||||
string outputPath = Path.Combine(BackgroundService.WebRootPath, Startup.BlacklistFileName);
|
||||
|
||||
// fetch file
|
||||
GitFile file;
|
||||
{
|
||||
context.WriteLine($"Fetching '{config.FilePath}' from {repo}{(config.GitRef != null ? $"@{config.GitRef}" : "")}...");
|
||||
|
||||
string? previousETag = File.Exists(outputPath)
|
||||
? BackgroundService.LastMalwareBlacklistETag
|
||||
: null;
|
||||
|
||||
file =
|
||||
await BackgroundService.GitHubClient.GetFileAsync(repo, config.FilePath, config.GitRef, previousETag)
|
||||
?? throw new InvalidOperationException($"Can't update the malware blacklist: file '{config.FilePath}' not found in GitHub repository '{repo}'. If the repository is private, the configured GitHub credentials may not have read access to it.");
|
||||
}
|
||||
|
||||
// skip if unchanged
|
||||
if (!file.IsModified)
|
||||
{
|
||||
context.WriteLine($"Skipped: file hasn't changed since the last sync (ETag {file.ETag}).");
|
||||
return;
|
||||
}
|
||||
|
||||
// convert to public format
|
||||
context.WriteLine($"Parsing blacklist (ETag {file.ETag})...");
|
||||
MalwareListModel rawBlacklist = MalwareBlacklistConverter.FromInternalFormat(file.Content);
|
||||
ModBlacklistModel blacklist = MalwareBlacklistConverter.ToPublicModel(rawBlacklist);
|
||||
string publicJson = MalwareBlacklistConverter.ToPublicJson(blacklist);
|
||||
context.WriteLine($"Updated malware blacklist ({blacklist.Blacklist.Length} mod entries and {blacklist.LooseFileBlacklist.Length} loose file entries).");
|
||||
|
||||
// save file
|
||||
{
|
||||
string tempPath = $"{outputPath}.tmp";
|
||||
await File.WriteAllTextAsync(tempPath, publicJson, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
||||
File.Move(tempPath, outputPath, overwrite: true);
|
||||
|
||||
context.WriteLine($"Saved to {Startup.BlacklistFileName}.");
|
||||
}
|
||||
|
||||
BackgroundService.LastMalwareBlacklistETag = file.ETag;
|
||||
context.WriteLine("Done!");
|
||||
}
|
||||
|
||||
/// <summary>Remove mods which haven't been requested in over 48 hours.</summary>
|
||||
public static Task RemoveStaleModsAsync()
|
||||
{
|
||||
if (!BackgroundService.IsStarted)
|
||||
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
|
||||
|
||||
// remove mods in mod cache
|
||||
BackgroundService.ModCache.RemoveStaleMods(TimeSpan.FromHours(48));
|
||||
|
||||
return Task.CompletedTask;
|
||||
this.JobServer?.Dispose();
|
||||
}
|
||||
|
||||
|
||||
/*********
|
||||
** Private method
|
||||
** Private methods
|
||||
*********/
|
||||
/// <summary>Initialize the background service if it's not already initialized.</summary>
|
||||
/// <exception cref="InvalidOperationException">The background service is already initialized.</exception>
|
||||
private void TryInit()
|
||||
/// <summary>Register a background job so it's enqueued immediately, and also runs on a recurring schedule.</summary>
|
||||
/// <typeparam name="TJob">The job type.</typeparam>
|
||||
private void RegisterJob<TJob>()
|
||||
where TJob : IBackgroundJob
|
||||
{
|
||||
if (BackgroundService.JobServer != null)
|
||||
throw new InvalidOperationException("The scheduler service is already started.");
|
||||
JobOptions options = TJob.Options;
|
||||
|
||||
BackgroundService.JobServer = new BackgroundJobServer();
|
||||
}
|
||||
|
||||
/// <summary>Update the cached mods export for a site.</summary>
|
||||
/// <typeparam name="TCacheRepository">The export cache repository type.</typeparam>
|
||||
/// <typeparam name="TExportApiClient">The export API client.</typeparam>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
/// <param name="cache">The export cache to update.</param>
|
||||
/// <param name="client">The export API with which to fetch data from the remote API.</param>
|
||||
/// <param name="fetchCacheHeadersAsync">Fetch the HTTP cache headers set by the remote API.</param>
|
||||
/// <param name="fetchDataAsync">Fetch the latest export file from the Nexus Mods export API.</param>
|
||||
/// <exception cref="InvalidOperationException">The <see cref="StartAsync"/> method wasn't called before running this task.</exception>
|
||||
private static async Task UpdateExportAsync<TCacheRepository, TExportApiClient>(PerformContext? context, TCacheRepository cache, TExportApiClient client, Func<TExportApiClient, Task<ApiCacheHeaders>> fetchCacheHeadersAsync, Func<TCacheRepository, TExportApiClient, Task> fetchDataAsync)
|
||||
where TCacheRepository : IExportCacheRepository
|
||||
{
|
||||
if (!BackgroundService.IsStarted)
|
||||
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
|
||||
|
||||
// log initial state
|
||||
context.WriteLine(cache.IsLoaded
|
||||
? $"The previous export is cached with data from {BackgroundService.FormatDateModified(cache.CacheHeaders.LastModified)}."
|
||||
: "No previous export is cached."
|
||||
);
|
||||
|
||||
// fetch cache headers
|
||||
context.WriteLine("Fetching cache headers...");
|
||||
ApiCacheHeaders serverCacheHeaders = await fetchCacheHeadersAsync(client);
|
||||
DateTimeOffset serverModified = serverCacheHeaders.LastModified;
|
||||
string? serverEntityTag = serverCacheHeaders.EntityTag;
|
||||
|
||||
// update data
|
||||
{
|
||||
// skip if no update needed
|
||||
if (cache.IsStale(serverModified, BackgroundService.ExportStaleAge))
|
||||
context.WriteLine($"Skipped data fetch: server was last modified {BackgroundService.FormatDateModified(serverModified)}, which exceeds the {BackgroundService.ExportStaleAge}-minute-stale limit.");
|
||||
else if (cache.IsLoaded && cache.CacheHeaders.LastModified >= serverModified)
|
||||
context.WriteLine($"Skipped data fetch: server was last modified {BackgroundService.FormatDateModified(serverModified)}, which {(serverModified == cache.CacheHeaders.LastModified ? "matches" : "is older than")} our cached data.");
|
||||
|
||||
// update cache headers if data unchanged
|
||||
else if (cache.IsLoaded && cache.CacheHeaders.EntityTag != null && cache.CacheHeaders.EntityTag == serverEntityTag)
|
||||
{
|
||||
context.WriteLine($"Skipped data fetch: server provided entity tag '{serverEntityTag}', which already matches the data we have.");
|
||||
cache.SetCacheHeaders(serverCacheHeaders);
|
||||
}
|
||||
|
||||
// else update data
|
||||
else
|
||||
{
|
||||
context.WriteLine("Fetching data...");
|
||||
await fetchDataAsync(cache, client);
|
||||
}
|
||||
}
|
||||
|
||||
// clear if stale
|
||||
if (cache.IsStale(BackgroundService.ExportStaleAge))
|
||||
{
|
||||
context.WriteLine("The cached data is stale, clearing cache...");
|
||||
cache.Clear();
|
||||
}
|
||||
|
||||
// log final result
|
||||
context.WriteLine(cache.IsLoaded
|
||||
? $"Done! The export is currently cached with data from {BackgroundService.FormatDateModified(cache.CacheHeaders.LastModified)}."
|
||||
: "Done! The export cache is currently disabled."
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>Format a 'date modified' value for the task logs.</summary>
|
||||
/// <param name="date">The date to log.</param>
|
||||
private static string FormatDateModified(DateTimeOffset? date)
|
||||
{
|
||||
if (!date.HasValue)
|
||||
return "<null>";
|
||||
|
||||
string ageLabel = (DateTimeOffset.UtcNow - date.Value).Humanize(precision: 2, minUnit: TimeUnit.Minute, maxUnit: TimeUnit.Hour);
|
||||
|
||||
return $"{date.Value:O} (age: {ageLabel})";
|
||||
BackgroundJob.Enqueue<TJob>(job => job.RunAsync(null));
|
||||
RecurringJob.AddOrUpdate<TJob>(options.Id, job => job.RunAsync(null), options.CronSchedule);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire.Server;
|
||||
|
||||
namespace StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
|
||||
/// <summary>A background job scheduled via <see cref="BackgroundService"/>.</summary>
|
||||
internal interface IBackgroundJob
|
||||
{
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <summary>The recurring job options.</summary>
|
||||
static abstract JobOptions Options { get; }
|
||||
|
||||
|
||||
/*********
|
||||
** Methods
|
||||
*********/
|
||||
/// <summary>Run the background job.</summary>
|
||||
/// <param name="context">Information about the context in which the job is performed. This is injected automatically by Hangfire.</param>
|
||||
Task RunAsync(PerformContext? context);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
|
||||
/// <summary>The Hangfire options for a recurring scheduled job.</summary>
|
||||
/// <param name="Id">The unique job ID.</param>
|
||||
/// <param name="CronSchedule">The CRON schedule expression.</param>
|
||||
internal record JobOptions(string Id, string CronSchedule);
|
||||
@@ -0,0 +1,131 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Hangfire.Console;
|
||||
using Hangfire.Server;
|
||||
using Humanizer;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients;
|
||||
using StardewModdingAPI.Web.Framework.Caching;
|
||||
using StardewModdingAPI.Web.Framework.ConfigModels;
|
||||
|
||||
namespace StardewModdingAPI.Web.Framework.BackgroundJobs;
|
||||
|
||||
/// <summary>The base implementation for a background job which updates the cached mod export for a mod site.</summary>
|
||||
/// <typeparam name="TCacheRepository">The export cache repository type.</typeparam>
|
||||
/// <typeparam name="TExportApiClient">The export API client type.</typeparam>
|
||||
internal abstract class ModSiteExportJob<TCacheRepository, TExportApiClient>
|
||||
where TCacheRepository : IExportCacheRepository
|
||||
{
|
||||
/*********
|
||||
** Fields
|
||||
*********/
|
||||
/// <summary>The config settings for mod update checks.</summary>
|
||||
private readonly IOptions<ModUpdateCheckConfig> UpdateCheckConfig;
|
||||
|
||||
/// <summary>The cache in which to store the mod export.</summary>
|
||||
protected readonly TCacheRepository Cache;
|
||||
|
||||
/// <summary>The HTTP client for fetching the mod export from the site's export API.</summary>
|
||||
protected readonly TExportApiClient Client;
|
||||
|
||||
|
||||
/*********
|
||||
** Accessors
|
||||
*********/
|
||||
/// <summary>Whether the job should be scheduled.</summary>
|
||||
public abstract bool IsEnabled { get; }
|
||||
|
||||
/// <summary>The number of minutes a site export should be considered valid based on its last-updated date before it's ignored.</summary>
|
||||
private int ExportStaleAge => this.UpdateCheckConfig.Value.SuccessCacheMinutes + 10;
|
||||
|
||||
|
||||
/*********
|
||||
** Public methods
|
||||
*********/
|
||||
/// <inheritdoc cref="IBackgroundJob.RunAsync" />
|
||||
public async Task RunAsync(PerformContext? context)
|
||||
{
|
||||
TCacheRepository cache = this.Cache;
|
||||
|
||||
// log initial state
|
||||
context.WriteLine(cache.IsLoaded
|
||||
? $"The previous export is cached with data from {this.FormatDateModified(cache.CacheHeaders.LastModified)}."
|
||||
: "No previous export is cached."
|
||||
);
|
||||
|
||||
// fetch cache headers
|
||||
context.WriteLine("Fetching cache headers...");
|
||||
ApiCacheHeaders serverCacheHeaders = await this.FetchCacheHeadersAsync();
|
||||
DateTimeOffset serverModified = serverCacheHeaders.LastModified;
|
||||
string? serverEntityTag = serverCacheHeaders.EntityTag;
|
||||
|
||||
// update data
|
||||
{
|
||||
// skip if no update needed
|
||||
if (cache.IsStale(serverModified, this.ExportStaleAge))
|
||||
context.WriteLine($"Skipped data fetch: server was last modified {this.FormatDateModified(serverModified)}, which exceeds the {this.ExportStaleAge}-minute-stale limit.");
|
||||
else if (cache.IsLoaded && cache.CacheHeaders.LastModified >= serverModified)
|
||||
context.WriteLine($"Skipped data fetch: server was last modified {this.FormatDateModified(serverModified)}, which {(serverModified == cache.CacheHeaders.LastModified ? "matches" : "is older than")} our cached data.");
|
||||
|
||||
// update cache headers if data unchanged
|
||||
else if (cache.IsLoaded && cache.CacheHeaders.EntityTag != null && cache.CacheHeaders.EntityTag == serverEntityTag)
|
||||
{
|
||||
context.WriteLine($"Skipped data fetch: server provided entity tag '{serverEntityTag}', which already matches the data we have.");
|
||||
cache.SetCacheHeaders(serverCacheHeaders);
|
||||
}
|
||||
|
||||
// else update data
|
||||
else
|
||||
{
|
||||
context.WriteLine("Fetching data...");
|
||||
await this.FetchDataAsync();
|
||||
}
|
||||
}
|
||||
|
||||
// clear if stale
|
||||
if (cache.IsStale(this.ExportStaleAge))
|
||||
{
|
||||
context.WriteLine("The cached data is stale, clearing cache...");
|
||||
cache.Clear();
|
||||
}
|
||||
|
||||
// log final result
|
||||
context.WriteLine(cache.IsLoaded
|
||||
? $"Done! The export is currently cached with data from {this.FormatDateModified(cache.CacheHeaders.LastModified)}."
|
||||
: "Done! The export cache is currently disabled."
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/*********
|
||||
** Protected methods
|
||||
*********/
|
||||
/// <summary>Construct an instance.</summary>
|
||||
/// <param name="cache"><inheritdoc cref="Cache" path="/summary"/></param>
|
||||
/// <param name="client"><inheritdoc cref="Client" path="/summary"/></param>
|
||||
/// <param name="updateCheckConfig"><inheritdoc cref="UpdateCheckConfig" path="/summary"/></param>
|
||||
protected ModSiteExportJob(TCacheRepository cache, TExportApiClient client, IOptions<ModUpdateCheckConfig> updateCheckConfig)
|
||||
{
|
||||
this.Cache = cache;
|
||||
this.Client = client;
|
||||
this.UpdateCheckConfig = updateCheckConfig;
|
||||
}
|
||||
|
||||
/// <summary>Fetch the HTTP cache headers set by the export API.</summary>
|
||||
protected abstract Task<ApiCacheHeaders> FetchCacheHeadersAsync();
|
||||
|
||||
/// <summary>Fetch the latest export from the export API, and save it to the <see cref="Cache"/>.</summary>
|
||||
protected abstract Task FetchDataAsync();
|
||||
|
||||
/// <summary>Format a 'date modified' value for the task logs.</summary>
|
||||
/// <param name="date">The date to log.</param>
|
||||
private string FormatDateModified(DateTimeOffset? date)
|
||||
{
|
||||
if (!date.HasValue)
|
||||
return "<null>";
|
||||
|
||||
string ageLabel = (DateTimeOffset.UtcNow - date.Value).Humanize(precision: 2, minUnit: TimeUnit.Minute, maxUnit: TimeUnit.Hour);
|
||||
|
||||
return $"{date.Value:O} (age: {ageLabel})";
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.ModDropExport;
|
||||
using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
|
||||
using StardewModdingAPI.Toolkit.Serialization;
|
||||
using StardewModdingAPI.Web.BackgroundJobs;
|
||||
using StardewModdingAPI.Web.Framework;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CompatibilityRepo;
|
||||
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
|
||||
@@ -133,6 +134,15 @@ internal class Startup
|
||||
});
|
||||
|
||||
// init background service
|
||||
// These are singletons since some of them track state between runs.
|
||||
services
|
||||
.AddSingleton<CompatibilityListJob>()
|
||||
.AddSingleton<CurseForgeExportJob>()
|
||||
.AddSingleton<MalwareBlacklistJob>()
|
||||
.AddSingleton<ModDatasetJob>()
|
||||
.AddSingleton<ModDropExportJob>()
|
||||
.AddSingleton<NexusExportJob>()
|
||||
.AddSingleton<RemoveStaleModsJob>();
|
||||
if (backgroundServicesConfig.Enabled)
|
||||
services.AddHostedService<BackgroundService>();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user