Merge pull request #940 from abhiaagarwal/arbitrary_files_modbuild

[ModBuildConfig] Add ability to include arbitrary mods in build output

docs/technical/mod-package-release-notes.md

@@ -1,6 +1,9 @@
 ← [mod build config](./mod-build-config.md)
 
 ## Release notes
+## Upcoming release
+* You can now [bundle content packs with your mod](mod-package.md#bundle-content-packs) (thanks to abhiaagarwal!).
+
 ### 4.2.0
 Released 05 September 2024 for SMAPI 3.13.0 or later.
 

docs/technical/mod-package.md

@@ -7,6 +7,9 @@ for SMAPI mods and related tools. The package is fully compatible with Linux, ma
 * [Use](#use)
 * [Features](#features)
 * [Configure](#configure)
+  * [How to set options](#how-to-set-options)
+  * [Available properties](#available-properties)
+* [Bundle content packs](#Bundle-content-packs)
 * [Code warnings](#code-warnings)
 * [FAQs](#faqs)
   * [How do I set the game path?](#custom-game-path)
@@ -295,6 +298,31 @@ For example, this excludes all `.txt` and `.pdf` files, as well as the `assets/p
 </li>
 </ul>
 
+## Bundle content packs
+You can bundle any number of [content packs](https://stardewvalleywiki.com/Modding:Content_pack_frameworks)
+with your main C# mod. They'll be grouped with the main mod into a parent folder automatically,
+which will be copied to the `Mods` folder and included in the release zip.
+
+To do that, add an `ItemGroup` with a `ContentPacks` line for each content pack you want to include:
+
+```xml
+<ItemGroup>
+    <ContentPacks Include="content packs/[CP] SomeContentPack" Version="$(Version)" />
+    <ContentPacks Include="content packs/[CP] AnotherContentPack" Version="$(Version)" />
+</ItemGroup>
+```
+
+You can use these properties for each line:
+
+property                | effect
+----------------------- | ------
+`Include`               | _(Required)_ The path to the content pack folder. This can be an absolute path, or relative to the current project.
+`Version`               | _(Required)_ The expected version of the content pack. This should usually be the same version as your main mod, to keep update alerts in sync. The package will validate that the included content pack's manifest version matches.
+`FolderName`            | _(Optional)_ The content pack folder name to create. Defaults to the folder name from `Include`.
+`ValidateManifest`      | _(Optional)_ Whether to validate that the included mod has a valid `manifest.json` file and version. Default `true`.
+`IgnoreModFilePaths`    | _(Optional)_ A list of file paths to ignore (relative to the content pack's directory); see `IgnoreModFilePaths` in the main settings. Default none.
+`IgnoreModFilePatterns` | _(Optional)_ A list of file regex patterns to ignore (relative to the content pack's directory); see `IgnoreModFilePatterns` in the main settings. Default none.
+
 ## Code warnings
 ### Overview
 The NuGet package adds code warnings in Visual Studio specific to Stardew Valley. For example:  

src/SMAPI.ModBuildConfig/DeployModTask.cs

@@ -63,6 +63,9 @@ namespace StardewModdingAPI.ModBuildConfig
         /// <summary>A comma-separated list of <see cref="ExtraAssemblyTypes"/> values which indicate which extra DLLs to bundle.</summary>
         public string BundleExtraAssemblies { get; set; }
 
+        /// <summary>A list of content pack folders to bundle with the mod.</summary>
+        public ITaskItem[] ContentPacks { get; set; }
+
 
         /*********
         ** Public methods
@@ -120,18 +123,51 @@ namespace StardewModdingAPI.ModBuildConfig
                 ExtraAssemblyTypes bundleAssemblyTypes = this.GetExtraAssembliesToBundleOption();
 
                 // parse ignore patterns
-                string[] ignoreFilePaths = this.GetCustomIgnoreFilePaths().ToArray();
-                Regex[] ignoreFilePatterns = this.GetCustomIgnorePatterns().ToArray();
+                string[] ignoreFilePaths = this.GetCustomIgnoreFilePaths(this.IgnoreModFilePatterns).ToArray();
+                Regex[] ignoreFilePatterns = this.GetCustomIgnorePatterns(this.IgnoreModFilePaths).ToArray();
 
-                // get mod info
-                ModFileManager package = new(this.ProjectDir, this.TargetDir, ignoreFilePaths, ignoreFilePatterns, bundleAssemblyTypes, this.ModDllName, validateRequiredModFiles: this.EnableModDeploy || this.EnableModZip);
+                var modPackages = new Dictionary<string, IModFileManager>
+                {
+                    [this.ModFolderName] = new MainModFileManager(this.ProjectDir, this.TargetDir, ignoreFilePaths, ignoreFilePatterns, bundleAssemblyTypes, this.ModDllName, validateRequiredModFiles: this.EnableModDeploy || this.EnableModZip)
+                };
 
+                if (this.ContentPacks != null)
+                {
+                    foreach (ITaskItem item in this.ContentPacks)
+                    {
+                        // get content folder path
+                        string contentPath = item.ItemSpec;
+                        if (string.IsNullOrWhiteSpace(contentPath))
+                            throw new UserErrorException("Content pack doesn't have the required 'Include' attribute.");
+
+                        // get folder name
+                        string folderName = item.GetMetadata("FolderName");
+                        if (string.IsNullOrWhiteSpace(folderName))
+                            folderName = Path.GetFileName(contentPath);
+
+                        // get version
+                        string version = item.GetMetadata("Version");
+
+                        // get options
+                        ignoreFilePaths = this.GetCustomIgnoreFilePaths(item.GetMetadata("IgnoreModFilePatterns")).ToArray();
+                        ignoreFilePatterns = this.GetCustomIgnorePatterns(this.IgnoreModFilePaths).ToArray();
+                        string rawValidateManifest = item.GetMetadata("ValidateManifest");
+                        bool validateManifest = string.IsNullOrEmpty(rawValidateManifest) || bool.Parse(rawValidateManifest);
+
+                        // apply
+                        this.Log.LogMessage(MessageImportance.High, $"[mod build package] Bundling content pack: {folderName} v{version} at {contentPath}.");
+                        modPackages.Add(
+                            folderName,
+                            new ContentPackFileManager(this.ProjectDir, contentPath, version, ignoreFilePaths, ignoreFilePatterns, validateManifest)
+                        );
+                    }
+                }
                 // deploy mod files
                 if (this.EnableModDeploy)
                 {
                     string outputPath = Path.Combine(this.GameModsDir, this.EscapeInvalidFilenameCharacters(this.ModFolderName));
                     this.Log.LogMessage(MessageImportance.High, $"[mod build package] Copying the mod files to {outputPath}...");
-                    this.CreateModFolder(package.GetFiles(), outputPath);
+                    this.CreateModFolder(modPackages, outputPath);
                 }
 
                 // create release zip
@@ -141,7 +177,7 @@ namespace StardewModdingAPI.ModBuildConfig
                     string zipPath = Path.Combine(this.ModZipPath, zipName);
 
                     this.Log.LogMessage(MessageImportance.High, $"[mod build package] Generating the release zip at {zipPath}...");
-                    this.CreateReleaseZip(package.GetFiles(), this.ModFolderName, zipPath);
+                    this.CreateReleaseZip(modPackages, zipPath);
                 }
 
                 return true;
@@ -171,6 +207,9 @@ namespace StardewModdingAPI.ModBuildConfig
                 if (property.Name == nameof(this.IgnoreModFilePatterns) && string.IsNullOrWhiteSpace(this.IgnoreModFilePatterns))
                     continue;
 
+                if (property.Name == nameof(this.ContentPacks))
+                    continue;
+
                 string name = property.Name;
 
                 string value = property.GetValue(this)?.ToString();
@@ -208,12 +247,13 @@ namespace StardewModdingAPI.ModBuildConfig
         }
 
         /// <summary>Get the custom ignore patterns provided by the user.</summary>
-        private IEnumerable<Regex> GetCustomIgnorePatterns()
+        /// <param name="patterns">The comma-separated regex patterns.</param>
+        private IEnumerable<Regex> GetCustomIgnorePatterns(string patterns)
         {
-            if (string.IsNullOrWhiteSpace(this.IgnoreModFilePatterns))
+            if (string.IsNullOrWhiteSpace(patterns))
                 yield break;
 
-            foreach (string raw in this.IgnoreModFilePatterns.Split(','))
+            foreach (string raw in patterns.Split(','))
             {
                 Regex regex;
                 try
@@ -222,7 +262,7 @@ namespace StardewModdingAPI.ModBuildConfig
                 }
                 catch (Exception ex)
                 {
-                    this.Log.LogWarning($"[mod build package] Ignored invalid <{nameof(this.IgnoreModFilePatterns)}> pattern {raw}:\n{ex}");
+                    this.Log.LogWarning($"[mod build package] Ignored invalid <{nameof(patterns)}> pattern {raw}:\n{ex}");
                     continue;
                 }
 
@@ -231,12 +271,13 @@ namespace StardewModdingAPI.ModBuildConfig
         }
 
         /// <summary>Get the custom relative file paths provided by the user to ignore.</summary>
-        private IEnumerable<string> GetCustomIgnoreFilePaths()
+        /// <param name="pattern">The comma-separated file paths.</param>
+        private IEnumerable<string> GetCustomIgnoreFilePaths(string pattern)
         {
-            if (string.IsNullOrWhiteSpace(this.IgnoreModFilePaths))
+            if (string.IsNullOrWhiteSpace(pattern))
                 yield break;
 
-            foreach (string raw in this.IgnoreModFilePaths.Split(','))
+            foreach (string raw in pattern.Split(','))
             {
                 string path;
                 try
@@ -245,7 +286,7 @@ namespace StardewModdingAPI.ModBuildConfig
                 }
                 catch (Exception ex)
                 {
-                    this.Log.LogWarning($"[mod build package] Ignored invalid <{nameof(this.IgnoreModFilePaths)}> path {raw}:\n{ex}");
+                    this.Log.LogWarning($"[mod build package] Ignored invalid <{nameof(pattern)}> path {raw}:\n{ex}");
                     continue;
                 }
 
@@ -254,48 +295,54 @@ namespace StardewModdingAPI.ModBuildConfig
         }
 
         /// <summary>Copy the mod files into the game's mod folder.</summary>
-        /// <param name="files">The files to include.</param>
-        /// <param name="modFolderPath">The folder path to create with the mod files.</param>
-        private void CreateModFolder(IDictionary<string, FileInfo> files, string modFolderPath)
+        /// <param name="modPackages">The mods to include.</param>
+        /// <param name="outputPath">The folder path to create with the mod files.</param>
+        private void CreateModFolder(IDictionary<string, IModFileManager> modPackages, string outputPath)
         {
-            foreach (var entry in files)
+            foreach (var mod in modPackages)
             {
-                string fromPath = entry.Value.FullName;
-                string toPath = Path.Combine(modFolderPath, entry.Key);
+                string relativePath = modPackages.Count == 1
+                    ? outputPath
+                    : Path.Combine(outputPath, this.EscapeInvalidFilenameCharacters(mod.Key));
 
-                Directory.CreateDirectory(Path.GetDirectoryName(toPath)!);
+                foreach (var file in mod.Value.GetFiles())
+                {
+                    string fromPath = file.Value.FullName;
+                    string toPath = Path.Combine(relativePath, file.Key);
 
-                File.Copy(fromPath, toPath, overwrite: true);
+                    Directory.CreateDirectory(Path.GetDirectoryName(toPath)!);
+
+                    File.Copy(fromPath, toPath, overwrite: true);
+                }
             }
         }
 
         /// <summary>Create a release zip in the recommended format for uploading to mod sites.</summary>
-        /// <param name="files">The files to include.</param>
-        /// <param name="modName">The name of the mod.</param>
+        /// <param name="modPackages">The mods to include.</param>
         /// <param name="zipPath">The absolute path to the zip file to create.</param>
-        private void CreateReleaseZip(IDictionary<string, FileInfo> files, string modName, string zipPath)
+        private void CreateReleaseZip(IDictionary<string, IModFileManager> modPackages, string zipPath)
         {
-            // get folder name within zip
-            string folderName = this.EscapeInvalidFilenameCharacters(modName);
-
             // create zip file
             Directory.CreateDirectory(Path.GetDirectoryName(zipPath)!);
             using Stream zipStream = new FileStream(zipPath, FileMode.Create, FileAccess.Write);
             using ZipArchive archive = new(zipStream, ZipArchiveMode.Create);
 
-            foreach (var fileEntry in files)
+            foreach (var mod in modPackages)
             {
-                string relativePath = fileEntry.Key;
-                FileInfo file = fileEntry.Value;
+                string modFolder = this.EscapeInvalidFilenameCharacters(mod.Key);
+                foreach (var file in mod.Value.GetFiles())
+                {
+                    string relativePath = file.Key;
+                    FileInfo fileInfo = file.Value;
 
-                // get file info
-                string filePath = file.FullName;
-                string entryName = folderName + '/' + relativePath.Replace(Path.DirectorySeparatorChar, '/');
+                    string archivePath = modPackages.Count == 1
+                        ? $"{modFolder}/{relativePath}"
+                        : $"{this.ModFolderName}/{modFolder}/{relativePath}";
 
-                // add to zip
-                using Stream fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
-                using Stream fileStreamInZip = archive.CreateEntry(entryName).Open();
-                fileStream.CopyTo(fileStreamInZip);
+                    using Stream fileStream = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read);
+                    using Stream fileStreamInZip = archive.CreateEntry(archivePath).Open();
+                    fileStream.CopyTo(fileStreamInZip);
+                }
             }
         }
 

src/SMAPI.ModBuildConfig/Framework/ContentPackFileManager.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Newtonsoft.Json;
+using StardewModdingAPI.Toolkit;
+using StardewModdingAPI.Toolkit.Framework;
+using StardewModdingAPI.Toolkit.Serialization;
+using StardewModdingAPI.Toolkit.Serialization.Models;
+using StardewModdingAPI.Toolkit.Utilities;
+
+namespace StardewModdingAPI.ModBuildConfig.Framework
+{
+    /// <summary>Manages the files that are part of a bundled content pack.</summary>
+    internal class ContentPackFileManager : IModFileManager
+    {
+        /*********
+        ** Fields
+        *********/
+        /// <summary>The name of the manifest file.</summary>
+        private readonly string ManifestFileName = "manifest.json";
+
+        /// <summary>The files that are part of the package.</summary>
+        private readonly Dictionary<string, FileInfo> Files = new(StringComparer.OrdinalIgnoreCase);
+
+
+        /*********
+        ** Public methods
+        *********/
+        /// <summary>Construct an instance.</summary>
+        /// <param name="projectDir">The folder containing the project files.</param>
+        /// <param name="contentPackDir">The absolute or relative path to the content pack folder.</param>
+        /// <param name="version">The mod version.</param>
+        /// <param name="ignoreFilePaths">The custom relative file paths provided by the user to ignore.</param>
+        /// <param name="ignoreFilePatterns">Custom regex patterns matching files to ignore when deploying or zipping the mod.</param>
+        /// <param name="validateManifest">Whether to validate that the content pack's manifest is valid.</param>
+        /// <exception cref="UserErrorException">The mod package isn't valid.</exception>
+        public ContentPackFileManager(string projectDir, string contentPackDir, string version, string[] ignoreFilePaths, Regex[] ignoreFilePatterns, bool validateManifest)
+        {
+            // get folders
+            DirectoryInfo projectDirInfo = new DirectoryInfo(Path.Combine(projectDir, contentPackDir));
+            if (!projectDirInfo.Exists)
+                throw GetError($"that folder doesn't exist at {projectDirInfo.FullName}");
+
+            // collect files
+            foreach (FileInfo entry in projectDirInfo.GetFiles("*", SearchOption.AllDirectories))
+            {
+                string relativePath = PathUtilities.GetRelativePath(projectDirInfo.FullName, entry.FullName);
+                FileInfo file = entry;
+
+                if (!this.ShouldIgnore(file, relativePath, ignoreFilePaths, ignoreFilePatterns))
+                    this.Files[relativePath] = file;
+            }
+
+            // validate manifest
+            if (validateManifest)
+            {
+                // get manifest file
+                if (!this.Files.TryGetValue(this.ManifestFileName, out FileInfo manifestFile))
+                    throw GetError($"it has no {this.ManifestFileName} file");
+
+                // parse file
+                Manifest manifest;
+                try
+                {
+                    new JsonHelper().ReadJsonFileIfExists(manifestFile.FullName, out Manifest rawManifest);
+                    manifest = rawManifest;
+                }
+                catch (JsonReaderException ex)
+                {
+                    throw GetError($"its {this.ManifestFileName} file isn't valid JSON: {ex.InnerException?.Message ?? ex.Message}");
+                }
+
+                // validate manifest fields
+                if (!ManifestValidator.TryValidateFields(manifest, out string error))
+                    throw GetError($"its {this.ManifestFileName} file is invalid: {error}");
+
+                // validate version
+                if (version == null)
+                    throw GetError("no Version value was provided");
+                if (!SemanticVersion.TryParse(version, out ISemanticVersion requiredVersion))
+                    throw GetError($"the provided Version value '{version}' isn't a valid semantic version");
+                if (manifest.Version.CompareTo(requiredVersion) != 0)
+                    throw GetError($"its {this.ManifestFileName} has version '{manifest.Version}' instead of the required '{requiredVersion}'");
+            }
+
+            UserErrorException GetError(string reasonPhrase)
+            {
+                return new UserErrorException($"The content pack at '{contentPackDir}' can't be loaded because {reasonPhrase}.");
+            }
+        }
+
+        ///<inheritdoc/>
+        public IDictionary<string, FileInfo> GetFiles()
+        {
+            return new Dictionary<string, FileInfo>(this.Files, StringComparer.OrdinalIgnoreCase);
+        }
+
+        /// <summary>Get whether a content file should be ignored.</summary>
+        /// <param name="file">The file to check.</param>
+        /// <param name="relativePath">The file's relative path in the package.</param>
+        /// <param name="ignoreFilePaths">The custom relative file paths provided by the user to ignore.</param>
+        /// <param name="ignoreFilePatterns">Custom regex patterns matching files to ignore when deploying or zipping the mod.</param>
+        private bool ShouldIgnore(FileInfo file, string relativePath, string[] ignoreFilePaths, Regex[] ignoreFilePatterns)
+        {
+            // apply custom patterns
+            if (ignoreFilePaths.Any(p => p == relativePath) || ignoreFilePatterns.Any(p => p.IsMatch(relativePath)))
+                return true;
+
+            // ignore special files
+            return
+                // release zips
+                this.EqualsInvariant(file.Extension, ".zip")
+
+                // OS metadata files
+                || this.EqualsInvariant(file.Name, ".DS_Store")
+                || this.EqualsInvariant(file.Name, "Thumbs.db");
+        }
+
+        // <summary>Get whether a string is equal to another case-insensitively.</summary>
+        /// <param name="str">The string value.</param>
+        /// <param name="other">The string to compare with.</param>
+        private bool EqualsInvariant(string str, string other)
+        {
+            if (str == null)
+                return other == null;
+            return str.Equals(other, StringComparison.OrdinalIgnoreCase);
+        }
+    }
+}

src/SMAPI.ModBuildConfig/Framework/IModFileManager.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+using System.IO;
+
+namespace StardewModdingAPI.ModBuildConfig.Framework
+{
+    /// <summary>Manages the files that are part of a mod in the release package.</summary>
+    public interface IModFileManager
+    {
+        /// <summary>Get the files in the mod package.</summary>
+        public IDictionary<string, FileInfo> GetFiles();
+    }
+}

src/SMAPI.ModBuildConfig/Framework/MainModFileManager.cs

@@ -7,8 +7,8 @@ using StardewModdingAPI.Toolkit.Utilities;
 
 namespace StardewModdingAPI.ModBuildConfig.Framework
 {
-    /// <summary>Manages the files that are part of a mod package.</summary>
-    internal class ModFileManager
+    /// <summary>Manages the files that are part of the main C# mod.</summary>
+    internal class MainModFileManager : IModFileManager
     {
         /*********
         ** Fields
@@ -17,7 +17,7 @@ namespace StardewModdingAPI.ModBuildConfig.Framework
         private readonly string ManifestFileName = "manifest.json";
 
         /// <summary>The files that are part of the package.</summary>
-        private readonly IDictionary<string, FileInfo> Files;
+        private readonly Dictionary<string, FileInfo> Files = new(StringComparer.OrdinalIgnoreCase);
 
         /// <summary>The file extensions used by assembly files.</summary>
         private readonly ISet<string> AssemblyFileExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
@@ -71,10 +71,8 @@ namespace StardewModdingAPI.ModBuildConfig.Framework
         /// <param name="modDllName">The name (without extension or path) for the current mod's DLL.</param>
         /// <param name="validateRequiredModFiles">Whether to validate that required mod files like the manifest are present.</param>
         /// <exception cref="UserErrorException">The mod package isn't valid.</exception>
-        public ModFileManager(string projectDir, string targetDir, string[] ignoreFilePaths, Regex[] ignoreFilePatterns, ExtraAssemblyTypes bundleAssemblyTypes, string modDllName, bool validateRequiredModFiles)
+        public MainModFileManager(string projectDir, string targetDir, string[] ignoreFilePaths, Regex[] ignoreFilePatterns, ExtraAssemblyTypes bundleAssemblyTypes, string modDllName, bool validateRequiredModFiles)
         {
-            this.Files = new Dictionary<string, FileInfo>(StringComparer.OrdinalIgnoreCase);
-
             // validate paths
             if (!Directory.Exists(projectDir))
                 throw new UserErrorException("Could not create mod package because the project folder wasn't found.");
@@ -105,7 +103,7 @@ namespace StardewModdingAPI.ModBuildConfig.Framework
             }
         }
 
-        /// <summary>Get the files in the mod package.</summary>
+        ///<inheritdoc/>
         public IDictionary<string, FileInfo> GetFiles()
         {
             return new Dictionary<string, FileInfo>(this.Files, StringComparer.OrdinalIgnoreCase);

src/SMAPI.ModBuildConfig/build/smapi.targets

@@ -99,6 +99,7 @@
       IgnoreModFilePaths="$(IgnoreModFilePaths)"
 
       BundleExtraAssemblies="$(BundleExtraAssemblies)"
+      ContentPacks="@(ContentPacks)"
     />
   </Target>
 </Project>