A tool to build a self-contained source tarball

The does some pruning of the dependencies' files to avoid a 100 MiB+ tarball.
Even with that, the final tarball size is 17 MiB with xz, 25 MiB with gzip.

See the file documentation at the top of `tools/make_src_dist.py` for
more information.

.gitignore

@@ -19,6 +19,9 @@ comparer-config.toml
 /build-*/
 .vscode/tasks.json
 
+# Extra files in the source distribution (see make_src_dist.py)
+/dist/
+
 # ELF object file, shared library and object archive.
 *.o
 *.so

CMakeLists.txt

@@ -10,6 +10,12 @@ if(POLICY CMP0111)
   cmake_policy(SET CMP0111 NEW)
 endif()
 
+if(IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/dist")
+  message("-- Detect a source distribution with the required FetchContent dependencies and devilutionx.mpq included")
+  set(SRC_DIST ON)
+  add_subdirectory(dist)
+endif()
+
 include(CMakeDependentOption)
 include(CMake/out_of_tree.cmake)
 include(CMake/genex.cmake)
@@ -42,7 +48,7 @@ option(STREAM_ALL_AUDIO "Stream all the audio. For extremely RAM-constrained pla
 mark_as_advanced(STREAM_ALL_AUDIO)
 
 # By default, devilutionx.mpq is built only if smpq is installed.
-if(NOT DEFINED BUILD_ASSETS_MPQ)
+if(NOT DEFINED BUILD_ASSETS_MPQ AND NOT SRC_DIST)
   find_program(SMPQ smpq)
 elseif(BUILD_ASSETS_MPQ)
   find_program(SMPQ smpq REQUIRED)
@@ -1148,7 +1154,7 @@ if(VITA)
   set(VITA_MKSFOEX_FLAGS "${VITA_MKSFOEX_FLAGS} -d PARENTAL_LEVEL=1")
   set(VITA_MKSFOEX_FLAGS "${VITA_MKSFOEX_FLAGS} -d ATTRIBUTE2=12")
   vita_create_self(devilutionx.self devilutionx UNSAFE)
-  if(BUILD_ASSETS_MPQ)
+  if(BUILD_ASSETS_MPQ OR SRC_DIST)
     vita_create_vpk(devilutionx.vpk ${VITA_TITLEID} devilutionx.self
       VERSION ${VITA_VERSION}
       NAME ${VITA_APP_NAME}
@@ -1208,7 +1214,7 @@ if(NINTENDO_3DS)
   add_dependencies(${APP_TARGET_PREFIX}_cia romfs_files)
 endif()
 
-if(CPACK AND (APPLE OR BUILD_ASSETS_MPQ))
+if(CPACK AND (APPLE OR BUILD_ASSETS_MPQ OR SRC_DIST))
   if(WIN32)
     if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
       set(SDL2_WIN32_DLLS_DIR "${CMAKE_BINARY_DIR}")

tools/make_src_dist.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+
+"""
+Makes a tarball suitable for distros.
+
+It contains the following:
+
+1. The repo source code.
+
+2. An additional `dist` directory with:
+
+   1. `FetchContent` dependencies that currently must be vendored.
+       These are stripped from especially heavy bloat.
+
+   2. `devilutionx.mpq`.
+      While this file can be generated by the build system, it requires
+      the `smpq` host dependency which may be missing in some distributions.
+
+   3. `CMakeFlags.txt` - a file the cmake flags containing the version,
+      the path to `devilutionx.mpq` and the dependency path.
+
+      This file is automatically used by the build system if present.
+
+The only stdout output of this script is the path to the generated tarball.
+"""
+
+import logging
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+
+# We only package the dependencies that are:
+# 1. Uncommon in package managers (sdl_audiolib and simpleini).
+# 2. Require devilutionx forks (all others).
+_DEPS = ['asio', 'libmpq', 'libsmackerdec',
+         'libzt', 'sdl_audiolib', 'simpleini']
+
+_ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent
+_BUILD_DIR = _ROOT_DIR.joinpath('build-src-dist')
+_ARCHIVE_DIR = _BUILD_DIR.joinpath('archive')
+_DIST_DIR = _ARCHIVE_DIR.joinpath('dist')
+
+_LOGGER = logging.getLogger()
+_LOGGER.setLevel(logging.INFO)
+_LOGGER.addHandler(logging.StreamHandler(sys.stderr))
+
+
+def main():
+	cmake(f'-S{_ROOT_DIR}', f'-B{_BUILD_DIR}', '-DBUILD_ASSETS_MPQ=ON')
+	cmake('--build', _BUILD_DIR, '--target', 'devilutionx_mpq')
+
+	if _ARCHIVE_DIR.exists():
+		shutil.rmtree(_ARCHIVE_DIR)
+
+	_LOGGER.info(f'Copying repo files...')
+	for src_bytes in git('ls-files', '-z').rstrip(b'\0').split(b'\0'):
+		src = src_bytes.decode()
+		dst_path = _ARCHIVE_DIR.joinpath(src)
+		dst_path.parent.mkdir(parents=True, exist_ok=True)
+		if re.search('(^|/)\.gitkeep$', src):
+			continue
+		shutil.copy2(_ROOT_DIR.joinpath(src), dst_path, follow_symlinks=False)
+
+	_LOGGER.info(f'Copying devilutionx.mpq...')
+	_DIST_DIR.mkdir(parents=True)
+	shutil.copy(_BUILD_DIR.joinpath('devilutionx.mpq'), _DIST_DIR)
+
+	for dep in _DEPS:
+		_LOGGER.info(f'Copying {dep}...')
+		shutil.copytree(
+			src=_BUILD_DIR.joinpath('_deps', f'{dep}-src'),
+			dst=_DIST_DIR.joinpath(f'{dep}-src'),
+			ignore=ignore_dep_src)
+
+	version_num, version_suffix = get_version()
+	write_dist_cmakelists(version_num, version_suffix)
+	print(make_archive(version_num, version_suffix))
+
+
+def cmake(*cmd_args):
+	_LOGGER.info(f'+ cmake {subprocess.list2cmdline(cmd_args)}')
+	subprocess.run(['cmake', *cmd_args], cwd=_ROOT_DIR, stdout=subprocess.DEVNULL)
+
+
+def git(*cmd_args):
+	_LOGGER.debug(f'+ git {subprocess.list2cmdline(cmd_args)}')
+	return subprocess.run(['git', *cmd_args], cwd=_ROOT_DIR, capture_output=True).stdout
+
+
+# Ignore files in dependencies that we don't need.
+# Examples of some heavy ones:
+# 48M libzt-src/ext
+# 9.8M asio-src/asio/src/doc
+_IGNORE_DEP_DIR_RE = re.compile(
+	r'(/|^)\.|(/|^)(tests?|other|vcx?proj|examples?|doxygen|docs?|asio-src/asio/src)(/|$)')
+_IGNORE_DEP_FILE_RE = re.compile(
+	r'(^\.|Makefile|vcx?proj|example|doxygen|docs?|\.(doxy|cmd|png|html|ico|icns)$)')
+
+
+def ignore_dep_src(src, names):
+	if 'sdl_audiolib' in src:
+		# SDL_audiolib currently fails to compile if any of the files are missing.
+		# TODO: Fix this in SDL_audiolib by making this optional:
+		# https://github.com/realnc/SDL_audiolib/blob/5a700ba556d3a5b5c531c2fa1f45fc0c3214a16b/CMakeLists.txt#L399-L401
+		return []
+
+	if _IGNORE_DEP_DIR_RE.search(src):
+		_LOGGER.debug(f'Excluded directory {src}')
+		return names
+
+	def ignore_name(name):
+		if _IGNORE_DEP_FILE_RE.search(name) or _IGNORE_DEP_DIR_RE.search(name):
+			_LOGGER.debug(f'Excluded file {src}/{name}')
+			return True
+		return False
+
+	return filter(ignore_name, names)
+
+
+def get_version():
+	git_tag = git('describe', '--abbrev=0', '--tags').rstrip()
+	git_commit_sha = git('rev-parse', '--short', 'HEAD').rstrip()
+	git_tag_sha = git('rev-parse', '--short', git_tag).rstrip()
+	return git_tag, (git_commit_sha if git_tag_sha != git_commit_sha else None)
+
+
+def write_dist_cmakelists(version_num, version_suffix):
+	version_num, version_suffix = get_version()
+	with open(_DIST_DIR.joinpath('CMakeLists.txt'), 'wb') as f:
+		f.write(b'# Generated by tools/make_src_dist.py\n')
+		f.write(b'set(VERSION_NUM "%s" PARENT_SCOPE)\n' % version_num)
+		if version_suffix:
+			f.write(b'set(VERSION_SUFFIX "%s" PARENT_SCOPE)\n' % version_suffix)
+
+		f.write(b'''
+# Pre-generated `devilutionx.mpq` is provided so that distributions do not have to depend on smpq.
+set(DEVILUTIONX_MPQ "${CMAKE_CURRENT_SOURCE_DIR}/devilutionx.mpq" PARENT_SCOPE)
+
+# This would ensure that CMake does not attempt to connect to network.
+# We do not set this to allow for builds for Windows and Android, which do fetch some
+# dependencies even with this source distribution.
+# set(FETCHCONTENT_FULLY_DISCONNECTED ON PARENT_SCOPE)
+
+# Set the path to each dependency that must be vendored:
+''')
+		for dep in _DEPS:
+			f.write(b'set(FETCHCONTENT_SOURCE_DIR_%s "${CMAKE_CURRENT_SOURCE_DIR}/%s-src" CACHE STRING "")\n' % (
+				dep.upper().encode(), dep.encode()))
+
+
+def make_archive(version_num, version_suffix):
+	archive_base_name = f'devilutionx-{version_num.decode()}'
+	if version_suffix:
+		archive_base_name += f'-{version_suffix.decode()}'
+	_LOGGER.info(f'Compressing {_ARCHIVE_DIR}')
+	return shutil.make_archive(
+            format='xztar',
+            logger=_LOGGER,
+            base_name=_BUILD_DIR.joinpath(archive_base_name),
+            root_dir=_ARCHIVE_DIR,
+            base_dir='.')
+
+
+main()