From 357907013f168e39aeee1a50dc0cb78e1417dcef Mon Sep 17 00:00:00 2001 From: Matt Borgerson Date: Sun, 30 May 2021 15:50:01 -0700 Subject: [PATCH] build: Support cross-building for Apple silicon --- .github/workflows/build.yml | 95 ++++++++++++++---- Info.plist | 36 +++++++ build.sh | 129 ++++++++++++++---------- scripts/download-macos-libs.py | 173 +++++++++++++++++++++++++++++++++ scripts/gen-license.py | 21 ++-- 5 files changed, 372 insertions(+), 82 deletions(-) create mode 100644 Info.plist create mode 100755 scripts/download-macos-libs.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bff9e053af..af8981cccc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,21 +123,34 @@ jobs: path: ${{ matrix.artifact_filename }} macOS: - name: Build for macOS + name: Build for ${{ matrix.arch }} macOS (${{ matrix.configuration }}) runs-on: macOS-latest needs: Init strategy: matrix: + arch: ["x86_64", "arm64"] configuration: ["Debug", "Release"] include: - - configuration: Debug - build_param: --debug - artifact_name: xemu-macos-debug - artifact_filename: xemu-macos-debug.zip - - configuration: Release - build_param: - artifact_name: xemu-macos-release - artifact_filename: xemu-macos-release.zip + - arch: x86_64 + configuration: Debug + build_param: --debug -a x86_64 + artifact_name: xemu-macos-x86_64-debug + artifact_filename: xemu-macos-x86_64-debug.zip + - arch: x86_64 + configuration: Release + build_param: -a x86_64 + artifact_name: xemu-macos-x86_64-release + artifact_filename: xemu-macos-x86_64-release.zip + - arch: arm64 + configuration: Debug + build_param: --debug -a arm64 + artifact_name: xemu-macos-arm64-debug + artifact_filename: xemu-macos-arm64-debug.zip + - arch: arm64 + configuration: Release + build_param: -a arm64 + artifact_name: xemu-macos-arm64-release + artifact_filename: xemu-macos-arm64-release.zip steps: - name: Clone Tree uses: actions/checkout@v2 @@ -162,11 +175,7 @@ jobs: ccache \ coreutils \ dylibbundler \ - libepoxy \ - pixman \ pkg-config \ - libsamplerate \ - sdl2 \ ninja - name: Initialize Compiler Cache id: cache @@ -174,8 +183,8 @@ jobs: uses: actions/cache@v1 with: path: /tmp/xemu-ccache - key: cache-${{ runner.os }}-${{ matrix.configuration }}-${{ github.sha }} - restore-keys: cache-${{ runner.os }}-${{ matrix.configuration }}- + key: cache-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.configuration }}-${{ github.sha }} + restore-keys: cache-${{ runner.os }}-${{ matrix.arch }}-${{ matrix.configuration }}- - name: Compile run: | export CCACHE_DIR=/tmp/xemu-ccache @@ -193,10 +202,54 @@ jobs: name: ${{ matrix.artifact_name }} path: ${{ matrix.artifact_filename }} + macOSBuildUniversal: + name: Build for Universal macOS (${{ matrix.configuration }}) + runs-on: macOS-latest + needs: [macOS] + strategy: + matrix: + configuration: ["debug", "release"] + env: + BUILD_TAG: + steps: + - name: Download x86_64 Build + uses: actions/download-artifact@v2 + with: + name: xemu-macos-x86_64-${{ matrix.configuration }} + path: xemu-macos-x86_64-${{ matrix.configuration }} + - name: Download arm64 Build + uses: actions/download-artifact@v2 + with: + name: xemu-macos-arm64-${{ matrix.configuration }} + path: xemu-macos-arm64-${{ matrix.configuration }} + - name: Build Universal bundle + run: | + mkdir dist + for arch in x86_64 arm64; do + pushd xemu-macos-${arch}-${{ matrix.configuration }} + unzip xemu-macos-${arch}-${{ matrix.configuration }}.zip + popd + pushd dist + unzip -o ../xemu-macos-${arch}-${{ matrix.configuration }}/xemu-macos-${arch}-${{ matrix.configuration }}.zip + popd + done + pushd dist + rm xemu.app/Contents/MacOS/xemu + lipo -create -output xemu.app/Contents/MacOS/xemu \ + ../xemu-macos-x86_64-${{ matrix.configuration }}/xemu.app/Contents/MacOS/xemu \ + ../xemu-macos-arm64-${{ matrix.configuration }}/xemu.app/Contents/MacOS/xemu + zip -r ../xemu-macos-universal-${{ matrix.configuration }}.zip * + popd + - name: Upload Build Artifact + uses: actions/upload-artifact@v2 + with: + name: xemu-macos-universal-${{ matrix.configuration }} + path: xemu-macos-universal-${{ matrix.configuration }}.zip + Release: if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/xemu-v')) runs-on: ubuntu-latest - needs: [Ubuntu, macOS, UbuntuWinCross] + needs: [Ubuntu, macOSBuildUniversal, UbuntuWinCross] env: BUILD_TAG: steps: @@ -244,8 +297,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_name: xemu-macos-release.zip - asset_path: dist/xemu-macos-release/xemu-macos-release.zip + asset_name: xemu-macos-universal-release.zip + asset_path: dist/xemu-macos-universal-release/xemu-macos-universal-release.zip asset_content_type: application/zip - name: Upload Release Assets (macOS Debug Build) id: upload-release-asset-macos-debug @@ -254,8 +307,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_name: xemu-macos-debug.zip - asset_path: dist/xemu-macos-debug/xemu-macos-debug.zip + asset_name: xemu-macos-universal-debug.zip + asset_path: dist/xemu-macos-universal-debug/xemu-macos-universal-debug.zip asset_content_type: application/zip # Sync archive version of source (including submodule code) to the @@ -264,7 +317,7 @@ jobs: # package creation. PushToPPA: if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/xemu-v')) - needs: [Ubuntu, macOS, UbuntuWinCross] + needs: [Ubuntu, macOSBuildUniversal, UbuntuWinCross] runs-on: ubuntu-latest steps: - name: Clone Tree diff --git a/Info.plist b/Info.plist new file mode 100644 index 0000000000..5303f0fa0e --- /dev/null +++ b/Info.plist @@ -0,0 +1,36 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + xemu + CFBundleIconFile + xemu.icns + CFBundleIdentifier + xemu.app.0 + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + xemu + CFBundlePackageType + APPL + CFBundleShortVersionString + 1 + CFBundleSignature + xemu + CFBundleVersion + 1 + LSApplicationCategoryType + public.app-category.games + LSMinimumSystemVersion + 10.6 + NSPrincipalClass + NSApplication + NSHighResolutionCapable + + com.apple.security.cs.allow-jit + + + diff --git a/build.sh b/build.sh index 814a5be6a3..2dbed2375d 100755 --- a/build.sh +++ b/build.sh @@ -6,6 +6,8 @@ set -o physical # Resolve symlinks when changing directory project_source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +target_arch=$(uname -m) + package_windows() { rm -rf dist mkdir -p dist @@ -26,20 +28,37 @@ package_wincross() { } package_macos() { - # - # Create bundle - # rm -rf dist # Copy in executable mkdir -p dist/xemu.app/Contents/MacOS/ - cp build/qemu-system-i386 dist/xemu.app/Contents/MacOS/xemu + exe_path=dist/xemu.app/Contents/MacOS/xemu + lib_path=dist/xemu.app/Contents/Libraries/${target_arch} + lib_rpath=../Libraries/${target_arch} + cp build/qemu-system-i386 ${exe_path} # Copy in in executable dylib dependencies - mkdir -p dist/xemu.app/Contents/Frameworks dylibbundler -cd -of -b -x dist/xemu.app/Contents/MacOS/xemu \ - -d dist/xemu.app/Contents/Frameworks/ \ - -p '@executable_path/../Frameworks/' + -d ${lib_path}/ \ + -p "@executable_path/${lib_rpath}/" \ + -s ${PWD}/macos-libs/${target_arch}/opt/local/lib/openssl-1.0/ \ + -s ${PWD}/macos-libs/${target_arch}/opt/local/lib/ + + # Fixup some paths dylibbundler missed + for dep in $(otool -L "$exe_path" | grep -e '/opt/local/' | cut -d' ' -f1); do + dep_basename="$(basename $dep)" + new_path="@executable_path/${lib_rpath}/${dep_basename}" + echo "Fixing $exe_path dependency $dep_basename -> $new_path" + install_name_tool -change "$dep" "$new_path" "$exe_path" + done + for lib_path in ${lib_path}/*.dylib; do + for dep in $(otool -L "$lib_path" | grep -e '/opt/local/' | cut -d' ' -f1); do + dep_basename="$(basename $dep)" + new_path="@rpath/${dep_basename}" + echo "Fixing $lib_path dependency $dep_basename -> $new_path" + install_name_tool -change "$dep" "$new_path" "$lib_path" + done + done # Copy in runtime resources mkdir -p dist/xemu.app/Contents/Resources @@ -50,45 +69,8 @@ package_macos() { for r in 16 32 128 256 512; do cp "${project_source_dir}/ui/icons/xemu_${r}x${r}.png" "xemu.iconset/icon_${r}x${r}.png"; done iconutil --convert icns --output dist/xemu.app/Contents/Resources/xemu.icns xemu.iconset - # Generate Info.plist file - cat < dist/xemu.app/Contents/Info.plist - - - - - CFBundleDevelopmentRegion - en - CFBundleExecutable - xemu - CFBundleIconFile - xemu.icns - CFBundleIdentifier - xemu.app.0 - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - xemu - CFBundlePackageType - APPL - CFBundleShortVersionString - 1 - CFBundleSignature - xemu - CFBundleVersion - 1 - LSApplicationCategoryType - public.app-category.games - LSMinimumSystemVersion - 10.6 - NSPrincipalClass - NSApplication - NSHighResolutionCapable - - - -EOF - - python3 ./scripts/gen-license.py > dist/LICENSE.txt + cp Info.plist dist/xemu.app/Contents/ + python3 ./scripts/gen-license.py --version-file=macos-libs/$target_arch/INSTALLED > dist/LICENSE.txt } package_linux() { @@ -162,6 +144,10 @@ do platform="${2}" shift 2 ;; + '-a'*) + target_arch="${2}" + shift 2 + ;; *) break ;; @@ -185,14 +171,51 @@ case "$platform" in # Adjust compilation options based on platform postbuild='package_linux' ;; Darwin) - echo 'Compiling for MacOS...' - sys_cflags='-march=ivybridge' + echo "Compiling for MacOS for $target_arch..." + sdk_base=/Library/Developer/CommandLineTools/SDKs/ + sdk_macos_10_14="${sdk_base}/MacOSX10.14.sdk" + sdk_macos_10_15="${sdk_base}/MacOSX10.15.sdk" + sdk_macos_11_1="${sdk_base}/MacOSX11.1.sdk" + if [ "$target_arch" == "arm64" ]; then + macos_min_ver=11.1 + if test -d "$sdk_macos_11_1"; then + sdk="$sdk_macos_11_1" + else + echo "SDK not found. Install Xcode Command Line Tools" + exit 1 + fi + elif [ "$target_arch" == "x86_64" ]; then + macos_min_ver=10.13 + if test -d "$sdk_macos_11_1"; then + sdk="$sdk_macos_11_1" + elif test -d "$sdk_macos_10_15"; then + sdk="$sdk_macos_10_15" + elif test -d "$sdk_macos_10_14"; then + sdk="$sdk_macos_10_14" + else + echo "SDK not found. Install Xcode Command Line Tools" + exit 1 + fi + else + echo "Unsupported arch $target_arch" + exit 1 + fi + python3 ./scripts/download-macos-libs.py ${target_arch} + lib_prefix=${PWD}/macos-libs/${target_arch}/opt/local + export CFLAGS="-arch ${target_arch} \ + -target ${target_arch}-apple-macos${macos_min_ver} \ + -isysroot ${sdk} \ + -I${lib_prefix}/include \ + -mmacosx-version-min=$macos_min_ver" + export LDFLAGS="-arch ${target_arch} \ + -isysroot ${sdk}" + if [ "$target_arch" == "x86_64" ]; then + sys_cflags='-march=ivybridge' + fi sys_ldflags='-headerpad_max_install_names' - opts="$opts --disable-cocoa" - # necessary to find libffi, which is required by gobject - export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}/usr/local/opt/libffi/lib/pkgconfig" - export PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig:${PKG_CONFIG_PATH}" - echo $PKG_CONFIG_PATH + export PKG_CONFIG_PATH="${lib_prefix}/lib/pkgconfig" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:${lib_prefix}/lib/openssl-1.0/pkgconfig/" + opts="$opts --disable-cocoa --cross-prefix=" postbuild='package_macos' ;; CYGWIN*|MINGW*|MSYS*) diff --git a/scripts/download-macos-libs.py b/scripts/download-macos-libs.py new file mode 100755 index 0000000000..5c0cb51604 --- /dev/null +++ b/scripts/download-macos-libs.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Downloads required libraries for xemu builds on macOS from MacPorts repositories +""" +# Based on https://github.com/tpoechtrager/osxcross/blob/master/tools/osxcross-macports +# which is based on https://github.com/maci0/pmmacports +from urllib.request import urlopen +import re +import os.path +from tarfile import TarFile +import subprocess + +# MIRROR = 'http://packages.macports.org/macports/packages' +MIRROR = 'http://nue.de.packages.macports.org/macports/packages' + +class LibInstaller: + DARWIN_TARGET_X64="darwin_17" # macOS 10.13 + DARWIN_TARGET_ARM64="darwin_20" # macOS 11.x + + def __init__(self, arch): + self._queue = [] + self._installed = [] + if arch == 'x86_64': + self._darwin_target = self.DARWIN_TARGET_X64 + elif arch == 'arm64': + self._darwin_target = self.DARWIN_TARGET_ARM64 + else: + assert False, "Add arch" + self._arch = arch + + self._extract_path = os.path.realpath(f'./macos-libs/{self._arch}') + if not os.path.exists(self._extract_path): + os.makedirs(self._extract_path) + self._installed_path = os.path.join(self._extract_path, 'INSTALLED') + self._pkgs_path = os.path.realpath(os.path.join(f'./macos-pkgs')) + if not os.path.exists(self._pkgs_path): + os.makedirs(self._pkgs_path) + + def get_latest_pkg_filename_url(self, pkg_name): + pkg_base_name = pkg_name.split('-')[0] + pkg_base_url = f'{MIRROR}/{pkg_base_name}' + pkg_list = urlopen(pkg_base_url).read().decode('utf-8') + pkgs = re.findall(pkg_name + r'[\w\.\-\_\+]*?\.' + self._darwin_target + r'\.' + self._arch + r'\.tbz2', pkg_list) + pkg_filename = pkgs[-1] + return pkg_filename, f'{pkg_base_url}/{pkg_filename}' + + def is_pkg_installed(self, pkg_name): + if not os.path.exists(self._installed_path): + return False + with open(self._installed_path) as f: + installed = [l.strip().split('=')[0] for l in f.readlines()] + return pkg_name in installed + + def mark_pkg_installed(self, pkg_name, pkg_version): + if self.is_pkg_installed(pkg_name): + return + with open(os.path.join(self._extract_path, 'INSTALLED'), 'a+') as f: + f.write(f'{pkg_name}={pkg_version}\n') + + def download_file(self, desc, url, dst): + if os.path.exists(dst): + print(f' [+] Already have {desc}') + else: + print(f' [+] Downloading {desc}') + with open(dst, 'wb') as f: + f.write(urlopen(url).read()) + + def verify_pkg(self, pkg_path, sig_path): + PUBKEYURL="https://svn.macports.org/repository/macports/trunk/base/macports-pubkey.pem" + PUBKEYRMD160="d3a22f5be7184d6575afcc1be6fdb82fd25562e8" + PUBKEYSHA1="214baa965af76ff71187e6c1ac91c559547f48ab" + key_filename = 'macports-pubkey.pem' + dst_key_filename = os.path.join(self._pkgs_path, key_filename) + self.download_file('MacPorts key', PUBKEYURL, dst_key_filename) + rmd160 = subprocess.run('openssl rmd160 "' + dst_key_filename + "\" | awk '{print $2}'", + capture_output=True, shell=True, + check=True).stdout.decode('utf-8').strip() + sha1 = subprocess.run('openssl sha1 "' + dst_key_filename + "\" | awk '{print $2}'", + capture_output=True, shell=True, + check=True).stdout.decode('utf-8').strip() + assert (rmd160 == PUBKEYRMD160 and sha1 == PUBKEYSHA1), 'Invalid MacPorts key' + sha1 = subprocess.run('openssl dgst -ripemd160 ' + f'-verify "{dst_key_filename}" ' + f'-signature "{sig_path}" "{pkg_path}"', + shell=True, check=True) + + def install_pkg(self, pkg_name): + if self.is_pkg_installed(pkg_name): + return + + print(f'[*] Fetching {pkg_name}') + pkg_filename, pkg_url = self.get_latest_pkg_filename_url(pkg_name) + pkg_version = re.match(r'^[\w_]+-([\w\.\-\_\+]*?)\.' + self._darwin_target, pkg_filename).groups(1)[0] + dst_pkg_filename = os.path.join(self._pkgs_path, pkg_filename) + print(f' [*] Found package {pkg_filename}') + self.download_file(pkg_filename, pkg_url, dst_pkg_filename) + + dst_pkg_sig_filename = dst_pkg_filename + '.rmd160' + pkg_sig_url = pkg_url + '.rmd160' + self.download_file('package signature', pkg_sig_url, dst_pkg_sig_filename) + + print(f' [+] Verifying package') + self.verify_pkg(dst_pkg_filename, dst_pkg_sig_filename) + + print(f' [+] Looking for dependencies') + tb = TarFile.open(dst_pkg_filename) + pkg_contents_file = tb.extractfile('./+CONTENTS').read().decode('utf-8') + for dep in re.findall(r'@pkgdep (.+)', pkg_contents_file): + print(f' [>] {dep}') + dep = dep.split('-')[0] + self._queue.append(dep) + for dep in re.findall(r'@pkgdep (.+)', pkg_contents_file): + print(f' [>] {dep}') + dep = dep.split('-')[0] + self._queue.append(dep) + + print(f' [*] Checking tarball...') + + for fpath in tb.getnames(): + extracted_path = os.path.realpath(os.path.join(self._extract_path, fpath)) + assert extracted_path.startswith(self._extract_path), f'tarball has a global file: {fname}' + + print(f' [*] Extracting to {self._extract_path}') + tb.extractall(self._extract_path, numeric_owner=True) + + for fpath in tb.getnames(): + extracted_path = os.path.realpath(os.path.join(self._extract_path, fpath)) + if extracted_path.endswith('.pc'): + print(f' [*] Fixing {extracted_path}') + with open(extracted_path, 'r') as f: + lines = f.readlines() + for i, l in enumerate(lines): + if l.strip().startswith('prefix'): + lines[i] = f'prefix={self._extract_path}/opt/local\n' + break + with open(extracted_path, 'w') as f: + f.write(''.join(lines)) + + if pkg_name == 'glib2': + fpath = './opt/local/include/glib-2.0/glib/gi18n.h' + extracted_path = os.path.realpath(os.path.join(self._extract_path, fpath)) + print(f' [*] Fixing {extracted_path}') + with open(extracted_path, 'r') as f: + lines = f.read() + s = '/opt/local/include/libintl.h' + lines = lines.replace(s, self._extract_path + s) + with open(extracted_path, 'w') as f: + f.write(lines) + + self.mark_pkg_installed(pkg_name, pkg_version) + + def install_pkgs(self, requested): + self._queue.extend(requested) + while len(self._queue) > 0: + pkg_name = self._queue.pop(0) + self.install_pkg(pkg_name) + +def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument('arch', choices=('arm64', 'x86_64')) + args = ap.parse_args() + li = LibInstaller(args.arch) + li.install_pkgs([ + 'libsdl2', + 'glib2', + 'libsamplerate', + 'libpixman', + 'libepoxy', + 'openssl10']) + +if __name__ == '__main__': + main() diff --git a/scripts/gen-license.py b/scripts/gen-license.py index 491be507ea..a146ad3698 100755 --- a/scripts/gen-license.py +++ b/scripts/gen-license.py @@ -29,6 +29,7 @@ linux = 'linux' all_platforms = { windows, macos, linux } current_platform = linux +versions = {} def banner(s): space = 1 @@ -81,9 +82,7 @@ class Lib: check=True).stdout.decode('utf-8').strip() return self._version elif current_platform == macos and self.pkg_mac: - self._version = subprocess.run(r"brew info " + self.pkg_mac + " | head -n1", - capture_output=True, shell=True, - check=True).stdout.decode('utf-8').strip() + self._version = versions[self.pkg_mac] return self._version elif current_platform == linux and self.pkg_ubuntu: self._version = subprocess.run(r"dpkg -s " + self.pkg_ubuntu + " | grep Version | cut -d: -f2", @@ -246,15 +245,15 @@ Lib('pcre', 'http://pcre.org/', # glib dep Lib('gettext', 'https://www.gnu.org/software/gettext/', lgplv2_1, 'https://git.savannah.gnu.org/gitweb/?p=gettext.git;a=blob_plain;f=gettext-runtime/intl/COPYING.LIB;hb=HEAD', - ships_static={windows}, ships_dynamic={macos}, platform={windows}, + ships_static={windows}, ships_dynamic={macos}, pkg_win='gettext', pkg_mac='gettext', ), # glib dep Lib('iconv', 'https://www.gnu.org/software/libiconv/', lgplv2_1, 'https://git.savannah.gnu.org/gitweb/?p=libiconv.git;a=blob_plain;f=COPYING.LIB;hb=HEAD', - ships_static={windows}, platform={windows}, - pkg_win='libiconv' + ships_static={windows}, ships_dynamic={macos}, + pkg_win='libiconv', pkg_mac='libiconv' ), Lib('libepoxy', 'https://github.com/anholt/libepoxy', @@ -292,7 +291,7 @@ Lib('openssl', 'https://www.openssl.org/', # openssl dep Lib('zlib', 'https://zlib.net/', zlib, 'https://raw.githubusercontent.com/madler/zlib/master/README', license_lines=(87,106), - ships_static={windows}, + ships_static={windows}, ships_dynamic={macos}, pkgconfig=PkgConfig('zlib'), pkg_win='zlib', pkg_mac='zlib', pkg_ubuntu='zlib1g-dev' ), @@ -304,7 +303,7 @@ Lib('libmingw32', 'http://mingw-w64.org/', Lib('gtk', 'https://www.gtk.org/', lgplv2_1, 'https://gitlab.gnome.org/GNOME/gtk/-/raw/master/COPYING', - platform=linux, + platform={linux}, pkgconfig=PkgConfig('gtk+-3.0'), pkg_ubuntu='libgtk-3-dev' ), ] @@ -344,12 +343,18 @@ def main(): import argparse ap = argparse.ArgumentParser() ap.add_argument('--platform', default='') + ap.add_argument('--version-file', default='') args = ap.parse_args() if args.platform == '': args.platform = sys.platform.lower() global current_platform current_platform = args.platform + if args.version_file != '': + with open(args.version_file, 'r') as f: + global versions + versions = {pkg: ver + for pkg, ver in map(lambda l: l.strip().split('='), f.readlines())} gen_license() if __name__ == '__main__':