From 9fb47d22b553f7f3ab79ebddb4ef5d632b02a7cd Mon Sep 17 00:00:00 2001 From: asarium Date: Fri, 11 Dec 2020 18:22:25 +0100 Subject: [PATCH] Extract build scripts into separate files This makes the workflows easier to manage, makes those build steps reusable, and allows for easier migration to other CI services should that be required at some point. --- .editorconfig | 4 ++ .github/workflows/build-dist.yaml | 87 +++++--------------------- .github/workflows/pr-workflow.yaml | 43 +------------ ci/linux/configure_cmake.sh | 35 +++++++++++ ci/linux/create_dist_pack.sh | 18 ++++++ ci/{github => linux}/dist_functions.sh | 0 ci/linux/generate_appimage.sh | 12 ++++ ci/linux/run_tests.sh | 13 ++++ ci/linux/upload_dist_package.sh | 12 ++++ ci/{travis => linux}/valgrind.supp | 0 10 files changed, 112 insertions(+), 112 deletions(-) create mode 100755 ci/linux/configure_cmake.sh create mode 100755 ci/linux/create_dist_pack.sh rename ci/{github => linux}/dist_functions.sh (100%) mode change 100644 => 100755 create mode 100755 ci/linux/generate_appimage.sh create mode 100755 ci/linux/run_tests.sh create mode 100755 ci/linux/upload_dist_package.sh rename ci/{travis => linux}/valgrind.supp (100%) diff --git a/.editorconfig b/.editorconfig index 50073de620..1fc5a82f75 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,3 +9,7 @@ indent_style = tab [*.{yml,yaml}] indent_style = space indent_size = 2 + +[*.sh] +indent_style = space +indent_size = 4 diff --git a/.github/workflows/build-dist.yaml b/.github/workflows/build-dist.yaml index 7fecb12816..b5783b73cc 100644 --- a/.github/workflows/build-dist.yaml +++ b/.github/workflows/build-dist.yaml @@ -39,38 +39,7 @@ jobs: env: CONFIGURATION: ${{ matrix.configuration }} COMPILER: gcc-5 - shell: bash - run: | - if [ "$COMPILER" = "gcc-5" ]; then - export CC=gcc-5 - export CXX=g++-5 - fi - if [ "$COMPILER" = "gcc-9" ]; then - export CC=gcc-9 - export CXX=g++-9 - fi - if [ "$COMPILER" = "clang-4.0" ]; then - export CC=clang-4.0 - export CXX=clang++-4.0 - fi - - LD_LIBRARY_PATH=$Qt5_DIR/lib:$LD_LIBRARY_PATH - CXXFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" - CFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" - - if [[ "$CC" =~ ^clang.*$ ]]; then - CXXFLAGS="$CXXFLAGS -stdlib=libc++" - fi - - mkdir build - cd build - - export CXXFLAGS - export CFLAGS - export LD_LIBRARY_PATH - CMAKE="cmake -G Ninja -DFSO_FATAL_WARNINGS=ON $CMAKE_OPTIONS" - eval $CMAKE -DFSO_INSTALL_DEBUG_FILES=ON -DCMAKE_BUILD_TYPE=$CONFIGURATION -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/build/install" \ - -DFSO_BUILD_APPIMAGE=ON -DFSO_BUILD_INCLUDED_LIBS=ON -DFFMPEG_USE_PRECOMPILED=ON -DFSO_BUILD_QTFRED=OFF -DFSO_BUILD_TESTS=ON .. + run: $GITHUB_WORKSPACE/ci/linux/configure_cmake.sh - name: Compile working-directory: ./build env: @@ -78,22 +47,16 @@ jobs: run: | LD_LIBRARY_PATH=$Qt5_DIR/lib:$LD_LIBRARY_PATH ninja -k 20 all - name: Run Tests - working-directory: ./build/bin - run: | - LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH ./unittests --gtest_shuffle + working-directory: ./build + env: + CONFIGURATION: ${{ matrix.configuration }} + XDG_RUNTIME_DIR: /root + run: $GITHUB_WORKSPACE/ci/linux/run_tests.sh - name: Generate AppImage working-directory: ./build env: CONFIGURATION: ${{ matrix.configuration }} - run: | - # Install Freespace2 targets - cmake -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/build/install -DCOMPONENT=Unspecified -P cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/build/install -DCOMPONENT=Freespace2 -P cmake_install.cmake - - # We need to be a bit creative for determining the AppImage name since we don't want to hard-code the name - FILENAME="$(find $GITHUB_WORKSPACE/build/install/bin -name 'fs2_open_*' -type f -printf "%f\n").AppImage" - appimagetool -n $GITHUB_WORKSPACE/build/install "$GITHUB_WORKSPACE/build/install/$FILENAME" - ls -al $GITHUB_WORKSPACE/build/install + run: $GITHUB_WORKSPACE/ci/linux/generate_appimage.sh $GITHUB_WORKSPACE/build/install - name: Upload build result uses: actions/upload-artifact@v2 with: @@ -122,24 +85,15 @@ jobs: path: builds - name: Create Distribution package working-directory: ./builds - shell: bash - run: | - . $GITHUB_WORKSPACE/ci/github/dist_functions.sh - ls -alR - tar -cvzf "$(get_package_name)-builds-Linux.tar.gz" * - ls -alR + run: $GITHUB_WORKSPACE/ci/linux/create_dist_pack.sh Linux - name: Upload result package working-directory: ./builds - shell: bash env: + INDIEGAMES_USER: ${{ secrets.INDIEGAMES_USER }} INDIEGAMES_SSHPASS: ${{ secrets.INDIEGAMES_PASSWORD }} + DATACORDER_USER: ${{ secrets.DATACORDER_USER }} DATACORDER_SSHPASS: ${{ secrets.DATACORDER_PASSWORD }} - run: | - . $GITHUB_WORKSPACE/ci/github/dist_functions.sh - - SSHPASS=$INDIEGAMES_SSHPASS upload_files_to_sftp "${{ secrets.INDIEGAMES_USER }}@scp.indiegames.us" "public_html/builds/nightly" - - SSHPASS=$DATACORDER_SSHPASS upload_files_to_sftp "${{ secrets.DATACORDER_USER }}@porphyrion.feralhosting.com" "/www/datacorder.porphyrion.feralhosting.com/public_html/builds/nightly" + run: $GITHUB_WORKSPACE/ci/linux/upload_dist_package.sh build_windows: strategy: @@ -245,24 +199,13 @@ jobs: env: ARCH: ${{ matrix.arch }} SIMD: ${{ matrix.simd }} - run: | - . $GITHUB_WORKSPACE/ci/github/dist_functions.sh - ls -alR - - 7z a -xr'!*.pdb' "$(get_package_name)-builds-$ARCH-$SIMD.zip" "*" - - 7z a "$(get_package_name)-debug-$ARCH-$SIMD.7z" "*.pdb" - - ls -alR + run: $GITHUB_WORKSPACE/ci/linux/create_dist_pack.sh Windows - name: Upload result package working-directory: ./builds shell: bash env: + INDIEGAMES_USER: ${{ secrets.INDIEGAMES_USER }} INDIEGAMES_SSHPASS: ${{ secrets.INDIEGAMES_PASSWORD }} + DATACORDER_USER: ${{ secrets.DATACORDER_USER }} DATACORDER_SSHPASS: ${{ secrets.DATACORDER_PASSWORD }} - run: | - . $GITHUB_WORKSPACE/ci/github/dist_functions.sh - - SSHPASS=$INDIEGAMES_SSHPASS upload_files_to_sftp "${{ secrets.INDIEGAMES_USER }}@scp.indiegames.us" "public_html/builds/nightly" - - SSHPASS=$DATACORDER_SSHPASS upload_files_to_sftp "${{ secrets.DATACORDER_USER }}@porphyrion.feralhosting.com" "/www/datacorder.porphyrion.feralhosting.com/public_html/builds/nightly" + run: $GITHUB_WORKSPACE/ci/linux/upload_dist_package.sh diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index ee9ff78f3f..9c44347714 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -44,43 +44,11 @@ jobs: submodules: true fetch-depth: 0 - name: Configure CMake - shell: bash env: CONFIGURATION: ${{ matrix.configuration }} COMPILER: ${{ matrix.compiler }} JOB_CMAKE_OPTIONS: ${{ matrix.cmake_options }} - run: | - if [ "$COMPILER" = "gcc-5" ]; then - export CC=gcc-5 - export CXX=g++-5 - fi - if [ "$COMPILER" = "gcc-10" ]; then - export CC=gcc-10 - export CXX=g++-10 - fi - if [ "$COMPILER" = "clang-9" ]; then - # Work around bug in clang that uses the wrong installation directory: https://bugs.llvm.org/show_bug.cgi?id=47460 - export CC=$(readlink -f $(which clang-9)) - export CXX=$(readlink -f $(which clang++-9)) - fi - - LD_LIBRARY_PATH=$Qt5_DIR/lib:$LD_LIBRARY_PATH - CXXFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" - CFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" - - CMAKE_OPTIONS="$JOB_CMAKE_OPTIONS" - if [[ "$COMPILER" =~ ^clang.*$ ]]; then - CMAKE_OPTIONS="$CMAKE_OPTIONS -DCLANG_USE_LIBCXX=ON" - fi - - mkdir build - cd build - - export CXXFLAGS - export CFLAGS - export LD_LIBRARY_PATH - CMAKE="cmake -G Ninja -DFSO_FATAL_WARNINGS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $CMAKE_OPTIONS" - eval $CMAKE -DCMAKE_BUILD_TYPE=$CONFIGURATION -DFSO_BUILD_TESTS=ON -DFSO_BUILD_INCLUDED_LIBS=ON -DFSO_BUILD_QTFRED=ON .. + run: $GITHUB_WORKSPACE/ci/linux/configure_cmake.sh - name: Compile working-directory: ./build run: LD_LIBRARY_PATH=$Qt5_DIR/lib:$LD_LIBRARY_PATH ninja -k 20 all @@ -89,13 +57,7 @@ jobs: env: CONFIGURATION: ${{ matrix.configuration }} XDG_RUNTIME_DIR: /root - run: | - if [ "$CONFIGURATION" = "Debug" ]; then - valgrind --leak-check=full --error-exitcode=1 --gen-suppressions=all \ - --suppressions="$GITHUB_WORKSPACE/ci/travis/valgrind.supp" ./bin/unittests --gtest_shuffle - else - ./bin/unittests --gtest_shuffle - fi + run: $GITHUB_WORKSPACE/ci/linux/run_tests.sh - name: Run Clang Tidy # Clang-tidy reuses the precompiled headers so this only makes sense for the clang compilers if: startsWith(matrix.compiler, 'clang-') @@ -105,6 +67,7 @@ jobs: uses: asarium/clang-tidy-action@v1 with: fixesFile: clang-fixes.yaml + build_windows: strategy: matrix: diff --git a/ci/linux/configure_cmake.sh b/ci/linux/configure_cmake.sh new file mode 100755 index 0000000000..3d449b5318 --- /dev/null +++ b/ci/linux/configure_cmake.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +if [ "$COMPILER" = "gcc-5" ]; then + export CC=gcc-5 + export CXX=g++-5 +fi +if [ "$COMPILER" = "gcc-10" ]; then + export CC=gcc-10 + export CXX=g++-10 +fi +if [ "$COMPILER" = "clang-9" ]; then + # Work around bug in clang that uses the wrong installation directory: https://bugs.llvm.org/show_bug.cgi?id=47460 + export CC=$(readlink -f $(which clang-9)) + export CXX=$(readlink -f $(which clang++-9)) +fi + +LD_LIBRARY_PATH=$Qt5_DIR/lib:$LD_LIBRARY_PATH +CXXFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" +CFLAGS="-m64 -mtune=generic -mfpmath=sse -msse -msse2 -pipe -Wno-unknown-pragmas" + +CMAKE_OPTIONS="$JOB_CMAKE_OPTIONS" +if [[ "$COMPILER" =~ ^clang.*$ ]]; then + CMAKE_OPTIONS="$CMAKE_OPTIONS -DCLANG_USE_LIBCXX=ON" +fi + +mkdir build +cd build + +export CXXFLAGS +export CFLAGS +export LD_LIBRARY_PATH + +cmake -G Ninja -DFSO_FATAL_WARNINGS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON $CMAKE_OPTIONS \ + -DCMAKE_INSTALL_PREFIX="$(pwd)/install" -DFSO_BUILD_APPIMAGE=ON -DCMAKE_BUILD_TYPE=$CONFIGURATION \ + -DFFMPEG_USE_PRECOMPILED=ON -DFSO_BUILD_TESTS=ON -DFSO_BUILD_INCLUDED_LIBS=ON -DFSO_BUILD_QTFRED=ON .. diff --git a/ci/linux/create_dist_pack.sh b/ci/linux/create_dist_pack.sh new file mode 100755 index 0000000000..11e0e30e87 --- /dev/null +++ b/ci/linux/create_dist_pack.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +SCRIPT=$(readlink -f "$0") +HERE=$(dirname "$SCRIPT") + +OS="$1" + +source $HERE/dist_functions.sh + +if [ "$OS" = "Linux" ]; then + tar -cvzf "$(get_package_name)-builds-Linux.tar.gz" * +elif [ "$OS" = "Windows" ]; then + 7z a -xr'!*.pdb' "$(get_package_name)-builds-$ARCH-$SIMD.zip" "*" + + 7z a "$(get_package_name)-debug-$ARCH-$SIMD.7z" "*.pdb" +else + echo "Invalid OS: $OS" +fi diff --git a/ci/github/dist_functions.sh b/ci/linux/dist_functions.sh old mode 100644 new mode 100755 similarity index 100% rename from ci/github/dist_functions.sh rename to ci/linux/dist_functions.sh diff --git a/ci/linux/generate_appimage.sh b/ci/linux/generate_appimage.sh new file mode 100755 index 0000000000..3dbf0251b6 --- /dev/null +++ b/ci/linux/generate_appimage.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +INSTALL_FOLDER=$1 + +# Install Freespace2 targets +cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER -DCOMPONENT=Unspecified -P cmake_install.cmake +cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_FOLDER -DCOMPONENT=Freespace2 -P cmake_install.cmake + +# We need to be a bit creative for determining the AppImage name since we don't want to hard-code the name +FILENAME="$(find $INSTALL_FOLDER/bin -name 'fs2_open_*' -type f -printf "%f\n").AppImage" +appimagetool -n $INSTALL_FOLDER "$INSTALL_FOLDER/$FILENAME" +ls -al $INSTALL_FOLDER diff --git a/ci/linux/run_tests.sh b/ci/linux/run_tests.sh new file mode 100755 index 0000000000..4b9670986d --- /dev/null +++ b/ci/linux/run_tests.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SCRIPT=$(readlink -f "$0") +HERE=$(dirname "$SCRIPT") + +export LD_LIBRARY_PATH=$(pwd)/bin/lib:$LD_LIBRARY_PATH + +if [ "$CONFIGURATION" = "Debug" ]; then + valgrind --leak-check=full --error-exitcode=1 --gen-suppressions=all \ + --suppressions="$HERE/valgrind.supp" ./bin/unittests --gtest_shuffle +else + ./bin/unittests --gtest_shuffle +fi diff --git a/ci/linux/upload_dist_package.sh b/ci/linux/upload_dist_package.sh new file mode 100755 index 0000000000..0de54fd57b --- /dev/null +++ b/ci/linux/upload_dist_package.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT=$(readlink -f "$0") +HERE=$(dirname "$SCRIPT") + +source $HERE/dist_functions.sh + +SSHPASS=$INDIEGAMES_SSHPASS upload_files_to_sftp "$INDIEGAMES_USER@scp.indiegames.us" "public_html/builds/nightly" + +SSHPASS=$DATACORDER_SSHPASS upload_files_to_sftp "$DATACORDER_USER@porphyrion.feralhosting.com" "www/datacorder.porphyrion.feralhosting.com/public_html/builds/nightly" diff --git a/ci/travis/valgrind.supp b/ci/linux/valgrind.supp similarity index 100% rename from ci/travis/valgrind.supp rename to ci/linux/valgrind.supp