From 8628993fb0ac204a83cc605494b7104b77f33d10 Mon Sep 17 00:00:00 2001 From: Margen67 Date: Fri, 7 May 2021 03:45:52 -1000 Subject: [PATCH] Build improvements Remove unneeded quotation marks. linux-workflow.yml: Checkout Submodules: Shorten --jobs to -j, use $(getconf _NPROCESSORS_ONLN). Make restore-keys into one line. Remove unneeded ./. macos-workflow.yml: Checkout Submodules: Shorten --jobs to -j. Use $(getconf _NPROCESSORS_ONLN) instead of 2. Move HOMEBREW_NO_INSTALL_CLEANUP to env: and add HOMEBREW_NO_ANALYTICS. Remove unneeded ./. pr-triage.yml: Make on: pull_request_target into one line. Properly capitalize GitHub. compile.sh: Add warning when running outside of GitHub Actions CI. Use $(getconf _NPROCESSORS_ONLN). validate-vs-filters.ps1: End of file newline. windows-workflow.yml: Rename config to configuration. Opt out of PowerShell telemetry. Checkout Submodules: Shorten --jobs to -j, use $env:NUMBER_OF_PROCESSORS. Remove unneeded shell from Verify VS Project Files. setup-msbuild: Use v1. (always the latest version) Remove useless vs-version parameter. Remove unneeded ./ and .\. Add -p to mkdir. .gitmodules: Make submodules shallow. build.sh: Move parameters from shebang to set. Consistent formatting for ifs. Use $(getconf _NPROCESSORS_ONLN). --- .github/workflows/lint-gamedb.yml | 6 +- .github/workflows/linux-workflow.yml | 64 ++++++++++---- .github/workflows/macos-workflow.yml | 71 ++++++++++++---- .github/workflows/pr-triage.yml | 11 +-- .github/workflows/scripts/linux/compile.sh | 4 + .../scripts/windows/validate-vs-filters.ps1 | 2 +- .github/workflows/windows-workflow.yml | 84 +++++++++++++------ .gitmodules | 5 ++ build.sh | 35 ++++---- 9 files changed, 195 insertions(+), 87 deletions(-) diff --git a/.github/workflows/lint-gamedb.yml b/.github/workflows/lint-gamedb.yml index 3479ed4c5..1fef2977d 100644 --- a/.github/workflows/lint-gamedb.yml +++ b/.github/workflows/lint-gamedb.yml @@ -5,12 +5,12 @@ on: branches: - master paths: - - "**/GameIndex.yaml" + - '**/GameIndex.yaml' pull_request: branches: - master paths: - - "**/GameIndex.yaml" + - '**/GameIndex.yaml' jobs: lint: @@ -21,7 +21,7 @@ jobs: - name: Validate GameDB env: - SCRIPT_DIR: "./.github/workflows/scripts/validation/lint-gamedb" + SCRIPT_DIR: .github/workflows/scripts/validation/lint-gamedb run: | pip install -r "${SCRIPT_DIR}/requirements.txt" python "${SCRIPT_DIR}/lint-gamedb.py" diff --git a/.github/workflows/linux-workflow.yml b/.github/workflows/linux-workflow.yml index 6c85d042f..083d1622a 100644 --- a/.github/workflows/linux-workflow.yml +++ b/.github/workflows/linux-workflow.yml @@ -7,22 +7,58 @@ on: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Windows/**" + - '**/*.md' + - '**/*.bat' + - '**/*.cmd' + - '**/*.props' + - '**/*.sln' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/macos-workflow.yml' + - '.github/workflows/pr-triage.yml' + - '.github/workflows/scripts/windows/**' + - '.github/workflows/scripts/validation/**' + - '.github/workflows/windows-workflow.yml' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'pcsx2/CDVD/Windows/**' + - 'pcsx2/DEV9/Win32/**' + - 'pcsx2/PAD/Windows/**' + - 'pcsx2/SPU2/Windows/**' + - 'pcsx2/USB/Win32/**' + - 'pcsx2/windows/**' pull_request: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Windows/**" + - '**/*.md' + - '**/*.bat' + - '**/*.cmd' + - '**/*.props' + - '**/*.sln' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/macos-workflow.yml' + - '.github/workflows/pr-triage.yml' + - '.github/workflows/scripts/windows/**' + - '.github/workflows/scripts/validation/**' + - '.github/workflows/windows-workflow.yml' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'pcsx2/CDVD/Windows/**' + - 'pcsx2/DEV9/Win32/**' + - 'pcsx2/PAD/Windows/**' + - 'pcsx2/SPU2/Windows/**' + - 'pcsx2/USB/Win32/**' + - 'pcsx2/windows/**' jobs: build: @@ -82,7 +118,7 @@ jobs: - name: Checkout Submodules if: steps.cache-submodules.outputs.cache-hit != 'true' - run: git submodule update --init --recursive --jobs 2 + run: git submodule update --init --recursive -j $(getconf _NPROCESSORS_ONLN) # -- SETUP CCACHE - https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - name: Prepare ccache timestamp @@ -139,7 +175,7 @@ jobs: run: .github/workflows/scripts/linux/generate-cmake.sh - name: Build PCSX2 - working-directory: ./build + working-directory: build run: ../.github/workflows/scripts/linux/compile.sh - name: Package AppImage diff --git a/.github/workflows/macos-workflow.yml b/.github/workflows/macos-workflow.yml index a16f4cc7a..f2803e7af 100644 --- a/.github/workflows/macos-workflow.yml +++ b/.github/workflows/macos-workflow.yml @@ -7,22 +7,60 @@ on: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Windows/**" + - '**/*.md' + - '**/*.bat' + - '**/*.cmd' + - '**/*.props' + - '**/*.sln' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/linux-workflow.yml' + - '.github/workflows/pr-triage.yml' + - '.github/workflows/scripts/windows/**' + - '.github/workflows/scripts/validation/**' + - '.github/workflows/windows-workflow.yml' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'debian-packager/**' + - 'pcsx2/CDVD/Windows/**' + - 'pcsx2/DEV9/Win32/**' + - 'pcsx2/PAD/Windows/**' + - 'pcsx2/SPU2/Windows/**' + - 'pcsx2/USB/Win32/**' + - 'pcsx2/windows/**' pull_request: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Windows/**" + - '**/*.md' + - '**/*.bat' + - '**/*.cmd' + - '**/*.props' + - '**/*.sln' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/linux-workflow.yml' + - '.github/workflows/pr-triage.yml' + - '.github/workflows/scripts/windows/**' + - '.github/workflows/scripts/validation/**' + - '.github/workflows/windows-workflow.yml' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'debian-packager/**' + - 'pcsx2/CDVD/Windows/**' + - 'pcsx2/DEV9/Win32/**' + - 'pcsx2/PAD/Windows/**' + - 'pcsx2/SPU2/Windows/**' + - 'pcsx2/USB/Win32/**' + - 'pcsx2/windows/**' jobs: build: @@ -53,13 +91,14 @@ jobs: uses: actions/checkout@v2 - name: Checkout Submodules - run: git submodule update --init --recursive --jobs 2 + run: git submodule update --init --recursive -j $(getconf _NPROCESSORS_ONLN) - name: Install Packages env: PLATFORM: ${{ matrix.platform }} + HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_ANALYTICS: 1 run: | - export HOMEBREW_NO_INSTALL_CLEANUP=1 brew update brew install sound-touch portaudio wxmac gtk+3 sdl2 libsamplerate brew install --cask xquartz # We use its OpenGL headers @@ -68,5 +107,5 @@ jobs: run: cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_PO=FALSE -B build . - name: Build PCSX2 - working-directory: ./build - run: make -j2 # macOS doesn't use make install + working-directory: build + run: make -j$(getconf _NPROCESSORS_ONLN) # macOS doesn't use make install diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index 71e1a0b16..ee507374f 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -1,10 +1,7 @@ # Runs steps to triage an incoming Pull Request, for example - applying labels. name: 🤔 Pull Request Triage -on: - pull_request_target: - branches: - - '*' +on: pull_request_target jobs: triage: @@ -12,18 +9,18 @@ jobs: steps: - uses: actions/labeler@main with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" + repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: xTVaser/first-interaction@v1.2.4 with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" + repo-token: ${{ secrets.GITHUB_TOKEN }} debug-mode: false pr-message: |- ## Thank you for submitting a contribution to PCSX2 As this is your first pull request, [please be aware of the contributing guidelines](https://github.com/PCSX2/pcsx2/blob/master/.github/CONTRIBUTING.md). - Additionally, as per recent changes in Github Actions, your pull request will need to be approved by a maintainer before Github Actions can run against it. [You can find more information about this change here.](https://github.blog/2021-04-22-github-actions-update-helping-maintainers-combat-bad-actors/) + Additionally, as per recent changes in GitHub Actions, your pull request will need to be approved by a maintainer before GitHub Actions can run against it. [You can find more information about this change here.](https://github.blog/2021-04-22-github-actions-update-helping-maintainers-combat-bad-actors/) Please be patient until this happens. In the meantime if you'd like to confirm the builds are passing, you have the option of opening a PR on your own fork, just make sure your fork's master branch is up to date! pr-labels: "First Time Contribution" diff --git a/.github/workflows/scripts/linux/compile.sh b/.github/workflows/scripts/linux/compile.sh index c0f805934..4a7201616 100755 --- a/.github/workflows/scripts/linux/compile.sh +++ b/.github/workflows/scripts/linux/compile.sh @@ -2,6 +2,10 @@ set -e +if [ -n "${GITHUB_ACTIONS}" ]; then + echo "Warning: Running this script outside of GitHub Actions isn't recommended." +fi + export CCACHE_BASEDIR=${GITHUB_WORKSPACE} export CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache export CCACHE_COMPRESS="true" diff --git a/.github/workflows/scripts/windows/validate-vs-filters.ps1 b/.github/workflows/scripts/windows/validate-vs-filters.ps1 index 09e7b43ba..42e099579 100644 --- a/.github/workflows/scripts/windows/validate-vs-filters.ps1 +++ b/.github/workflows/scripts/windows/validate-vs-filters.ps1 @@ -16,4 +16,4 @@ foreach ($file in $filterFiles) { if ($failed) { exit 1 -} \ No newline at end of file +} diff --git a/.github/workflows/windows-workflow.yml b/.github/workflows/windows-workflow.yml index 385df474e..5562b573b 100644 --- a/.github/workflows/windows-workflow.yml +++ b/.github/workflows/windows-workflow.yml @@ -7,22 +7,54 @@ on: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Linux/**" + - '**/*.md' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/linux-workflow.yml' + - '.github/workflows/macos-workflow.yml' + - '.github/workflows/scripts/linux/**' + - '.github/workflows/scripts/validation/**' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'debian-packager/**' + - 'linux_various/**' + - 'mscompile.cmd' + - 'pcsx2/CDVD/Linux/**' + - 'pcsx2/DEV9/Linux/**' + - 'pcsx2/Linux/**' + - 'pcsx2/PAD/Linux/**' + - 'pcsx2/SPU2/Linux/**' + - 'pcsx2/USB/linux/**' pull_request: branches: - master paths-ignore: - - .gitignore - - "**/*.md" - - .clang-format - - debian-packager/ - - bin/PCSX2_keys.ini.default - - "pcsx2/PAD/Linux/**" + - '**/*.md' + - '.clang-format' + - '.codacy.yaml' + - '.github/*' + - '.github/workflows/lint-gamedb.yml' + - '.github/workflows/linux-workflow.yml' + - '.github/workflows/macos-workflow.yml' + - '.github/workflows/scripts/linux/**' + - '.github/workflows/scripts/validation/**' + - '.gitignore' + - 'bin/PCSX2_keys.ini.default' + - 'build.sh' + - 'buildbot.xml' + - 'debian-packager/**' + - 'linux_various/**' + - 'mscompile.cmd' + - 'pcsx2/CDVD/Linux/**' + - 'pcsx2/DEV9/Linux/**' + - 'pcsx2/Linux/**' + - 'pcsx2/PAD/Linux/**' + - 'pcsx2/SPU2/Linux/**' + - 'pcsx2/USB/linux/**' workflow_dispatch: inputs: retainDebugArtifacts: @@ -38,52 +70,50 @@ jobs: matrix: os: [windows-2019] platform: [Win32, x64] - config: [Release, Release AVX2] + configuration: [Release, Release AVX2] experimental: [false] - name: "${{ matrix.platform }} | ${{ matrix.config }}" + name: ${{ matrix.platform }} | ${{ matrix.configuration }} runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.experimental }} # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. timeout-minutes: 30 + env: + POWERSHELL_TELEMETRY_OPTOUT: 1 + steps: # NOTE - useful for debugging # - name: Dump GitHub context # env: # GITHUB_CONTEXT: ${{ toJson(github) }} - # run: | - # echo "$GITHUB_CONTEXT" + # run: echo "$GITHUB_CONTEXT" - name: Checkout Repository uses: actions/checkout@v2 - name: Checkout Submodules if: steps.cache-submodules.outputs.cache-hit != 'true' - run: git submodule update --init --recursive --jobs 2 + run: git submodule update --init --recursive -j $env:NUMBER_OF_PROCESSORS - name: Verify VS Project Files - shell: powershell - run: .\.github\workflows\scripts\windows\validate-vs-filters.ps1 + run: .github\workflows\scripts\windows\validate-vs-filters.ps1 - name: Setup msbuild - uses: microsoft/setup-msbuild@v1.0.2 - with: - vs-version: 16.7 + uses: microsoft/setup-msbuild@v1 - name: Build PCSX2 env: # Set to 'true' to retain the .pdb / .exp / .lib, etc files which can be useful for repro'ing issues that only occur in the compiled .exe RetainDebuggingArtifacts: ${{ github.event.inputs.retainDebugArtifacts == 'true' }} - run: msbuild "PCSX2_suite.sln" /m /v:m /p:Configuration="${{ matrix.config }}" /p:Platform="${{ matrix.platform }}" + run: msbuild "PCSX2_suite.sln" /m /v:m /p:Configuration="${{ matrix.configuration }}" /p:Platform="${{ matrix.platform }}" - name: Prepare Artifact Metadata id: artifact-metadata shell: bash run: | ARCH=$([ "${{ matrix.platform }}" == Win32 ] && echo "32bit" || echo "64bit") - SIMD=$([ "${{ matrix.config }}" == Release ] && echo "SSE4" || echo "AVX2") - ARTIFACT_NAME="" + SIMD=$([ "${{ matrix.configuration }}" == Release ] && echo "SSE4" || echo "AVX2") if [ ${{ github.event_name }} == "pull_request" ]; then PR_SHA=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}") ARTIFACT_NAME="PCSX2-${ARCH}-${SIMD}" @@ -106,7 +136,7 @@ jobs: - name: Prepare Artifact Folder shell: bash - working-directory: ./bin + working-directory: bin run: | retainDebugArtifacts=$([ "${{ github.event.inputs.retainDebugArtifacts }}" == "true" ] && echo "true" || echo "false") if [ "${retainDebugArtifacts}" == "false" ]; then @@ -114,7 +144,7 @@ jobs: rm -f *.bsc *.exp *.ilk *.iobj *.ipdb *.lib *.pdb fi shopt -s extglob dotglob - mkdir "${{ steps.artifact-metadata.outputs.name }}" + mkdir -p "${{ steps.artifact-metadata.outputs.name }}" mv !("${{ steps.artifact-metadata.outputs.name }}") "${{ steps.artifact-metadata.outputs.name }}" shopt -u dotglob diff --git a/.gitmodules b/.gitmodules index 3b76fea7c..29aab7944 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,15 +1,20 @@ [submodule "3rdparty/xz/xz"] path = 3rdparty/xz/xz url = https://github.com/PCSX2/xz.git + shallow = true [submodule "3rdparty/gtest"] path = 3rdparty/gtest url = https://github.com/google/googletest.git + shallow = true [submodule "3rdparty/fmt/fmt"] path = 3rdparty/fmt/fmt url = https://github.com/fmtlib/fmt.git + shallow = true [submodule "3rdparty/yaml-cpp/yaml-cpp"] path = 3rdparty/yaml-cpp/yaml-cpp url = https://github.com/jbeder/yaml-cpp.git + shallow = true [submodule "3rdparty/libchdr/libchdr"] path = 3rdparty/libchdr/libchdr url = https://github.com/rtissera/libchdr.git + shallow = true diff --git a/build.sh b/build.sh index a23b5931d..919dc222d 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ -#!/bin/sh -u +#!/bin/sh # PCSX2 - PS2 Emulator for PCs # Copyright (C) 2002-2014 PCSX2 Dev Team @@ -15,25 +15,22 @@ # If not, see . #set -e # This terminates the script in case of any error +set -u # Function declarations set_ncpu_toolfile() { + ncpu=$(getconf _NPROCESSORS_ONLN) if [ "$(uname -s)" = 'Darwin' ]; then - ncpu="$(sysctl -n hw.ncpu)" - # Get the major Darwin/OSX version. if [ "$(sysctl -n kern.osrelease | cut -d . -f 1)" -lt 13 ]; then - echo "This old OSX version is not supported! Build will fail." - toolfile=cmake/darwin-compiler-i386-clang.cmake + echo "This old OSX version is not supported! Build will fail." + toolfile=cmake/darwin-compiler-i386-clang.cmake else - echo "Using Mavericks build with C++11 support." - toolfile=cmake/darwin13-compiler-i386-clang.cmake + echo "Using Mavericks build with C++11 support." + toolfile=cmake/darwin13-compiler-i386-clang.cmake fi - elif [ "$(uname -s)" = 'FreeBSD' ]; then - ncpu="$(sysctl -n hw.ncpu)" - else - ncpu=$(grep -w -c processor /proc/cpuinfo) + elif [ "$(uname -s)" != 'FreeBSD' ]; then toolfile=cmake/linux-compiler-i386-multilib.cmake fi } @@ -73,17 +70,17 @@ set_compiler() { if [ "$useClang" -eq 1 ]; then if [ "$useCross" -eq 0 ]; then - CC=clang CXX=clang++ cmake $flags "$root" 2>&1 | tee -a "$log" + CC=clang CXX=clang++ cmake $flags "$root" 2>&1 | tee -a "$log" else - CC="clang -m32" CXX="clang++ -m32" cmake $flags "$root" 2>&1 | tee -a "$log" + CC="clang -m32" CXX="clang++ -m32" cmake $flags "$root" 2>&1 | tee -a "$log" fi else if [ "$useIcc" -eq 1 ]; then - if [ "$useCross" -eq 0 ]; then - CC="icc" CXX="icpc" cmake $flags "$root" 2>&1 | tee -a "$log" - else - CC="icc -m32" CXX="icpc -m32" cmake $flags "$root" 2>&1 | tee -a "$log" - fi + if [ "$useCross" -eq 0 ]; then + CC="icc" CXX="icpc" cmake $flags "$root" 2>&1 | tee -a "$log" + else + CC="icc -m32" CXX="icpc -m32" cmake $flags "$root" 2>&1 | tee -a "$log" + fi else # Default compiler AKA GCC cmake $flags "$root" 2>&1 | tee -a "$log" @@ -113,7 +110,7 @@ run_cppcheck() log=cpp_check__${flat_d}.log rm -f "$log" - cppcheck $check -j $ncpu --platform=unix32 $define "$root/$d" 2>&1 | tee "$log" + cppcheck $check -j "$ncpu" --platform=unix32 "$define" "$root/$d" 2>&1 | tee "$log" # Create a small summary (warning it might miss some issues) fgrep -e "(warning)" -e "(error)" -e "(style)" -e "(performance)" -e "(portability)" "$log" >> $summary done