From 0d37525d753801a8429e13069b7ba82865541a09 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Thu, 25 Jun 2020 16:36:28 -0700 Subject: [PATCH] [vcpkg] Commonize pipelines build powershell and don't use binary caching in CI (rather than PR) builds (#12082) --- .../azure-pipelines/linux/azure-pipelines.yml | 26 ++--- .../azure-pipelines/osx/azure-pipelines.yml | 25 ++--- .../azure-pipelines/test-modified-ports.ps1 | 102 ++++++++++++++++++ .../windows/azure-pipelines.yml | 31 ++---- toolsrc/src/vcpkg/commands.ci.cpp | 27 +++-- 5 files changed, 139 insertions(+), 72 deletions(-) create mode 100755 scripts/azure-pipelines/test-modified-ports.ps1 diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml index 1e2a653e0..77842437f 100644 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -32,28 +32,16 @@ jobs: VCPKG_DOWNLOADS: '/mnt/vcpkg-ci/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - - powershell: | - $env:VCPKG_DOWNLOADS = '/mnt/vcpkg-ci/downloads' - $commonArgs = @( - '--binarycaching', - '--x-binarysource=clear;files,/archives,upload', - '--x-buildtrees-root=/mnt/vcpkg-ci/buildtrees', - '--x-install-root=/mnt/vcpkg-ci/install', - '--x-packages-root=/mnt/vcpkg-ci/packages' - ) - - ./vcpkg x-ci-clean @commonArgs - $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" - mkdir $(System.ArtifactsDirectory)/xml-results - $env:HOME = '/home/agent' - ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList @commonArgs - displayName: '** Test Modified Ports **' - task: PowerShell@2 - displayName: 'Analyze results and prepare test logs' + displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true - filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "/archives/fail" -triplets "x64-linux" -errorOnRegression' + filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' + arguments: '-Triplet x64-linux -BuildReason $(Build.Reason) -ArchivesRoot /archives -WorkingRoot /mnt/vcpkg-ci -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-linux.xml' + condition: always() - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index 85fe543c8..8008df45f 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -36,27 +36,16 @@ jobs: VCPKG_DOWNLOADS: '/Users/vagrant/Data/downloads' - bash: toolsrc/build.rel/vcpkg-test displayName: 'Run vcpkg tests' - - powershell: | - $env:VCPKG_DOWNLOADS = '/Users/vagrant/Data/downloads' - $commonArgs = @( - '--binarycaching', - '--x-binarysource=clear;files,/Users/vagrant/Data/archives,upload', - '--x-buildtrees-root=/Users/vagrant/Data/buildtrees', - '--x-install-root=/Users/vagrant/Data/installed', - '--x-packages-root=/Users/vagrant/Data/packages' - ) - - ./vcpkg x-ci-clean @commonArgs - $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" - mkdir $(System.ArtifactsDirectory)/xml-results - ./vcpkg ci x64-osx --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-osx.xml" --exclude=$skipList @commonArgs - displayName: '** Test Modified Ports **' - task: PowerShell@2 - displayName: 'Analyze results and prepare test logs' + displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true - filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir /Users/vagrant/Data/archives/fail -triplets "x64-osx" -errorOnRegression' + filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' + arguments: '-Triplet x64-osx -BuildReason $(Build.Reason) -ArchivesRoot /Users/vagrant/Data/archives -WorkingRoot /Users/vagrant/Data -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/x64-osx.xml' + condition: always() - bash: | df -h displayName: 'Report on Disk Space After Build' diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 new file mode 100755 index 000000000..e38115b32 --- /dev/null +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -0,0 +1,102 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +<# +.SYNOPSIS +Runs the 'Test Modified Ports' part of the vcpkg CI system for all platforms. + +.PARAMETER Triplet +The triplet to test. + +.PARAMETER ArchivesRoot +The location where the binary caching archives are stored. Shared across runs of this script. + +.PARAMETER WorkingRoot +The location used as scratch space for 'installed', 'packages', and 'buildtrees' vcpkg directories. + +.PARAMETER ArtifactStagingDirectory +The Azure Pipelines artifact staging directory. If not supplied, defaults to the current directory. + +.PARAMETER ArtifactsDirectory +The Azure Pipelines artifacts directory. If not supplied, defaults to the current directory. + +.PARAMETER BuildReason +The reason Azure Pipelines is running this script (controls whether Binary Caching is used). If not +supplied, binary caching will be used. +#> + +[CmdletBinding()] +Param( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string]$Triplet, + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $ArchivesRoot, + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + $WorkingRoot, + [ValidateNotNullOrEmpty()] + $ArtifactStagingDirectory = '.', + [ValidateNotNullOrEmpty()] + $ArtifactsDirectory = '.', + $BuildReason = $null +) + +if (-Not (Test-Path "triplets/$Triplet.cmake")) { + Write-Error "Incorrect triplet '$Triplet', please supply a valid triplet." +} + +$env:VCPKG_DOWNLOADS = Join-Path $WorkingRoot 'downloads' +$buildtreesRoot = Join-Path $WorkingRoot 'buildtrees' +$installRoot = Join-Path $WorkingRoot 'installed' +$packagesRoot = Join-Path $WorkingRoot 'packages' +$commonArgs = @( + "--x-buildtrees-root=$buildtreesRoot", + "--x-install-root=$installRoot", + "--x-packages-root=$packagesRoot" +) + +$binaryCaching = $false +if ([string]::IsNullOrWhiteSpace($BuildReason)) { + Write-Host 'Build reason not specified, defaulting to using binary caching.' + $binaryCaching = $true +} elseif ($BuildReason -eq 'PullRequest') { + Write-Host 'Build reason was Pull Request, using binary caching.' + $binaryCaching = $true +} + +if ($binaryCaching) { + $commonArgs += @( + '--binarycaching', + "--x-binarysource=clear;files,$ArchivesRoot,upload" + ) +} + +if ($Triplet -eq 'x64-linux') { + $env:HOME = '/home/agent' + $executableExtension = [string]::Empty +} elseif ($Triplet -eq 'x64-osx') { + $executableExtension = [string]::Empty +} else { + $executableExtension = '.exe' +} + +$xmlResults = Join-Path $ArtifactsDirectory 'xml-results' +mkdir $xmlResults +$xmlFile = Join-Path $xmlResults "$Triplet.xml" + +& "./vcpkg$executableExtension" x-ci-clean @commonArgs +$skipList = . "$PSScriptRoot/generate-skip-list.ps1" -Triplet $Triplet -BaselineFile "$PSScriptRoot/../ci.baseline.txt" + +# WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed. +# Install them so the CI succeeds: +if ($Triplet -in @('x64-uwp', 'arm64-windows', 'arm-uwp')) { + .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs +} + +& "./vcpkg$executableExtension" ci $Triplet --x-xunit=$xmlFile --exclude=$skipList @commonArgs +& "$PSScriptRoot/analyze-test-results.ps1" -logDir $xmlResults -outputDir $ArtifactStagingDirectory ` + -failureLogDir (Join-Path $ArchivesRoot 'fail') -triplets $Triplet -errorOnRegression ` + -baselineFile .\scripts\ci.baseline.txt diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 274df268f..0f2120746 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -38,33 +38,16 @@ jobs: ninja.exe -C build.x86.debug build.x86.debug\vcpkg-test.exe failOnStderr: true - - powershell: | - $env:VCPKG_DOWNLOADS='D:\downloads' - $commonArgs = @( - '--binarycaching', - '--x-binarysource=clear;files,W:\,upload', - '--x-buildtrees-root=D:\buildtrees', - '--x-install-root=D:\install', - '--x-packages-root=D:\packages' - ) - - .\vcpkg.exe x-ci-clean @commonArgs - $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile .\scripts\ci.baseline.txt - mkdir $(System.ArtifactsDirectory)\xml-results - # WORKAROUND: the x86-windows flavors of these are needed for all cross-compilation, but they are not auto-installed. - # Install them so the CI succeeds: - if ( "${{ parameters.triplet }}" -notmatch "x86-windows" -and "${{ parameters.triplet }}" -notmatch "x64-windows" ) { - .\vcpkg.exe install protobuf:x86-windows boost-build:x86-windows sqlite3:x86-windows @commonArgs - } - - .\vcpkg.exe ci ${{ parameters.triplet }} --x-xunit=`"$(System.ArtifactsDirectory)\xml-results\${{ parameters.triplet }}.xml`" --exclude=$skipList @commonArgs - displayName: '** Test Modified Ports **' - task: PowerShell@2 - displayName: 'Analyze results and prepare test logs' + displayName: '*** Test Modified Ports and Prepare Test Logs ***' inputs: failOnStderr: true - filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile .\scripts\ci.baseline.txt -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "W:\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"' + filePath: 'scripts/azure-pipelines/test-modified-ports.ps1' + arguments: '-Triplet ${{ parameters.triplet }} -BuildReason $(Build.Reason) -ArchivesRoot W:\ -WorkingRoot D:\ -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) -ArtifactsDirectory $(System.ArtifactsDirectory)' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(System.ArtifactsDirectory)/xml-results/${{ parameters.triplet }}.xml' + condition: always() - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs' inputs: diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 76f823795..af9611931 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -391,8 +391,14 @@ namespace vcpkg::Commands::CI System::print2(System::Color::warning, "Warning: Running ci without binary caching!\n"); } - auto binaryprovider = - create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + std::unique_ptr binaryproviderStorage; + if (args.binary_caching_enabled()) + { + binaryproviderStorage = + create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO); + } + + IBinaryProvider& binaryprovider = binaryproviderStorage ? *binaryproviderStorage : null_binary_provider(); const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); @@ -458,14 +464,13 @@ namespace vcpkg::Commands::CI return FullPackageSpec{spec, std::move(default_features)}; }); - auto split_specs = - find_unknown_ports_for_ci(paths, - exclusions_set, - provider, - var_provider, - all_default_full_specs, - purge_tombstones, - args.binary_caching_enabled() ? *binaryprovider : null_binary_provider()); + auto split_specs = find_unknown_ports_for_ci(paths, + exclusions_set, + provider, + var_provider, + all_default_full_specs, + purge_tombstones, + binaryprovider); PortFileProvider::MapPortFileProvider new_default_provider(split_specs->default_feature_provider); Dependencies::CreateInstallPlanOptions serialize_options; @@ -510,7 +515,7 @@ namespace vcpkg::Commands::CI { auto collection_timer = Chrono::ElapsedTimer::create_started(); auto summary = Install::perform( - action_plan, Install::KeepGoing::YES, paths, status_db, *binaryprovider, var_provider); + action_plan, Install::KeepGoing::YES, paths, status_db, binaryprovider, var_provider); auto collection_time_elapsed = collection_timer.elapsed(); // Adding results for ports that were built or pulled from an archive