diff --git a/docs/maintainers/control-files.md b/docs/maintainers/control-files.md index ea44c2b8a..43c1c70a7 100644 --- a/docs/maintainers/control-files.md +++ b/docs/maintainers/control-files.md @@ -173,7 +173,7 @@ Build-Depends: qt5 Feature: mpi Description: MPI functionality for VTK -Build-Depends: msmpi, hdf5[parallel] +Build-Depends: mpi, hdf5[parallel] Feature: python Description: Python functionality for VTK diff --git a/ports/msmpi/CONTROL b/ports/msmpi/CONTROL index baa44b8fe..9640f564b 100644 --- a/ports/msmpi/CONTROL +++ b/ports/msmpi/CONTROL @@ -1,4 +1,5 @@ Source: msmpi -Version: 10.0-2 +Version: 10.1 +Homepage: https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi Description: Microsoft MPI -Supports: windows \ No newline at end of file +Supports: windows diff --git a/ports/msmpi/portfile.cmake b/ports/msmpi/portfile.cmake index 10482cb0a..58c3661c7 100644 --- a/ports/msmpi/portfile.cmake +++ b/ports/msmpi/portfile.cmake @@ -1,23 +1,21 @@ -include(vcpkg_common_functions) +vcpkg_fail_port_install(ON_TARGET "Linux" "OSX" "UWP") -if(VCPKG_CMAKE_SYSTEM_NAME) - message(FATAL_ERROR "This port is only for building msmpi on Windows Desktop") -endif() - -set(MSMPI_VERSION "10.0.12498") +set(MSMPI_VERSION "10.1.12498") set(SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src/msmpi-${MSMPI_VERSION}) vcpkg_download_distfile(SDK_ARCHIVE - URLS "https://download.microsoft.com/download/A/E/0/AE002626-9D9D-448D-8197-1EA510E297CE/msmpisdk.msi" + URLS "https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisdk.msi" FILENAME "msmpisdk-${MSMPI_VERSION}.msi" - SHA512 36a31b2516f45fbc26167b31d2d6419f1928aef1591033f0430d36570159205e1a3134557a4ac0462f2d879add1fc6fee87a6997032e4438b528cd42a8bbe6b1 + SHA512 330fad53c1979dfae786abed228d82c82207f0d61148e2efc1f37c8931838b806eb50554e7f006a56962f748a21f7017c540bd70444b7a93a72c313a64e9254c ) + +#to enable CI, you should modify the following URL also in ${VCPKG_ROOT}/scripts/azure-pipelines/windows/provision-image.ps1 macro(download_msmpi_redistributable_package) vcpkg_download_distfile(REDIST_ARCHIVE - URLS "https://download.microsoft.com/download/A/E/0/AE002626-9D9D-448D-8197-1EA510E297CE/msmpisetup.exe" + URLS "https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe" FILENAME "msmpisetup-${MSMPI_VERSION}.exe" - SHA512 c272dc842eb1e693f25eb580e1caf0c1fdb385611a12c20cdc6a40cf592ccbdba434a1c16edb63eef14b1a2ac6e678ac1cd561ec5fd003a5d17191a0fad281ae + SHA512 1ee463e7dfc3e55a7ac048fdfde13fef09a5eea4b74d8fd7c22a7aad667a025b467ce939e5de308e25bbc186c3fe66e0e24ac03a3741656fc7558f2af2fa132a ) endmacro() diff --git a/ports/openmpi/CONTROL b/ports/openmpi/CONTROL index 86af5ffd5..4da1df53e 100644 --- a/ports/openmpi/CONTROL +++ b/ports/openmpi/CONTROL @@ -1,5 +1,5 @@ Source: openmpi -Version: 4.0.1 +Version: 4.0.3 Homepage: https://www.open-mpi.org/ Description: The Open MPI Project is an open source Message Passing Interface implementation that is developed and maintained by a consortium of academic, research, and industry partners. Open MPI is therefore able to combine the expertise, technologies, and resources from all across the High Performance Computing community in order to build the best MPI library available. Open MPI offers advantages for system and software vendors, application developers and computer science researchers. -Supports: !(windows|uwp) \ No newline at end of file +Supports: !(windows|uwp) diff --git a/ports/openmpi/portfile.cmake b/ports/openmpi/portfile.cmake index 9f69f9f9b..6dbb01127 100644 --- a/ports/openmpi/portfile.cmake +++ b/ports/openmpi/portfile.cmake @@ -1,100 +1,37 @@ -include(vcpkg_common_functions) - -if(VCPKG_CMAKE_SYSTEM_NAME STREQUAL "WindowsStore" OR NOT VCPKG_CMAKE_SYSTEM_NAME) - message(FATAL_ERROR "This port is only for openmpi on Unix-like systems") -endif() +vcpkg_fail_port_install(ON_TARGET "Windows" "UWP") vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY) -set(OpenMPI_FULL_VERSION "4.0.1") +set(OpenMPI_FULL_VERSION "4.0.3") set(OpenMPI_SHORT_VERSION "4.0") vcpkg_download_distfile(ARCHIVE URLS "https://download.open-mpi.org/release/open-mpi/v${OpenMPI_SHORT_VERSION}/openmpi-${OpenMPI_FULL_VERSION}.tar.gz" FILENAME "openmpi-${OpenMPI_FULL_VERSION}.tar.gz" - SHA512 760716974cb6b25ad820184622e1ee7926bc6fda87db6b574f76792bc1ca99522e52195866c14b7cb2df5a4981efdaf9f71d2c5533cc0e8e45c2c4b3b74cbacc + SHA512 23a9dfb7f4a63589b82f4e073a825550d3bc7e6b34770898325323ef4a28ed90b47576acaae6be427eb2007b37a88e18c1ea44d929b8ca083fe576ef1111fef6 ) -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - list(APPEND BUILD_TYPES "release") -endif() -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - list(APPEND BUILD_TYPES "debug") -endif() - -set(SOURCE_PATH_DEBUG ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-debug/openmpi-${OpenMPI_FULL_VERSION}) -set(SOURCE_PATH_RELEASE ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-release/openmpi-${OpenMPI_FULL_VERSION}) -set(OUT_PATH_DEBUG ${SOURCE_PATH_RELEASE}/../../make-build-${TARGET_TRIPLET}-debug) -set(OUT_PATH_RELEASE ${SOURCE_PATH_RELEASE}/../../make-build-${TARGET_TRIPLET}-release) -file(REMOVE_RECURSE ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-debug/) -file(REMOVE_RECURSE ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-release/) -file(MAKE_DIRECTORY ${OUT_PATH_DEBUG}) -file(MAKE_DIRECTORY ${OUT_PATH_RELEASE}) - -foreach(BUILD_TYPE IN LISTS BUILD_TYPES) - vcpkg_extract_source_archive(${ARCHIVE} ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-${BUILD_TYPE}) - #vcpkg_apply_patches(SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src-${TARGET_TRIPLET}-${BUILD_TYPE}/openmpi-${OpenMPI_FULL_VERSION} PATCHES patch.file) -endforeach() +vcpkg_extract_source_archive_ex( + OUT_SOURCE_PATH SOURCE_PATH + ARCHIVE ${ARCHIVE} +) vcpkg_find_acquire_program(PERL) get_filename_component(PERL_PATH ${PERL} DIRECTORY) vcpkg_add_to_path(${PERL_PATH}) -set(BASH bash) +vcpkg_configure_make( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS + --with-hwloc=internal + --with-libevent=internal + OPTIONS_DEBUG + --enable-debug +) -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - message(STATUS "Configuring ${TARGET_TRIPLET}-dbg") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "${SOURCE_PATH_DEBUG}/configure --prefix=${OUT_PATH_DEBUG} --enable-debug" - WORKING_DIRECTORY "${SOURCE_PATH_DEBUG}" - LOGNAME "config-${TARGET_TRIPLET}-dbg" - ) - message(STATUS "Building ${TARGET_TRIPLET}-dbg") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "make -j ${VCPKG_CONCURRENCY}" - NO_PARALLEL_COMMAND ${BASH} --noprofile --norc -c "make" - WORKING_DIRECTORY "${SOURCE_PATH_DEBUG}" - LOGNAME "make-build-${TARGET_TRIPLET}-dbg" - ) - message(STATUS "Installing ${TARGET_TRIPLET}-dbg") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "make install" - WORKING_DIRECTORY "${SOURCE_PATH_DEBUG}" - LOGNAME "make-install-${TARGET_TRIPLET}-dbg" - ) - file(COPY ${OUT_PATH_DEBUG}/lib DESTINATION ${CURRENT_PACKAGES_DIR}/debug) - message(STATUS "Installing ${TARGET_TRIPLET}-dbg done") -endif() +vcpkg_install_make() -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - message(STATUS "Configuring ${TARGET_TRIPLET}-rel") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "${SOURCE_PATH_RELEASE}/configure --prefix=${OUT_PATH_RELEASE}" - WORKING_DIRECTORY "${SOURCE_PATH_RELEASE}" - LOGNAME "config-${TARGET_TRIPLET}-rel" - ) - message(STATUS "Building ${TARGET_TRIPLET}-rel") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "make -j ${VCPKG_CONCURRENCY}" - NO_PARALLEL_COMMAND ${BASH} --noprofile --norc -c "make" - WORKING_DIRECTORY "${SOURCE_PATH_RELEASE}" - LOGNAME "make-build-${TARGET_TRIPLET}-rel" - ) - message(STATUS "Installing ${TARGET_TRIPLET}-rel") - vcpkg_execute_required_process( - COMMAND ${BASH} --noprofile --norc -c "make install" - WORKING_DIRECTORY "${SOURCE_PATH_RELEASE}" - LOGNAME "make-install-${TARGET_TRIPLET}-rel" - ) - file(COPY ${OUT_PATH_RELEASE}/lib DESTINATION ${CURRENT_PACKAGES_DIR}) - file(COPY ${OUT_PATH_RELEASE}/include DESTINATION ${CURRENT_PACKAGES_DIR}) - file(COPY ${OUT_PATH_RELEASE}/share DESTINATION ${CURRENT_PACKAGES_DIR}) - file(COPY ${OUT_PATH_RELEASE}/bin DESTINATION ${CURRENT_PACKAGES_DIR}) - message(STATUS "Installing ${TARGET_TRIPLET}-rel done") -endif() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(INSTALL ${SOURCE_PATH_DEBUG}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/openmpi RENAME copyright) -else() - file(INSTALL ${SOURCE_PATH_RELEASE}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/openmpi RENAME copyright) -endif() +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/sdl2/CONTROL b/ports/sdl2/CONTROL index e20991fb6..f9bdadb69 100644 --- a/ports/sdl2/CONTROL +++ b/ports/sdl2/CONTROL @@ -1,5 +1,5 @@ Source: sdl2 -Version: 2.0.12 +Version: 2.0.12-1 Homepage: https://www.libsdl.org/download-2.0.php Description: Simple DirectMedia Layer is a cross-platform development library designed to provide low level access to audio, keyboard, mouse, joystick, and graphics hardware via OpenGL and Direct3D. diff --git a/ports/sdl2/fix-EventToken-header-reference.patch b/ports/sdl2/fix-EventToken-header-reference.patch new file mode 100644 index 000000000..5e34dc2f2 --- /dev/null +++ b/ports/sdl2/fix-EventToken-header-reference.patch @@ -0,0 +1,17 @@ + +diff -r 398b67c0a7d3 -r 7c2f418f701b src/video/winrt/SDL_winrtgamebar.cpp +--- a/src/video/winrt/SDL_winrtgamebar.cpp Wed Jun 10 09:36:10 2020 -0700 ++++ b/src/video/winrt/SDL_winrtgamebar.cpp Wed Jun 10 09:38:43 2020 -0700 +@@ -25,7 +25,7 @@ + /* Windows includes */ + #include + #include +-#include ++#include + + + /* SDL includes */ + + + + diff --git a/ports/sdl2/portfile.cmake b/ports/sdl2/portfile.cmake index 4bed5c0a6..366ad1e3a 100644 --- a/ports/sdl2/portfile.cmake +++ b/ports/sdl2/portfile.cmake @@ -14,6 +14,7 @@ vcpkg_extract_source_archive_ex( disable-hidapi-for-uwp.patch fix-space-in-path.patch disable-wcslcpy-and-wcslcat-for-windows.patch + fix-EventToken-header-reference.patch ) string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" SDL_STATIC) diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index f97a3a01e..7efc2df73 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,52 +2,88 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2020-04-28' - linux-pool: 'PrLin-2020-05-07' + windows-pool: 'PrWin-2020-06-04' + linux-pool: 'PrLin-2020-06-08' + delete-tombstones-first: false -jobs: - - template: windows/run-tests.yml - parameters: - poolName: $(windows-pool) +stages: + - stage: delete_tombstones + displayName: 'Delete Tombstones' + jobs: + - job: windows + displayName: 'Clean Windows Tombstones' + pool: $(windows-pool) + condition: eq(variables['delete-tombstones-first'], 'true') + steps: + - task: PowerShell@2 + displayName: 'Initialize Environment' + inputs: + filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' + - script: rmdir /s /q W:\fail + displayName: 'Delete W:\fail' + - job: linux + displayName: 'Clean Linux Tombstones' + pool: $(linux-pool) + condition: eq(variables['delete-tombstones-first'], 'true') + steps: + - bash: rm -rf /archives/fail + displayName: 'Delete /archives/fail' + - job: osx + displayName: 'Clean MacOS Tombstones' + condition: eq(variables['delete-tombstones-first'], 'true') + pool: + name: vcpkgAgentPool + demands: Agent.OS -equals Darwin + steps: + - bash: rm -rf /Users/vagrant/Data/archives/fail + displayName: 'Delete /Users/vagrant/Data/archives/fail' - - template: windows/azure-pipelines.yml - parameters: - triplet: x86-windows - jobName: x86_windows - poolName: $(windows-pool) + - stage: run_pr_test + displayName: 'Run PR Test' + dependsOn: delete_tombstones + jobs: + - template: windows/run-tests.yml + parameters: + poolName: $(windows-pool) - - template: windows/azure-pipelines.yml - parameters: - triplet: x64-windows - jobName: x64_windows - poolName: $(windows-pool) + - template: windows/azure-pipelines.yml + parameters: + triplet: x86-windows + jobName: x86_windows + poolName: $(windows-pool) - - template: windows/azure-pipelines.yml - parameters: - triplet: x64-windows-static - jobName: x64_windows_static - poolName: $(windows-pool) + - template: windows/azure-pipelines.yml + parameters: + triplet: x64-windows + jobName: x64_windows + poolName: $(windows-pool) - - template: windows/azure-pipelines.yml - parameters: - triplet: x64-uwp - jobName: x64_uwp - poolName: $(windows-pool) + - template: windows/azure-pipelines.yml + parameters: + triplet: x64-windows-static + jobName: x64_windows_static + poolName: $(windows-pool) - - template: windows/azure-pipelines.yml - parameters: - triplet: arm64-windows - jobName: arm64_windows - poolName: $(windows-pool) + - template: windows/azure-pipelines.yml + parameters: + triplet: x64-uwp + jobName: x64_uwp + poolName: $(windows-pool) - - template: windows/azure-pipelines.yml - parameters: - triplet: arm-uwp - jobName: arm_uwp - poolName: $(windows-pool) + - template: windows/azure-pipelines.yml + parameters: + triplet: arm64-windows + jobName: arm64_windows + poolName: $(windows-pool) - - template: osx/azure-pipelines.yml + - template: windows/azure-pipelines.yml + parameters: + triplet: arm-uwp + jobName: arm_uwp + poolName: $(windows-pool) - - template: linux/azure-pipelines.yml - parameters: - poolName: $(linux-pool) + - template: osx/azure-pipelines.yml + + - template: linux/azure-pipelines.yml + parameters: + poolName: $(linux-pool) diff --git a/scripts/azure-pipelines/clean-tombstones.yml b/scripts/azure-pipelines/clean-tombstones.yml deleted file mode 100644 index dc601bcfc..000000000 --- a/scripts/azure-pipelines/clean-tombstones.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT -# -variables: - windows-pool: 'PrWin-2020-04-28' - linux-pool: 'PrLin-2020-05-07' - -jobs: - - job: windows - displayName: 'Clean Windows Tombstones' - pool: $(windows-pool) - steps: - - task: PowerShell@2 - displayName: 'Initialize Environment' - inputs: - filePath: 'scripts/azure-pipelines/windows/initialize-environment.ps1' - - script: rmdir /s /q W:\fail - displayName: 'Delete W:\fail' - - job: linux - displayName: 'Clean Linux Tombstones' - pool: $(linux-pool) - steps: - - bash: rm -rf /archives/fail - displayName: 'Delete /archives/fail' - - job: osx - displayName: 'Clean MacOS Tombstones' - pool: - name: vcpkgAgentPool - demands: Agent.OS -equals Darwin - steps: - - bash: rm -rf /Users/vagrant/Data/archives/fail - displayName: 'Delete /Users/vagrant/Data/archives/fail' diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 index de89c2207..e9fc79e7b 100755 --- a/scripts/azure-pipelines/linux/create-vmss.ps1 +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -158,7 +158,8 @@ Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 10 #################################################################################################### Write-Progress ` - -Activity 'Creating prototype VM' ` + -Activity $ProgressActivity ` + -Status 'Creating prototype VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $NicName = $ResourceGroupName + 'NIC' diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index dccf28691..0fa7edbf7 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -5,15 +5,41 @@ sudo apt -y update sudo apt -y dist-upgrade -# Install common build dependencies and partitioning tools -sudo apt -y install at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ gfortran zip libx11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config -# Required by qt5-x11extras -sudo apt -y install libxkbcommon-dev libxkbcommon-x11-dev -# Required by libhdfs3 -sudo apt -y install libkrb5-dev -# Required by mesa -sudo apt -y install python3-setuptools +# Install common build dependencies +APT_PACKAGES="at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils \ + build-essential g++ gfortran zip libx11-dev libxkbcommon-x11-dev libxi-dev \ + libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev \ + libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml \ + flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev \ + xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config" +# Additionally required by qt5-base +APT_PACKAGES="$APT_PACKAGES libxext-dev libxfixes-dev libxrender-dev \ + libxcb1-dev libx11-xcb-dev libxcb-glx0-dev" + +# Additionally required by qt5-base for qt5-x11extras +APT_PACKAGES="$APT_PACKAGES libxkbcommon-dev libxcb-keysyms1-dev \ + libxcb-image0-dev libxcb-shm0-dev libxcb-icccm4-dev libxcb-sync0-dev \ + libxcb-xfixes0-dev libxcb-shape0-dev libxcb-randr0-dev libxcb-render-util0-dev" + +# Additionally required by libhdfs3 +APT_PACKAGES="$APT_PACKAGES libkrb5-dev" + +# Additionally required by mesa +APT_PACKAGES="$APT_PACKAGES python3-setuptools" + +# Additionally required/installed by Azure DevOps Scale Set Agents +APT_PACKAGES="$APT_PACKAGES liblttng-ust0 libkrb5-3 zlib1g libicu60" + +sudo apt -y install $APT_PACKAGES + +# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from +# removing some of the above +sudo apt-mark hold libcurl4 +sudo apt-mark hold liblttng-ust0 +sudo apt-mark hold libkrb5-3 +sudo apt-mark hold zlib1g +sudo apt-mark hold libicu60 # Install newer version of nasm than the apt package, required by intel-ipsec mkdir /tmp/nasm @@ -37,7 +63,6 @@ sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb sudo apt -y update sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 sudo apt install -y --no-install-recommends libcudnn7-dev -sudo ln -s /usr/local/cuda-10.1/lib64/stubs/libcuda.so /usr/local/cuda-10.1/lib64/stubs/libcuda.so.1 # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb @@ -46,6 +71,11 @@ sudo apt update sudo add-apt-repository universe sudo apt install -y powershell +if [ -z "$StorageAccountName" ]; then +echo "No storage account supplied, skipping." +else +echo "Mapping storage account" + # Write SMB credentials sudo mkdir /etc/smbcredentials smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred @@ -56,19 +86,4 @@ sudo chmod 600 $smbCredentialFile # Mount the archives SMB share to /archives sudo mkdir /archives -m=777 echo "//$StorageAccountName.file.core.windows.net/archives /archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0" | sudo tee -a /etc/fstab - -# Create 'home' directory for haskell stack bits that want this -sudo mkdir -p /home/root -m=777 - -# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above -sudo rm /etc/debian_version - -# Install dependencies that the Azure Pipelines agent will want later to make launching VMs faster -# https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux -# (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs) -apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60 - -# Create work trees for the Azure Pipelines agent so that it puts its work tree into temporary storage. -sudo chmod 777 /mnt -sudo mkdir /agent -m=777 -sudo ln -s /mnt /agent/_work +fi diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 24089b674..2c0afa920 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -27,9 +27,9 @@ jobs: $commonArgs = @( '--binarycaching', '--x-binarysource=clear;files,W:\,upload', - '--x-buildtrees-root=E:\buildtrees', - '--x-install-root=E:\install', - '--x-packages-root=E:\packages' + '--x-buildtrees-root=D:\buildtrees', + '--x-install-root=D:\install', + '--x-packages-root=D:\packages' ) .\vcpkg.exe x-ci-clean @commonArgs diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 1417c298e..cf742d2ff 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -18,11 +18,10 @@ or are running from Azure Cloud Shell. $Location = 'westus2' $Prefix = 'PrWin-' + (Get-Date -Format 'yyyy-MM-dd') -$VMSize = 'Standard_F16s_v2' +$VMSize = 'Standard_D16a_v4' $ProtoVMName = 'PROTOTYPE' $LiveVMPrefix = 'BUILD' $WindowsServerSku = '2019-Datacenter' -$InstalledDiskSizeInGB = 1024 $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' @@ -160,7 +159,8 @@ Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 20 #################################################################################################### Write-Progress ` - -Activity 'Creating prototype VM' ` + -Activity $ProgressActivity ` + -Status 'Creating prototype VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) $NicName = $ResourceGroupName + 'NIC' @@ -186,16 +186,6 @@ $VM = Set-AzVMSourceImage ` -Skus $WindowsServerSku ` -Version latest -$InstallDiskName = $ProtoVMName + "InstallDisk" -$VM = Add-AzVMDataDisk ` - -Vm $VM ` - -Name $InstallDiskName ` - -Lun 0 ` - -Caching ReadWrite ` - -CreateOption Empty ` - -DiskSizeInGB $InstalledDiskSizeInGB ` - -StorageAccountType 'StandardSSD_LRS' - $VM = Set-AzVMBootDiagnostic -VM $VM -Disable New-AzVm ` -ResourceGroupName $ResourceGroupName ` @@ -274,7 +264,6 @@ Write-Progress ` Remove-AzVM -Id $VM.ID -Force Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $PrototypeOSDiskName -Force -Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $InstallDiskName -Force #################################################################################################### Write-Progress ` diff --git a/scripts/azure-pipelines/windows/provision-image.ps1 b/scripts/azure-pipelines/windows/provision-image.ps1 index 9a33461ee..e992f94a4 100644 --- a/scripts/azure-pipelines/windows/provision-image.ps1 +++ b/scripts/azure-pipelines/windows/provision-image.ps1 @@ -90,8 +90,7 @@ if (-not [string]::IsNullOrEmpty($AdminUserPassword)) { } Write-Host "Executing $PsExecPath " + @PsExecArgs - - $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru + & $PsExecPath @PsExecArgs > C:\ProvisionLog.txt Write-Host 'Cleaning up...' Remove-Item $PsExecPath exit $proc.ExitCode @@ -109,10 +108,16 @@ $Workloads = @( 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', 'Microsoft.VisualStudio.Component.Windows10SDK.18362', 'Microsoft.Net.Component.4.8.SDK', - 'Microsoft.Component.NetFX.Native' + 'Microsoft.Component.NetFX.Native', + 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', + 'Microsoft.VisualStudio.Component.VC.Llvm.Clang' ) -$MpiUrl = 'https://download.microsoft.com/download/A/E/0/AE002626-9D9D-448D-8197-1EA510E297CE/msmpisetup.exe' +$WindowsSDKUrl = 'https://download.microsoft.com/download/1/c/3/1c3d5161-d9e9-4e4b-9b43-b70fe8be268c/windowssdk/winsdksetup.exe' + +$WindowsWDKUrl = 'https://download.microsoft.com/download/1/a/7/1a730121-7aa7-46f7-8978-7db729aa413d/wdk/wdksetup.exe' + +$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' $CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe' $CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + ` @@ -278,6 +283,76 @@ Function InstallZip { } } +<# +.SYNOPSIS +Installs Windows SDK version 2004 + +.DESCRIPTION +Downloads the Windows SDK installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallWindowsSDK { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading Windows SDK...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing Windows SDK...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + } + } + catch { + Write-Error "Failed to install Windows SDK! $($_.Exception.Message)" + } +} + +<# +.SYNOPSIS +Installs Windows WDK version 2004 + +.DESCRIPTION +Downloads the Windows WDK installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallWindowsWDK { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading Windows WDK...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing Windows WDK...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + } + } + catch { + Write-Error "Failed to install Windows WDK! $($_.Exception.Message)" + } +} + <# .SYNOPSIS Installs MPI @@ -352,94 +427,43 @@ Function InstallCuda { } } -<# -.SYNOPSIS -Partitions a new physical disk. - -.DESCRIPTION -Takes the disk $DiskNumber, turns it on, then partitions it for use with label -$Label and drive letter $Letter. - -.PARAMETER DiskNumber -The number of the disk to set up. - -.PARAMETER Letter -The drive letter at which to mount the disk. - -.PARAMETER Label -The label to give the disk. -#> -Function New-PhysicalDisk { - Param( - [int]$DiskNumber, - [string]$Letter, - [string]$Label - ) - - if ($Letter.Length -ne 1) { - throw "Bad drive letter $Letter, expected only one letter. (Did you accidentially add a : ?)" - } - - try { - Write-Host "Attempting to online physical disk $DiskNumber" - [string]$diskpartScriptPath = Get-TempFilePath -Extension 'txt' - [string]$diskpartScriptContent = - "SELECT DISK $DiskNumber`r`n" + - "ONLINE DISK`r`n" - - Write-Host "Writing diskpart script to $diskpartScriptPath with content:" - Write-Host $diskpartScriptContent - Set-Content -Path $diskpartScriptPath -Value $diskpartScriptContent - Write-Host 'Invoking DISKPART...' - & diskpart.exe /s $diskpartScriptPath - - Write-Host "Provisioning physical disk $DiskNumber as drive $Letter" - [string]$diskpartScriptContent = - "SELECT DISK $DiskNumber`r`n" + - "ATTRIBUTES DISK CLEAR READONLY`r`n" + - "CREATE PARTITION PRIMARY`r`n" + - "FORMAT FS=NTFS LABEL=`"$Label`" QUICK`r`n" + - "ASSIGN LETTER=$Letter`r`n" - Write-Host "Writing diskpart script to $diskpartScriptPath with content:" - Write-Host $diskpartScriptContent - Set-Content -Path $diskpartScriptPath -Value $diskpartScriptContent - Write-Host 'Invoking DISKPART...' - & diskpart.exe /s $diskpartScriptPath - } - catch { - Write-Error "Failed to provision physical disk $DiskNumber as drive $Letter! $($_.Exception.Message)" - } -} - Write-Host "AdminUser password not supplied; assuming already running as AdminUser" -New-PhysicalDisk -DiskNumber 2 -Letter 'E' -Label 'install disk' - Write-Host 'Disabling pagefile...' wmic computersystem set AutomaticManagedPagefile=False wmic pagefileset delete -Write-Host 'Configuring AntiVirus exclusions...' -Add-MPPreference -ExclusionPath C:\ -Add-MPPreference -ExclusionPath D:\ -Add-MPPreference -ExclusionPath E:\ -Add-MPPreference -ExclusionProcess ninja.exe -Add-MPPreference -ExclusionProcess clang-cl.exe -Add-MPPreference -ExclusionProcess cl.exe -Add-MPPreference -ExclusionProcess link.exe -Add-MPPreference -ExclusionProcess python.exe +$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue +if ($null -eq $av) { + Write-Host 'AntiVirus not installed, skipping exclusions.' +} else { + Write-Host 'Configuring AntiVirus exclusions...' + Add-MPPreference -ExclusionPath C:\ + Add-MPPreference -ExclusionPath D:\ + Add-MPPreference -ExclusionProcess ninja.exe + Add-MPPreference -ExclusionProcess clang-cl.exe + Add-MPPreference -ExclusionProcess cl.exe + Add-MPPreference -ExclusionProcess link.exe + Add-MPPreference -ExclusionProcess python.exe +} InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' +InstallWindowsSDK -Url $WindowsSDKUrl +InstallWindowsWDK -Url $WindowsWDKUrl InstallMpi -Url $MpiUrl InstallCuda -Url $CudaUrl -Features $CudaFeatures InstallZip -Url $BinSkimUrl -Name 'BinSkim' -Dir 'C:\BinSkim' -if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountName))) { +if ([string]::IsNullOrWhiteSpace($StorageAccountName)) { + Write-Host 'No storage account name configured.' +} else { Write-Host 'Storing storage account name to environment' Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` -Name StorageAccountName ` -Value $StorageAccountName } -if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountKey))) { +if ([string]::IsNullOrWhiteSpace($StorageAccountKey)) { + Write-Host 'No storage account key configured.' +} else { Write-Host 'Storing storage account key to environment' Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` -Name StorageAccountKey ` diff --git a/scripts/boost/generate-ports.ps1 b/scripts/boost/generate-ports.ps1 index 619ba0061..992cf37a2 100644 --- a/scripts/boost/generate-ports.ps1 +++ b/scripts/boost/generate-ports.ps1 @@ -394,7 +394,7 @@ foreach ($library in $libraries) if ($libraries_in_boost_port.length -gt 1) { # Generate master boost control file which depends on each individual library - # mpi is excluded due to it having a dependency on msmpi + # mpi is excluded due to it having a dependency on msmpi/openmpi $boostDependsList = @($libraries_in_boost_port | % { "boost-$_" } | ? { $_ -notmatch "boost-mpi" }) -join ", " @( diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index a85965091..728225004 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -143,11 +143,8 @@ blosc:x64-uwp=fail bond:arm-uwp=fail bond:x64-osx=fail bond:x64-uwp=fail -boost-context:arm-uwp=fail -boost-context:x64-uwp=fail boost-coroutine:arm-uwp=fail boost-coroutine:arm64-windows=fail -boost-coroutine:x64-uwp=fail boost-fiber:arm-uwp=fail boost-fiber:arm64-windows=fail boost-fiber:x64-osx=fail @@ -446,8 +443,6 @@ fdlibm:arm-uwp=fail fdlibm:x64-uwp=fail fftw3:arm-uwp=fail fftw3:x64-uwp=fail -# ffmpeg on arm64 is currently failing due to an internal compiler error -ffmpeg:arm64-windows=fail field3d:x64-windows=fail field3d:x64-windows-static=fail field3d:x86-windows=fail @@ -606,8 +601,8 @@ hwloc:arm-uwp=fail hwloc:x64-uwp=fail hyperscan:x64-linux=ignore # hypre has a conflict with 'superlu' port -hypre:x64-linux=fail -hypre:x64-osx=fail +hypre:x64-linux=skip +hypre:x64-osx=skip icu:arm64-windows=fail icu:arm-uwp=fail icu:x64-uwp=fail @@ -1204,6 +1199,7 @@ nativefiledialog:x64-uwp=fail netcdf-cxx4:x64-linux=ignore nethost:x64-uwp=fail nethost:arm-uwp=fail +nettle:x64-windows-static=skip nettle:x64-windows=skip nettle:x64-osx=fail networkdirect-sdk:arm64-windows=fail @@ -1464,8 +1460,6 @@ portmidi:arm-uwp=fail portmidi:x64-linux=fail portmidi:x64-osx=fail portmidi:x64-uwp=fail -ppconsul:arm-uwp=fail -ppconsul:x64-uwp=fail ppconsul:x64-linux=ignore pqp:arm-uwp=fail pqp:x64-uwp=fail @@ -1536,6 +1530,9 @@ qt5-x11extras:x64-osx=fail qt5-x11extras:x86-windows=fail qt5-x11extras:x64-windows=fail qt5-x11extras:x64-windows-static=fail +# Broken by VS2019 16.6 and throws a ton of dialogs attempting to build +# fixed by https://github.com/microsoft/vcpkg/pull/11596 +qt5-translations:x64-windows-static=skip quickfast:x64-linux=ignore quickfix:x64-linux=ignore quickfix:x64-windows=ignore