diff --git a/utils/docker/build_docker_image.sh b/utils/docker/build_docker_image.sh index 0e129636ca4..89b98aaaf7c 100755 --- a/utils/docker/build_docker_image.sh +++ b/utils/docker/build_docker_image.sh @@ -13,6 +13,8 @@ IMAGE_SOURCE="" DOCKER_REPOSITORY="" DOCKER_TAG="" BUILDSCRIPT_ARGS="" +CHECKOUT_ARGS="" +CMAKE_ENABLED_PROJECTS="" function show_usage() { cat << EOF @@ -25,7 +27,7 @@ Available options: -s|--source image source dir (i.e. debian8, nvidia-cuda, etc) -d|--docker-repository docker repository for the image -t|--docker-tag docker tag for the image - LLVM-specific: + Checkout arguments: -b|--branch svn branch to checkout, i.e. 'trunk', 'branches/release_40' (default: 'trunk') @@ -40,11 +42,12 @@ Available options: Project 'llvm' is always included and ignored, if specified. Can be specified multiple times. - -i|--install-target name of a cmake install target to build and include in - the resulting archive. Can be specified multiple times. -c|--checksums name of a file, containing checksums of llvm checkout. Script will fail if checksums of the checkout do not match. + Build-specific: + -i|--install-target name of a cmake install target to build and include in + the resulting archive. Can be specified multiple times. Required options: --source and --docker-repository, at least one --install-target. @@ -75,6 +78,7 @@ EOF CHECKSUMS_FILE="" SEEN_INSTALL_TARGET=0 +SEEN_CMAKE_ARGS=0 while [[ $# -gt 0 ]]; do case "$1" in -h|--help) @@ -96,11 +100,26 @@ while [[ $# -gt 0 ]]; do DOCKER_TAG="$1" shift ;; - -i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project) - if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then - SEEN_INSTALL_TARGET=1 - fi + -r|--revision|-c|-cherrypick|-b|--branch) + CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $2" + shift 2 + ;; + -i|--install-target) + SEEN_INSTALL_TARGET=1 BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2" + shift 2 + ;; + -p|--llvm-project) + PROJ="$2" + if [ "$PROJ" == "cfe" ]; then + PROJ="clang" + fi + + CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $PROJ" + if [ "$PROJ" != "clang-tools-extra" ]; then + CMAKE_ENABLED_PROJECTS="$CMAKE_ENABLED_PROJECTS:$PROJ" + fi + shift 2 ;; -c|--checksums) @@ -111,6 +130,7 @@ while [[ $# -gt 0 ]]; do --) shift BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*" + SEEN_CMAKE_ARGS=1 shift $# ;; *) @@ -120,6 +140,17 @@ while [[ $# -gt 0 ]]; do esac done + +if [ "$CMAKE_ENABLED_PROJECTS" != "" ]; then + # Remove the leading ':' character. + CMAKE_ENABLED_PROJECTS="${CMAKE_ENABLED_PROJECTS:1}" + + if [[ $SEEN_CMAKE_ARGS -eq 0 ]]; then + BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS --" + fi + BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -DLLVM_ENABLE_PROJECTS=$CMAKE_ENABLED_PROJECTS" +fi + command -v docker >/dev/null || { echo "Docker binary cannot be found. Please install Docker to use this script." @@ -165,6 +196,7 @@ fi echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE" docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \ + --build-arg "checkout_args=$CHECKOUT_ARGS" \ --build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \ -f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \ "$BUILD_DIR" diff --git a/utils/docker/debian8/Dockerfile b/utils/docker/debian8/Dockerfile index fd3bf9f49f0..fe12ec55e52 100644 --- a/utils/docker/debian8/Dockerfile +++ b/utils/docker/debian8/Dockerfile @@ -41,10 +41,13 @@ RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \ ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. -ARG buildscript_args + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} # Run the build. Results of the build will be available at /tmp/clang-install/. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +ARG buildscript_args +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. diff --git a/utils/docker/example/Dockerfile b/utils/docker/example/Dockerfile index bb42a4df6bf..d875ed96d90 100644 --- a/utils/docker/example/Dockerfile +++ b/utils/docker/example/Dockerfile @@ -19,10 +19,13 @@ LABEL maintainer "Maintainer " ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} +# Run the build. Results of the build will be available at /tmp/clang-install/. ARG buildscript_args -# Run the build. Results of the build will be available as /tmp/clang-install. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. diff --git a/utils/docker/nvidia-cuda/Dockerfile b/utils/docker/nvidia-cuda/Dockerfile index 6a354c88c7c..32804d77ee9 100644 --- a/utils/docker/nvidia-cuda/Dockerfile +++ b/utils/docker/nvidia-cuda/Dockerfile @@ -17,10 +17,13 @@ RUN apt-get update && \ ADD checksums /tmp/checksums ADD scripts /tmp/scripts -# Arguments passed to build_install_clang.sh. -ARG buildscript_args + +# Checkout the source code. +ARG checkout_args +RUN /tmp/scripts/checkout.sh ${checkout_args} # Run the build. Results of the build will be available at /tmp/clang-install/. -RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args} +ARG buildscript_args +RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args} # Stage 2. Produce a minimal release image with build results. diff --git a/utils/docker/scripts/build_install_llvm.sh b/utils/docker/scripts/build_install_llvm.sh index 7e5ac1e5807..0b9c0837203 100755 --- a/utils/docker/scripts/build_install_llvm.sh +++ b/utils/docker/scripts/build_install_llvm.sh @@ -14,114 +14,37 @@ function show_usage() { cat << EOF Usage: build_install_llvm.sh [options] -- [cmake-args] -Checkout svn sources and run cmake with the specified arguments. Used -inside docker container. +Run cmake with the specified arguments. Used inside docker container. Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into -/tmp/clang-install/ directory. +the directory specified by --to option. Available options: -h|--help show this help message - -b|--branch svn branch to checkout, i.e. 'trunk', - 'branches/release_40' - (default: 'trunk') - -r|--revision svn revision to checkout - -c|--cherrypick revision to cherry-pick. Can be specified multiple times. - Cherry-picks are performed in the sorted order using the - following command: - 'svn patch <(svn diff -c \$rev)'. - -p|--llvm-project name of an svn project to checkout. Will also add the - project to a list LLVM_ENABLE_PROJECTS, passed to CMake. - For clang, please use 'clang', not 'cfe'. - Project 'llvm' is always included and ignored, if - specified. - Can be specified multiple times. -i|--install-target name of a cmake install target to build and include in the resulting archive. Can be specified multiple times. -Required options: At least one --install-target. + --to destination directory where to install the targets. +Required options: --to, at least one --install-target. All options after '--' are passed to CMake invocation. EOF } -LLVM_SVN_REV="" -CHERRYPICKS="" -LLVM_BRANCH="" CMAKE_ARGS="" CMAKE_INSTALL_TARGETS="" -# We always checkout llvm -LLVM_PROJECTS="llvm" -CMAKE_LLVM_ENABLE_PROJECTS="" -CLANG_TOOLS_EXTRA_ENABLED=0 - -function contains_project() { - local TARGET_PROJ="$1" - local PROJ - for PROJ in $LLVM_PROJECTS; do - if [ "$PROJ" == "$TARGET_PROJ" ]; then - return 0 - fi - done - return 1 -} - -function append_project() { - local PROJ="$1" - - LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" - if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then - CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ" - else - CMAKE_LLVM_ENABLE_PROJECTS="$PROJ" - fi -} +CLANG_INSTALL_DIR="" while [[ $# -gt 0 ]]; do case "$1" in - -r|--revision) - shift - LLVM_SVN_REV="$1" - shift - ;; - -c|--cherrypick) - shift - CHERRYPICKS="$CHERRYPICKS $1" - shift - ;; - -b|--branch) - shift - LLVM_BRANCH="$1" - shift - ;; - -p|--llvm-project) - shift - PROJ="$1" - shift - - if [ "$PROJ" == "cfe" ]; then - PROJ="clang" - fi - - if [ "$PROJ" == "clang-tools-extra" ]; then - if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences." - else - CLANG_TOOLS_EXTRA_ENABLED=1 - fi - - continue - fi - - if ! contains_project "$PROJ" ; then - append_project "$PROJ" - else - echo "Project '$PROJ' is already enabled, ignoring extra occurrences." - fi - ;; -i|--install-target) shift CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1" shift ;; + --to) + shift + CLANG_INSTALL_DIR="$1" + shift + ;; --) shift CMAKE_ARGS="$*" @@ -142,102 +65,22 @@ if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then exit 1 fi -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - if ! contains_project "clang"; then - echo "Project 'clang-tools-extra' was enabled without 'clang'." - echo "Adding 'clang' to a list of projects." - - append_project "clang" - fi +if [ "$CLANG_INSTALL_DIR" == "" ]; then + echo "No install directory. Please specify the --to argument." + exit 1 fi -if [ "$LLVM_BRANCH" == "" ]; then - LLVM_BRANCH="trunk" -fi - -if [ "$LLVM_SVN_REV" != "" ]; then - SVN_REV_ARG="-r$LLVM_SVN_REV" - echo "Checking out svn revision r$LLVM_SVN_REV." -else - SVN_REV_ARG="" - echo "Checking out latest svn revision." -fi - -# Sort cherrypicks and remove duplicates. -CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)" - -function apply_cherrypicks() { - local CHECKOUT_DIR="$1" - - [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks" - pushd "$CHECKOUT_DIR" - - # This function is always called on a sorted list of cherrypicks. - for CHERRY_REV in $CHERRYPICKS; do - echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR" - - local PATCH_FILE="$(mktemp)" - svn diff -c $CHERRY_REV > "$PATCH_FILE" - svn patch "$PATCH_FILE" - rm "$PATCH_FILE" - done - - popd -} - CLANG_BUILD_DIR=/tmp/clang-build -CLANG_INSTALL_DIR=/tmp/clang-install -mkdir "$CLANG_BUILD_DIR" +mkdir -p "$CLANG_INSTALL_DIR" -# Get the sources from svn. -echo "Checking out sources from svn" -mkdir "$CLANG_BUILD_DIR/src" -for LLVM_PROJECT in $LLVM_PROJECTS; do - if [ "$LLVM_PROJECT" == "clang" ]; then - SVN_PROJECT="cfe" - else - SVN_PROJECT="$LLVM_PROJECT" - fi - - echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT" - svn co -q $SVN_REV_ARG \ - "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" - - # We apply cherrypicks to all repositories regardless of whether the revision - # changes this repository or not. For repositories not affected by the - # cherrypick, applying the cherrypick is a no-op. - apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT" -done - -if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then - echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra" - svn co -q $SVN_REV_ARG \ - "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \ - "$CLANG_BUILD_DIR/src/clang/tools/extra" - - apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra" -fi - -CHECKSUMS_FILE="/tmp/checksums/checksums.txt" - -if [ -f "$CHECKSUMS_FILE" ]; then - echo "Validating checksums for LLVM checkout..." - python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \ - --partial --multi_dir "$CLANG_BUILD_DIR/src" -else - echo "Skipping checksumming checks..." -fi - -mkdir "$CLANG_BUILD_DIR/build" +mkdir -p "$CLANG_BUILD_DIR/build" pushd "$CLANG_BUILD_DIR/build" # Run the build as specified in the build arguments. echo "Running build" cmake -GNinja \ -DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \ - -DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \ $CMAKE_ARGS \ "$CLANG_BUILD_DIR/src/llvm" ninja $CMAKE_INSTALL_TARGETS @@ -245,6 +88,6 @@ ninja $CMAKE_INSTALL_TARGETS popd # Cleanup. -rm -rf "$CLANG_BUILD_DIR" +rm -rf "$CLANG_BUILD_DIR/build" echo "Done" diff --git a/utils/docker/scripts/checkout.sh b/utils/docker/scripts/checkout.sh new file mode 100755 index 00000000000..1bd034d1844 --- /dev/null +++ b/utils/docker/scripts/checkout.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +#===- llvm/utils/docker/scripts/checkout.sh ---------------------===// +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===-----------------------------------------------------------------------===// + +set -e + +function show_usage() { + cat << EOF +Usage: checkout.sh [options] + +Checkout svn sources into /tmp/clang-build/src. Used inside a docker container. + +Available options: + -h|--help show this help message + -b|--branch svn branch to checkout, i.e. 'trunk', + 'branches/release_40' + (default: 'trunk') + -r|--revision svn revision to checkout + -c|--cherrypick revision to cherry-pick. Can be specified multiple times. + Cherry-picks are performed in the sorted order using the + following command: + 'svn patch <(svn diff -c \$rev)'. + -p|--llvm-project name of an svn project to checkout. + For clang, please use 'clang', not 'cfe'. + Project 'llvm' is always included and ignored, if + specified. + Can be specified multiple times. +EOF +} + +LLVM_SVN_REV="" +CHERRYPICKS="" +LLVM_BRANCH="" +# We always checkout llvm +LLVM_PROJECTS="llvm" + +function contains_project() { + local TARGET_PROJ="$1" + local PROJ + for PROJ in $LLVM_PROJECTS; do + if [ "$PROJ" == "$TARGET_PROJ" ]; then + return 0 + fi + done + return 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -r|--revision) + shift + LLVM_SVN_REV="$1" + shift + ;; + -c|--cherrypick) + shift + CHERRYPICKS="$CHERRYPICKS $1" + shift + ;; + -b|--branch) + shift + LLVM_BRANCH="$1" + shift + ;; + -p|--llvm-project) + shift + PROJ="$1" + shift + + if [ "$PROJ" == "cfe" ]; then + PROJ="clang" + fi + + if ! contains_project "$PROJ" ; then + if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then + echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first." + LLVM_PROJECTS="$LLVM_PROJECTS clang" + fi + LLVM_PROJECTS="$LLVM_PROJECTS $PROJ" + else + echo "Project '$PROJ' is already enabled, ignoring extra occurrences." + fi + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + esac +done + +if [ "$LLVM_BRANCH" == "" ]; then + LLVM_BRANCH="trunk" +fi + +if [ "$LLVM_SVN_REV" != "" ]; then + SVN_REV_ARG="-r$LLVM_SVN_REV" + echo "Checking out svn revision r$LLVM_SVN_REV." +else + SVN_REV_ARG="" + echo "Checking out latest svn revision." +fi + +# Sort cherrypicks and remove duplicates. +CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)" + +function apply_cherrypicks() { + local CHECKOUT_DIR="$1" + + [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks" + pushd "$CHECKOUT_DIR" + + # This function is always called on a sorted list of cherrypicks. + for CHERRY_REV in $CHERRYPICKS; do + echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR" + + local PATCH_FILE="$(mktemp)" + svn diff -c $CHERRY_REV > "$PATCH_FILE" + svn patch "$PATCH_FILE" + rm "$PATCH_FILE" + done + + popd +} + +CLANG_BUILD_DIR=/tmp/clang-build + +# Get the sources from svn. +echo "Checking out sources from svn" +mkdir -p "$CLANG_BUILD_DIR/src" +for LLVM_PROJECT in $LLVM_PROJECTS; do + if [ "$LLVM_PROJECT" == "clang" ]; then + SVN_PROJECT="cfe" + else + SVN_PROJECT="$LLVM_PROJECT" + fi + + if [ "$SVN_PROJECT" != "clang-tools-extra" ]; then + CHECKOUT_DIR="$CLANG_BUILD_DIR/src/$LLVM_PROJECT" + else + CHECKOUT_DIR="$CLANG_BUILD_DIR/src/clang/tools/extra" + fi + + echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CHECKOUT_DIR" + svn co -q $SVN_REV_ARG \ + "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \ + "$CHECKOUT_DIR" + + # We apply cherrypicks to all repositories regardless of whether the revision + # changes this repository or not. For repositories not affected by the + # cherrypick, applying the cherrypick is a no-op. + apply_cherrypicks "$CHECKOUT_DIR" +done + +CHECKSUMS_FILE="/tmp/checksums/checksums.txt" + +if [ -f "$CHECKSUMS_FILE" ]; then + echo "Validating checksums for LLVM checkout..." + python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \ + --partial --multi_dir "$CLANG_BUILD_DIR/src" +else + echo "Skipping checksumming checks..." +fi + +echo "Done"