RPCS3/llvm-mirror
1
0
Fork 0
mirror of https://github.com/RPCS3/llvm-mirror.git synced 2024-11-26 21:00:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Dockerfiles] Split checkout and build scripts into separate files.

Browse Source 
Summary:
This is a small refactoring to extract the svn checkout code from the
build script used inside the docker image.
This would give more flexibility if more than a single invocation of
cmake is needed inside the docker image.

User-facing interface (build_docker_image.sh) hasn't changed, only the
internal scripts running inside the build container are affected.

Reviewers: ioeric

Reviewed By: ioeric

Subscribers: mehdi_amini, llvm-commits

Differential Revision: https://reviews.llvm.org/D45868

llvm-svn: 330412
This commit is contained in:
Ilya Biryukov 2018-04-20 10:19:38 +00:00
parent dd48f29ba7
commit 30ee746f6f
6 changed files with 246 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
utils/docker/build_docker_image.sh
View File

@ -13,6 +13,8 @@ IMAGE_SOURCE=""
DOCKER_REPOSITORY=""
DOCKER_TAG=""
BUILDSCRIPT_ARGS=""
CHECKOUT_ARGS=""
CMAKE_ENABLED_PROJECTS=""
function show_usage() {
cat << EOF
@ -25,7 +27,7 @@ Available options:
-s|--source image source dir (i.e. debian8, nvidia-cuda, etc)
-d|--docker-repository docker repository for the image
-t|--docker-tag docker tag for the image
LLVM-specific:
Checkout arguments:
-b|--branch svn branch to checkout, i.e. 'trunk',
'branches/release_40'
(default: 'trunk')
@ -40,11 +42,12 @@ Available options:
Project 'llvm' is always included and ignored, if
specified.
Can be specified multiple times.
-i|--install-target name of a cmake install target to build and include in
the resulting archive. Can be specified multiple times.
-c|--checksums name of a file, containing checksums of llvm checkout.
Script will fail if checksums of the checkout do not
match.
Build-specific:
-i|--install-target name of a cmake install target to build and include in
the resulting archive. Can be specified multiple times.
Required options: --source and --docker-repository, at least one
--install-target.
@ -75,6 +78,7 @@ EOF
CHECKSUMS_FILE=""
SEEN_INSTALL_TARGET=0
SEEN_CMAKE_ARGS=0
while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help)
@ -96,11 +100,26 @@ while [[ $# -gt 0 ]]; do
DOCKER_TAG="$1"
shift
;;
-i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project)
if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then
SEEN_INSTALL_TARGET=1
fi
-r|--revision|-c|-cherrypick|-b|--branch)
CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $2"
shift 2
;;
-i|--install-target)
SEEN_INSTALL_TARGET=1
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS $1 $2"
shift 2
;;
-p|--llvm-project)
PROJ="$2"
if [ "$PROJ" == "cfe" ]; then
PROJ="clang"
fi
CHECKOUT_ARGS="$CHECKOUT_ARGS $1 $PROJ"
if [ "$PROJ" != "clang-tools-extra" ]; then
CMAKE_ENABLED_PROJECTS="$CMAKE_ENABLED_PROJECTS:$PROJ"
fi
shift 2
;;
-c|--checksums)
@ -111,6 +130,7 @@ while [[ $# -gt 0 ]]; do
--)
shift
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -- $*"
SEEN_CMAKE_ARGS=1
shift $#
;;
*)
@ -120,6 +140,17 @@ while [[ $# -gt 0 ]]; do
esac
done
if [ "$CMAKE_ENABLED_PROJECTS" != "" ]; then
# Remove the leading ':' character.
CMAKE_ENABLED_PROJECTS="${CMAKE_ENABLED_PROJECTS:1}"
if [[ $SEEN_CMAKE_ARGS -eq 0 ]]; then
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS --"
fi
BUILDSCRIPT_ARGS="$BUILDSCRIPT_ARGS -DLLVM_ENABLE_PROJECTS=$CMAKE_ENABLED_PROJECTS"
fi
command -v docker >/dev/null ||
{
echo "Docker binary cannot be found. Please install Docker to use this script."
@ -165,6 +196,7 @@ fi
echo "Building ${DOCKER_REPOSITORY}${DOCKER_TAG} from $IMAGE_SOURCE"
docker build -t "${DOCKER_REPOSITORY}${DOCKER_TAG}" \
--build-arg "checkout_args=$CHECKOUT_ARGS" \
--build-arg "buildscript_args=$BUILDSCRIPT_ARGS" \
-f "$BUILD_DIR/$IMAGE_SOURCE/Dockerfile" \
"$BUILD_DIR"

9
utils/docker/debian8/Dockerfile
View File

@ -41,10 +41,13 @@ RUN mkdir /tmp/cmake-install && cd /tmp/cmake-install && \
ADD checksums /tmp/checksums
ADD scripts /tmp/scripts
# Arguments passed to build_install_clang.sh.
ARG buildscript_args
# Checkout the source code.
ARG checkout_args
RUN /tmp/scripts/checkout.sh ${checkout_args}
# Run the build. Results of the build will be available at /tmp/clang-install/.
RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
ARG buildscript_args
RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
# Stage 2. Produce a minimal release image with build results.

9
utils/docker/example/Dockerfile
View File

@ -19,10 +19,13 @@ LABEL maintainer "Maintainer <maintainer@email>"
ADD checksums /tmp/checksums
ADD scripts /tmp/scripts
# Arguments passed to build_install_clang.sh.
# Checkout the source code.
ARG checkout_args
RUN /tmp/scripts/checkout.sh ${checkout_args}
# Run the build. Results of the build will be available at /tmp/clang-install/.
ARG buildscript_args
# Run the build. Results of the build will be available as /tmp/clang-install.
RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
# Stage 2. Produce a minimal release image with build results.

9
utils/docker/nvidia-cuda/Dockerfile
View File

@ -17,10 +17,13 @@ RUN apt-get update && \
ADD checksums /tmp/checksums
ADD scripts /tmp/scripts
# Arguments passed to build_install_clang.sh.
ARG buildscript_args
# Checkout the source code.
ARG checkout_args
RUN /tmp/scripts/checkout.sh ${checkout_args}
# Run the build. Results of the build will be available at /tmp/clang-install/.
RUN /tmp/scripts/build_install_llvm.sh ${buildscript_args}
ARG buildscript_args
RUN /tmp/scripts/build_install_llvm.sh --to /tmp/clang-install ${buildscript_args}
# Stage 2. Produce a minimal release image with build results.

189
utils/docker/scripts/build_install_llvm.sh
View File

@ -14,114 +14,37 @@ function show_usage() {
cat << EOF
Usage: build_install_llvm.sh [options] -- [cmake-args]
Checkout svn sources and run cmake with the specified arguments. Used
inside docker container.
Run cmake with the specified arguments. Used inside docker container.
Passes additional -DCMAKE_INSTALL_PREFIX and puts the build results into
/tmp/clang-install/ directory.
the directory specified by --to option.
Available options:
-h|--help show this help message
-b|--branch svn branch to checkout, i.e. 'trunk',
'branches/release_40'
(default: 'trunk')
-r|--revision svn revision to checkout
-c|--cherrypick revision to cherry-pick. Can be specified multiple times.
Cherry-picks are performed in the sorted order using the
following command:
'svn patch <(svn diff -c \$rev)'.
-p|--llvm-project name of an svn project to checkout. Will also add the
project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
For clang, please use 'clang', not 'cfe'.
Project 'llvm' is always included and ignored, if
specified.
Can be specified multiple times.
-i|--install-target name of a cmake install target to build and include in
the resulting archive. Can be specified multiple times.
Required options: At least one --install-target.
--to destination directory where to install the targets.
Required options: --to, at least one --install-target.
All options after '--' are passed to CMake invocation.
EOF
}
LLVM_SVN_REV=""
CHERRYPICKS=""
LLVM_BRANCH=""
CMAKE_ARGS=""
CMAKE_INSTALL_TARGETS=""
# We always checkout llvm
LLVM_PROJECTS="llvm"
CMAKE_LLVM_ENABLE_PROJECTS=""
CLANG_TOOLS_EXTRA_ENABLED=0
function contains_project() {
local TARGET_PROJ="$1"
local PROJ
for PROJ in $LLVM_PROJECTS; do
if [ "$PROJ" == "$TARGET_PROJ" ]; then
return 0
fi
done
return 1
}
function append_project() {
local PROJ="$1"
LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
if [ "$CMAKE_LLVM_ENABLE_PROJECTS" != "" ]; then
CMAKE_LLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS;$PROJ"
else
CMAKE_LLVM_ENABLE_PROJECTS="$PROJ"
fi
}
CLANG_INSTALL_DIR=""
while [[ $# -gt 0 ]]; do
case "$1" in
-r|--revision)
shift
LLVM_SVN_REV="$1"
shift
;;
-c|--cherrypick)
shift
CHERRYPICKS="$CHERRYPICKS $1"
shift
;;
-b|--branch)
shift
LLVM_BRANCH="$1"
shift
;;
-p|--llvm-project)
shift
PROJ="$1"
shift
if [ "$PROJ" == "cfe" ]; then
PROJ="clang"
fi
if [ "$PROJ" == "clang-tools-extra" ]; then
if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
echo "Project 'clang-tools-extra' is already enabled, ignoring extra occurrences."
else
CLANG_TOOLS_EXTRA_ENABLED=1
fi
continue
fi
if ! contains_project "$PROJ" ; then
append_project "$PROJ"
else
echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
fi
;;
-i|--install-target)
shift
CMAKE_INSTALL_TARGETS="$CMAKE_INSTALL_TARGETS $1"
shift
;;
--to)
shift
CLANG_INSTALL_DIR="$1"
shift
;;
--)
shift
CMAKE_ARGS="$*"
@ -142,102 +65,22 @@ if [ "$CMAKE_INSTALL_TARGETS" == "" ]; then
exit 1
fi
if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
if ! contains_project "clang"; then
echo "Project 'clang-tools-extra' was enabled without 'clang'."
echo "Adding 'clang' to a list of projects."
append_project "clang"
fi
if [ "$CLANG_INSTALL_DIR" == "" ]; then
echo "No install directory. Please specify the --to argument."
exit 1
fi
if [ "$LLVM_BRANCH" == "" ]; then
LLVM_BRANCH="trunk"
fi
if [ "$LLVM_SVN_REV" != "" ]; then
SVN_REV_ARG="-r$LLVM_SVN_REV"
echo "Checking out svn revision r$LLVM_SVN_REV."
else
SVN_REV_ARG=""
echo "Checking out latest svn revision."
fi
# Sort cherrypicks and remove duplicates.
CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
function apply_cherrypicks() {
local CHECKOUT_DIR="$1"
[ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
pushd "$CHECKOUT_DIR"
# This function is always called on a sorted list of cherrypicks.
for CHERRY_REV in $CHERRYPICKS; do
echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
local PATCH_FILE="$(mktemp)"
svn diff -c $CHERRY_REV > "$PATCH_FILE"
svn patch "$PATCH_FILE"
rm "$PATCH_FILE"
done
popd
}
CLANG_BUILD_DIR=/tmp/clang-build
CLANG_INSTALL_DIR=/tmp/clang-install
mkdir "$CLANG_BUILD_DIR"
mkdir -p "$CLANG_INSTALL_DIR"
# Get the sources from svn.
echo "Checking out sources from svn"
mkdir "$CLANG_BUILD_DIR/src"
for LLVM_PROJECT in $LLVM_PROJECTS; do
if [ "$LLVM_PROJECT" == "clang" ]; then
SVN_PROJECT="cfe"
else
SVN_PROJECT="$LLVM_PROJECT"
fi
echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CLANG_BUILD_DIR/src/$LLVM_PROJECT"
svn co -q $SVN_REV_ARG \
"https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
"$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
# We apply cherrypicks to all repositories regardless of whether the revision
# changes this repository or not. For repositories not affected by the
# cherrypick, applying the cherrypick is a no-op.
apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
done
if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
echo "Checking out https://llvm.org/svn/llvm-project/clang-tools-extra to $CLANG_BUILD_DIR/src/clang/tools/extra"
svn co -q $SVN_REV_ARG \
"https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
"$CLANG_BUILD_DIR/src/clang/tools/extra"
apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra"
fi
CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
if [ -f "$CHECKSUMS_FILE" ]; then
echo "Validating checksums for LLVM checkout..."
python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
--partial --multi_dir "$CLANG_BUILD_DIR/src"
else
echo "Skipping checksumming checks..."
fi
mkdir "$CLANG_BUILD_DIR/build"
mkdir -p "$CLANG_BUILD_DIR/build"
pushd "$CLANG_BUILD_DIR/build"
# Run the build as specified in the build arguments.
echo "Running build"
cmake -GNinja \
-DCMAKE_INSTALL_PREFIX="$CLANG_INSTALL_DIR" \
-DLLVM_ENABLE_PROJECTS="$CMAKE_LLVM_ENABLE_PROJECTS" \
$CMAKE_ARGS \
"$CLANG_BUILD_DIR/src/llvm"
ninja $CMAKE_INSTALL_TARGETS
@ -245,6 +88,6 @@ ninja $CMAKE_INSTALL_TARGETS
popd
# Cleanup.
rm -rf "$CLANG_BUILD_DIR"
rm -rf "$CLANG_BUILD_DIR/build"
echo "Done"

173
utils/docker/scripts/checkout.sh Executable file
View File

@ -0,0 +1,173 @@
#!/usr/bin/env bash
#===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
#
# The LLVM Compiler Infrastructure
#
# This file is distributed under the University of Illinois Open Source
# License. See LICENSE.TXT for details.
#
#===-----------------------------------------------------------------------===//
set -e
function show_usage() {
cat << EOF
Usage: checkout.sh [options]
Checkout svn sources into /tmp/clang-build/src. Used inside a docker container.
Available options:
-h|--help show this help message
-b|--branch svn branch to checkout, i.e. 'trunk',
'branches/release_40'
(default: 'trunk')
-r|--revision svn revision to checkout
-c|--cherrypick revision to cherry-pick. Can be specified multiple times.
Cherry-picks are performed in the sorted order using the
following command:
'svn patch <(svn diff -c \$rev)'.
-p|--llvm-project name of an svn project to checkout.
For clang, please use 'clang', not 'cfe'.
Project 'llvm' is always included and ignored, if
specified.
Can be specified multiple times.
EOF
}
LLVM_SVN_REV=""
CHERRYPICKS=""
LLVM_BRANCH=""
# We always checkout llvm
LLVM_PROJECTS="llvm"
function contains_project() {
local TARGET_PROJ="$1"
local PROJ
for PROJ in $LLVM_PROJECTS; do
if [ "$PROJ" == "$TARGET_PROJ" ]; then
return 0
fi
done
return 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
-r|--revision)
shift
LLVM_SVN_REV="$1"
shift
;;
-c|--cherrypick)
shift
CHERRYPICKS="$CHERRYPICKS $1"
shift
;;
-b|--branch)
shift
LLVM_BRANCH="$1"
shift
;;
-p|--llvm-project)
shift
PROJ="$1"
shift
if [ "$PROJ" == "cfe" ]; then
PROJ="clang"
fi
if ! contains_project "$PROJ" ; then
if [ "$PROJ" == "clang-tools-extra" ] && [ ! contains_project "clang" ]; then
echo "Project 'clang-tools-extra' specified before 'clang'. Adding 'clang' to a list of projects first."
LLVM_PROJECTS="$LLVM_PROJECTS clang"
fi
LLVM_PROJECTS="$LLVM_PROJECTS $PROJ"
else
echo "Project '$PROJ' is already enabled, ignoring extra occurrences."
fi
;;
-h|--help)
show_usage
exit 0
;;
*)
echo "Unknown option: $1"
exit 1
esac
done
if [ "$LLVM_BRANCH" == "" ]; then
LLVM_BRANCH="trunk"
fi
if [ "$LLVM_SVN_REV" != "" ]; then
SVN_REV_ARG="-r$LLVM_SVN_REV"
echo "Checking out svn revision r$LLVM_SVN_REV."
else
SVN_REV_ARG=""
echo "Checking out latest svn revision."
fi
# Sort cherrypicks and remove duplicates.
CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
function apply_cherrypicks() {
local CHECKOUT_DIR="$1"
[ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
pushd "$CHECKOUT_DIR"
# This function is always called on a sorted list of cherrypicks.
for CHERRY_REV in $CHERRYPICKS; do
echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
local PATCH_FILE="$(mktemp)"
svn diff -c $CHERRY_REV > "$PATCH_FILE"
svn patch "$PATCH_FILE"
rm "$PATCH_FILE"
done
popd
}
CLANG_BUILD_DIR=/tmp/clang-build
# Get the sources from svn.
echo "Checking out sources from svn"
mkdir -p "$CLANG_BUILD_DIR/src"
for LLVM_PROJECT in $LLVM_PROJECTS; do
if [ "$LLVM_PROJECT" == "clang" ]; then
SVN_PROJECT="cfe"
else
SVN_PROJECT="$LLVM_PROJECT"
fi
if [ "$SVN_PROJECT" != "clang-tools-extra" ]; then
CHECKOUT_DIR="$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
else
CHECKOUT_DIR="$CLANG_BUILD_DIR/src/clang/tools/extra"
fi
echo "Checking out https://llvm.org/svn/llvm-project/$SVN_PROJECT to $CHECKOUT_DIR"
svn co -q $SVN_REV_ARG \
"https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
"$CHECKOUT_DIR"
# We apply cherrypicks to all repositories regardless of whether the revision
# changes this repository or not. For repositories not affected by the
# cherrypick, applying the cherrypick is a no-op.
apply_cherrypicks "$CHECKOUT_DIR"
done
CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
if [ -f "$CHECKSUMS_FILE" ]; then
echo "Validating checksums for LLVM checkout..."
python "$(dirname $0)/llvm_checksum/llvm_checksum.py" -c "$CHECKSUMS_FILE" \
--partial --multi_dir "$CLANG_BUILD_DIR/src"
else
echo "Skipping checksumming checks..."
fi
echo "Done"