langchain-ai/datafusion
1
0
Fork 0
mirror of https://github.com/langchain-ai/datafusion.git synced 2026-07-19 23:35:20 -04:00
Code Issues 3 Packages Projects Releases Wiki Activity

ARROW-7346: [CI] Explicit usage of ccache across the builds

Browse Source 
- ccache configuration is centralised in the docker-compose.yml
- stricter build directory paths
- java jni tests were not running, since the vendoring of flatbuffers there was a hidden linking error

Closes #5982 from kszucs/explicit_use_ccache and squashes the following commits:

90fcd0c05 <Krisztián Szűcs> Sort env definitions alphabetically
30a64dbb5 <Krisztián Szűcs> Lint CMake
d76282bde <Krisztián Szűcs> enable plasma java client; fix flatbuffers link error because it is bundled now
b6d47db8c <Krisztián Szűcs> Fix PLASMA_STORE binary path
c4648afa4 <Krisztián Szűcs> Explicit ccache; centralized config in docker-compose

Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
This commit is contained in:
Krisztián Szűcs
2019-12-08 13:07:08 +01:00
parent 58e127ebb8
commit 24fbf74691
31 changed files with 277 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/cpp.yml
+4 -4
View File
@@ -119,10 +119,10 @@ jobs:
run: brew bundle --file=cpp/Brewfile
- name: Build
shell: bash
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
windows:
name: AMD64 ${{ matrix.name }} C++
@@ -165,7 +165,7 @@ jobs:
submodules: true
- name: Build
shell: bash
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/cpp/build
run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
.github/workflows/java.yml
+2 -2
View File
@@ -79,7 +79,7 @@ jobs:
submodules: true
- name: Build
shell: bash
run: ci/scripts/java_build.sh $(pwd)
run: ci/scripts/java_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/java_test.sh $(pwd)
run: ci/scripts/java_test.sh $(pwd) $(pwd)/build
.github/workflows/python.yml
+3 -3
View File
@@ -178,8 +178,8 @@ jobs:
shell: bash
run: |
export PYTHON=python3
ci/scripts/cpp_build.sh $(pwd)
ci/scripts/python_build.sh $(pwd)
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
ci/scripts/python_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/python_test.sh $(pwd)
run: ci/scripts/python_test.sh $(pwd) $(pwd)/build
.github/workflows/ruby.yml
+3 -3
View File
@@ -102,11 +102,11 @@ jobs:
- name: Build
shell: bash
run: |
ci/scripts/cpp_build.sh $(pwd) $(pwd)/cpp/build
ci/scripts/c_glib_build.sh $(pwd) $(pwd)/c_glib/build
ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build
- name: Test GLib
shell: bash
run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/c_glib/build
run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build
- name: Test Ruby
shell: bash
run: ci/scripts/ruby_test.sh $(pwd)
.github/workflows/rust.yml
+2 -2
View File
@@ -80,7 +80,7 @@ jobs:
submodules: true
- name: Build
shell: bash
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/rust_test.sh $(pwd)
@@ -108,7 +108,7 @@ jobs:
submodules: true
- name: Build
shell: bash
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/rust/target
run: ci/scripts/rust_build.sh $(pwd) $(pwd)/build
- name: Test
shell: bash
run: ci/scripts/rust_test.sh $(pwd)
ci/docker/conda-cpp.dockerfile
+12 -11
View File
@@ -59,19 +59,20 @@ SHELL [ "/bin/bash", "-c", "-l" ]
# use login shell when running the container
ENTRYPOINT [ "/bin/bash", "-c", "-l" ]
ENV ARROW_S3=ON \
ARROW_ORC=ON \
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=CONDA \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=ON \
ARROW_HOME=$CONDA_PREFIX \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_FLIGHT=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_S3=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=CONDA \
ARROW_HOME=$CONDA_PREFIX \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
PARQUET_HOME=$CONDA_PREFIX
ci/docker/cuda-10.0-cpp.dockerfile
+9 -7
View File
@@ -72,15 +72,17 @@ RUN apt-get update -y -q && \
# - libgtest-dev only provide sources
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ENV ARROW_CUDA=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_CUDA=ON \
ARROW_HOME=/usr/local \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_PLASMA=ON \
ARROW_USE_ASAN=ON \
ARROW_USE_CCACHE=ON \
ARROW_USE_UBSAN=ON \
ARROW_NO_DEPRECATED_API=ON \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_HOME=/usr/local \
GTest_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED
PATH=/usr/lib/ccache/:$PATH \
Thrift_SOURCE=BUNDLED
ci/docker/debian-10-cpp.dockerfile
+17 -15
View File
@@ -58,23 +58,25 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ENV CC=gcc \
CXX=g++ \
cares_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_BUILD_TESTS=ON \
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=ON \
ARROW_HOME=/usr/local \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_ORC=ON \
ARROW_GANDIVA=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_HOME=/usr/local
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
cares_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED
ci/docker/fedora-29-cpp.dockerfile
+18 -16
View File
@@ -58,23 +58,25 @@ RUN dnf update -y && \
# * c-ares cmake config is not installed on Fedora but gRPC needs it
# when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844
# * protobuf libraries in Fedora 29 are too old for gRPC
ENV CC=gcc \
CXX=g++ \
ARROW_ORC=ON \
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=ON \
ARROW_GANDIVA_JAVA=ON \
ARROW_GANDIVA=OFF \
ARROW_HOME=/usr/local \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
cares_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_PARQUET=ON \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=OFF \
ARROW_GANDIVA_JAVA=ON \
ARROW_BUILD_TESTS=ON \
ARROW_HOME=/usr/local
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED
ci/docker/linux-apt-jni.dockerfile
+14 -11
View File
@@ -54,16 +54,19 @@ ARG cmake=3.11.4
RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt
ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH
ENV CC=gcc \
ENV ARROW_BUILD_TESTS=OFF \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA_JAVA=ON \
ARROW_GANDIVA=ON \
ARROW_HOME=/usr/local \
ARROW_JNI=ON \
ARROW_ORC=ON \
ARROW_PARQUET=OFF \
ARROW_PLASMA_JAVA_CLIENT=ON \
ARROW_PLASMA=ON \
ARROW_USE_CCACHE=ON \
CC=gcc \
CXX=g++ \
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
ARROW_BUILD_TESTS=OFF \
ARROW_FLIGHT=OFF \
ARROW_PARQUET=OFF \
ARROW_PLASMA=ON \
ARROW_ORC=ON \
ARROW_JNI=ON \
ARROW_GANDIVA=ON \
ARROW_GANDIVA_JAVA=ON \
ARROW_HOME=/usr/local
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED
ci/docker/ubuntu-14.04-cpp.dockerfile
+27 -25
View File
@@ -56,34 +56,36 @@ RUN apt-get update -y -q && \
# - libgflags-dev is too old
# - libre2-dev unavailable
# - liblz4-dev is too old
ENV CC=gcc \
CXX=g++ \
ARROW_BUILD_TESTS=ON \
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
Thrift_SOURCE=BUNDLED \
BOOST_SOURCE=BUNDLED \
benchmark_SOURCE=BUNDLED \
Brotli_SOURCE=BUNDLED \
RE2_SOURCE=BUNDLED \
ZSTD_SOURCE=BUNDLED \
Lz4_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
gflags_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
ARROW_FLIGHT=OFF \
ARROW_WITH_OPENSSL=OFF \
ARROW_GANDIVA=OFF \
ARROW_GANDIVA_JAVA=OFF \
ARROW_GANDIVA=OFF \
ARROW_HOME=/usr/local \
ARROW_ORC=OFF \
ARROW_PARQUET=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_HOME=/usr/local
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_OPENSSL=OFF \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
benchmark_SOURCE=BUNDLED \
BOOST_SOURCE=BUNDLED \
Brotli_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
gflags_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Lz4_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
RE2_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
ZSTD_SOURCE=BUNDLED
ci/docker/ubuntu-16.04-cpp.dockerfile
+16 -14
View File
@@ -65,23 +65,25 @@ RUN apt-get update -y -q && \
ENV ARROW_BUILD_BENCHMARKS=OFF \
ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
BOOST_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
ARROW_GANDIVA_JAVA=OFF \
ARROW_GANDIVA=ON \
ARROW_HOME=/usr/local \
ARROW_PARQUET=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=OFF \
ARROW_WITH_SNAPPY=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=OFF \
BOOST_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
CC=gcc
gRPC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED \
RapidJSON_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED
ci/docker/ubuntu-18.04-cpp.dockerfile
+9 -7
View File
@@ -95,6 +95,7 @@ RUN apt-get update -y -q && \
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=ON \
ARROW_HDFS=ON \
@@ -105,14 +106,15 @@ ENV ARROW_BUILD_TESTS=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_USE_ASAN=OFF \
ARROW_USE_CCACHE=ON \
ARROW_USE_UBSAN=OFF \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_ZSTD=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_BROTLI=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ORC_SOURCE=BUNDLED \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
GTest_SOURCE=BUNDLED \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Thrift_SOURCE=BUNDLED
ci/docker/ubuntu-18.04-r-sanitizer.dockerfile
+11 -9
View File
@@ -73,26 +73,28 @@ RUN echo "MAKEFLAGS=-j$(R --slave -e 'cat(parallel::detectCores())')" >> /usr/lo
# - libgtest-dev only provide sources
# - libprotobuf-dev only provide sources
# - thrift is too old
ENV gRPC_SOURCE=BUNDLED \
Protobuf_SOURCE=BUNDLED \
cares_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
Thrift_SOURCE=BUNDLED \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=OFF \
ARROW_GANDIVA=OFF \
ARROW_HDFS=OFF \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_ORC=OFF \
ARROW_PARQUET=ON \
ARROW_PLASMA=OFF \
ARROW_USE_ASAN=OFF \
ARROW_USE_CCACHE=ON \
ARROW_USE_UBSAN=OFF \
ARROW_NO_DEPRECATED_API=ON \
ARROW_INSTALL_NAME_RPATH=OFF \
ARROW_WITH_BZ2=OFF \
ARROW_WITH_ZSTD=OFF \
cares_SOURCE=BUNDLED \
gRPC_SOURCE=BUNDLED \
GTest_SOURCE=BUNDLED \
LC_ALL=en_US.UTF-8 \
R_BIN=RDsan
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED \
R_BIN=RDsan \
Thrift_SOURCE=BUNDLED
COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
ci/scripts/c_glib_build.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -ex
source_dir=${1}/c_glib
build_dir=${2:-${source_dir}/build}
build_dir=${2}/c_glib
: ${ARROW_GLIB_GTK_DOC:=false}
: ${ARROW_GLIB_DEVELOPMENT_MODE:=false}
ci/scripts/c_glib_test.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -ex
source_dir=${1}/c_glib
build_dir=${2:-${source_dir}/build}
build_dir=${2}/c_glib
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
ci/scripts/cpp_build.sh
+4 -19
View File
@@ -20,9 +20,11 @@
set -ex
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
build_dir=${2}/cpp
with_docs=${3:-false}
: ${ARROW_USE_CCACHE:=OFF}
# TODO(kszucs): consider to move these to CMake
if [ ! -z "${CONDA_PREFIX}" ]; then
export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR} -DCMAKE_RANLIB=${RANLIB}"
@@ -31,25 +33,7 @@ elif [ -x "$(command -v xcrun)" ]; then
export ARROW_GANDIVA_PC_CXX_FLAGS="-isysroot;$(xcrun --show-sdk-path)"
fi
# We know ccache is installed by default in most Linux development builds
# (and docker containers)
uname=$(uname -s)
case "${uname}" in
Linux*) ccache_default=ON;;
*) ccache_default=OFF;;
esac
export ARROW_USE_CCACHE=${ARROW_USE_CCACHE:-$ccache_default}
if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=1
export CCACHE_COMPRESSLEVEL=5
# Typically /build/ccache
export CCACHE_DIR=${build_dir}/../ccache
export CCACHE_MAXSIZE=500M
export PATH=/usr/lib/ccache/:$PATH
echo -e "===\n=== ccache statistics before build\n==="
ccache -s
fi
@@ -92,6 +76,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
-DARROW_ORC=${ARROW_ORC:-OFF} \
-DARROW_PARQUET=${ARROW_PARQUET:-OFF} \
-DARROW_PLASMA=${ARROW_PLASMA:-OFF} \
-DARROW_PLASMA_JAVA_CLIENT=${ARROW_PLASMA_JAVA_CLIENT:-OFF} \
-DARROW_PYTHON=${ARROW_PYTHON:-OFF} \
-DARROW_S3=${ARROW_S3:-OFF} \
-DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
ci/scripts/cpp_test.sh
+1 -1
View File
@@ -21,7 +21,7 @@ set -ex
arrow_dir=${1}
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
build_dir=${2}/cpp
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data
ci/scripts/docs_build.sh
+1 -1
View File
@@ -19,7 +19,7 @@
set -ex
arrow_dir=${1}
build_dir=${2}
build_dir=${2}/docs
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH}
ci/scripts/fuzzit_build.sh
+1 -1
View File
@@ -21,7 +21,7 @@ set -euxo pipefail
export TARGET_ID=apache-arrow/arrow-ipc-fuzzing
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
build_dir=${2}/cpp
pushd ${build_dir}/relwithdebinfo
ci/scripts/integration_hdfs.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -e
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
build_dir=${2}/cpp
export CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath --glob`
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
ci/scripts/integration_hiveserver2.sh
+1 -1
View File
@@ -20,7 +20,7 @@ set -e
arrow_dir=${1}
source_dir=${1}/cpp
build_dir=${2:-${source_dir}/build}
build_dir=${2}/cpp
${arrow_dir}/ci/scripts/util_wait_for_it.sh impala:21050 -t 300 -s -- echo "impala is up"
ci/scripts/integration_turbodbc.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -e
source_dir=${1}
build_dir=${2:-${source_dir}/build}
build_dir=${2}/turbodbc
# check that optional pyarrow modules are available
# because pytest would just skip the pyarrow tests
ci/scripts/java_build.sh
+1 -1
View File
@@ -20,7 +20,7 @@ set -ex
arrow_dir=${1}
source_dir=${1}/java
cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug}
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
with_docs=${3:-false}
mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
ci/scripts/java_test.sh
+2 -2
View File
@@ -20,11 +20,11 @@ set -ex
arrow_dir=${1}
source_dir=${1}/java
cpp_build_dir=${2:-${arrow_dir}/cpp/build/debug}
cpp_build_dir=${2}/cpp/${ARROW_BUILD_TYPE:-debug}
# for jni and plasma tests
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PLASMA_STORE=${ARROW_HOME}/bin/plasma_store_server
export PLASMA_STORE=${ARROW_HOME}/bin/plasma-store-server
mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
ci/scripts/python_build.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -ex
source_dir=${1}/python
build_dir=${2:-${source_dir}/build}
build_dir=${2}/python
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
ci/scripts/python_test.sh
+1 -1
View File
@@ -25,4 +25,4 @@ export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
py.test -r s --pyargs pyarrow
pytest -r s --pyargs pyarrow
ci/scripts/ruby_test.sh
-1
View File
@@ -20,7 +20,6 @@
set -ex
source_dir=${1}/ruby
build_dir=${2:-${source_dir}/build}
export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
ci/scripts/rust_build.sh
+1 -1
View File
@@ -20,7 +20,7 @@
set -ex
source_dir=${1}/rust
build_dir=${2:-${source_dir}/target}
build_dir=${2}/rust
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
ci/scripts/rust_test.sh
+1 -1
View File
@@ -21,7 +21,7 @@ set -ex
arrow_dir=${1}
source_dir=${1}/rust
build_dir=${2:-${source_dir}/target}
build_dir=${2}/rust
export ARROW_TEST_DATA=${arrow_dir}/testing/data
export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
docker-compose.yml
+111 -50
View File
@@ -40,6 +40,13 @@ volumes:
amd64-fedora-29-cache:
maven-cache:
x-ccache: &ccache
CCACHE_COMPILERCHECK: content
CCACHE_COMPRESS: 1
CCACHE_COMPRESSLEVEL: 5
CCACHE_MAXSIZE: 500M
CCACHE_DIR: /build/ccache
services:
################################# C++ #######################################
@@ -73,12 +80,14 @@ services:
arch: ${ARCH}
prefix: /opt/conda
shm_size: &shm-size 2G
environment:
<<: *ccache
volumes: &conda-volumes
- .:/arrow:delegated
- ${ARCH}-conda-cache:/build:delegated
command: &cpp-conda-command
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/cpp_test.sh /arrow /build/cpp"]
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/cpp_test.sh /arrow /build"]
cuda-cpp:
# Usage:
@@ -94,13 +103,15 @@ services:
cache_from:
- ${REPO}:${ARCH}-cuda-${CUDA}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: &cuda-volumes
- .:/arrow:delegated
- ${ARCH}-cuda-${CUDA}-cache:/build:delegated
command: &cpp-command >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/cpp_test.sh /arrow /build/cpp"
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/cpp_test.sh /arrow /build"
debian-cpp:
# Usage:
@@ -118,6 +129,8 @@ services:
args:
arch: ${ARCH}
shm_size: *shm-size
environment:
<<: *ccache
volumes: &debian-volumes
- .:/arrow:delegated
- ${ARCH}-debian-${DEBIAN}-cache:/build:delegated
@@ -139,6 +152,8 @@ services:
args:
arch: ${ARCH}
shm_size: *shm-size
environment:
<<: *ccache
volumes: &ubuntu-volumes
- .:/arrow:delegated
- ${ARCH}-ubuntu-${UBUNTU}-cache:/build:delegated
@@ -165,6 +180,7 @@ services:
shm_size: *shm-size
volumes: *ubuntu-volumes
environment:
<<: *ccache
CC: clang-7
CXX: clang++-7
ARROW_JEMALLOC: "OFF"
@@ -189,6 +205,8 @@ services:
args:
arch: ${ARCH}
shm_size: *shm-size
environment:
<<: *ccache
volumes: &fedora-volumes
- .:/arrow:delegated
- ${ARCH}-fedora-${FEDORA}-cache:/build:delegated
@@ -215,12 +233,14 @@ services:
environment:
ARROW_GLIB_GTK_DOC: "true"
shm_size: *shm-size
environment:
<<: *ccache
volumes: *debian-volumes
command: &c-glib-command >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib &&
/arrow/ci/scripts/c_glib_test.sh /arrow /build/c_glib"
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build &&
/arrow/ci/scripts/c_glib_test.sh /arrow /build"
ubuntu-c-glib:
# Usage:
@@ -239,6 +259,7 @@ services:
args:
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
environment:
<<: *ccache
ARROW_GLIB_GTK_DOC: "true"
shm_size: *shm-size
volumes: *ubuntu-volumes
@@ -268,12 +289,14 @@ services:
args:
base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
shm_size: *shm-size
environment:
<<: *ccache
volumes: *debian-volumes
command: &ruby-command >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib &&
/arrow/ci/scripts/c_glib_test.sh /arrow /build/c_glib &&
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build &&
/arrow/ci/scripts/c_glib_test.sh /arrow /build &&
/arrow/ci/scripts/ruby_test.sh /arrow"
ubuntu-ruby:
@@ -294,6 +317,8 @@ services:
args:
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
shm_size: *shm-size
environment:
<<: *ccache
volumes: *ubuntu-volumes
command: *ruby-command
@@ -318,10 +343,12 @@ services:
arch: ${ARCH}
python: ${PYTHON}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command: &python-conda-command
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/python_test.sh /arrow"]
cuda-python:
@@ -341,11 +368,13 @@ services:
args:
base: ${REPO}:${ARCH}-cuda-${CUDA}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: *cuda-volumes
command: &python-command >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/python_test.sh /arrow"
debian-python:
@@ -365,6 +394,8 @@ services:
args:
base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: *debian-volumes
command: *python-command
@@ -385,6 +416,8 @@ services:
args:
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: *ubuntu-volumes
command: *python-command
@@ -405,6 +438,8 @@ services:
args:
base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: *fedora-volumes
command: *python-command
@@ -425,6 +460,7 @@ services:
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
cmake: 3.2.3
environment:
<<: *ccache
# Vendor boost to avoid dealing with stale FindBoost.
ARROW_BOOST_SOURCE: "BUNDLED"
ARROW_ORC: "OFF"
@@ -456,6 +492,8 @@ services:
python: ${PYTHON}
pandas: ${PANDAS}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command: *python-conda-command
@@ -481,10 +519,12 @@ services:
python: ${PYTHON}
dask: ${DASK}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/integration_dask.sh"]
conda-python-turbodbc:
@@ -509,11 +549,13 @@ services:
python: ${PYTHON}
turbodbc: ${TURBODBC}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
/arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build/turbodbc"]
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/integration_turbodbc.sh /turbodbc /build"]
########################## Python Wheels ####################################
@@ -573,9 +615,11 @@ services:
arch: ${ARCH}
r: ${R}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/r_test.sh /arrow"]
ubuntu-r:
@@ -593,10 +637,12 @@ services:
r: ${R}
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
shm_size: *shm-size
environment:
<<: *ccache
volumes: *ubuntu-volumes
command: >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/r_test.sh /arrow"
ubuntu-r-sanitizer:
@@ -613,10 +659,12 @@ services:
dockerfile: ci/docker/ubuntu-18.04-r-sanitizer.dockerfile
cache_from:
- ${REPO}:amd64-ubuntu-18.04-r-sanitizer
environment:
<<: *ccache
volumes: *ubuntu-volumes
command: >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/r_sanitize.sh /arrow"
################################ Rust #######################################
@@ -639,8 +687,8 @@ services:
command: &rust-command >
/bin/bash -c "
echo ${RUST} > /arrow/rust/rust-toolchain &&
/arrow/ci/scripts/rust_build.sh /arrow /build/rust &&
/arrow/ci/scripts/rust_test.sh /arrow /build/rust"
/arrow/ci/scripts/rust_build.sh /arrow /build &&
/arrow/ci/scripts/rust_test.sh /arrow /build"
################################# Go ########################################
@@ -730,8 +778,8 @@ services:
- maven-cache:/root/.m2:delegated
command: &java-command >
/bin/bash -c "
/arrow/ci/scripts/java_build.sh /arrow &&
/arrow/ci/scripts/java_test.sh /arrow"
/arrow/ci/scripts/java_build.sh /arrow /build &&
/arrow/ci/scripts/java_test.sh /arrow /build"
debian-java-jni:
# Includes plasma test and jni for gandiva and orc.
@@ -748,15 +796,17 @@ services:
args:
base: ${REPO}:${ARCH}-debian-9-java-${JDK}-maven-${MAVEN}
shm_size: *shm-size
environment:
<<: *ccache
volumes:
- .:/arrow:delegated
- maven-cache:/root/.m2:delegated
- ${ARCH}-debian-9-cache:/build:delegated
command:
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/java_build.sh /arrow /build/cpp/debug
/arrow/ci/scripts/java_test.sh /arrow /build/cpp/debug"
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/java_build.sh /arrow /build &&
/arrow/ci/scripts/java_test.sh /arrow /build"
############################## Integration ##################################
@@ -782,13 +832,14 @@ services:
go: ${GO}
volumes: *conda-volumes
environment:
<<: *ccache
# tell archery where the arrow binaries are located
ARROW_CPP_EXE_PATH: /build/cpp/debug
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/go_build.sh /arrow &&
/arrow/ci/scripts/java_build.sh /arrow &&
/arrow/ci/scripts/js_build.sh /arrow &&
/arrow/ci/scripts/java_build.sh /arrow /build &&
/arrow/ci/scripts/js_build.sh /arrow /build &&
pip install -e /arrow/dev/archery &&
archery integration --with-all --run-flight"]
@@ -811,16 +862,17 @@ services:
node: ${NODE}
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
environment:
<<: *ccache
ARROW_GLIB_GTK_DOC: "true"
volumes: *ubuntu-volumes
command:
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp true &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build/c_glib &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
/arrow/ci/scripts/java_build.sh /arrow /build/cpp/debug true &&
/arrow/ci/scripts/cpp_build.sh /arrow /build true &&
/arrow/ci/scripts/c_glib_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/java_build.sh /arrow /build true &&
/arrow/ci/scripts/js_build.sh /arrow true &&
/arrow/ci/scripts/docs_build.sh /arrow /build/docs"
/arrow/ci/scripts/docs_build.sh /arrow /build"
################################# Tools #####################################
@@ -838,6 +890,8 @@ services:
args:
rust: ${RUST}
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
environment:
<<: *ccache
volumes: *ubuntu-volumes
command: >
/bin/bash -c "pip install -e /arrow/dev/archery && archery lint"
@@ -859,11 +913,13 @@ services:
FUZZIT_HOST: bionic-llvm7
CI_ARROW_SHA: ${CI_ARROW_SHA:-UNSET}
CI_ARROW_BRANCH: ${CI_ARROW_BRANCH:-UNSET}
environment:
<<: *ccache
volumes: *debian-volumes
command: >
/bin/bash -c "
/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/fuzzit_build.sh /arrow /build/cpp"
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/fuzzit_build.sh /arrow /build"
######################### Integration Tests #################################
@@ -914,10 +970,12 @@ services:
- ARROW_HIVESERVER2=ON
- ARROW_HIVESERVER2_TEST_HOST=impala
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/integration_hiveserver2.sh /arrow /build/cpp"]
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/integration_hiveserver2.sh /arrow /build"]
conda-python-hdfs:
# Usage:
@@ -943,18 +1001,19 @@ services:
links:
- impala:impala
environment:
- ARROW_HDFS_TEST_HOST=impala
- ARROW_HDFS_TEST_PORT=8020
- ARROW_HDFS_TEST_USER=hdfs
<<: *ccache
ARROW_HDFS_TEST_HOST: impala
ARROW_HDFS_TEST_PORT: 8020
ARROW_HDFS_TEST_USER: hdfs
shm_size: *shm-size
volumes: &conda-maven-volumes
- .:/arrow:delegated
- maven-cache:/root/.m2:delegated
- ${ARCH}-conda-cache:/build:delegated
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
/arrow/ci/scripts/integration_hdfs.sh /arrow /build/cpp"]
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/integration_hdfs.sh /arrow /build"]
conda-python-spark:
# Usage:
@@ -978,9 +1037,11 @@ services:
maven: 3.5
spark: ${SPARK}
shm_size: *shm-size
environment:
<<: *ccache
volumes: *conda-maven-volumes
command:
["/arrow/ci/scripts/cpp_build.sh /arrow /build/cpp &&
/arrow/ci/scripts/python_build.sh /arrow /build/python &&
/arrow/ci/scripts/java_build.sh /arrow &&
["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/java_build.sh /arrow /build &&
/arrow/ci/scripts/integration_spark.sh /arrow /spark"]