vmtest: build and run bpf kernel selftests against various kernels

Run kernel selftests in vmtest with the goal to test libbpf backward
compatibility with older kernels.
The list of kernels should be specified in .travis.yml config in
`jobs` section, e.g. KERNEL=5.5.0.
Enlisted kernel releases
- 5.5.0 # built from main
- 5.5.0-rc6 # built from bpf-next
- LATEST

The kernel specified as 'LATEST' in .travis.yml is built from bpf-next kernel
tree, the rest of the kernels are downloaded from the specified in INDEX file.
The kernel sources from bpf-next are manually patched with [1] from bpf tree to
fix ranqslower build. This workaround should be removed after the patch is merged
from bpf to bpf-next tree.
Due to kernel sources being checked out the duration of the LATEST kernel test is
~30m.

bpf selftests are built from tools/testing/selftests/bpf/ of bpf-next tree with
HEAD revision set to CHECKPOINT-COMMIT specified in libbpf so selftests and
libbpf are in sync.
Currently only programs are tested with test_progs program, test_maps and
test_verifier should follow.
test_progs are run with blacklist required due to:
- some features, e.g. fentry/fexit are not supported in older kernels
- environment limitations, e.g an absence of the recent pahole in Debian
- incomplete disk image

The blacklist is passed to test_progs with -b option as specified in [2]
patch set.

Most of the preceeding tests are disabled due to incomplete disk image currenly
lacking proper networking settings.
For the LATEST kernel fome fentry/fexit tests are disabled due to pahole v1.16
is not abailible in Debian yet.

Next steps are resolving issues with blacklisted tests, enabling maps and
verifier testing, expanding the list of tested kernels.

[1] https://lore.kernel.org/bpf/908498f794661c44dca54da9e09dc0c382df6fcb.1580425879.git.hex@fb.com/t.mbox.gz
[2] https://www.spinics.net/lists/netdev/msg625192.html

.travis.yml

@@ -9,26 +9,43 @@ env:
         - PROJECT_NAME='libbpf'
         - AUTHOR_EMAIL="$(git log -1 --pretty=\"%aE\")"
         - CI_MANAGERS="$TRAVIS_BUILD_DIR/travis-ci/managers"
-        - VMTEST_DIR="$TRAVIS_BUILD_DIR/travis-ci/vmtest"
+        - VMTEST_ROOT="$TRAVIS_BUILD_DIR/travis-ci/vmtest"
         - REPO_ROOT="$TRAVIS_BUILD_DIR"
-        # Default setup command run on VM boot.
-        - VMTEST_SETUPCMD='echo 42'
+        - GIT_FETCH_DEPTH=64
+        - VMTEST_SETUPCMD="PROJECT_NAME=${PROJECT_NAME} ./${PROJECT_NAME}/travis-ci/vmtest/run_selftests.sh"
     jobs:
         # Setup command override.
-        - KERNEL=5.4 VMTEST_SETUPCMD="PROJECT_NAME=${PROJECT_NAME} ./${PROJECT_NAME}/travis-ci/vmtest/setup_example.sh"
-        - KERNEL=5.3
-        - KERNEL=4.19.88
+        # 5.5.0-rc6 is built from bpf-next; TODO(hex@): remove when pahole v1.16 is available
+        - KERNEL=5.5.0-rc6
+        - KERNEL=5.5.0
+        - KERNEL=LATEST
 
 addons:
     apt:
         packages:
             - qemu-kvm
             - zstd
+            - binutils-dev
+            - elfutils
+            - libcap-dev
+            - libelf-dev
 install: sudo adduser "${USER}" kvm
 before_script:
+    - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+    - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" | sudo tee -a /etc/apt/sources.list
+    - echo "deb http://archive.ubuntu.com/ubuntu eoan main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
+    - sudo apt-get -qq update
+    - sudo apt-get -y install dwarves=1.15-1
+    - sudo apt-get -qq -y install clang-10 lld-10 llvm-10
+    - if [[ "${KERNEL}" = 'LATEST' ]]; then ${VMTEST_ROOT}/build_latest_kernel.sh travis-ci/vmtest/bpf-next; fi
+    - ${VMTEST_ROOT}/prepare_selftests.sh travis-ci/vmtest/bpf-next
     # Escape whitespace characters.
     - setup_cmd=$(sed 's/\([[:space:]]\)/\\\1/g' <<< "${VMTEST_SETUPCMD}")
-    - sudo -E sudo -E -u "${USER}" "${VMTEST_DIR}/run.sh" -k "${KERNEL}"'*' -o -d ~ -s "${setup_cmd}" ~/root.img; exitstatus=$?
+    - if [[ "${KERNEL}" = 'LATEST' ]]; then
+        sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -b travis-ci/vmtest/bpf-next -o -d ~ -s "${setup_cmd}" ~/root.img;
+      else
+        sudo -E sudo -E -u "${USER}" "${VMTEST_ROOT}/run.sh" -k "${KERNEL}*" -o -d ~ -s "${setup_cmd}" ~/root.img;
+      fi; exitstatus=$?
     - test $exitstatus -le 1
 script:
     - test $exitstatus -eq 0

travis-ci/vmtest/build_latest_kernel.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eux
+
+GIT_FETCH_DEPTH="${GIT_FETCH_DEPTH}" ${VMTEST_ROOT}/checkout_latest_kernel.sh $1
+cd $1
+cp ${VMTEST_ROOT}/configs/latest.config .config
+make -j $((4*$(nproc))) olddefconfig all

travis-ci/vmtest/build_selftests.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+LIBBPF_PATH="${REPO_ROOT}"
+REPO_PATH="travis-ci/vmtest/bpf-next"
+make \
+	CLANG=clang-10 \
+	LLC=llc-10 \
+	LLVM_STRIP=llvm-strip-10 \
+	VMLINUX_BTF="${VMLINUX_BTF}" \
+	-C "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \
+	-j $((4*$(nproc)))
+mkdir ${LIBBPF_PATH}/selftests
+cp -R "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" \
+	${LIBBPF_PATH}/selftests
+cd ${LIBBPF_PATH}
+rm selftests/bpf/.gitignore
+git add selftests
+
+blacklist_path="${VMTEST_ROOT}/configs/blacklist"
+git add "${blacklist_path}"

travis-ci/vmtest/checkout_latest_kernel.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -eux
+
+CWD=$(pwd)
+LIBBPF_PATH=$(pwd)
+REPO_PATH=$1
+
+BPF_NEXT_ORIGIN=https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git
+LINUX_SHA=$(cat ${LIBBPF_PATH}/CHECKPOINT-COMMIT)
+
+echo REPO_PATH = ${REPO_PATH}
+echo LINUX_SHA = ${LINUX_SHA}
+
+if [ ! -d "${REPO_PATH}" ]; then
+	mkdir -p ${REPO_PATH}
+	cd ${REPO_PATH}
+	git init
+	git remote add bpf-next ${BPF_NEXT_ORIGIN}
+	git fetch --depth ${GIT_FETCH_DEPTH} bpf-next
+	git reset --hard ${LINUX_SHA}
+else
+	cd ${REPO_PATH}
+fi

travis-ci/vmtest/prepare_selftests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eux
+GIT_FETCH_DEPTH="${GIT_FETCH_DEPTH}" ${VMTEST_ROOT}/checkout_latest_kernel.sh $1
+
+# Fix runqslower build
+# TODO(hex@): remove after the patch is merged from bpf to bpf-next tree
+cd $1
+wget https://lore.kernel.org/bpf/908498f794661c44dca54da9e09dc0c382df6fcb.1580425879.git.hex@fb.com/t.mbox.gz
+gunzip t.mbox.gz
+git apply t.mbox

travis-ci/vmtest/run.sh

@@ -175,11 +175,10 @@ cache_urls() {
 	if ! declare -p URLS &> /dev/null; then
 		# This URL contains a mapping from file names to URLs where
 		# those files can be downloaded.
-		local INDEX='https://libbpf-vmtest.s3-us-west-1.amazonaws.com/x86_64/INDEX'
 		declare -gA URLS
 		while IFS=$'\t' read -r name url; do
 			URLS["$name"]="$url"
-		done < <(curl -LfsS "$INDEX")
+		done < <(cat "${VMTEST_ROOT}/configs/INDEX")
 	fi
 }
 
@@ -367,6 +366,11 @@ else
 	sudo chmod 644 "$vmlinux"
 fi
 
+LIBBPF_PATH="${REPO_ROOT}" \
+	REPO_PATH="travis-ci/vmtest/bpf-next" \
+	VMTEST_ROOT="${VMTEST_ROOT}" \
+	VMLINUX_BTF=${vmlinux} ${VMTEST_ROOT}/build_selftests.sh
+
 if (( SKIPSOURCE )); then
 	echo "Not copying source files..." >&2
 else
@@ -393,13 +397,17 @@ chmod 644 /exitstatus"
 if [[ ! -z SETUPCMD ]]; then
 	# Unescape whitespace characters.
 	setup_cmd=$(sed 's/\(\\\)\([[:space:]]\)/\2/g' <<< "${SETUPCMD}")
+	kernel="${KERNELRELEASE}"
+	if [[ -v BUILDDIR ]]; then kernel='latest'; fi
+	setup_envvars="export KERNEL=${kernel}"
 	setup_script=$(printf "#!/bin/sh
 set -e
 
 echo 'Running setup commands'
 %s
+%s
 echo $? > /exitstatus
-chmod 644 /exitstatus" "${setup_cmd}")
+chmod 644 /exitstatus" "${setup_envvars}" "${setup_cmd}")
 fi
 
 echo "${setup_script}" | sudo tee "$mnt/etc/rcS.d/S50-run-tests" > /dev/null

travis-ci/vmtest/run_selftests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eux
+
+configs_path='libbpf/travis-ci/vmtest/configs'
+blacklist_path="$configs_path/blacklist/BLACKLIST-${KERNEL}"
+if [[ -s "${blacklist_path}" ]]; then
+	BLACKLIST=$(cat "${blacklist_path}" | tr '\n' ',')
+fi
+
+whitelist_path="$configs_path/whitelist/WHITELIST-${KERNEL}"
+if [[ -s "${whitelist_path}" ]]; then
+	WHITELIST=$(cat "${whitelist_path}" | tr '\n' ',')
+fi
+
+cd libbpf/selftests/bpf
+
+echo TEST_PROGS
+./test_progs ${BLACKLIST:+-b$BLACKLIST} ${WHITELIST:+-t$WHITELIST}