From b75e7687f430672a7f20c6e64df77c67e371680a Mon Sep 17 00:00:00 2001 From: "Markus F.X.J. Oberhumer" Date: Fri, 30 Jun 2023 01:20:48 +0200 Subject: [PATCH] all: CI and misc updates --- .github/workflows/ci.yml | 5 +- .github/workflows/test-alpine-linux.yml | 50 +++++++- .github/workflows/weekly-ci-zigcc.yml | 4 +- CMakeLists.txt | 9 +- .../10-create-image.sh | 4 +- .../11-list-packages.sh | 2 +- .../20-image-run-shell.sh | 4 +- misc/cross-compile-upx-with-podman/Dockerfile | 6 +- .../build-all-inside-container.sh | 1 + .../packages.txt | 20 +++- .../10-create-image.sh | 2 +- .../11-list-packages.sh | 2 +- .../20-image-run-shell.sh | 4 +- misc/rebuild-stubs-with-podman/Dockerfile | 26 ++-- misc/rebuild-stubs-with-podman/packages.txt | 20 +++- misc/scripts/build_upx_by_hand.sh | 113 +++++++++++------- 16 files changed, 192 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4be52be3..9b1be8c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,8 +11,8 @@ on: [push, workflow_dispatch] env: DEBIAN_FRONTEND: noninteractive UPX_CMAKE_BUILD_FLAGS: --verbose - # 2023-06-27 - ZIG_DIST_VERSION: 0.11.0-dev.3859+88284c124 + # 2023-06-29 + ZIG_DIST_VERSION: 0.11.0-dev.3886+0c1bfe271 jobs: job-rebuild-and-verify-stubs: @@ -144,6 +144,7 @@ jobs: run: | # testsuite needs working "readlink -en" and "sha256sum" if ! test -e /usr/local/opt/coreutils/libexec/gnubin/readlink; then + echo "===== brew leaves:"; brew leaves; echo "===== brew list:"; brew list brew update brew install coreutils fi diff --git a/.github/workflows/test-alpine-linux.yml b/.github/workflows/test-alpine-linux.yml index bb05b1e6..2ebcd79f 100644 --- a/.github/workflows/test-alpine-linux.yml +++ b/.github/workflows/test-alpine-linux.yml @@ -58,8 +58,10 @@ jobs: name: ${{ env.artifact_name }} path: 'upx*/build/*/upx' - job-alpine-by-hand: # uses shell - # and also uses a subdirectory "upx with space" in order to detect possible quoting issues + job-alpine-by-hand: # uses a POSIX-compliant shell + # ...and also uses a subdirectory "upx with space" in order to detect possible quoting issues + # ...and also uses ccache as we are running the same build-script again and again + if: ${{ true }} strategy: { matrix: { container: ['alpine:3.9','alpine:edge'] } } name: ${{ format('gcc by-hand {0}', matrix.container) }} runs-on: ubuntu-latest @@ -67,7 +69,15 @@ jobs: steps: - name: ${{ format('Install packages {0}', matrix.container) }} run: | - apk update && apk upgrade && apk add bash dash g++ git mksh zsh + # install ccache, g++, git and various POSIX shells + shells="bash dash loksh mksh zsh" + case ${{ matrix.container }} in + *:edge) shells="$shells oksh yash" ;; + esac + echo "installing shells: $shells" + apk update && apk upgrade && apk add ccache g++ git $shells + # enable ccache + echo -e "CC=ccache gcc\nCXX=ccache g++ -std=gnu++17" >> $GITHUB_ENV # this seems to be needed when running in a container (beause of UID mismatch??) git config --global --add safe.directory '*' - name: 'Check out code' @@ -77,11 +87,45 @@ jobs: path: 'upx with space' - name: 'Build by-hand with bash' run: 'bash "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with bash --posix' + run: 'bash --posix "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with busybox ash' + run: 'busybox ash "./upx with space/misc/scripts/build_upx_by_hand.sh"' - name: 'Build by-hand with busybox sh' run: 'busybox sh "./upx with space/misc/scripts/build_upx_by_hand.sh"' - name: 'Build by-hand with dash' run: 'dash "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with loksh' + run: 'ksh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with loksh -o posix' + run: 'ksh -o posix "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with loksh -o sh' + run: 'ksh -o sh "./upx with space/misc/scripts/build_upx_by_hand.sh"' - name: 'Build by-hand with mksh' run: 'mksh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with mksh -o posix' + run: 'mksh -o posix "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with mksh -o sh' + run: 'mksh -o sh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with oksh' + if: ${{ contains(matrix.container, ':edge') }} + run: 'oksh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with oksh -o posix' + if: ${{ contains(matrix.container, ':edge') }} + run: 'oksh -o posix "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with oksh -o sh' + if: ${{ contains(matrix.container, ':edge') }} + run: 'oksh -o sh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with yash' + if: ${{ contains(matrix.container, ':edge') }} + run: 'yash "./upx with space/misc/scripts/build_upx_by_hand.sh"' - name: 'Build by-hand with zsh' run: 'zsh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with zsh --emulate ksh' + run: 'zsh --emulate ksh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Build by-hand with zsh --emulate sh' + run: 'zsh --emulate sh "./upx with space/misc/scripts/build_upx_by_hand.sh"' + - name: 'Show ccache stats' + run: | + ccache -s + ccache -p diff --git a/.github/workflows/weekly-ci-zigcc.yml b/.github/workflows/weekly-ci-zigcc.yml index 526f2655..4005519a 100644 --- a/.github/workflows/weekly-ci-zigcc.yml +++ b/.github/workflows/weekly-ci-zigcc.yml @@ -8,8 +8,8 @@ on: workflow_dispatch: env: - # 2023-06-27 - ZIG_DIST_VERSION: 0.11.0-dev.3859+88284c124 + # 2023-06-29 + ZIG_DIST_VERSION: 0.11.0-dev.3886+0c1bfe271 jobs: job-linux-zigcc: # uses cmake + make diff --git a/CMakeLists.txt b/CMakeLists.txt index 177ad8b2..ebc14620 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -416,10 +416,6 @@ endif() # UPX_CONFIG_CMAKE_DISABLE_INSTALL # finally print some info about the build configuration #*********************************************************************** -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/maint/make/CMakeLists.maint.txt") -include("${CMAKE_CURRENT_SOURCE_DIR}/maint/make/CMakeLists.maint.txt") -endif() - function(print_var) foreach(v ${ARGV}) if(${v}) @@ -427,6 +423,11 @@ function(print_var) endif() endforeach() endfunction() + +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/maint/make/CMakeLists.maint.txt") +include("${CMAKE_CURRENT_SOURCE_DIR}/maint/make/CMakeLists.maint.txt") +endif() + print_var(CMAKE_VERSION UPX_CONFIG_CMAKE_MINIMUM_REQUIRED_VERSION) if(NOT UPX_CONFIG_CMAKE_DISABLE_PRINT_INFO) print_var(CMAKE_HOST_SYSTEM_NAME CMAKE_HOST_SYSTEM_VERSION) diff --git a/misc/cross-compile-upx-with-podman/10-create-image.sh b/misc/cross-compile-upx-with-podman/10-create-image.sh index b04c14ea..a755bd6b 100755 --- a/misc/cross-compile-upx-with-podman/10-create-image.sh +++ b/misc/cross-compile-upx-with-podman/10-create-image.sh @@ -6,10 +6,10 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # create the image from Dockerfile # using a rootless Podman container -# NOTE: this image is based on rebuild-stubs-with-podman/upx-stubtools-20221212-v5, +# NOTE: this image is based on rebuild-stubs-with-podman/upx-stubtools-20221212-v6, # so you have to create that image first # WARNING: we install many packages, so the resulting image needs A LOT of disk space! -image=upx-cross-compile-20230115-v3 +image=upx-cross-compile-20230115-v4 podman build -t "$image" -f "$argv0dir/Dockerfile" "$argv0dir" diff --git a/misc/cross-compile-upx-with-podman/11-list-packages.sh b/misc/cross-compile-upx-with-podman/11-list-packages.sh index bf2a8c0d..9150c112 100755 --- a/misc/cross-compile-upx-with-podman/11-list-packages.sh +++ b/misc/cross-compile-upx-with-podman/11-list-packages.sh @@ -6,7 +6,7 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # list all system packages that are installed in the image # using a rootless Podman container -image=upx-cross-compile-20230115-v3 +image=upx-cross-compile-20230115-v4 podman image list "$image" echo diff --git a/misc/cross-compile-upx-with-podman/20-image-run-shell.sh b/misc/cross-compile-upx-with-podman/20-image-run-shell.sh index 93ab4270..9d4d0ad4 100755 --- a/misc/cross-compile-upx-with-podman/20-image-run-shell.sh +++ b/misc/cross-compile-upx-with-podman/20-image-run-shell.sh @@ -6,7 +6,7 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # run an interactive shell in the image # using a rootless Podman container -image=upx-cross-compile-20230115-v3 +image=upx-cross-compile-20230115-v4 flags=( --read-only --rm --pull=never ) flags+=( --cap-drop=all ) # drop all capabilities @@ -23,6 +23,8 @@ if [[ 1 == 1 ]]; then # INFO: SELinux users *may* have to add ":z" to the volume mount flags; check the docs! flags+=( -v "${argv0dir}/../..:/home/upx/src/upx" ) flags+=( -w /home/upx/src/upx ) # set working directory + flags+=( --tmpfs /home/upx/.cache:rw,exec ) # mount a writeable tmpfs + flags+=( --tmpfs /home/upx/.local:rw,exec ) # mount a writeable tmpfs else # run as user root 0:0 # ONLY FOR DEBUGGING THE IMAGE diff --git a/misc/cross-compile-upx-with-podman/Dockerfile b/misc/cross-compile-upx-with-podman/Dockerfile index c55e2c44..1cedc3cb 100644 --- a/misc/cross-compile-upx-with-podman/Dockerfile +++ b/misc/cross-compile-upx-with-podman/Dockerfile @@ -1,8 +1,8 @@ -# NOTE: this image is based on rebuild-stubs-with-podman/upx-stubtools-20221212-v5, +# NOTE: this image is based on rebuild-stubs-with-podman/upx-stubtools-20221212-v6, # so you have to create that image first # WARNING: we install many packages, so the resulting image needs A LOT of disk space! -FROM localhost/upx-stubtools-20221212-v5 -ENV UPX_CONTAINER_IMAGE_NAME=upx-cross-compile-20230115-v3 +FROM localhost/upx-stubtools-20221212-v6 +ENV UPX_CONTAINER_IMAGE_NAME=upx-cross-compile-20230115-v4 ARG DEBIAN_FRONTEND=noninteractive USER root diff --git a/misc/cross-compile-upx-with-podman/build-all-inside-container.sh b/misc/cross-compile-upx-with-podman/build-all-inside-container.sh index bd5ca62a..9b5d0b57 100755 --- a/misc/cross-compile-upx-with-podman/build-all-inside-container.sh +++ b/misc/cross-compile-upx-with-podman/build-all-inside-container.sh @@ -15,6 +15,7 @@ fi # go to upx top-level directory cd "$argv0dir/../.." || exit 1 pwd +test -f doc/upx.pod || exit 1 # sanity check test -f src/version.h || exit 1 # sanity check function run_config_and_build { diff --git a/misc/cross-compile-upx-with-podman/packages.txt b/misc/cross-compile-upx-with-podman/packages.txt index 4cf6bab6..96672396 100644 --- a/misc/cross-compile-upx-with-podman/packages.txt +++ b/misc/cross-compile-upx-with-podman/packages.txt @@ -41,9 +41,11 @@ ii binutils-sparc64-linux-gnu 2.38-4ubuntu2.2 ii binutils-x86-64-linux-gnu 2.38-4ubuntu2.2 amd64 GNU binary utilities, for x86-64-linux-gnu target ii binutils-x86-64-linux-gnux32 2.38-4ubuntu2.2 amd64 GNU binary utilities, for x86-64-linux-gnux32 target ii bsdutils 1:2.37.2-4ubuntu3 amd64 basic utilities from 4.4BSD-Lite +ii busybox 1:1.30.1-7ubuntu3 amd64 Tiny utilities for small and embedded systems ii bzip2 1.0.8-5build1 amd64 high-quality block-sorting file compressor - utilities ii ca-certificates 20230311ubuntu0.22.04.1 all Common CA certificates ii cabextract 1.9-3 amd64 Microsoft Cabinet file unpacker +ii ccache 4.5.1-1 amd64 Compiler cache for fast recompilation of C/C++ code ii chrpath 0.16-2 amd64 Tool to edit the rpath in ELF binaries ii clang-14 1:14.0.0-1ubuntu1 amd64 C, C++ and Objective-C compiler ii clang-format-14 1:14.0.0-1ubuntu1 amd64 Tool to format C/C++/Obj-C code @@ -166,6 +168,7 @@ ii g++-s390x-linux-gnu 4:11.2.0-1ubuntu1 ii g++-sh4-linux-gnu 4:11.2.0--1ubuntu1 amd64 GNU C++ compiler for the sh4 architecture ii g++-sparc64-linux-gnu 4:11.2.0--1ubuntu1 amd64 GNU C++ compiler for the sparc64 architecture ii g++-x86-64-linux-gnux32 4:11.2.0--1ubuntu1 amd64 GNU C++ compiler for the x32 architecture +ii gawk 1:5.1.0-1build3 amd64 GNU awk, a pattern scanning and processing language ii gcc 4:11.2.0-1ubuntu1 amd64 GNU C compiler ii gcc-10-cross-base-mipsen 10.3.0-1ubuntu1cross2 all GCC, the GNU Compiler Collection (library base package) ii gcc-10-mips-linux-gnu 10.3.0-1ubuntu1cross2 amd64 GNU C compiler (cross compiler for mips architecture) @@ -628,6 +631,7 @@ ii libgudev-1.0-0:amd64 1:237-2build1 ii libgudev-1.0-0:i386 1:237-2build1 i386 GObject-based wrapper library for libudev ii libharfbuzz0b:amd64 2.7.4-1ubuntu3.1 amd64 OpenType text shaping engine (shared library) ii libharfbuzz0b:i386 2.7.4-1ubuntu3.1 i386 OpenType text shaping engine (shared library) +ii libhiredis0.14:amd64 0.14.1-2 amd64 minimalistic C client library for Redis ii libhogweed6:amd64 3.7.3-1build2 amd64 low level cryptographic library (public-key cryptos) ii libhogweed6:i386 3.7.3-1build2 i386 low level cryptographic library (public-key cryptos) ii libhwasan0-arm64-cross 12.1.0-2ubuntu1~22.04cross1 all AddressSanitizer -- a fast memory error detector @@ -820,6 +824,7 @@ ii libsensors5:i386 1:3.6.0-7ubuntu1 ii libsepol2:amd64 3.3-1build1 amd64 SELinux library for manipulating binary security policies ii libshout3:amd64 2.4.5-1build3 amd64 MP3/Ogg Vorbis broadcast streaming library ii libshout3:i386 2.4.5-1build3 i386 MP3/Ogg Vorbis broadcast streaming library +ii libsigsegv2:amd64 2.13-1ubuntu3 amd64 Library for handling page faults in a portable way ii libslang2:amd64 2.3.2-5build4 amd64 S-Lang programming library - runtime version ii libslang2:i386 2.3.2-5build4 i386 S-Lang programming library - runtime version ii libslirp0:amd64 4.6.1-1build1 amd64 General purpose TCP-IP emulator library @@ -1070,7 +1075,7 @@ ii linux-libc-dev-s390x-cross 5.15.0-22.22cross3 ii linux-libc-dev-sh4-cross 5.15.0-22.22cross4 all Linux Kernel Headers for development (for cross-compiling) ii linux-libc-dev-sparc64-cross 5.15.0-22.22cross4 all Linux Kernel Headers for development (for cross-compiling) ii linux-libc-dev-x32-cross 5.15.0-22.22cross4 all Linux Kernel Headers for development (for cross-compiling) -ii linux-libc-dev:amd64 5.15.0-75.82 amd64 Linux Kernel Headers for development +ii linux-libc-dev:amd64 5.15.0-76.83 amd64 Linux Kernel Headers for development ii lldb-14 1:14.0.0-1ubuntu1 amd64 Next generation, high-performance debugger ii llvm-14 1:14.0.0-1ubuntu1 amd64 Modular compiler and toolchain technologies ii llvm-14-dev 1:14.0.0-1ubuntu1 amd64 Modular compiler and toolchain technologies, libraries and headers @@ -1091,6 +1096,7 @@ ii mesa-vulkan-drivers:i386 22.2.5-0ubuntu0.1~22.04.3 ii mingw-w64-common 8.0.0-1 all Common files for Mingw-w64 ii mingw-w64-i686-dev 8.0.0-1 all Development files for MinGW-w64 targeting Win32 ii mingw-w64-x86-64-dev 8.0.0-1 all Development files for MinGW-w64 targeting Win64 +ii mksh 59c-16 amd64 MirBSD Korn Shell ii moreutils 0.66-1 amd64 additional Unix utilities ii mount 2.37.2-4ubuntu3 amd64 tools for mounting and manipulating filesystems ii msr-tools 1.3-4 amd64 Utilities for modifying MSRs from userspace @@ -1161,6 +1167,7 @@ ii systemd-sysv 249.11-0ubuntu3.9 ii systemd-timesyncd 249.11-0ubuntu3.9 amd64 minimalistic service to synchronize local time with NTP servers ii sysvinit-utils 3.01-1ubuntu1 amd64 System-V-like utilities ii tar 1.34+dfsg-1ubuntu0.1.22.04.1 amd64 GNU version of the tar archiving utility +ii time 1.9-0.1build2 amd64 GNU time program for measuring CPU resource usage ii ubuntu-keyring 2021.03.26 all GnuPG keys of the Ubuntu archive ii ubuntu-mono 20.10-0ubuntu2 all Ubuntu Mono Icon theme ii ucf 3.0043 all Update Configuration File(s): preserve user changes to config files @@ -1181,6 +1188,7 @@ ii xdg-user-dirs 0.17-2ubuntu4 ii xkb-data 2.33-1 all X Keyboard Extension (XKB) configuration data ii xxd 2:8.2.3995-1ubuntu2.8 amd64 tool to make (or reverse) a hex dump ii xz-utils 5.2.5-2ubuntu1 amd64 XZ-format compression utilities +ii yash 2.51-1 amd64 yet another shell ii zip 3.0-12build2 amd64 Archiver for .zip files ii zlib1g-dev:amd64 1:1.2.11.dfsg-2ubuntu9.2 amd64 compression library - development ii zlib1g:amd64 1:1.2.11.dfsg-2ubuntu9.2 amd64 compression library - runtime @@ -1193,7 +1201,7 @@ ii zstd 1.4.8+dfsg-3build1 ||/ Name Version Architecture Description Packages sorted by Installed-Size: - 7715290 ===== TOTAL (1187 packages) + 7722308 ===== TOTAL (1195 packages) 474522 libwine i386 270993 llvm-14-dev amd64 264247 qemu-efi-aarch64 all @@ -1549,11 +1557,13 @@ Packages sorted by Installed-Size: 1746 libunistring2 amd64 1730 libsdl2-2.0-0 amd64 1695 libgcc-11-dev-alpha-cross all + 1680 gawk amd64 1655 zstd amd64 1628 libslang2 amd64 1602 libsqlite3-0 amd64 1534 libcairo2 i386 1516 e2fsprogs amd64 + 1508 mksh amd64 1487 libstdc++6-armhf-cross all 1484 libsamplerate0 amd64 1483 libsamplerate0 i386 @@ -1562,6 +1572,7 @@ Packages sorted by Installed-Size: 1455 libx11-6 i386 1445 hyperfine amd64 1436 libobjc-11-dev amd64 + 1429 yash amd64 1429 libx11-data all 1425 libp11-kit0 i386 1408 llvm-14-runtime amd64 @@ -1571,6 +1582,7 @@ Packages sorted by Installed-Size: 1373 libslang2 i386 1354 libgcrypt20 amd64 1351 libcairo2 amd64 + 1343 ccache amd64 1328 libzstd-dev amd64 1292 libp11-kit0 amd64 1285 libasound2 i386 @@ -1619,6 +1631,7 @@ Packages sorted by Installed-Size: 806 libsoup2.4-1 i386 794 rsync amd64 793 libbrotli1 i386 + 791 busybox amd64 787 libcurl4 amd64 784 libbrotli1 amd64 783 libcups2 amd64 @@ -2035,6 +2048,7 @@ Packages sorted by Installed-Size: 130 dbus-user-session amd64 128 libtasn1-6 i386 128 libdrm2 amd64 + 126 time amd64 125 libmpc3 amd64 125 libitm1-alpha-cross all 123 libtimedate-perl all @@ -2103,6 +2117,7 @@ Packages sorted by Installed-Size: 94 libcairo-gobject2 i386 93 libsasl2-modules-db amd64 92 libsasl2-modules-db i386 + 92 libhiredis0.14 amd64 92 libgsm1 i386 92 libgcc-s1-sparc64-cross all 92 libgcc-s1-riscv64-cross all @@ -2206,6 +2221,7 @@ Packages sorted by Installed-Size: 50 libkeyutils1 i386 50 glib-networking-services amd64 50 gcc amd64 + 49 libsigsegv2 amd64 48 libxtst6 amd64 48 libatomic1-armel-cross all 48 glib-networking-common all diff --git a/misc/rebuild-stubs-with-podman/10-create-image.sh b/misc/rebuild-stubs-with-podman/10-create-image.sh index 300f7fbf..fd0ab06e 100755 --- a/misc/rebuild-stubs-with-podman/10-create-image.sh +++ b/misc/rebuild-stubs-with-podman/10-create-image.sh @@ -6,7 +6,7 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # create the image from Dockerfile # using a rootless Podman container -image=upx-stubtools-20221212-v5 +image=upx-stubtools-20221212-v6 podman build -t "$image" -f "$argv0dir/Dockerfile" "$argv0dir" diff --git a/misc/rebuild-stubs-with-podman/11-list-packages.sh b/misc/rebuild-stubs-with-podman/11-list-packages.sh index 8c18dca7..64cd04e4 100755 --- a/misc/rebuild-stubs-with-podman/11-list-packages.sh +++ b/misc/rebuild-stubs-with-podman/11-list-packages.sh @@ -6,7 +6,7 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # list all system packages that are installed in the image # using a rootless Podman container -image=upx-stubtools-20221212-v5 +image=upx-stubtools-20221212-v6 podman image list "$image" echo diff --git a/misc/rebuild-stubs-with-podman/20-image-run-shell.sh b/misc/rebuild-stubs-with-podman/20-image-run-shell.sh index 6958cf4c..0e49a74b 100755 --- a/misc/rebuild-stubs-with-podman/20-image-run-shell.sh +++ b/misc/rebuild-stubs-with-podman/20-image-run-shell.sh @@ -6,7 +6,7 @@ argv0=$0; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" # run an interactive shell in the image # using a rootless Podman container -image=upx-stubtools-20221212-v5 +image=upx-stubtools-20221212-v6 flags=( --read-only --rm --pull=never ) flags+=( --cap-drop=all ) # drop all capabilities @@ -23,6 +23,8 @@ if [[ 1 == 1 ]]; then # INFO: SELinux users *may* have to add ":z" to the volume mount flags; check the docs! flags+=( -v "${argv0dir}/../..:/home/upx/src/upx" ) flags+=( -w /home/upx/src/upx ) # set working directory + flags+=( --tmpfs /home/upx/.cache:rw,exec ) # mount a writeable tmpfs + flags+=( --tmpfs /home/upx/.local:rw,exec ) # mount a writeable tmpfs else # run as user root 0:0 # ONLY FOR DEBUGGING THE IMAGE diff --git a/misc/rebuild-stubs-with-podman/Dockerfile b/misc/rebuild-stubs-with-podman/Dockerfile index 4c561f92..4b64dbf6 100644 --- a/misc/rebuild-stubs-with-podman/Dockerfile +++ b/misc/rebuild-stubs-with-podman/Dockerfile @@ -1,5 +1,5 @@ FROM docker.io/library/ubuntu:22.04 -ENV UPX_CONTAINER_IMAGE_NAME=upx-stubtools-20221212-v5 +ENV UPX_CONTAINER_IMAGE_NAME=upx-stubtools-20221212-v6 ARG DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 @@ -7,16 +7,16 @@ ENV LANG=C.UTF-8 RUN dpkg --add-architecture i386 \ && apt-get update && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ - aria2 bash-completion ca-certificates git less libmpc3 libncurses5 \ - make ncurses-term perl-base python2-minimal wget xz-utils \ + aria2 bash bash-completion ca-certificates dash git less libmpc3 libncurses5 \ + make ncurses-term perl-base python2-minimal time wget xz-utils \ libc6:i386 zlib1g:i386 \ # the following packages are not required for rebuilding the stubs, but - # they do make the image much more convenient and also allow building + # they do make the image *much* more convenient and also allow building # the full UPX binary inside the container via CMake: - 7zip bfs bzip2 cabextract chrpath cmake cpio curl elfutils fd-find file fzf g++ gdb gojq \ - ht htop hyperfine jq libzstd-dev lsb-release lz4 lzip lzop moreutils ninja-build \ - p7zip parallel patch patchelf pax-utils paxctl re2c ripgrep rsync \ - screen universal-ctags unzip vim zip zlib1g-dev zsh zstd \ + 7zip bfs busybox bzip2 cabextract ccache chrpath cmake cpio curl elfutils fd-find file fzf \ + g++ gawk gdb gojq ht htop hyperfine jq libzstd-dev lsb-release lz4 lzip lzop \ + mksh moreutils ninja-build p7zip parallel patch patchelf pax-utils paxctl \ + re2c ripgrep rsync screen universal-ctags unzip vim yash zip zlib1g-dev zsh zstd \ # extra packages for compiling with "gcc -m32" and and "gcc -mx32": g++-multilib gcc-multilib \ && true @@ -52,8 +52,12 @@ RUN cd /root \ # create default user upx 2000:2000 RUN useradd upx -U --uid 2000 --shell /bin/bash -m \ - && mkdir -p /home/upx/.cache/zig /home/upx/.local/bin /home/upx/.wine /home/upx/src/upx \ - && ln -s /usr/local/bin/bin-upx-20221212 /home/upx/.local/bin/bin-upx \ - && chown -R upx:upx /home/upx \ + && cd /home/upx && chmod 700 . \ + && mkdir -p .cache .local src/upx \ + && for d in ccache go-build tmp wine zig; do mkdir -p .cache/$d; done \ + && for d in bin share state; do mkdir -p .local/$d; done \ + && ln -s .cache/wine .wine && ln -s .cache/tmp tmp \ + && ln -s /usr/local/bin/bin-upx-20221212 .local/bin/bin-upx \ + && chown -R upx:upx . \ && true USER upx diff --git a/misc/rebuild-stubs-with-podman/packages.txt b/misc/rebuild-stubs-with-podman/packages.txt index c63ea907..cceda5ed 100644 --- a/misc/rebuild-stubs-with-podman/packages.txt +++ b/misc/rebuild-stubs-with-podman/packages.txt @@ -14,9 +14,11 @@ ii binutils 2.38-4ubuntu2.2 amd64 ii binutils-common:amd64 2.38-4ubuntu2.2 amd64 Common files for the GNU assembler, linker and binary utilities ii binutils-x86-64-linux-gnu 2.38-4ubuntu2.2 amd64 GNU binary utilities, for x86-64-linux-gnu target ii bsdutils 1:2.37.2-4ubuntu3 amd64 basic utilities from 4.4BSD-Lite +ii busybox 1:1.30.1-7ubuntu3 amd64 Tiny utilities for small and embedded systems ii bzip2 1.0.8-5build1 amd64 high-quality block-sorting file compressor - utilities ii ca-certificates 20230311ubuntu0.22.04.1 all Common CA certificates ii cabextract 1.9-3 amd64 Microsoft Cabinet file unpacker +ii ccache 4.5.1-1 amd64 Compiler cache for fast recompilation of C/C++ code ii chrpath 0.16-2 amd64 Tool to edit the rpath in ELF binaries ii cmake 3.22.1-1ubuntu1.22.04.1 amd64 cross-platform, open-source make system ii cmake-data 3.22.1-1ubuntu1.22.04.1 all CMake data files (modules, templates and documentation) @@ -43,6 +45,7 @@ ii g++ 4:11.2.0-1ubuntu1 amd64 ii g++-11 11.3.0-1ubuntu1~22.04.1 amd64 GNU C++ compiler ii g++-11-multilib 11.3.0-1ubuntu1~22.04.1 amd64 GNU C++ compiler (multilib support) ii g++-multilib 4:11.2.0-1ubuntu1 amd64 GNU C++ compiler (multilib files) +ii gawk 1:5.1.0-1build3 amd64 GNU awk, a pattern scanning and processing language ii gcc 4:11.2.0-1ubuntu1 amd64 GNU C compiler ii gcc-11 11.3.0-1ubuntu1~22.04.1 amd64 GNU C compiler ii gcc-11-base:amd64 11.3.0-1ubuntu1~22.04.1 amd64 GCC, the GNU Compiler Collection (base package) @@ -135,6 +138,7 @@ ii libgomp1:amd64 12.1.0-2ubuntu1~22.04 amd64 ii libgpg-error0:amd64 1.43-3 amd64 GnuPG development runtime library ii libgpm2:amd64 1.20.7-10build1 amd64 General Purpose Mouse - shared library ii libgssapi-krb5-2:amd64 1.19.2-2ubuntu0.2 amd64 MIT Kerberos runtime libraries - krb5 GSS-API Mechanism +ii libhiredis0.14:amd64 0.14.1-2 amd64 minimalistic C client library for Redis ii libhogweed6:amd64 3.7.3-1build2 amd64 low level cryptographic library (public-key cryptos) ii libicu70:amd64 70.1-2 amd64 International Components for Unicode ii libidn2-0:amd64 2.3.2-2build1 amd64 Internationalized domain names (IDNA2008/TR46) library @@ -202,6 +206,7 @@ ii libsemanage2:amd64 3.3-1build2 amd64 ii libsensors-config 1:3.6.0-7ubuntu1 all lm-sensors configuration files ii libsensors5:amd64 1:3.6.0-7ubuntu1 amd64 library to read temperature/voltage/fan sensors ii libsepol2:amd64 3.3-1build1 amd64 SELinux library for manipulating binary security policies +ii libsigsegv2:amd64 2.13-1ubuntu3 amd64 Library for handling page faults in a portable way ii libsmartcols1:amd64 2.37.2-4ubuntu3 amd64 smart column output alignment library ii libsodium23:amd64 1.0.18-1build2 amd64 Network communication, cryptography and signaturing library ii libsource-highlight-common 3.1.9-4.1build2 all architecture-independent files for source highlighting library @@ -249,7 +254,7 @@ ii libxxhash0:amd64 0.8.1-1 amd64 ii libyaml-0-2:amd64 0.2.2-1build2 amd64 Fast YAML 1.1 parser and emitter library ii libzstd-dev:amd64 1.4.8+dfsg-3build1 amd64 fast lossless compression algorithm -- development files ii libzstd1:amd64 1.4.8+dfsg-3build1 amd64 fast lossless compression algorithm -ii linux-libc-dev:amd64 5.15.0-75.82 amd64 Linux Kernel Headers for development +ii linux-libc-dev:amd64 5.15.0-76.83 amd64 Linux Kernel Headers for development ii login 1:4.8.1-2ubuntu2.1 amd64 system login tools ii logsave 1.46.5-2ubuntu1.1 amd64 save the output of a command in a log file ii lsb-base 11.1.0ubuntu4 all Linux Standard Base init script functionality @@ -260,6 +265,7 @@ ii lzop 1.04-2build2 amd64 ii make 4.3-4.1build1 amd64 utility for directing compilation ii mawk 1.3.4.20200120-3 amd64 Pattern scanning and text processing language ii media-types 7.0.0 all List of standard media types and their usual file extension +ii mksh 59c-16 amd64 MirBSD Korn Shell ii moreutils 0.66-1 amd64 additional Unix utilities ii mount 2.37.2-4ubuntu3 amd64 tools for mounting and manipulating filesystems ii ncurses-base 6.3-2ubuntu0.1 all basic terminal type definitions @@ -295,6 +301,7 @@ ii sensible-utils 0.0.17 all ii sysstat 12.5.2-2ubuntu0.2 amd64 system performance tools for Linux ii sysvinit-utils 3.01-1ubuntu1 amd64 System-V-like utilities ii tar 1.34+dfsg-1ubuntu0.1.22.04.1 amd64 GNU version of the tar archiving utility +ii time 1.9-0.1build2 amd64 GNU time program for measuring CPU resource usage ii ubuntu-keyring 2021.03.26 all GnuPG keys of the Ubuntu archive ii ucf 3.0043 all Update Configuration File(s): preserve user changes to config files ii universal-ctags 5.9.20210829.0-1 amd64 build tag file indexes of source code definitions @@ -307,6 +314,7 @@ ii vim-runtime 2:8.2.3995-1ubuntu2.8 all ii wget 1.21.2-2ubuntu1 amd64 retrieves files from the web ii xxd 2:8.2.3995-1ubuntu2.8 amd64 tool to make (or reverse) a hex dump ii xz-utils 5.2.5-2ubuntu1 amd64 XZ-format compression utilities +ii yash 2.51-1 amd64 yet another shell ii zip 3.0-12build2 amd64 Archiver for .zip files ii zlib1g-dev:amd64 1:1.2.11.dfsg-2ubuntu9.2 amd64 compression library - development ii zlib1g:amd64 1:1.2.11.dfsg-2ubuntu9.2 amd64 compression library - runtime @@ -319,7 +327,7 @@ ii zstd 1.4.8+dfsg-3build1 amd64 ||/ Name Version Architecture Description Packages sorted by Installed-Size: - 741791 ===== TOTAL (313 packages) + 748809 ===== TOTAL (321 packages) 52577 gcc-11 amd64 34444 libicu70 amd64 32780 vim-runtime all @@ -401,16 +409,20 @@ Packages sorted by Installed-Size: 1854 aria2 amd64 1750 libdb5.3 amd64 1746 libunistring2 amd64 + 1680 gawk amd64 1655 zstd amd64 1602 libsqlite3-0 amd64 1516 e2fsprogs amd64 + 1508 mksh amd64 1472 sysstat amd64 1464 bash-completion all 1445 hyperfine amd64 + 1429 yash amd64 1429 libx11-data all 1388 procps amd64 1386 libx11-6 amd64 1354 libgcrypt20 amd64 + 1343 ccache amd64 1328 libzstd-dev amd64 1292 libp11-kit0 amd64 1138 libpam-modules amd64 @@ -424,6 +436,7 @@ Packages sorted by Installed-Size: 876 libarchive13 amd64 846 libzstd1 amd64 794 rsync amd64 + 791 busybox amd64 787 libcurl4 amd64 784 libbrotli1 amd64 771 libcurl3-gnutls amd64 @@ -558,6 +571,7 @@ Packages sorted by Installed-Size: 133 init-system-helpers all 132 libipt2 amd64 131 libprocps8 amd64 + 126 time amd64 125 libmpc3 amd64 123 libtimedate-perl all 123 libnsl2 amd64 @@ -583,6 +597,7 @@ Packages sorted by Installed-Size: 96 libmspack0 amd64 95 libpsl5 amd64 93 libsasl2-modules-db amd64 + 92 libhiredis0.14 amd64 91 libjansson4 amd64 90 python3 amd64 83 sysvinit-utils amd64 @@ -608,6 +623,7 @@ Packages sorted by Installed-Size: 51 libutempter0 amd64 51 hostname amd64 50 gcc amd64 + 49 libsigsegv2 amd64 47 libkeyutils1 amd64 45 libgdbm-compat4 amd64 45 libcap-ng0 amd64 diff --git a/misc/scripts/build_upx_by_hand.sh b/misc/scripts/build_upx_by_hand.sh index a06784a3..cc8a1fe0 100755 --- a/misc/scripts/build_upx_by_hand.sh +++ b/misc/scripts/build_upx_by_hand.sh @@ -3,49 +3,78 @@ set -e # -# build UPX "by hand", using POSIX shell and a minimal number of compilation flags +# build UPX "by hand", using a POSIX-compliant shell and +# a minimal number of compilation flags # Copyright (C) Markus Franz Xaver Johannes Oberhumer # +# uses optional environment variables: AR, CC, CXX, OPTIMIZE, top_srcdir + # shell init ### set -x # enable logging DUALCASE=1; export DUALCASE # for MKS sh test -n "${ZSH_VERSION+set}" && emulate sh # for zsh -argv0="$0"; argv0abs="$(readlink -fn "$argv0")"; argv0dir="$(dirname "$argv0abs")" -# HINT: set "argv0dir" manually if your system does not have "readlink" +my_argv0="$0" -# toolchain settings and flags +# toolchain settings and compilation flags +AR="${AR:-ar}" CC="${CC:-cc}" CXX="${CXX:-c++ -std=gnu++17}" -AR="${AR:-ar}" # HINT: use "export AR=false" if "$AR rcs" does not work on your system; see below if test "x$AR" = "x0" || test "x$AR" = "xfalse" || test "x$AR" = "x/bin/false"; then AR="" # do not use $AR fi # protect against security threats caused by misguided compiler "optimizations" mandatory_flags="-fno-strict-aliasing -fno-strict-overflow -funsigned-char" -# not mandatory and not minimal, but usually a good idea: -### mandatory_flags="-Wall -O2 $mandatory_flags" +if test "x$OPTIMIZE" != "x" && test "x$OPTIMIZE" != "x0"; then + # not mandatory and not minimal, but usually a good idea: + mandatory_flags="-Wall -O2 $mandatory_flags" +fi CC="$CC $mandatory_flags" CXX="$CXX $mandatory_flags" # go to upx top-level directory -cd "$argv0dir/../.." || exit 1 -pwd +# HINT: set "top_srcdir" manually if your system does not have "readlink" +if test "x$top_srcdir" = "x"; then + my_argv0abs="$(readlink -fn "$my_argv0")" + my_argv0dir="$(dirname "$my_argv0abs")" + cd "$my_argv0dir/../.." || exit 1 +else + cd "$top_srcdir" || exit 1 +fi +top_srcdir="$(pwd)" # absolute +rel_top_srcdir=. # relative top_srcdir +echo "# current directory: '$(pwd)'" +test -f doc/upx.pod || exit 1 # sanity check test -f src/version.h || exit 1 # sanity check -top_srcdir="$PWD" rm -rf ./build/by-hand # WARNING: existing build-directory gets deleted! -mkdir -p ./build/by-hand + +# helper function +run() { + if test 0 = 1; then + # DEBUG dry-run: print command, but don't actually run unless $1 is "+" + if test "x$1" = "x+"; then + shift; echo "$@"; "$@"; return + fi + shift; echo "$@"; return + fi + # print short info and run command + test "x$1" != "x" && test "x$1" != "x+" && echo "$1" + shift; "$@" +} # helper function check_submodule() { - local f - for f in COPYING LICENSE LICENSE.txt; do - if test -f "$top_srcdir/vendor/$1/$f"; then - # create and enter build directory - mkdir -p "$top_srcdir/build/by-hand/$1" - cd "$top_srcdir/build/by-hand/$1" || exit 1 - echo "===== build $1 =====" + #local ff # "local" seems unsupported by some versions of ksh + for ff in COPYING LICENSE LICENSE.txt; do + if test -f "$rel_top_srcdir/vendor/$1/$ff"; then + # create and enter build directory; updates global $rel_top_srcdir + run "+" cd "$rel_top_srcdir" || exit 1 + rel_top_srcdir=. + echo "#==== build $1 =====" + run "+" mkdir "build/by-hand/$1" + run "+" cd "build/by-hand/$1" || exit 1 + rel_top_srcdir=../../.. return 0 fi done @@ -53,48 +82,44 @@ check_submodule() { } # build +run "+" mkdir -p "build/by-hand" if check_submodule bzip2; then - for f in "$top_srcdir"/vendor/bzip2/*.c; do - echo "CC $f" - $CC -c "$f" + for f in "$rel_top_srcdir"/vendor/bzip2/*.c; do + run "CC $f" $CC -c "$f" done fi if check_submodule ucl; then - for f in "$top_srcdir"/vendor/ucl/src/*.c; do - echo "CC $f" - $CC -I"$top_srcdir"/vendor/ucl/include -I"$top_srcdir"/vendor/ucl -c "$f" + for f in "$rel_top_srcdir"/vendor/ucl/src/*.c; do + run "CC $f" $CC -I"$rel_top_srcdir"/vendor/ucl/include -I"$rel_top_srcdir"/vendor/ucl -c "$f" done fi if check_submodule zlib; then - for f in "$top_srcdir"/vendor/zlib/*.c; do - echo "CC $f" - $CC -DHAVE_STDARG_H -DHAVE_VSNPRINTF -DHAVE_UNISTD_H -c "$f" + for f in "$rel_top_srcdir"/vendor/zlib/*.c; do + run "CC $f" $CC -DHAVE_STDARG_H -DHAVE_VSNPRINTF -DHAVE_UNISTD_H -c "$f" done fi if check_submodule zstd; then - for f in "$top_srcdir"/vendor/zstd/lib/*/*.c; do - echo "CC $f" - $CC -DDYNAMIC_BMI2=0 -DZSTD_DISABLE_ASM -c "$f" + for f in "$rel_top_srcdir"/vendor/zstd/lib/*/*.c; do + run "CC $f" $CC -DDYNAMIC_BMI2=0 -DZSTD_DISABLE_ASM -c "$f" done fi -echo "===== build UPX =====" -cd "$top_srcdir"/build/by-hand || exit 1 -for f in "$top_srcdir"/src/*.cpp "$top_srcdir"/src/*/*.cpp; do - echo "CXX $f" - $CXX -I"$top_srcdir"/vendor -c "$f" +run "+" cd "$rel_top_srcdir" || exit 1 +rel_top_srcdir=. +echo "#==== build UPX =====" +run "+" cd "build/by-hand" || exit 1 +rel_top_srcdir=../.. +for f in "$rel_top_srcdir"/src/*.cpp "$rel_top_srcdir"/src/*/*.cpp; do + run "CXX $f" $CXX -I"$rel_top_srcdir"/vendor -c "$f" done -# echo "===== link UPX =====" +# echo "#==== link UPX =====" if test "x$AR" = "x"; then # link without using $AR - echo "CXX upx" - $CXX -o upx *.o */*.o + run "CXX upx" $CXX -o upx *.o */*.o else - echo "AR libupx" - $AR rcs libupx_submodules.a */*.o - echo "CXX upx" - $CXX -o upx *.o -L. -lupx_submodules + run "AR libupx" $AR rcs libupx_submodules.a */*.o + run "CXX upx" $CXX -o upx *.o -L. -lupx_submodules fi -pwd +echo "# current directory: '$(pwd)'" ls -l upx* -echo "All done." +echo "# All done."