From fc4dcd46dbce950bcaacfc1bd207949b22f5cdc2 Mon Sep 17 00:00:00 2001 From: "Markus F.X.J. Oberhumer" Date: Sat, 22 Jul 2023 16:12:13 +0200 Subject: [PATCH] CI and build updates --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/weekly-ci-alpine-linux.yml | 12 +++++++++++ .github/workflows/weekly-ci-zigcc.yml | 4 ++-- CMakeLists.txt | 2 +- README.SRC | 2 +- misc/scripts/build_upx_by_hand.sh | 2 +- misc/testsuite/upx_testsuite_1.sh | 21 +++++++++++--------- src/compress/compress_zlib.cpp | 5 +++-- 8 files changed, 37 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f88c6aeb..19af28c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ env: CMAKE_REQUIRED_QUIET: OFF DEBIAN_FRONTEND: noninteractive UPX_CMAKE_BUILD_FLAGS: --verbose - # 2023-07-19 - ZIG_DIST_VERSION: 0.11.0-dev.4009+11695745e + # 2023-07-20 + ZIG_DIST_VERSION: 0.11.0-dev.4059+17255bed4 jobs: job-rebuild-and-verify-stubs: @@ -149,7 +149,7 @@ jobs: - name: 'Install brew packages' if: ${{ matrix.testsuite }} run: | - # testsuite needs working "readlink -en" and "sha256sum" + # testsuite needs working "readlink -en" and "sha256sum -b" if ! test -e /usr/local/opt/coreutils/libexec/gnubin/readlink; then echo "===== brew leaves:"; brew leaves; echo "===== brew list:"; brew list # only run "brew update" if needed @@ -320,12 +320,12 @@ jobs: %RUN_LIB% -out:ucl.lib *.obj @REM ===== build zlib ===== cd %BDIR%\zlib - %RUN_CL% -J -O2 -W3 -WX %DEFS% -c %H%\vendor\zlib\*.c + %RUN_CL% -J -O2 -W3 -WX %DEFS% -DHAVE_VSNPRINTF -c %H%\vendor\zlib\*.c %RUN_LIB% -out:zlib.lib *.obj @REM ===== build zstd ===== cd %BDIR%\zstd set s=%H%\vendor\zstd\lib - @rem %RUN_CL% -J -O2 -W4 -WX -DDYNAMIC_BMI2=0 -DZSTD_DISABLE_ASM %DEFS% -c %s%\common\*.c %s%\compress\*.c %s%\decompress\*.c + @rem %RUN_CL% -J -O2 -W4 -WX %DEFS% -DDYNAMIC_BMI2=0 -DZSTD_DISABLE_ASM -c %s%\common\*.c %s%\compress\*.c %s%\decompress\*.c @rem %RUN_LIB% -out:zstd.lib *.obj @REM ===== build UPX ===== cd %BDIR%\upx diff --git a/.github/workflows/weekly-ci-alpine-linux.yml b/.github/workflows/weekly-ci-alpine-linux.yml index 543e0141..6a5b45b7 100644 --- a/.github/workflows/weekly-ci-alpine-linux.yml +++ b/.github/workflows/weekly-ci-alpine-linux.yml @@ -15,6 +15,7 @@ on: env: CMAKE_REQUIRED_QUIET: OFF DEBIAN_FRONTEND: noninteractive + UPX_TESTSUITE_LEVEL: 2 jobs: job-alpine-cmake: # uses cmake + make @@ -141,4 +142,15 @@ jobs: (cd "upx with space"/build/xtarget/gcc-static/release && DESTDIR="$PWD/Install with cmake" cmake --install .) (cd "upx with space"/build/xtarget/gcc-static/release && DESTDIR="$PWD/Install with make" make install) + # test suite + - name: ${{ format('Run test suite level {0}', env.UPX_TESTSUITE_LEVEL) }} + run: | + # use a directory that contains whitespace to detect possible quoting issues + git clone --depth=1 https://github.com/upx/upx-testsuite "upx-testsuite with space" + export upx_testsuite_SRCDIR="$(readlink -fn "upx-testsuite with space")" + testsuite_1="$(readlink -fn "upx with space"/misc/testsuite/upx_testsuite_1.sh)" + # testsuite needs bash and working "readlink -en" and "sha256sum -b" + apk add bash coreutils + (cd "upx with space"/build/xtarget/gcc-static/release && upx_exe=./upx bash "$testsuite_1") + # vim:set ts=2 sw=2 et: diff --git a/.github/workflows/weekly-ci-zigcc.yml b/.github/workflows/weekly-ci-zigcc.yml index c61aff80..fc0ea416 100644 --- a/.github/workflows/weekly-ci-zigcc.yml +++ b/.github/workflows/weekly-ci-zigcc.yml @@ -8,8 +8,8 @@ on: env: CMAKE_REQUIRED_QUIET: OFF DEBIAN_FRONTEND: noninteractive - # 2023-07-19 - ZIG_DIST_VERSION: 0.11.0-dev.4009+11695745e + # 2023-07-20 + ZIG_DIST_VERSION: 0.11.0-dev.4059+17255bed4 jobs: job-linux-zigcc: # uses cmake + make diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f8307fd..dfe53806 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -371,7 +371,7 @@ endif() set(t upx_vendor_zlib) upx_compile_target_debug_with_O2(${t}) upx_sanitize_target(${t}) -target_compile_definitions(${t} PRIVATE HAVE_STDARG_H=1 HAVE_VSNPRINTF=1) +target_compile_definitions(${t} PRIVATE HAVE_VSNPRINTF=1) if(MSVC_FRONTEND) target_compile_options(${t} PRIVATE -W3 ${warn_WX}) else() diff --git a/README.SRC b/README.SRC index 684998e8..0c8e168b 100644 --- a/README.SRC +++ b/README.SRC @@ -100,7 +100,7 @@ Developer quick start 12) $ cd src # my-upx/upx/src 12a) On MacOS: install homebrew (https:://brew.sh), then "brew install coreutils" and put /opt/coreutils/libexec/gnubin into PATH so that - "readlink -en" and 'sha256sum' work. + "readlink -en" and "sha256sum -b" work. 13) $ make run-testsuite 2>&1 | tee testsuite.log diff --git a/misc/scripts/build_upx_by_hand.sh b/misc/scripts/build_upx_by_hand.sh index b71e8eeb..90de21e9 100755 --- a/misc/scripts/build_upx_by_hand.sh +++ b/misc/scripts/build_upx_by_hand.sh @@ -97,7 +97,7 @@ if check_submodule ucl; then fi if check_submodule zlib; then for f in "$rel_top_srcdir"/vendor/zlib/*.c; do - run "CC $f" $CC -DHAVE_STDARG_H -DHAVE_VSNPRINTF -DHAVE_UNISTD_H -c "$f" + run "CC $f" $CC -DHAVE_UNISTD_H -DHAVE_VSNPRINTF -c "$f" done fi if check_submodule zstd; then diff --git a/misc/testsuite/upx_testsuite_1.sh b/misc/testsuite/upx_testsuite_1.sh index 2b62242d..b4ee521a 100755 --- a/misc/testsuite/upx_testsuite_1.sh +++ b/misc/testsuite/upx_testsuite_1.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash ## vim:set ts=4 sw=4 et: set -e; set -o pipefail -argv0=$0; argv0abs=$(readlink -en -- "$argv0"); argv0dir=$(dirname "$argv0abs") +argv0=$0; argv0abs=$(readlink -fn "$argv0"); argv0dir=$(dirname "$argv0abs") # very first version of the upx-testsuite; requires: # $upx_exe (required, but with convenience fallback "./upx") @@ -23,7 +23,8 @@ argv0=$0; argv0abs=$(readlink -en -- "$argv0"); argv0dir=$(dirname "$argv0abs") [[ -z $upx_exe && -f ./upx && -x ./upx ]] && upx_exe=./upx # convenience fallback if [[ -z $upx_exe ]]; then echo "UPX-ERROR: please set \$upx_exe"; exit 1; fi if [[ ! -f $upx_exe ]]; then echo "UPX-ERROR: file '$upx_exe' does not exist"; exit 1; fi -upx_exe=$(readlink -en -- "$upx_exe") # make absolute +upx_exe=$(readlink -fn "$upx_exe") # make absolute +[[ -f $upx_exe ]] || exit 1 upx_run=() if [[ -n $upx_exe_runner ]]; then # usage examples: @@ -57,14 +58,16 @@ if [[ ! -d "$upx_testsuite_SRCDIR/files/packed" ]]; then echo " and set (export) the envvar upx_testsuite_SRCDIR to the local file path" exit 1 fi -upx_testsuite_SRCDIR=$(readlink -en -- "$upx_testsuite_SRCDIR") # make absolute +upx_testsuite_SRCDIR=$(readlink -fn "$upx_testsuite_SRCDIR") # make absolute +[[ -d $upx_testsuite_SRCDIR ]] || exit 1 # upx_testsuite_BUILDDIR if [[ -z $upx_testsuite_BUILDDIR ]]; then upx_testsuite_BUILDDIR="./tmp-upx-testsuite" fi mkdir -p "$upx_testsuite_BUILDDIR" || exit 1 -upx_testsuite_BUILDDIR=$(readlink -en -- "$upx_testsuite_BUILDDIR") # make absolute +upx_testsuite_BUILDDIR=$(readlink -fn "$upx_testsuite_BUILDDIR") # make absolute +[[ -d $upx_testsuite_BUILDDIR ]] || exit 1 cd / && cd "$upx_testsuite_BUILDDIR" || exit 1 @@ -137,7 +140,7 @@ testsuite_split_f() { } testsuite_check_sha() { - (cd "$1" && sha256sum -b -- */* | LC_ALL=C sort -k2) > $1/.sha256sums.current + (cd "$1" && sha256sum -b [0-9a-zA-Z]*/* | LC_ALL=C sort -k2) > $1/.sha256sums.current echo cat $1/.sha256sums.current if ! cmp -s $1/.sha256sums.expected $1/.sha256sums.current; then @@ -152,7 +155,7 @@ testsuite_check_sha() { } testsuite_check_sha_decompressed() { - (cd "$1" && sha256sum -b -- */* | LC_ALL=C sort -k2) > $1/.sha256sums.current + (cd "$1" && sha256sum -b [0-9a-zA-Z]*/* | LC_ALL=C sort -k2) > $1/.sha256sums.current if ! cmp -s $1/.sha256sums.expected $1/.sha256sums.current; then cat $1/.sha256sums.current echo "UPX-ERROR: FATAL: $1 FAILED: decompressed checksum mismatch" @@ -301,12 +304,12 @@ fi # recreate checksums from current version for an easy update in case of changes recreate_expected_sha256sums .sha256sums.recreate -testsuite_header "UPX testsuite summary" +testsuite_header "UPX testsuite summary: level $UPX_TESTSUITE_LEVEL" run_upx --version-short echo echo "upx_exe='$upx_exe'" -if [[ -f $upx_exe ]]; then - ls -l "$upx_exe" +ls -l "$upx_exe" +if command -v file >/dev/null; then file "$upx_exe" || true fi echo "upx_run='${upx_run[*]}'" diff --git a/src/compress/compress_zlib.cpp b/src/compress/compress_zlib.cpp index 72d85f0b..91b89779 100644 --- a/src/compress/compress_zlib.cpp +++ b/src/compress/compress_zlib.cpp @@ -28,6 +28,7 @@ #include "compress.h" #include "../util/membuffer.h" // NOLINTBEGIN(clang-analyzer-optin.performance.Padding) +#define ZLIB_CONST 1 #include #include // NOLINTEND(clang-analyzer-optin.performance.Padding) @@ -101,7 +102,7 @@ int upx_zlib_compress(const upx_bytep src, unsigned src_len, upx_bytep dst, unsi z_stream s; s.zalloc = (alloc_func) nullptr; s.zfree = (free_func) nullptr; - s.next_in = ACC_UNCONST_CAST(upx_bytep, src); + s.next_in = src; s.avail_in = src_len; s.next_out = dst; s.avail_out = *dst_len; @@ -150,7 +151,7 @@ int upx_zlib_decompress(const upx_bytep src, unsigned src_len, upx_bytep dst, un z_stream s; s.zalloc = (alloc_func) nullptr; s.zfree = (free_func) nullptr; - s.next_in = ACC_UNCONST_CAST(upx_bytep, src); + s.next_in = src; s.avail_in = src_len; s.next_out = dst; s.avail_out = *dst_len;