From 6ad2608dcbf9556d56db252602a7ce179ab440f0 Mon Sep 17 00:00:00 2001 From: "@Antelox" Date: Mon, 18 Nov 2024 12:10:27 +0100 Subject: [PATCH] Python package building rework (#2538) * - Refactored setup.py to remove hacks regarding packaging of wheels for different platforms, improve and cleanup the code - Updated README.txt - Removed old Makefile and build_wheel.sh scripts - Created a new workflow that takes care of building and testing python packages for different platforms/architectures/python versions * Added SPDX headers to the setup.py * - cstest_py: Fixed positional argument since it doesn't accept a `required` flag. It turns to have a mandatory tests folder path - integration_tests.py: Use pathlib to determine the required path - GitHub action: Simplified the tests execution command * GitHub Actions: Run python 3.8 (lowest) and 3.13 (current highest) for native runners only during testings and the rest during tag release * GitHub Action: - Fixed the cibw_build matrix element - Added a step to prepare artifact name * GitHub Action: Added run_tests.py script to run all tests during CI workflow * - Added SPDX headers to the run_tests.py script and to the build-wheels-publish.yml workflow file - Minor fixes to the workflow as pointed out in the PR review - Updated MANIFEST.in to reflect the actual libraries built during python wheel creation process - Use subprocess.run in place of os.system in run_tests.py script * GitHub Action: - Run qemu step only if non-native Linux runner - Added arch:universal2 matrix element for macos-latest runner * Python bindings: Refreshed the list of files needed to be copied for sdist archive * GitHub Action: Commented out arch:x86 matrix elements * GitHub Action: Run qemu step only if non-native Linux runner * GitHub Action: Minor fixes * Python bindings: Added missing .in pattern when collecting src files for sdist archive --- .github/workflows/build-wheels-publish.yml | 279 ++++++++++++++++++ .github/workflows/python-publish-release.yml | 90 ------ .github/workflows/python-tests.yml | 62 ---- bindings/python/MANIFEST.in | 11 +- bindings/python/Makefile | 47 --- bindings/python/README.txt | 6 +- bindings/python/build_wheel.sh | 16 - bindings/python/cstest_py/pyproject.toml | 6 +- .../python/cstest_py/src/cstest_py/cstest.py | 31 +- bindings/python/pyproject.toml | 2 +- bindings/python/setup.py | 140 ++++----- suite/cstest/test/integration_tests.py | 10 +- suite/run_tests.py | 25 ++ 13 files changed, 383 insertions(+), 342 deletions(-) create mode 100644 .github/workflows/build-wheels-publish.yml delete mode 100644 .github/workflows/python-publish-release.yml delete mode 100644 .github/workflows/python-tests.yml delete mode 100644 bindings/python/Makefile delete mode 100755 bindings/python/build_wheel.sh create mode 100644 suite/run_tests.py diff --git a/.github/workflows/build-wheels-publish.yml b/.github/workflows/build-wheels-publish.yml new file mode 100644 index 000000000..ffed6b362 --- /dev/null +++ b/.github/workflows/build-wheels-publish.yml @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: 2024 Antelox +# SPDX-License-Identifier: BSD-3 + +name: Build and publish wheels with cibuildwheel + +on: + workflow_dispatch: + inputs: + debugMode: + description: 'Debug Mode' + required: false + default: '' + type: choice + options: + - '0' + - '1' + + push: + paths-ignore: + - ".gitignore" + - "CREDITS.TXT" + - "ChangeLog" + - "README.md" + - "docs/**" + pull_request: + +env: + # Enable DEBUG flag either according to the tag release or manual override + CAPSTONE_DEBUG: ${{ inputs.debugMode != '' && inputs.debugMode || startsWith(github.ref, 'refs/tags') && '0' || '1' }} + +jobs: + # job to be executed for every push - testing purpose + build_wheels_always: + name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # NOTE: Making this to parallelize and speed up workflow + # i686 - manylinux + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' } + # i686 - musllinux + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' } + # x86_64 - manylinux + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' } + # x86_64 - musllinux + # - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' } + # aarch64 - manylinux + # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' } + # aarch64 - musllinux + # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' } + # macos - x86_64 + - { os: macos-13, arch: x86_64, cibw_build: 'cp38* cp313*', cibw_skip: '' } + # macos - arm64 + # - { os: macos-latest, arch: arm64, cibw_build: 'cp38* cp313*', cibw_skip: '' } + # - { os: macos-latest, arch: universal2, cibw_build: 'cp38* cp313*', cibw_skip: '' } + # windows - x86_64 + - { os: windows-latest, arch: AMD64, cibw_build: 'cp38* cp313*', cibw_skip: '' } + # windows - amd64 + # - { os: windows-latest, arch: x86, cibw_build: 'cp38* cp313*', cibw_skip: '' } + # windows - arm64 + # - { os: windows-latest, arch: ARM64, cibw_build: 'cp39* cp313*', cibw_skip: '' } + + steps: + - uses: actions/checkout@v4 + + # https://github.com/actions/upload-artifact/issues/22 + - name: Prepare a unique name for Artifacts + shell: bash + run: | + # replace not-allowed chars with dash + name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}" + name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//') + echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV + + # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 + - uses: actions/setup-python@v5 + if: runner.os == 'macOS' && runner.arch == 'ARM64' + with: + python-version: 3.8 + + - name: '🛠️ Win MSVC 32 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'x86' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + - name: '🛠️ Win MSVC 64 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'AMD64' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: '🛠️ Win MSVC ARM64 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'ARM64' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: amd64_arm64 + + - name: '🛠️ Set up QEMU' + if: runner.os == 'Linux' && matrix.arch != 'x86_64' + uses: docker/setup-qemu-action@v3 + + - name: '🚧 cibuildwheel run' + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD_FRONTEND: build + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_SKIP: ${{ matrix.cibw_skip }} + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }} + CIBW_ENVIRONMENT_PASS_LINUX: DEBUG + # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64 + # https://github.com/pypa/cibuildwheel/pull/1169 + CIBW_TEST_SKIP: "*-win_arm64 cp38-macosx_*:arm64" + CIBW_TEST_COMMAND: > + python -m pip install {package}/cstest_py && + python {project}/suite/run_tests.py + with: + package-dir: bindings/python + output-dir: wheelhouse + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ./wheelhouse/*.whl + + # To be executed only in case of a tag release + build_wheels_all: + name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }} + runs-on: ${{ matrix.os }} + if: startsWith(github.ref, 'refs/tags') + strategy: + fail-fast: false + matrix: + include: + # NOTE: Making this to parallelize and speed up workflow + # i686 - manylinux + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-manylinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-manylinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-manylinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-manylinux*', cibw_skip: '' } + # i686 - musllinux + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-musllinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-musllinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-musllinux*', cibw_skip: '' } + # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-musllinux*', cibw_skip: '' } + # x86_64 - manylinux + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-manylinux*', cibw_skip: '' } + # x86_64 - musllinux + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-musllinux*', cibw_skip: '' } + # aarch64 - manylinux + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-manylinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-manylinux*', cibw_skip: '' } + # aarch64 - musllinux + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-musllinux*', cibw_skip: '' } + - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-musllinux*', cibw_skip: '' } + # macos - x86_64 + - { os: macos-13, arch: x86_64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' } + # macos - arm64 + - { os: macos-latest, arch: arm64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' } + - { os: macos-latest, arch: universal2, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' } + # windows - amd64 + - { os: windows-latest, arch: AMD64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' } + # windows - x86 + # - { os: windows-latest, arch: x86, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' } + # windows - arm64 + - { os: windows-latest, arch: ARM64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' } + + steps: + - uses: actions/checkout@v4 + + # https://github.com/actions/upload-artifact/issues/22 + - name: Prepare a unique name for Artifacts + shell: bash + run: | + # replace not-allowed chars with dash + name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}" + name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//') + echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV + + - name: '🛠️ Win MSVC 32 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'x86' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + - name: '🛠️ Win MSVC 64 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'AMD64' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: '🛠️ Win MSVC ARM64 dev cmd setup' + if: runner.os == 'Windows' && matrix.arch == 'ARM64' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: amd64_arm64 + + - name: '🛠️ Set up QEMU' + if: runner.os == 'Linux' && matrix.arch != 'x86_64' + uses: docker/setup-qemu-action@v3 + + - name: '🚧 cibuildwheel run' + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD_FRONTEND: build + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_SKIP: ${{ matrix.cibw_skip }} + CIBW_ARCHS: ${{ matrix.arch }} + CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }} + CIBW_ENVIRONMENT_PASS_LINUX: DEBUG + # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64 + CIBW_TEST_SKIP: "*-win_arm64" + CIBW_TEST_COMMAND: > + python -m pip install {package}/cstest_py && + python {project}/suite/run_tests.py + with: + package-dir: bindings/python + output-dir: wheelhouse + + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ./wheelhouse/*.whl + + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + + - name: Build SDist + run: | + cd bindings/python + python3 -m pip install -U pip build + python3 -m build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: sdist-archive + path: bindings/python/dist/*.tar.gz + + publish: + needs: [ build_wheels_always, build_wheels_all, make_sdist ] + environment: pypi + permissions: + id-token: write + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags') + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist + + - name: Show downloaded artifacts + run: ls -laR dist + + - name: '📦 Publish distribution to PyPI' + uses: pypa/gh-action-pypi-publish@release/v1 + if: ${{ success() }} + with: + user: __token__ + password: ${{ secrets.pypi_pass }} diff --git a/.github/workflows/python-publish-release.yml b/.github/workflows/python-publish-release.yml deleted file mode 100644 index d1c15cd5f..000000000 --- a/.github/workflows/python-publish-release.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: RELEASE BUILD - PyPI 📦 Distribution - -on: [push, pull_request, release, workflow_dispatch] - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - steps: - - uses: actions/checkout@v4 - - - name: Set up MSVC x64 - if: matrix.os == 'windows-latest' - uses: ilammy/msvc-dev-cmd@v1 - - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 - env: - CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" - CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow - CIBW_ARCHS_WINDOWS: "AMD64" # ARM64 Seems ARM64 will rebuild amd64 wheel for unknow reason. - CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" - CIBW_SKIP: "" - with: - package-dir: bindings/python - - - uses: actions/upload-artifact@v4 - with: - path: ./wheelhouse/*.whl - name: artifacts-${{ matrix.os }} - - - name: Check binaries (Windows) - if: matrix.os == 'windows-latest' - run: | - python3.exe suite/check_wheel_bin_arch.py ./wheelhouse/ - - - name: Check binaries (Unix) - if: matrix.os != 'windows-latest' - run: | - ./suite/check_wheel_bin_arch.py ./wheelhouse/ - - make_sdist: - name: Make SDist - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Optional, use if you use setuptools_scm - submodules: true # Optional, use if you have submodules - - - name: Build SDist - run: | - cd bindings/python - pipx run build --sdist - - - uses: actions/upload-artifact@v4 - with: - path: bindings/python/dist/*.tar.gz - - publish: - needs: [build_wheels] - runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'released' - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - merge-multiple: true - path: dist - - - name: Show downloaded artifacts - run: ls -laR dist - - - name: Publish distribution 📦 to PyPI - if: ${{ success() }} - uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true - user: __token__ - password: ${{ secrets.pypi_pass }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml deleted file mode 100644 index 67dcde37f..000000000 --- a/.github/workflows/python-tests.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Python Package CI - -on: - push: - pull_request: - -# Stop previous runs on the same branch on new push -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, windows-2022, macOS-14] - python-version: ["3.8", "3.12"] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup MSVC - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 - - - name: Build and install capstone - run: pip install ./bindings/python - - - name: Install cstest_py - run: pip install ./bindings/python/cstest_py - - - name: Run legacy tests - run: python ./bindings/python/tests/test_all.py - - - name: cstest_py integration tests - run: | - cd suite/cstest/test/ - python3 ./integration_tests.py cstest_py - cd ../../../ - - - name: cstest_py MC - run: | - cstest_py tests/MC/ - - - name: cstest_py details - run: | - cstest_py tests/details/ - - - name: cstest_py issues - run: | - cstest_py tests/issues/ - - - name: cstest_py features - run: | - cstest_py tests/features/ diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index 98776c7d9..6246aac05 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -1,5 +1,8 @@ recursive-include src * -include LICENSE.TXT -include README.txt -include BUILDING.txt -include Makefile +recursive-include prebuilt * +include BUILDING.md +graft capstone/lib +graft capstone/include +global-include *.dll +global-include *.dylib +global-include *.so.* diff --git a/bindings/python/Makefile b/bindings/python/Makefile deleted file mode 100644 index 9b44705e4..000000000 --- a/bindings/python/Makefile +++ /dev/null @@ -1,47 +0,0 @@ -PYTHON3 ?= python3 - -.PHONY: gen_const install sdist bdist clean check - -gen_const: - cd .. && $(PYTHON3) const_generator.py python - -install: - rm -rf src/ - if test -n "${DESTDIR}"; then \ - $(PYTHON3) setup.py build install --root="${DESTDIR}"; \ - else \ - $(PYTHON3) setup.py build install; \ - fi - -# build & upload PyPi package with source code of the core -sdist: - rm -rf src/ dist/ - $(PYTHON3) setup.py sdist register upload - -# build & upload PyPi package with prebuilt core -bdist: - rm -rf src/ dist/ - $(PYTHON3) setup.py bdist_wheel register upload - -clean: - rm -rf build/ src/ dist/ *.egg-info - rm -rf capstone/lib capstone/include pyx/lib pyx/include - rm -f pyx/*.c pyx/__init__.py - for f in capstone/*.py; do rm -f pyx/$$(basename $$f)x; done - rm -f MANIFEST - rm -f *.pyc capstone/*.pyc - - -TESTS = test_basic.py test_detail.py test_arm.py test_aarch64.py test_m68k.py test_mips.py -TESTS += test_ppc.py test_sparc.py test_systemz.py test_x86.py test_xcore.py test_tms320c64x.py -TESTS += test_m680x.py test_skipdata.py test_mos65xx.py test_bpf.py test_riscv.py -TESTS += test_evm.py test_tricore.py test_wasm.py test_sh.py test_hppa.py -TESTS += test_lite.py test_iter.py test_customized_mnem.py test_alpha.py test_xtensa.py - -check: - @for t in $(TESTS); do \ - echo Check $$t ... ; \ - ./tests/$$t > /dev/null; \ - if [ $$? -eq 0 ]; then echo OK; else echo FAILED; exit 1; fi \ - done - diff --git a/bindings/python/README.txt b/bindings/python/README.txt index ed7f711a6..9fda7d881 100644 --- a/bindings/python/README.txt +++ b/bindings/python/README.txt @@ -1,12 +1,12 @@ To install Capstone, you should run `pip install capstone`. -If you would like to build Capstone with just the source distribution, without -pip, just run `python setup.py install` in the folder with setup.py in it. +If you would like to build and install Capstone with just the source distribution, +just run `python -m pip install .`, considering you are in the folder with setup.py in it. In order to use this source distribution, you will need an environment that can compile C code. On Linux, this is usually easy, but on Windows, this involves installing Visual Studio and using the "Developer Command Prompt" to perform the -installation. See BUILDING.txt for more information. +installation. See BUILDING.md for more information. By default, attempting to install the python bindings will trigger a build of the capstone native core. If this is undesirable for whatever reason, for diff --git a/bindings/python/build_wheel.sh b/bindings/python/build_wheel.sh deleted file mode 100755 index 38a9ceccf..000000000 --- a/bindings/python/build_wheel.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e -x - -cd bindings/python -if [ -f /opt/python/cp311-cp311/bin/python3 ];then - # Use manylinux Python - /opt/python/cp311-cp311/bin/python3 -m pip install wheel - /opt/python/cp311-cp311/bin/python3 setup.py bdist_wheel -else - python3 -m pip install wheel - python3 setup.py bdist_wheel -fi - -cd dist -auditwheel repair *.whl -mv -f wheelhouse/*.whl . \ No newline at end of file diff --git a/bindings/python/cstest_py/pyproject.toml b/bindings/python/cstest_py/pyproject.toml index 34b27c5aa..5df6317fe 100644 --- a/bindings/python/cstest_py/pyproject.toml +++ b/bindings/python/cstest_py/pyproject.toml @@ -5,14 +5,14 @@ name = "cstest_py" version = "0.1.0" dependencies = [ - "pyyaml >= 6.0.2", - "capstone >= 5.0.0", + "pyyaml >= 6.0.2", + "capstone >= 5.0.0", ] requires-python = ">= 3.8" [tool.setuptools] packages = ["cstest_py"] -package-dir = {"" = "src"} +package-dir = { "" = "src" } [project.scripts] cstest_py = "cstest_py.cstest:main" diff --git a/bindings/python/cstest_py/src/cstest_py/cstest.py b/bindings/python/cstest_py/src/cstest_py/cstest.py index 4fd244914..6e3590e2c 100755 --- a/bindings/python/cstest_py/src/cstest_py/cstest.py +++ b/bindings/python/cstest_py/src/cstest_py/cstest.py @@ -7,7 +7,6 @@ from __future__ import annotations import argparse import logging -import subprocess as sp import sys import os import yaml @@ -405,34 +404,16 @@ class CSTest: self.stats.print_evaluate() -def get_repo_root() -> str | None: - res = sp.run(["git", "rev-parse", "--show-toplevel"], capture_output=True) - if res.stderr: - log.error("Could not get repository root directory.") - return None - return res.stdout.decode("utf8").strip() - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( prog="Python CSTest", - description="Pyton binding cstest implementation.", + description="Python binding cstest implementation.", + ) + parser.add_argument( + dest="search_dir", + help="Directory to search for .yaml test files.", + type=Path, ) - repo_root = get_repo_root() - if repo_root: - parser.add_argument( - dest="search_dir", - help="Directory to search for .yaml test files.", - default=Path(f"{repo_root}/tests/"), - type=Path, - ) - else: - parser.add_argument( - dest="search_dir", - help="Directory to search for .yaml test files.", - required=True, - type=Path, - ) parser.add_argument( "-e", dest="exclude", diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index fed528d4a..3906c8614 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "build"] build-backend = "setuptools.build_meta" diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 63f245123..f3e4593f4 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,33 +1,25 @@ -#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 Antelox +# SPDX-License-Identifier: BSD-3 import glob +import logging import os import shutil import sys -import platform - -import logging from setuptools import setup -from sysconfig import get_platform -from setuptools.command.build import build +from setuptools.command.build_py import build_py from setuptools.command.sdist import sdist -from setuptools.command.bdist_egg import bdist_egg logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -SYSTEM = sys.platform - -# adapted from commit e504b81 of Nguyen Tan Cong -# Reference: https://docs.python.org/2/library/platform.html#cross-platform -IS_64BITS = sys.maxsize > 2**32 - # are we building from the repository or from a source distribution? ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) LIBS_DIR = os.path.join(ROOT_DIR, 'capstone', 'lib') HEADERS_DIR = os.path.join(ROOT_DIR, 'capstone', 'include') SRC_DIR = os.path.join(ROOT_DIR, 'src') BUILD_DIR = SRC_DIR if os.path.exists(SRC_DIR) else os.path.join(ROOT_DIR, '../..') +BUILD_PYTHON = os.path.join(BUILD_DIR, 'build_python') # Parse version from pkgconfig.mk VERSION_DATA = {} @@ -50,8 +42,8 @@ with open(os.path.join(BUILD_DIR, 'pkgconfig.mk')) as fp: VERSION_DATA[k] = v if 'PKG_MAJOR' not in VERSION_DATA or \ - 'PKG_MINOR' not in VERSION_DATA or \ - 'PKG_EXTRA' not in VERSION_DATA: + 'PKG_MINOR' not in VERSION_DATA or \ + 'PKG_EXTRA' not in VERSION_DATA: raise Exception("Malformed pkgconfig.mk") if 'PKG_TAG' in VERSION_DATA: @@ -59,10 +51,10 @@ if 'PKG_TAG' in VERSION_DATA: else: VERSION = '{PKG_MAJOR}.{PKG_MINOR}.{PKG_EXTRA}'.format(**VERSION_DATA) -if SYSTEM == 'darwin': +if sys.platform == 'darwin': VERSIONED_LIBRARY_FILE = "libcapstone.{PKG_MAJOR}.dylib".format(**VERSION_DATA) LIBRARY_FILE = "libcapstone.dylib" -elif SYSTEM in ('win32', 'cygwin'): +elif sys.platform in ('win32', 'cygwin'): VERSIONED_LIBRARY_FILE = "capstone.dll" LIBRARY_FILE = "capstone.dll" else: @@ -76,7 +68,8 @@ def clean_bins(): def copy_sources(): - """Copy the C sources into the source directory. + """ + Copy the C sources into the source directory. This rearranges the source files under the python distribution directory. """ @@ -91,11 +84,11 @@ def copy_sources(): shutil.copytree(os.path.join(BUILD_DIR, "include"), os.path.join(SRC_DIR, "include")) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.[ch]"))) - + src.extend(glob.glob(os.path.join(BUILD_DIR, "*.m[dk]"))) + src.extend(glob.glob(os.path.join(BUILD_DIR, "*.in"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "LICENSES/*"))) - src.extend(glob.glob(os.path.join(BUILD_DIR, "README"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "*.TXT"))) - src.extend(glob.glob(os.path.join(BUILD_DIR, "RELEASE_NOTES"))) + src.extend(glob.glob(os.path.join(BUILD_DIR, "ChangeLog"))) src.extend(glob.glob(os.path.join(BUILD_DIR, "CMakeLists.txt"))) for filename in src: @@ -125,85 +118,64 @@ def build_libraries(): shutil.copy(os.path.join(ROOT_DIR, 'prebuilt', LIBRARY_FILE), LIBS_DIR) return - os.chdir(BUILD_DIR) + if not os.path.exists(BUILD_PYTHON): + os.mkdir(BUILD_PYTHON) - # Windows build: this process requires few things: - # - MSVC installed - # - Run this command in an environment setup for MSVC - if not os.path.exists("build_py"): - os.mkdir("build_py") - os.chdir("build_py") - print("Build Directory: {}\n".format(os.getcwd())) - # Only build capstone.dll / libcapstone.dylib - if SYSTEM in ('win32', 'cygwin'): - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_STATIC_LIBS=OFF -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..') + logger.info("Build Directory: {}\n".format(BUILD_PYTHON)) + + conf = 'Debug' if int(os.getenv('DEBUG', 0)) else 'Release' + cmake_args = ['cmake', + '-DCAPSTONE_BUILD_SHARED_LIBS=ON', + '-DCAPSTONE_BUILD_STATIC_LIBS=OFF', + '-DCAPSTONE_BUILD_LEGACY_TESTS=OFF', + '-DCAPSTONE_BUILD_CSTOOL=OFF' + ] + cmake_build = ['cmake', + '--build', + '.' + ] + os.chdir(BUILD_PYTHON) + + if sys.platform in ('win32', 'cygwin'): + # Windows build: this process requires few things: + # - MSVC installed + # - Run this command in an environment setup for MSVC + cmake_args += ['-DCMAKE_BUILD_TYPE=' + conf, + '-G "NMake Makefiles"' + ] elif 'AFL_NOOPT' in os.environ: # build for test_corpus - os.system('cmake -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF ..') + pass else: - os.system('cmake -DCMAKE_BUILD_TYPE=Release -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..') - os.system("cmake --build .") + cmake_args += ['-DCMAKE_BUILD_TYPE=' + conf, + '-G "Unix Makefiles"' + ] + cmake_build += ['-j', str(os.getenv("THREADS", "4"))] + + os.system(' '.join(cmake_args + ['..'])) + os.system(' '.join(cmake_build)) shutil.copy(VERSIONED_LIBRARY_FILE, os.path.join(LIBS_DIR, LIBRARY_FILE)) os.chdir(cwd) -class custom_sdist(sdist): +class CustomSDist(sdist): def run(self): clean_bins() copy_sources() - return sdist.run(self) + return super().run() -class custom_build(build): +class CustomBuild(build_py): def run(self): if 'LIBCAPSTONE_PATH' in os.environ: logger.info('Skipping building C extensions since LIBCAPSTONE_PATH is set') else: logger.info('Building C extensions') build_libraries() - return build.run(self) + return super().run() -class custom_bdist_egg(bdist_egg): - def run(self): - self.run_command('build') - return bdist_egg.run(self) - - -cmdclass = {} -cmdclass['build'] = custom_build -cmdclass['sdist'] = custom_sdist -cmdclass['bdist_egg'] = custom_bdist_egg - -try: - from setuptools.command.develop import develop - - class custom_develop(develop): - def run(self): - logger.info("Building C extensions") - build_libraries() - return develop.run(self) - - cmdclass['develop'] = custom_develop -except ImportError: - print("Proper 'develop' support unavailable.") - -if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv: - # Inject the platform identifier into argv. - # Platform tags are described here: - # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags - # - # I couldn't really find out in time why we need to inject the platform here? - # The cibuildwheel doesn't need it for the Windows job. But for Mac and Linux. - # This here is very dirty and will maybe break in the future. - # Sorry if this is the case and you read this. - # See: https://github.com/capstone-engine/capstone/issues/2445 - idx = sys.argv.index('bdist_wheel') + 1 - sys.argv.insert(idx, '--plat-name') - name = get_platform() - sys.argv.insert(idx + 1, name.replace('.', '_').replace('-', '_')) - setup( provides=['capstone'], packages=['capstone'], @@ -218,14 +190,18 @@ setup( python_requires='>=3.8', classifiers=[ 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], - cmdclass=cmdclass, - zip_safe=False, - include_package_data=True, + cmdclass={'build_py': CustomBuild, 'sdist': CustomSDist}, package_data={ "capstone": ["lib/*", "include/capstone/*"], }, + has_ext_modules=lambda: True, # It's not a Pure Python wheel install_requires=[ "importlib_resources;python_version<'3.9'", ], diff --git a/suite/cstest/test/integration_tests.py b/suite/cstest/test/integration_tests.py index 6ca6d5711..d2354a39c 100755 --- a/suite/cstest/test/integration_tests.py +++ b/suite/cstest/test/integration_tests.py @@ -5,10 +5,8 @@ # Typing for Python3.8 from __future__ import annotations - import sys import subprocess as sp - from pathlib import Path @@ -34,13 +32,7 @@ def check(cmd: list[str], expected_stdout: str, expected_stderr: str, fail_msg: def run_tests(cmd: str): - p = ( - sp.run(["git", "rev-parse", "--show-toplevel"], check=True, capture_output=True) - .stdout.decode("utf8") - .strip() - ) - path = Path(p).joinpath("suite").joinpath("cstest").joinpath("test") - + path = Path(__file__).parent.resolve() cmd = cmd.split(" ") check( cmd + [f"{path.joinpath('empty_test_file.yaml')}"], diff --git a/suite/run_tests.py b/suite/run_tests.py new file mode 100644 index 000000000..b97556423 --- /dev/null +++ b/suite/run_tests.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: 2024 Antelox +# SPDX-License-Identifier: BSD-3 + +import logging +import subprocess +import sys +from pathlib import Path + +logger = logging.getLogger('tests') +logging.basicConfig(level=logging.INFO) +root_dir = Path(__file__).parent.parent.resolve() +tests = [ + f"{sys.executable} {root_dir}/bindings/python/tests/test_all.py", + f"{sys.executable} {root_dir}/suite/cstest/test/integration_tests.py cstest_py", + f"cstest_py {root_dir}/tests/MC/", + f"cstest_py {root_dir}/tests/details/", + f"cstest_py {root_dir}/tests/issues/", + f"cstest_py {root_dir}/tests/features/", +] + +for test in tests: + logger.info(f'Running {test}') + logger.info("#######################") + subprocess.run(test.split(" "), check=True) + logger.info("-----------------------")