Python package building rework (#2538)

* - Refactored setup.py to remove hacks regarding packaging of wheels for different platforms, improve and cleanup the code
- Updated README.txt
- Removed old Makefile and build_wheel.sh scripts
- Created a new workflow that takes care of building and testing python packages for different platforms/architectures/python versions

* Added SPDX headers to the setup.py

* - cstest_py: Fixed positional argument since it doesn't accept a `required` flag. It turns to have a mandatory tests folder path
- integration_tests.py: Use pathlib to determine the required path
- GitHub action: Simplified the tests execution command

* GitHub Actions: Run python 3.8 (lowest) and 3.13 (current highest) for native runners only during testings and the rest during tag release

* GitHub Action:
- Fixed the cibw_build matrix element
- Added a step to prepare artifact name

* GitHub Action: Added run_tests.py script to run all tests during CI workflow

* - Added SPDX headers to the run_tests.py script and to the build-wheels-publish.yml workflow file
- Minor fixes to the workflow as pointed out in the PR review
- Updated MANIFEST.in to reflect the actual libraries built during python wheel creation process
- Use subprocess.run in place of os.system in run_tests.py script

* GitHub Action:
- Run qemu step only if non-native Linux runner
- Added arch:universal2 matrix element for macos-latest runner

* Python bindings: Refreshed the list of files needed to be copied for sdist archive

* GitHub Action: Commented out arch:x86 matrix elements

* GitHub Action: Run qemu step only if non-native Linux runner

* GitHub Action: Minor fixes

* Python bindings: Added missing .in pattern when collecting src files for sdist archive

.github/workflows/build-wheels-publish.yml

@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: 2024 Antelox <anteloxrce@gmail.com>
+# SPDX-License-Identifier: BSD-3
+
+name: Build and publish wheels with cibuildwheel
+
+on:
+  workflow_dispatch:
+    inputs:
+      debugMode:
+        description: 'Debug Mode'
+        required: false
+        default: ''
+        type: choice
+        options:
+          - '0'
+          - '1'
+
+  push:
+    paths-ignore:
+      - ".gitignore"
+      - "CREDITS.TXT"
+      - "ChangeLog"
+      - "README.md"
+      - "docs/**"
+  pull_request:
+
+env:
+  # Enable DEBUG flag either according to the tag release or manual override
+  CAPSTONE_DEBUG: ${{ inputs.debugMode != '' && inputs.debugMode || startsWith(github.ref, 'refs/tags') && '0' || '1' }}
+
+jobs:
+  # job to be executed for every push - testing purpose
+  build_wheels_always:
+    name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # NOTE: Making this to parallelize and speed up workflow
+          # i686 - manylinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # i686 - musllinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # x86_64 - manylinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # x86_64 - musllinux
+          # - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # aarch64 - manylinux
+          # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-manylinux* cp313-manylinux*', cibw_skip: '' }
+          # aarch64 - musllinux
+          # - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp38-musllinux* cp313-musllinux*', cibw_skip: '' }
+          # macos - x86_64
+          - { os: macos-13, arch: x86_64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # macos - arm64
+          # - { os: macos-latest, arch: arm64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # - { os: macos-latest, arch: universal2, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - x86_64
+          - { os: windows-latest, arch: AMD64, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - amd64
+          # - { os: windows-latest, arch: x86, cibw_build: 'cp38* cp313*', cibw_skip: '' }
+          # windows - arm64
+          # - { os: windows-latest, arch: ARM64, cibw_build: 'cp39* cp313*', cibw_skip: '' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/actions/upload-artifact/issues/22
+      - name: Prepare a unique name for Artifacts
+        shell: bash
+        run: |
+          # replace not-allowed chars with dash
+          name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}"
+          name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//')
+          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
+
+      # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64
+      - uses: actions/setup-python@v5
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        with:
+          python-version: 3.8
+
+      - name: '🛠️ Win MSVC 32 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'x86'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x86
+
+      - name: '🛠️ Win MSVC 64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'AMD64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: '🛠️ Win MSVC ARM64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'ARM64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: amd64_arm64
+
+      - name: '🛠️ Set up QEMU'
+        if: runner.os == 'Linux' && matrix.arch != 'x86_64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: '🚧 cibuildwheel run'
+        uses: pypa/cibuildwheel@v2.21.3
+        env:
+          CIBW_BUILD_FRONTEND: build
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: ${{ matrix.cibw_skip }}
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }}
+          CIBW_ENVIRONMENT_PASS_LINUX: DEBUG
+          # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64
+          # https://github.com/pypa/cibuildwheel/pull/1169
+          CIBW_TEST_SKIP: "*-win_arm64 cp38-macosx_*:arm64"
+          CIBW_TEST_COMMAND: >
+            python -m pip install {package}/cstest_py &&
+            python {project}/suite/run_tests.py
+        with:
+          package-dir: bindings/python
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ./wheelhouse/*.whl
+
+  # To be executed only in case of a tag release
+  build_wheels_all:
+    name: Building on ${{ matrix.os }} - ${{ matrix.arch }} - ${{ matrix.cibw_build }}
+    runs-on: ${{ matrix.os }}
+    if: startsWith(github.ref, 'refs/tags')
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # NOTE: Making this to parallelize and speed up workflow
+          # i686 - manylinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # i686 - musllinux
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          # - { os: ubuntu-latest, arch: i686, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # x86_64 - manylinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # x86_64 - musllinux
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # aarch64 - manylinux
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-manylinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-manylinux*', cibw_skip: '' }
+          # aarch64 - musllinux
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp39-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp310-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp311-musllinux*', cibw_skip: '' }
+          - { os: ubuntu-latest, arch: aarch64, cibw_build: 'cp312-musllinux*', cibw_skip: '' }
+          # macos - x86_64
+          - { os: macos-13, arch: x86_64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # macos - arm64
+          - { os: macos-latest, arch: arm64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          - { os: macos-latest, arch: universal2, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' }
+          # windows - amd64
+          - { os: windows-latest, arch: AMD64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # windows - x86
+          # - { os: windows-latest, arch: x86, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *313*' }
+          # windows - arm64
+          - { os: windows-latest, arch: ARM64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38* *39* *313*' }
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # https://github.com/actions/upload-artifact/issues/22
+      - name: Prepare a unique name for Artifacts
+        shell: bash
+        run: |
+          # replace not-allowed chars with dash
+          name="cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.cibw_build }}"
+          name=$(echo -n "$name" | sed -e 's/[ \t:\/\\"<>|*?]/-/g' -e 's/--*/-/g' | sed -e 's/\-$//')
+          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
+
+      - name: '🛠️ Win MSVC 32 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'x86'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x86
+
+      - name: '🛠️ Win MSVC 64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'AMD64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: '🛠️ Win MSVC ARM64 dev cmd setup'
+        if: runner.os == 'Windows' && matrix.arch == 'ARM64'
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: amd64_arm64
+
+      - name: '🛠️ Set up QEMU'
+        if: runner.os == 'Linux' && matrix.arch != 'x86_64'
+        uses: docker/setup-qemu-action@v3
+
+      - name: '🚧 cibuildwheel run'
+        uses: pypa/cibuildwheel@v2.21.3
+        env:
+          CIBW_BUILD_FRONTEND: build
+          CIBW_BUILD: ${{ matrix.cibw_build }}
+          CIBW_SKIP: ${{ matrix.cibw_skip }}
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_ENVIRONMENT: DEBUG=${{ env.CAPSTONE_DEBUG }}
+          CIBW_ENVIRONMENT_PASS_LINUX: DEBUG
+          # https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64
+          CIBW_TEST_SKIP: "*-win_arm64"
+          CIBW_TEST_COMMAND: >
+            python -m pip install {package}/cstest_py &&
+            python {project}/suite/run_tests.py
+        with:
+          package-dir: bindings/python
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: ./wheelhouse/*.whl
+
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Build SDist
+        run: |
+          cd bindings/python
+          python3 -m pip install -U pip build
+          python3 -m build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist-archive
+          path: bindings/python/dist/*.tar.gz
+
+  publish:
+    needs: [ build_wheels_always, build_wheels_all, make_sdist ]
+    environment: pypi
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: dist
+
+      - name: Show downloaded artifacts
+        run: ls -laR dist
+
+      - name: '📦 Publish distribution to PyPI'
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: ${{ success() }}
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_pass }}

.github/workflows/python-publish-release.yml

@@ -1,90 +0,0 @@
-name: RELEASE BUILD - PyPI 📦 Distribution
-
-on: [push, pull_request, release, workflow_dispatch]
-
-jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up MSVC x64
-        if: matrix.os == 'windows-latest'
-        uses: ilammy/msvc-dev-cmd@v1
-
-      - name: Set up QEMU
-        if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.20.0
-        env:
-          CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"
-          CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" # ppc64le s390x really slow
-          CIBW_ARCHS_WINDOWS: "AMD64" # ARM64  Seems ARM64 will rebuild amd64 wheel for unknow reason.
-          CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*"
-          CIBW_SKIP: ""
-        with:
-          package-dir: bindings/python
-
-      - uses: actions/upload-artifact@v4
-        with:
-          path: ./wheelhouse/*.whl
-          name: artifacts-${{ matrix.os }}
-
-      - name: Check binaries (Windows)
-        if: matrix.os == 'windows-latest'
-        run: |
-          python3.exe suite/check_wheel_bin_arch.py ./wheelhouse/
-
-      - name: Check binaries (Unix)
-        if: matrix.os != 'windows-latest'
-        run: |
-          ./suite/check_wheel_bin_arch.py ./wheelhouse/
-
-  make_sdist:
-    name: Make SDist
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0  # Optional, use if you use setuptools_scm
-        submodules: true  # Optional, use if you have submodules
-
-    - name: Build SDist
-      run: |
-        cd bindings/python
-        pipx run build --sdist
-
-    - uses: actions/upload-artifact@v4
-      with:
-        path: bindings/python/dist/*.tar.gz
-
-  publish:
-    needs: [build_wheels]
-    runs-on: ubuntu-latest
-    if: github.event_name == 'release' && github.event.action == 'released'
-    permissions:
-      id-token: write
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          merge-multiple: true
-          path: dist
-
-      - name: Show downloaded artifacts
-        run: ls -laR dist
-
-      - name: Publish distribution 📦 to PyPI
-        if: ${{ success() }}
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          verbose: true
-          user: __token__
-          password: ${{ secrets.pypi_pass }}

.github/workflows/python-tests.yml

@@ -1,62 +0,0 @@
-name: Python Package CI
-
-on:
-  push:
-  pull_request:
-
-# Stop previous runs on the same branch on new push
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-24.04, windows-2022, macOS-14]
-        python-version: ["3.8", "3.12"]
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Setup MSVC
-      if: runner.os == 'Windows'
-      uses: ilammy/msvc-dev-cmd@v1
-
-    - name: Build and install capstone
-      run: pip install ./bindings/python
-
-    - name: Install cstest_py
-      run: pip install ./bindings/python/cstest_py
-
-    - name: Run legacy tests
-      run: python ./bindings/python/tests/test_all.py
-
-    - name: cstest_py integration tests
-      run: |
-        cd suite/cstest/test/
-        python3 ./integration_tests.py cstest_py
-        cd ../../../
-
-    - name: cstest_py MC
-      run: |
-        cstest_py tests/MC/
-
-    - name: cstest_py details
-      run: |
-        cstest_py tests/details/
-
-    - name: cstest_py issues
-      run: |
-        cstest_py tests/issues/
-
-    - name: cstest_py features
-      run: |
-        cstest_py tests/features/

bindings/python/MANIFEST.in

@@ -1,5 +1,8 @@
 recursive-include src *
-include LICENSE.TXT
+recursive-include prebuilt *
-include README.txt
+include BUILDING.md
-include BUILDING.txt
+graft capstone/lib
-include Makefile
+graft capstone/include
+global-include *.dll
+global-include *.dylib
+global-include *.so.*

bindings/python/Makefile

@@ -1,47 +0,0 @@
-PYTHON3 ?= python3
-
-.PHONY: gen_const install sdist bdist clean check
-
-gen_const:
-	cd .. && $(PYTHON3) const_generator.py python
-
-install:
-	rm -rf src/
-	if test -n "${DESTDIR}"; then \
-		$(PYTHON3) setup.py build install --root="${DESTDIR}"; \
-	else \
-		$(PYTHON3) setup.py build install; \
-	fi
-
-# build & upload PyPi package with source code of the core
-sdist:
-	rm -rf src/ dist/
-	$(PYTHON3) setup.py sdist register upload
-
-# build & upload PyPi package with prebuilt core
-bdist:
-	rm -rf src/ dist/
-	$(PYTHON3) setup.py bdist_wheel register upload
-
-clean:
-	rm -rf build/ src/ dist/ *.egg-info
-	rm -rf capstone/lib capstone/include pyx/lib pyx/include
-	rm -f pyx/*.c pyx/__init__.py
-	for f in capstone/*.py; do rm -f pyx/$$(basename $$f)x; done
-	rm -f MANIFEST
-	rm -f *.pyc capstone/*.pyc
-
-
-TESTS = test_basic.py test_detail.py test_arm.py test_aarch64.py test_m68k.py test_mips.py
-TESTS += test_ppc.py test_sparc.py test_systemz.py test_x86.py test_xcore.py test_tms320c64x.py
-TESTS += test_m680x.py test_skipdata.py test_mos65xx.py test_bpf.py test_riscv.py
-TESTS += test_evm.py test_tricore.py test_wasm.py test_sh.py test_hppa.py
-TESTS += test_lite.py test_iter.py test_customized_mnem.py test_alpha.py test_xtensa.py
-
-check:
-	@for t in $(TESTS); do \
-		echo Check $$t ... ; \
-		./tests/$$t > /dev/null; \
-		if [ $$? -eq 0 ]; then echo OK; else echo FAILED; exit 1; fi \
-	done
-

bindings/python/README.txt

@@ -1,12 +1,12 @@
 To install Capstone, you should run `pip install capstone`.
 
-If you would like to build Capstone with just the source distribution, without
+If you would like to build and install Capstone with just the source distribution,
-pip, just run `python setup.py install` in the folder with setup.py in it.
+just run `python -m pip install .`, considering you are in the folder with setup.py in it.
 
 In order to use this source distribution, you will need an environment that can
 compile C code. On Linux, this is usually easy, but on Windows, this involves
 installing Visual Studio and using the "Developer Command Prompt" to perform the
-installation. See BUILDING.txt for more information.
+installation. See BUILDING.md for more information.
 
 By default, attempting to install the python bindings will trigger a build of
 the capstone native core. If this is undesirable for whatever reason, for

bindings/python/build_wheel.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -e -x
-
-cd bindings/python
-if [ -f /opt/python/cp311-cp311/bin/python3 ];then
-  # Use manylinux Python
-  /opt/python/cp311-cp311/bin/python3 -m pip install wheel
-  /opt/python/cp311-cp311/bin/python3 setup.py bdist_wheel
-else
-  python3 -m pip install wheel
-  python3 setup.py bdist_wheel
-fi
-
-cd dist
-auditwheel repair *.whl
-mv -f wheelhouse/*.whl .

bindings/python/cstest_py/pyproject.toml

@@ -5,14 +5,14 @@
 name = "cstest_py"
 version = "0.1.0"
 dependencies = [
-  "pyyaml >= 6.0.2",
+    "pyyaml >= 6.0.2",
-  "capstone >= 5.0.0",
+    "capstone >= 5.0.0",
 ]
 requires-python = ">= 3.8"
 
 [tool.setuptools]
 packages = ["cstest_py"]
-package-dir = {"" = "src"}
+package-dir = { "" = "src" }
 
 [project.scripts]
 cstest_py = "cstest_py.cstest:main"

bindings/python/cstest_py/src/cstest_py/cstest.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 import argparse
 import logging
-import subprocess as sp
 import sys
 import os
 import yaml
@@ -405,34 +404,16 @@ class CSTest:
         self.stats.print_evaluate()
 
 
-def get_repo_root() -> str | None:
-    res = sp.run(["git", "rev-parse", "--show-toplevel"], capture_output=True)
-    if res.stderr:
-        log.error("Could not get repository root directory.")
-        return None
-    return res.stdout.decode("utf8").strip()
-
-
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         prog="Python CSTest",
-        description="Pyton binding cstest implementation.",
+        description="Python binding cstest implementation.",
+    )
+    parser.add_argument(
+        dest="search_dir",
+        help="Directory to search for .yaml test files.",
+        type=Path,
     )
-    repo_root = get_repo_root()
-    if repo_root:
-        parser.add_argument(
-            dest="search_dir",
-            help="Directory to search for .yaml test files.",
-            default=Path(f"{repo_root}/tests/"),
-            type=Path,
-        )
-    else:
-        parser.add_argument(
-            dest="search_dir",
-            help="Directory to search for .yaml test files.",
-            required=True,
-            type=Path,
-        )
     parser.add_argument(
         "-e",
         dest="exclude",

bindings/python/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["setuptools"]
+requires = ["setuptools", "build"]
 build-backend = "setuptools.build_meta"

bindings/python/setup.py

@@ -1,33 +1,25 @@
-#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2024 Antelox <anteloxrce@gmail.com>
+# SPDX-License-Identifier: BSD-3
 
 import glob
+import logging
 import os
 import shutil
 import sys
-import platform
-
-import logging
 from setuptools import setup
-from sysconfig import get_platform
+from setuptools.command.build_py import build_py
-from setuptools.command.build import build
 from setuptools.command.sdist import sdist
-from setuptools.command.bdist_egg import bdist_egg
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-SYSTEM = sys.platform
-
-# adapted from commit e504b81 of Nguyen Tan Cong
-# Reference: https://docs.python.org/2/library/platform.html#cross-platform
-IS_64BITS = sys.maxsize > 2**32
-
 # are we building from the repository or from a source distribution?
 ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
 LIBS_DIR = os.path.join(ROOT_DIR, 'capstone', 'lib')
 HEADERS_DIR = os.path.join(ROOT_DIR, 'capstone', 'include')
 SRC_DIR = os.path.join(ROOT_DIR, 'src')
 BUILD_DIR = SRC_DIR if os.path.exists(SRC_DIR) else os.path.join(ROOT_DIR, '../..')
+BUILD_PYTHON = os.path.join(BUILD_DIR, 'build_python')
 
 # Parse version from pkgconfig.mk
 VERSION_DATA = {}
@@ -50,8 +42,8 @@ with open(os.path.join(BUILD_DIR, 'pkgconfig.mk')) as fp:
         VERSION_DATA[k] = v
 
 if 'PKG_MAJOR' not in VERSION_DATA or \
-        'PKG_MINOR' not in VERSION_DATA or \
+    'PKG_MINOR' not in VERSION_DATA or \
-        'PKG_EXTRA' not in VERSION_DATA:
+    'PKG_EXTRA' not in VERSION_DATA:
     raise Exception("Malformed pkgconfig.mk")
 
 if 'PKG_TAG' in VERSION_DATA:
@@ -59,10 +51,10 @@ if 'PKG_TAG' in VERSION_DATA:
 else:
     VERSION = '{PKG_MAJOR}.{PKG_MINOR}.{PKG_EXTRA}'.format(**VERSION_DATA)
 
-if SYSTEM == 'darwin':
+if sys.platform == 'darwin':
     VERSIONED_LIBRARY_FILE = "libcapstone.{PKG_MAJOR}.dylib".format(**VERSION_DATA)
     LIBRARY_FILE = "libcapstone.dylib"
-elif SYSTEM in ('win32', 'cygwin'):
+elif sys.platform in ('win32', 'cygwin'):
     VERSIONED_LIBRARY_FILE = "capstone.dll"
     LIBRARY_FILE = "capstone.dll"
 else:
@@ -76,7 +68,8 @@ def clean_bins():
 
 
 def copy_sources():
-    """Copy the C sources into the source directory.
+    """
+    Copy the C sources into the source directory.
     This rearranges the source files under the python distribution
     directory.
     """
@@ -91,11 +84,11 @@ def copy_sources():
     shutil.copytree(os.path.join(BUILD_DIR, "include"), os.path.join(SRC_DIR, "include"))
 
     src.extend(glob.glob(os.path.join(BUILD_DIR, "*.[ch]")))
-
+    src.extend(glob.glob(os.path.join(BUILD_DIR, "*.m[dk]")))
+    src.extend(glob.glob(os.path.join(BUILD_DIR, "*.in")))
     src.extend(glob.glob(os.path.join(BUILD_DIR, "LICENSES/*")))
-    src.extend(glob.glob(os.path.join(BUILD_DIR, "README")))
     src.extend(glob.glob(os.path.join(BUILD_DIR, "*.TXT")))
-    src.extend(glob.glob(os.path.join(BUILD_DIR, "RELEASE_NOTES")))
+    src.extend(glob.glob(os.path.join(BUILD_DIR, "ChangeLog")))
     src.extend(glob.glob(os.path.join(BUILD_DIR, "CMakeLists.txt")))
 
     for filename in src:
@@ -125,85 +118,64 @@ def build_libraries():
         shutil.copy(os.path.join(ROOT_DIR, 'prebuilt', LIBRARY_FILE), LIBS_DIR)
         return
 
-    os.chdir(BUILD_DIR)
+    if not os.path.exists(BUILD_PYTHON):
+        os.mkdir(BUILD_PYTHON)
 
-    # Windows build: this process requires few things:
+    logger.info("Build Directory: {}\n".format(BUILD_PYTHON))
-    #    - MSVC installed
+
-    #    - Run this command in an environment setup for MSVC
+    conf = 'Debug' if int(os.getenv('DEBUG', 0)) else 'Release'
-    if not os.path.exists("build_py"):
+    cmake_args = ['cmake',
-        os.mkdir("build_py")
+                  '-DCAPSTONE_BUILD_SHARED_LIBS=ON',
-    os.chdir("build_py")
+                  '-DCAPSTONE_BUILD_STATIC_LIBS=OFF',
-    print("Build Directory: {}\n".format(os.getcwd()))
+                  '-DCAPSTONE_BUILD_LEGACY_TESTS=OFF',
-    # Only build capstone.dll / libcapstone.dylib
+                  '-DCAPSTONE_BUILD_CSTOOL=OFF'
-    if SYSTEM in ('win32', 'cygwin'):
+                  ]
-        os.system('cmake -DCMAKE_BUILD_TYPE=Release -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_STATIC_LIBS=OFF -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "NMake Makefiles" ..')
+    cmake_build = ['cmake',
+                   '--build',
+                   '.'
+                   ]
+    os.chdir(BUILD_PYTHON)
+
+    if sys.platform in ('win32', 'cygwin'):
+        # Windows build: this process requires few things:
+        #    - MSVC installed
+        #    - Run this command in an environment setup for MSVC
+        cmake_args += ['-DCMAKE_BUILD_TYPE=' + conf,
+                       '-G "NMake Makefiles"'
+                       ]
     elif 'AFL_NOOPT' in os.environ:
         # build for test_corpus
-        os.system('cmake -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF ..')
+        pass
     else:
-        os.system('cmake -DCMAKE_BUILD_TYPE=Release -DCAPSTONE_BUILD_SHARED_LIBS=ON -DCAPSTONE_BUILD_LEGACY_TESTS=OFF -DCAPSTONE_BUILD_CSTOOL=OFF -G "Unix Makefiles" ..')
+        cmake_args += ['-DCMAKE_BUILD_TYPE=' + conf,
-    os.system("cmake --build .")
+                       '-G "Unix Makefiles"'
+                       ]
+        cmake_build += ['-j', str(os.getenv("THREADS", "4"))]
+
+    os.system(' '.join(cmake_args + ['..']))
+    os.system(' '.join(cmake_build))
 
     shutil.copy(VERSIONED_LIBRARY_FILE, os.path.join(LIBS_DIR, LIBRARY_FILE))
     os.chdir(cwd)
 
 
-class custom_sdist(sdist):
+class CustomSDist(sdist):
     def run(self):
         clean_bins()
         copy_sources()
-        return sdist.run(self)
+        return super().run()
 
 
-class custom_build(build):
+class CustomBuild(build_py):
     def run(self):
         if 'LIBCAPSTONE_PATH' in os.environ:
             logger.info('Skipping building C extensions since LIBCAPSTONE_PATH is set')
         else:
             logger.info('Building C extensions')
             build_libraries()
-        return build.run(self)
+        return super().run()
 
 
-class custom_bdist_egg(bdist_egg):
-    def run(self):
-        self.run_command('build')
-        return bdist_egg.run(self)
-
-
-cmdclass = {}
-cmdclass['build'] = custom_build
-cmdclass['sdist'] = custom_sdist
-cmdclass['bdist_egg'] = custom_bdist_egg
-
-try:
-    from setuptools.command.develop import develop
-
-    class custom_develop(develop):
-        def run(self):
-            logger.info("Building C extensions")
-            build_libraries()
-            return develop.run(self)
-
-    cmdclass['develop'] = custom_develop
-except ImportError:
-    print("Proper 'develop' support unavailable.")
-
-if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv:
-    # Inject the platform identifier into argv.
-    # Platform tags are described here:
-    # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags
-    #
-    # I couldn't really find out in time why we need to inject the platform here?
-    # The cibuildwheel doesn't need it for the Windows job. But for Mac and Linux.
-    # This here is very dirty and will maybe break in the future.
-    # Sorry if this is the case and you read this.
-    # See: https://github.com/capstone-engine/capstone/issues/2445
-    idx = sys.argv.index('bdist_wheel') + 1
-    sys.argv.insert(idx, '--plat-name')
-    name = get_platform()
-    sys.argv.insert(idx + 1, name.replace('.', '_').replace('-', '_'))
-
 setup(
     provides=['capstone'],
     packages=['capstone'],
@@ -218,14 +190,18 @@ setup(
     python_requires='>=3.8',
     classifiers=[
         'License :: OSI Approved :: BSD License',
-        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
     ],
-    cmdclass=cmdclass,
+    cmdclass={'build_py': CustomBuild, 'sdist': CustomSDist},
-    zip_safe=False,
-    include_package_data=True,
     package_data={
         "capstone": ["lib/*", "include/capstone/*"],
     },
+    has_ext_modules=lambda: True,  # It's not a Pure Python wheel
     install_requires=[
         "importlib_resources;python_version<'3.9'",
     ],

suite/cstest/test/integration_tests.py

@@ -5,10 +5,8 @@
 
 # Typing for Python3.8
 from __future__ import annotations
-
 import sys
 import subprocess as sp
-
 from pathlib import Path
 
 
@@ -34,13 +32,7 @@ def check(cmd: list[str], expected_stdout: str, expected_stderr: str, fail_msg:
 
 
 def run_tests(cmd: str):
-    p = (
+    path = Path(__file__).parent.resolve()
-        sp.run(["git", "rev-parse", "--show-toplevel"], check=True, capture_output=True)
-        .stdout.decode("utf8")
-        .strip()
-    )
-    path = Path(p).joinpath("suite").joinpath("cstest").joinpath("test")
-
     cmd = cmd.split(" ")
     check(
         cmd + [f"{path.joinpath('empty_test_file.yaml')}"],

suite/run_tests.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: 2024 Antelox <anteloxrce@gmail.com>
+# SPDX-License-Identifier: BSD-3
+
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+logger = logging.getLogger('tests')
+logging.basicConfig(level=logging.INFO)
+root_dir = Path(__file__).parent.parent.resolve()
+tests = [
+    f"{sys.executable} {root_dir}/bindings/python/tests/test_all.py",
+    f"{sys.executable} {root_dir}/suite/cstest/test/integration_tests.py cstest_py",
+    f"cstest_py {root_dir}/tests/MC/",
+    f"cstest_py {root_dir}/tests/details/",
+    f"cstest_py {root_dir}/tests/issues/",
+    f"cstest_py {root_dir}/tests/features/",
+]
+
+for test in tests:
+    logger.info(f'Running {test}')
+    logger.info("#######################")
+    subprocess.run(test.split(" "), check=True)
+    logger.info("-----------------------")