ci: install pkg-config on macOS worker

Change `apt` sources because manylinux_2_24 uses Debian Stretch and the repos are not available anymore

.github/workflows/ci.yml

@@ -1,13 +1,13 @@
 name: run-rz-bindgen
 on:
   push:
-    paths:
-    - 'src/**.py'
-    - 'src/**.pyi'
-    - 'src/**.i'
-    - 'meson.build'
-    - 'meson_options.txt'
   workflow_dispatch:
+  workflow_call:
+    inputs:
+      rizin_ref:
+        required: false
+        type: string
+        default: 'dev'
 
 jobs:
   run-bindgen:
@@ -17,32 +17,37 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-latest]
+        os: [ubuntu-latest, windows-2019, macos-latest]
 
     steps:
     - name: Checkout rz-bindgen
       uses: actions/checkout@v3
 
-    - name: Run rz-bindgen
-      uses: pypa/[email protected]
+    - name: Checkout rizin
+      uses: actions/checkout@v3
+      with:
+        repository: rizinorg/rizin
+        path: rizin
+        ref: ${{ inputs.rizin_ref }}
+      
+    - name: Install pkg-config with Homebrew
+      if: matrix.os == 'macos-latest'
+      run: brew install pkg-config
 
-    - name: Upload artifacts
+    - name: Run rz-bindgen
+      uses: pypa/[email protected]
+      env:
+        CIBW_ARCHS_MACOS: x86_64 arm64 universal2
+        CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x
+
+    - name: Upload sdist
       uses: actions/upload-artifact@v3
       with:
-        path: wheelhouse/*.whl
+        name: sdist
+        path: dist/*.tar.gz
 
-  publish-pypi:
-    name: Publish PyPi
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-    needs: [run-bindgen]
-    steps:
-    - name: Download artifacts
-      uses: actions/download-artifact@v3
+    - name: Upload wheels
+      uses: actions/upload-artifact@v3
       with:
-        name: artifact
-        path: dist
-    - name: Publish wheels
-      uses: pypa/[email protected]
-      with:
-        password: ${{ secrets.PYPI_TOKEN }}
+        name: wheels
+        path: wheelhouse/*.whl

.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: release-rz-bindgen
+on:
+  push:
+    tags:
+      - rizin-v*_*
+  workflow_dispatch:
+
+jobs:
+  extract-rizin-version:
+    runs-on: ubuntu-latest
+    outputs:
+      rizin_version: ${{ steps.extract_version.outputs.result }}
+    steps:
+      - id: extract_version
+        run: echo "result=$(echo ${{ github.ref }} | cut -d'_' -f1 | cut -d'-' -f2-)" >> "$GITHUB_OUTPUT"
+
+  run-bindgen:
+    uses: ./.github/workflows/ci.yml
+    needs: [extract-rizin-version]
+    with:
+      rizin_ref: ${{ needs.extract-rizin-version.outputs.rizin_version }}
+
+  publish-pypi:
+    name: Publish PyPi
+    runs-on: ubuntu-latest
+    needs: [run-bindgen]
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheels
+        path: dist
+    - name: Download sdist
+      uses: actions/download-artifact@v3
+      with:
+        name: sdist
+        path: dist
+    - name: Publish wheels
+      uses: pypa/[email protected]
+      with:
+        password: ${{ secrets.PYPI_TOKEN }}

cibw_before_all.sh

@@ -4,16 +4,23 @@
 set -ex
 
 # Install deps
-pip3 install meson ninja meson-python
+pip3 install meson ninja meson-python build
 
 if command -v apt; then
+    set +e
+    grep VERSION_CODENAME=stretch /etc/os-release >/dev/null 2>/dev/null
+    if [[ $? == 0 ]] ; then
+	# Required because manylinux_2_24 uses Debian Stretch and the repos are not available anymore
+        echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list
+        echo "deb http://archive.debian.org/debian-security stretch/updates main" >> /etc/apt/sources.list
+        echo "Acquire::Check-Valid-Until no;" > /etc/apt/apt.conf.d/99no-check-valid-until
+    fi
+    set -e
     apt update && apt install --assume-yes libclang-7-dev clang-7 llvm-7
 elif command -v apk; then
     apk update && apk add clang-dev
 fi
 
-# Install rizin
-git clone --depth 1 https://github.com/wingdeans/rizin.git -b header-types
 pushd rizin
 
 if [[ "$OSTYPE" =~ msys* ]]; then
@@ -24,3 +31,9 @@ fi
 
 meson install -C build
 popd
+
+if command -v apt; then
+    if [ ! -e dist/*.tar.gz ] ; then
+        python3 -m build --sdist
+    fi
+fi

meson.build

@@ -1,9 +1,12 @@
 project('rz-bindgen', 'c', 'cpp',
-  version: 'v0.5.0dev',
-  license: 'LGPL3')
+  version: '0.6.0',
+  license: 'LGPL3',
+  meson_version: '>=0.53.0',
+  default_options: ['python.install_env=auto'])
 
 pymod = import('python')
-py = pymod.find_installation('python')
+fs = import('fs')
+py = pymod.find_installation()
 
 clang_path = get_option('clang_path')
 clang_args = get_option('clang_args')
@@ -18,7 +21,7 @@ doxygen_path = get_option('doxygen_path')
 if clang_path == ''
   llvm = dependency('llvm', required: false)
   if llvm.found()
-    clang_path = llvm.get_variable(configtool: 'libdir', default_value: 'none')
+    clang_path = llvm.get_variable(cmake: 'LLVM_LIBRARY_DIRS', configtool: 'libdir', default_value: 'none')
   elif build_machine.system() == 'darwin'
     clang_path = '/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/'
   elif build_machine.system() == 'windows'
@@ -28,12 +31,17 @@ if clang_path == ''
   endif
 endif
 
+if not fs.exists(clang_path)
+  error('CLang path does not exist')
+endif
+
+rz_core = disabler()
 if rizin_include_path == ''
-  rz_main = dependency('rz_main')
-  if rz_main.type_name() == 'pkgconfig'
-    rizin_include_path = rz_main.get_variable(pkgconfig: 'includedir') / 'librz'
-  elif rz_main.type_name() == 'cmake'
-    rizin_include_path = rz_main.get_variable(cmake: 'PACKAGE_INCLUDE_DIRS').split(';')[0]
+  rz_core = dependency('rz_core')
+  if rz_core.type_name() == 'pkgconfig'
+    rizin_include_path = rz_core.get_variable(pkgconfig: 'includedir') / 'librz'
+  elif rz_core.type_name() == 'cmake'
+    rizin_include_path = rz_core.get_variable(cmake: 'PACKAGE_INCLUDE_DIRS').split(';')[0]
   endif
 endif
 
@@ -64,19 +72,40 @@ summary({
   'Clang path': clang_path,
 })
 
-bindgen_outputs = custom_target(
-  'bindgen_outputs',
-  input: 'src' / 'main.py',
-  output: bindgen_output_names,
-  command: [
-    py, '@INPUT@',
-    '-o', '@OUTDIR@',
-    '--clang-path', clang_path,
-    '--clang-args', clang_args,
-    '--rizin-include-path', rizin_include_path,
-    '--targets', ','.join(targets)
-  ] + (doxygen_path != '' ? ['--doxygen-path', doxygen_path] : [])
-)
+if target_swig or target_sphinx
+  src_files = files(
+    'src/binding_class.py',
+    'src/binding_director.py',
+    'src/binding_enum.py',
+    'src/binding_func.py',
+    'src/binding_generic.py',
+    'src/binding_generic_specializations.py',
+    'src/binding_typemap.py',
+    'src/bindings.py',
+    'src/cparser_header.py',
+    'src/cparser_types.py',
+    'src/generator_sphinx.py',
+    'src/generator_swig.py',
+    'src/lint.py',
+    'src/main.py',
+    'src/writer.py',
+  )
+
+  bindgen_outputs = custom_target(
+    'bindgen_outputs',
+    input: 'src' / 'main.py',
+    output: bindgen_output_names,
+    depend_files: src_files,
+    command: [
+      py, '@INPUT@',
+      '-o', '@OUTDIR@',
+      '--clang-path', clang_path,
+      '--clang-args', clang_args,
+      '--rizin-include-path', rizin_include_path,
+      '--targets', ','.join(targets)
+    ] + (doxygen_path != '' ? ['--doxygen-path', doxygen_path] : [])
+  )
+endif
 
 if target_swig
   swig_output = custom_target(
@@ -94,19 +123,18 @@ if target_swig
   swig_py = swig_output[0]
   swig_wrap = swig_output[1]
 
+  if not rz_core.found()
+    rz_core = dependency('rz_core')
+  endif
+
   py.extension_module(
     '_rizin',
     swig_wrap,
     dependencies: [
       py.dependency(),
-      dependency('openssl', required: false),
-      dependency('rz_main')
+      rz_core,
     ],
-    include_directories: include_directories(
-      rizin_include_path,
-      rizin_include_path / 'sdb'
-    ),
-    install: true
+    install: true,
   )
 endif
 
@@ -127,3 +155,4 @@ endif
 if get_option('plugin').enabled()
   subdir('plugin')
 endif
+

plugin/meson.build

@@ -8,12 +8,17 @@ swig_runtime = custom_target(
   ]
 )
 
+rz_core_dep = dependency('rz_core')
+rizin_plugdir = rz_core_dep.get_variable(pkgconfig: 'plugindir', cmake: 'rz_core_PLUGINDIR')
+
 library(
   'rz-bindings-plugin',
   ['plugin.c', swig_runtime],
   dependencies: [
     py.dependency(embed: true),
     dependency('openssl', required: false),
-    dependency('rz_main')
+    rz_core_dep,
   ],
+  install: true,
+  install_dir: rizin_plugdir,
 )

plugin/plugin.c

@@ -3,6 +3,7 @@
 
 #include <Python.h>
 #include <rz_core.h>
+#include <rz_util/rz_file.h>
 #include <swig_runtime.h>
 
 static PyObject *rizin_module = NULL;
@@ -28,6 +29,11 @@ static PyObject *rizin_module = NULL;
         }                                                                      \
         return result;
 
+static int rz_bindings_run_file(RzLang *lang, const char *filename) {
+        FILE *file = fopen(filename, "rb");
+        return PyRun_SimpleFile(file, filename) == 0;
+}
+
 static int rz_bindings_init(RzLang *lang) {
         DEFER_SETUP;
 
@@ -69,12 +75,23 @@ static int rz_bindings_init(RzLang *lang) {
         Py_INCREF(py_rz_core);
         Py_INCREF(rizin_module);
 
+        char *globpath = rz_file_path_join(bindings_dir, "*.py");
+        RzList *files = rz_file_globsearch(globpath, 1);
+        RzListIter *it;
+        char *filename;
+
+        rz_list_foreach(files, it, filename) {
+                rz_bindings_run_file(lang, filename);
+        }
+        free(globpath);
+        rz_list_free(files);
+
         DEFER_CLEANUP;
 }
 
 static int rz_bindings_fini(RzLang *lang) {
         Py_XDECREF(rizin_module);
-        return Py_FinalizeEx();
+        return Py_FinalizeEx() == 0;
 }
 
 static int rz_bindings_prompt(RzLang *lang) {
@@ -153,11 +170,6 @@ static int rz_bindings_prompt(RzLang *lang) {
         DEFER_CLEANUP;
 }
 
-static int rz_bindings_run_file(RzLang *lang, const char *filename) {
-        FILE *file = fopen(filename, "rb");
-        return PyRun_SimpleFile(file, filename) == 0;
-}
-
 RzLangPlugin rz_lang_plugin_bindings = {
     .name = "python",
     .desc = "Python SWIG bindings",
@@ -174,5 +186,4 @@ RzLibStruct rizin_plugin = {
     .data = &rz_lang_plugin_bindings,
     .version = RZ_VERSION,
     .free = NULL,
-    .pkgname = "rz-bindings",
 };

pylintrc

@@ -4,4 +4,5 @@ disable=too-many-instance-attributes,
 	too-many-branches,
 	too-many-return-statements,
 	too-many-locals,
-	fixme
+	broad-exception-raised,
+	fixme

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
-  "meson==0.62.2",
-  "meson-python==0.7.0"
+  "meson>=0.62.2",
+  "meson-python>=0.7.0"
 ]
 build-backend = "mesonpy"
 
@@ -18,6 +18,8 @@ archs = ["AMD64"]
 
 [project]
 name = "rz-bindings"
-version = "0.5.0"
+dynamic = [
+  'version',
+]
 requires-python = ">=3.8"
 readme = "README.md"

src/binding_typemap.py

@@ -55,17 +55,3 @@ class Typemap:
             f"Function {func.cfunc.cursor.spelling} did not match typemap `{self.args}`. "
             f"Contents were: {args}"
         )
-
-
-buffer_len_typemap = Typemap(
-    "buffer_len",
-    [TypemapArg("unsigned char *", "buf"), TypemapArg("unsigned long long", "len")],
-)
-
-const_buffer_len_typemap = Typemap(
-    "const_buffer_len",
-    [
-        TypemapArg("const unsigned char *", "buf"),
-        TypemapArg("unsigned long long", "len"),
-    ],
-)

src/bindings.py

@@ -12,7 +12,6 @@ from binding_class import Class
 from binding_director import Director
 from binding_enum import Enum, MacroEnum
 from binding_generic import Generic
-from binding_typemap import buffer_len_typemap, const_buffer_len_typemap
 
 HeaderFunc = Callable[[Header], None]
 threaded_headers: OrderedDict[str, HeaderFunc] = OrderedDict()
@@ -169,6 +168,13 @@ def bind_analysis(analysis_h: Header) -> None:
     Class(analysis_h, typedef="RzAnalysisEsil")
     Class(analysis_h, typedef="RzAnalysisPlugin")
 
+    Class(
+        analysis_h,
+        typedef="RzAnalysisXRef",
+        rename_fields={"from": "from_addr", "to": "to_addr"},
+    )
+    Enum(analysis_h, typedef="RzAnalysisXRefType")
+
 
 @threaded_header("rz_asm.h")
 def bind_asm(asm_h: Header) -> None:
@@ -214,27 +220,7 @@ def bind_buf(buf_h: Header) -> None:
     RzBuf
     """
     rz_buf = Class(buf_h, typedef="RzBuffer")
-
-    # const ut8 *buffer, ut64 len
-    for name in [
-        "append_bytes",
-        "prepend_bytes",
-        "set_bytes",
-        "insert_bytes",
-        "write",
-        "write_at",
-    ]:
-        rz_buf.add_method(
-            f"rz_buf_{name}",
-            rename=name,
-            typemaps=[const_buffer_len_typemap],
-        )
-
-    # ut8 *buffer, ut64 len
-    rz_buf.add_method("rz_buf_read", rename="read", typemaps=[buffer_len_typemap])
-    rz_buf.add_method("rz_buf_read_at", rename="read_at", typemaps=[buffer_len_typemap])
-
-    rz_buf.add_method("rz_buf_seek", rename="seek")
+    rz_buf.add_prefixed_methods("rz_buf_")
 
     MacroEnum(buf_h, "RZ_BUF_SET", "RZ_BUF_CUR", "RZ_BUF_END")
 
@@ -377,20 +363,6 @@ def bind_io(io_h: Header) -> None:
     Class(io_h, typedef="RzIOPlugin")
 
 
-@threaded_header("rz_main.h")
-def bind_main(main_h: Header) -> None:
-    """
-    RzMain
-    """
-    main_h.ignore("rz_main_version")
-
-    rz_main = Class(main_h, typedef="RzMain")
-    rz_main.add_constructor("rz_main_new")
-    rz_main.add_destructor("rz_main_free")
-    rz_main.add_prefixed_methods("rz_main_")
-    rz_main.add_prefixed_funcs("rz_main_")
-
-
 @threaded_header("rz_util/rz_num.h")
 def bind_num(num_h: Header) -> None:
     """

src/cparser_header.py

@@ -12,6 +12,7 @@ from clang.cindex import TranslationUnit, Cursor, CursorKind
 
 from cparser_types import CType, wrap_type
 
+
 ### Cursor wrappers ###
 class AttrCursor:
     """

src/snippets_swig/prologue.i

@@ -21,22 +21,10 @@ void rizin_try_warn_deprecate(const char *name, const char *c_name) {
 
 // Buffer typemaps
 %include <pybuffer.i>
-
-%define %buffer_len_activate(TYPE, SIZE)
-%pybuffer_mutable_binary(TYPE, SIZE);
-%enddef
-
-%define %const_buffer_len_activate(TYPE, SIZE)
-%pybuffer_binary(TYPE, SIZE);
-%enddef
-
-%define %buffer_len_deactivate(TYPE, SIZE)
-%typemap(in) (TYPE, SIZE);
-%enddef
-
-%define %const_buffer_len_deactivate(TYPE, SIZE)
-%typemap(in) (TYPE, SIZE);
-%enddef
+%pybuffer_mutable_binary(unsigned char *buf, unsigned long long len);
+%pybuffer_binary(const unsigned char *buf, unsigned long long len);
+%pybuffer_mutable_binary(unsigned char *buf, int len);
+%pybuffer_binary(const unsigned char *buf, int len);
 
 // CArrays
 %include <carrays.i>

src/snippets_swig/register_command.py

@@ -4,11 +4,8 @@ def register_group(self, cmd, summary):
     help_desc.summary = summary
     self.rcmd.register_swig_command(cmd, None, None, help_desc)
 
-
-import inspect
-
-
 def register_command(self, cmd, fn):
+    import inspect
     params = list(inspect.signature(fn).parameters.values())
 
     core_arg = None
@@ -61,19 +58,31 @@ def register_command(self, cmd, fn):
                 if core_arg:
                     args[core_arg] = core
                 for i, param in enumerate(params):
-                    arg = args_array[i + 1]
+                    orig_arg = args_array[i + 1]
+                    arg = orig_arg
                     if param.annotation == RzNumArg:
+                        if not core.num.is_valid_input(arg):
+                            raise ValueError(f"Could not convert '{orig_arg}' to RzNum value")
+
                         arg = core.num.math(arg)
                     elif param.annotation == int:
-                        arg = int(arg)
+                        try:
+                            arg = int(arg)
+                        except ValueError:
+                            raise ValueError(f"Could not convert '{orig_arg}' to int") from e
                     elif param.annotation == RzFlagItem:
                         arg = core.flags.get(arg)
+                        if arg is None:
+                            raise ValueError(f"Could not find a flag named '{orig_arg}'")
                     elif param.annotation == RzAnalysisFunction:
                         arg = core.analysis.get_function_byname(arg)
+                        if arg is None:
+                            raise ValueError(f"Could not find a function named '{orig_arg}'")
                     args[param.name] = arg
                 return fn(**args)
             except Exception as e:
                 print(e)
+                return False
 
     director = wrapper()
     director.__disown__()