Release 0.5.9

This ensures that the order remains in ascending order

.github/workflows/semgrep.yml

@@ -0,0 +1,23 @@
+on:
+  pull_request: {}
+  push:
+    branches:
+    - main
+    - master
+    paths:
+    - .github/workflows/semgrep.yml
+  schedule:
+  # random HH:MM to avoid a load spike on GitHub Actions at 00:00
+  - cron: 34 10 * * *
+name: Semgrep
+jobs:
+  semgrep:
+    name: Scan
+    runs-on: ubuntu-20.04
+    env:
+      SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}
+    container:
+      image: returntocorp/semgrep
+    steps:
+    - uses: actions/checkout@v3
+    - run: semgrep ci

Makefile

@@ -7,7 +7,9 @@ all:
 pub:
 	rm -rf dist
 	$(PYTHON) setup.py sdist
-	twine upload --repository=pypi --verbose dist/*
+	twine check dist/*
+	twine upload -u __token__ --repository-url https://upload.pypi.org/legacy/ --verbose dist/*
+	#twine upload --repository=pypi --verbose dist/*
 
 install:
 	$(PIP) install .

README.md

@@ -64,6 +64,7 @@ versions       - list installed packages
 list           - list all packages available to r2env
 shell          - enter a new shell with PATH env var set
 purge          - remove ~/.r2env
+update         - update list of available profiles
 
 Environment
 

r2env/__main__.py

@@ -0,0 +1,3 @@
+from r2env.repl import main
+
+main()

r2env/config/profiles.json

@@ -1,235 +0,0 @@
-{
-  "r2frida": {
-    "source": "https://github.com/nowsecure/r2frida",
-    "versions": [
-      {
-        "id": "git",
-        "packages": {}
-      }
-    ]
-  },
-  "r2dec": {
-    "source": "https://github.com/wargio/r2dec-js",
-    "versions": [
-      {
-        "id": "git",
-        "packages": {}
-      }
-    ]
-  },
-  "r2ghidra": {
-    "source": "https://github.com/radareorg/r2ghidra",
-    "versions": [
-      {
-        "id": "git",
-        "packages": {}
-      }
-    ]
-  },
-  "r0": {
-    "source": "https://github.com/radareorg/r0",
-    "versions": [
-      {
-        "id": "git",
-        "packages": {}
-      }
-    ]
-  },
-  "radare2": {
-    "source": "https://github.com/radareorg/radare2",
-    "versions": [
-      {
-        "id": "5.3.0",
-        "packages": {
-          "w64": "radare2-5.3.0-w64.zip",
-          "deb_x64": "radare2_5.3.0_amd64.deb"
-        }
-      },
-      {
-        "id": "5.3.1",
-        "packages": {
-          "w64": "radare2-5.3.1-w64.zip",
-          "mac_x64": "radare2-5.3.1.pkg",
-          "deb_x64": "radare2_5.3.1_amd64.deb"
-        }
-      },
-      {
-        "id": "5.4.0",
-        "packages": {
-          "w64": "radare2-5.4.0-w64.zip",
-          "mac_x64": "radare2-5.4.0.pkg",
-          "deb_x64": "radare2_5.4.0_amd64.deb"
-        }
-      },
-      {
-        "id": "5.4.2",
-        "packages": {
-          "w32": "radare2-5.4.2-w32.zip",
-          "w64": "radare2-5.4.2-w64.zip",
-          "mac_x64": "radare2-5.4.2-amd64.pkg",
-          "mac_arm64": "radare2-5.4.2-m1.pkg",
-          "deb_i386": "radare2_5.4.2_i386.deb",
-          "deb_x64": "radare2_5.4.2_amd64.deb"
-        }
-      },
-      {
-        "id": "5.5.0",
-        "packages": {
-          "w32": "radare2-5.5.0-w32.zip",
-          "w64": "radare2-5.5.0-w64.zip",
-          "mac_x64": "radare2-5.5.0-amd64.pkg",
-          "mac_arm64": "radare2-5.5.0-m1.pkg",
-          "deb_i386": "radare2_5.5.0_i386.deb",
-          "deb_x64": "radare2_5.5.0_amd64.deb"
-        }
-      },
-      {
-        "id": "5.5.2",
-        "packages": {
-          "w32": "radare2-5.5.2-w32.zip",
-          "w64": "radare2-5.5.2-w64.zip",
-          "mac_x64": "radare2-5.5.2-amd64.pkg",
-          "mac_arm64": "radare2-5.5.2-m1.pkg",
-          "deb_i386": "radare2_5.5.2_i386.deb",
-          "deb_x64": "radare2_5.5.2_amd64.deb"
-        }
-      },
-      {
-        "id": "5.5.4",
-        "packages": {
-          "w32": "radare2-5.5.4-w32.zip",
-          "w64": "radare2-5.5.4-w64.zip",
-          "mac_x64": "radare2-5.5.4-amd64.pkg",
-          "mac_arm64": "radare2-5.5.4-m1.pkg",
-          "deb_i386": "radare2_5.5.4_i386.deb",
-          "deb_x64": "radare2_5.5.4_amd64.deb"
-        }
-      },
-      {
-        "id": "5.6.0",
-        "packages": {
-          "w32": "radare2-5.6.0-w32.zip",
-          "w64": "radare2-5.6.0-w64.zip",
-          "mac_x64": "radare2-5.6.0-amd64.pkg",
-          "mac_arm64": "radare2-5.6.0-m1.pkg",
-          "deb_i386": "radare2_5.6.0_i386.deb",
-          "deb_x64": "radare2_5.6.0_amd64.deb"
-        }
-      },
-      {
-        "id": "5.6.2",
-        "packages": {
-          "w32": "radare2-5.6.2-w32.zip",
-          "w64": "radare2-5.6.2-w64.zip",
-          "mac_x64": "radare2-5.6.2-amd64.pkg",
-          "mac_arm64": "radare2-5.6.2-m1.pkg",
-          "deb_i386": "radare2_5.6.2_i386.deb",
-          "deb_x64": "radare2_5.6.2_amd64.deb"
-        }
-      },
-      {
-        "id": "5.6.4",
-        "packages": {
-          "w32": "radare2-5.6.4-w32.zip",
-          "w64": "radare2-5.6.4-w64.zip",
-          "mac_x64": "radare2-5.6.4-amd64.pkg",
-          "mac_arm64": "radare2-5.6.4-m1.pkg",
-          "deb_i386": "radare2_5.6.4_i386.deb",
-          "deb_x64": "radare2_5.6.4_amd64.deb"
-        }
-      },
-      {
-        "id": "5.6.6",
-        "packages": {
-          "w32": "radare2-5.6.6-w32.zip",
-          "w64": "radare2-5.6.6-w64.zip",
-          "mac_x64": "radare2-5.6.6-amd64.pkg",
-          "mac_arm64": "radare2-5.6.6-m1.pkg",
-          "deb_i386": "radare2_5.6.6_i386.deb",
-          "deb_x64": "radare2_5.6.6_amd64.deb"
-        }
-      },
-      {
-        "id": "5.6.8",
-        "packages": {
-          "w32": "radare2-5.6.8-w32.zip",
-          "w64": "radare2-5.6.8-w64.zip",
-          "mac_x64": "radare2-5.6.8-amd64.pkg",
-          "mac_arm64": "radare2-5.6.8-m1.pkg",
-          "deb_i386": "radare2_5.6.8_i386.deb",
-          "deb_x64": "radare2_5.6.8_amd64.deb"
-        }
-      },
-      {
-        "id": "5.7.0",
-        "packages": {
-          "w32": "radare2-5.7.0-w32.zip",
-          "w64": "radare2-5.7.0-w64.zip",
-          "mac_x64": "radare2-5.7.0-amd64.pkg",
-          "mac_arm64": "radare2-5.7.0-m1.pkg",
-          "deb_i386": "radare2_5.7.0_i386.deb",
-          "deb_x64": "radare2_5.7.0_amd64.deb"
-        }
-      },
-      {
-        "id": "5.7.2",
-        "packages": {
-          "w32": "radare2-5.7.2-w32.zip",
-          "w64": "radare2-5.7.2-w64.zip",
-          "mac_x64": "radare2-5.7.2-amd64.pkg",
-          "mac_arm64": "radare2-5.7.2-m1.pkg",
-          "deb_i386": "radare2_5.7.2_i386.deb",
-          "deb_x64": "radare2_5.7.2_amd64.deb"
-        }
-      },
-      {
-        "id": "5.7.4",
-        "packages": {
-          "w32": "radare2-5.7.4-w32.zip",
-          "w64": "radare2-5.7.4-w64.zip",
-          "mac_x64": "radare2-5.7.4-amd64.pkg",
-          "mac_arm64": "radare2-5.7.4-m1.pkg",
-          "deb_i386": "radare2_5.7.4_i386.deb",
-          "deb_x64": "radare2_5.7.4_amd64.deb"
-        }
-      },
-      {
-        "id": "5.7.6",
-        "packages": {
-          "w32": "radare2-5.7.6-w32.zip",
-          "w64": "radare2-5.7.6-w64.zip",
-          "mac_x64": "radare2-5.7.6-amd64.pkg",
-          "mac_arm64": "radare2-5.7.6-m1.pkg",
-          "deb_i386": "radare2_5.7.6_i386.deb",
-          "deb_x64": "radare2_5.7.6_amd64.deb"
-        }
-      },
-      {
-        "id": "5.7.8",
-        "packages": {
-          "w32": "radare2-5.7.8-w32.zip",
-          "w64": "radare2-5.7.8-w64.zip",
-          "mac_x64": "radare2-5.7.8-amd64.pkg",
-          "mac_arm64": "radare2-5.7.8-m1.pkg",
-          "deb_i386": "radare2_5.7.8_i386.deb",
-          "deb_x64": "radare2_5.7.8_amd64.deb"
-        }
-      },
-      {
-        "id": "5.8.0",
-        "packages": {
-          "w32": "radare2-5.8.0-w32.zip",
-          "w64": "radare2-5.8.0-w64.zip",
-          "mac_x64": "radare2-5.8.0-amd64.pkg",
-          "mac_arm64": "radare2-5.8.0-m1.pkg",
-          "deb_i386": "radare2_5.8.0_i386.deb",
-          "deb_x64": "radare2_5.8.0_amd64.deb"
-        }
-      },
-      {
-        "id": "git",
-        "packages": {}
-      }
-    ]
-  }
-}

r2env/core.py

@@ -170,6 +170,9 @@ class R2Env:
         with open(version_file, 'w', encoding="utf-8") as file_desc:
             file_desc.write(package)
 
+    def update_version_profiles(self):
+        PackageManager.update_radare2_profiles(self._r2env_path)
+
     def check_if_r2env_initialized(self):
         if not os.path.isdir(self._r2env_path):
             raise R2EnvException("Not r2env initialized. Execute 'r2env init' first.")

r2env/package_manager.py

@@ -5,8 +5,10 @@ import shutil
 import time
 from urllib.error import URLError
 import zipfile
+import json
 
 import wget
+import requests
 
 from r2env.exceptions import PackageManagerException
 from r2env.tools import load_json_file, git_fetch, git_clean, print_console, ERROR, exit_if_not_exists, host_distname
@@ -19,7 +21,7 @@ class PackageManager:
     DOWNLOAD_URL = "https://github.com/radareorg/radare2/releases/download/"
 
     def __init__(self, r2env_path):
-        self._packages = self._load_profiles()
+        self._packages = self._load_profiles(r2env_path)
         self._r2env_path = r2env_path
 
     def get_package_path(self, package_name):
@@ -38,13 +40,110 @@ class PackageManager:
         return res
 
     @staticmethod
-    def _load_profiles():
-        filename = os.path.join(os.path.dirname(__file__), 'config', 'profiles.json')
+    def _load_profiles(r2env_path):
+        filename = os.path.join(r2env_path, 'profiles.json')
+        # if profiles.json doesn't exist, generate it now
+        if not os.path.exists(filename):
+            print_console(f"NOTE: {filename} does not exist. Generating it now.")
+            PackageManager.update_radare2_profiles(r2env_path)
         profiles = load_json_file(filename)
         if not profiles:
             raise PackageManagerException("No profiles configured.")
         return profiles
 
+    @staticmethod
+    def update_radare2_profiles(r2env_path):
+        gh_release_info_req = requests.get(
+            "https://api.github.com/repos/radareorg/radare2/releases",
+            {"headers": ["X-GitHub-Api-Version: 2022-11-28", "Accept: application/vnd.github+json"]},
+            timeout=10  # 10-second timeout for API request
+        )
+        gh_release_info_req.raise_for_status()  # raise an error if the request got a bad return code
+        gh_release_info = gh_release_info_req.json()
+        radare2_versions = [{'id': 'git', 'packages': {}}]  # include the git version by default
+        for release in gh_release_info:
+            version = {'id': release['tag_name'], 'packages': {}}
+            # generate a list of downloadable assets for this release that r2env can use
+            for asset in release['assets']:
+                # note: you're about to see 100 lines of hackery to deal with inconsistent naming schemes
+                # the goal is to include as many valid assets as is reasonable without including any that are not valid
+                # separate file extension
+                asset_name, asset_ext = os.path.splitext(asset['name'])
+
+                # normalize 'seperators' before splitting into components
+                asset_ref = asset_name.replace('_', '-').split('-')
+
+                # filter out things like r2ios_sdk and r2blob
+                if asset_ref[0] != 'radare2':
+                    continue
+
+                # Some assets don't have the version number in the name, which breaks the following processing.
+                # None of them are of interest anyway, so just skip them.
+                if release['tag_name'] not in asset_ref:
+                    continue
+
+                # remove the beginning 'radare2'
+                asset_ref.remove('radare2')
+
+                # we don't need to care about the release id when checking, so remove it
+                asset_ref.remove(release['tag_name'])
+
+                # once "radare2" and the version number have been removed from asset_ref,
+                # if all that's left matches a filter, save it.
+                _VERSION_FILTERS = {
+                    '.zip': {  # used for Windows builds among other things
+                        'w32': [['w32']],
+                        'w64': [['w64']]
+                    },
+                    '.deb': {
+                        'deb_i386': [['i386']],
+                        'deb_x64': [['amd64']]
+                    },
+                    '.pkg': {  # macOS bundle
+                        'mac_arm64': [['m1']],
+                        'mac_x64': [[], ['amd64'], ['x64'], ['macos']]
+                    }
+                }
+
+                if asset_ext in _VERSION_FILTERS:
+                    for platform, refs in _VERSION_FILTERS[asset_ext].items():
+                        if asset_ref in refs:
+                            version['packages'][platform] = asset['name']
+                            break
+
+            radare2_versions.append(version)
+
+        profiles = {
+            'r2frida': {
+                'source': 'https://github.com/nowsecure/r2frida',
+                'versions': [{'id': 'git', 'packages': {}}]
+            },
+            'r2dec': {
+                'source': 'https://github.com/wargio/r2dec-js',
+                'versions': [{'id': 'git', 'packages': {}}]
+            },
+            'r2yara': {
+                'source': 'https://github.com/radareorg/r2yara',
+                'versions': [{'id': 'git', 'packages': {}}]
+            },
+            'r2ghidra': {
+                'source': 'https://github.com/radareorg/r2ghidra',
+                'versions': [{'id': 'git', 'packages': {}}]
+            },
+            'r0': {
+                'source': 'https://github.com/radareorg/r0',
+                'versions': [{'id': 'git', 'packages': {}}]
+            },
+            'radare2': {
+                'source': 'https://github.com/radareorg/radare2',
+                'versions': radare2_versions[::-1]
+            }
+        }
+        if not os.path.isdir(r2env_path):
+            os.makedirs(r2env_path)
+        with open(os.path.join(r2env_path, 'profiles.json'), 'w', encoding='utf-8') as f:
+            json.dump(profiles, f)
+
     def install_package(self, profile, version, use_meson, use_dist):
         log_path = os.path.join(self._r2env_path, self.LOG_DIR)
         timestamp = round(time.monotonic() * 1000)
@@ -57,11 +156,17 @@ class PackageManager:
             os.makedirs(source_path)
         if not os.path.isdir(dst_dir):
             os.makedirs(dst_dir)
-        self._exit_if_package_not_available(profile, version)
+        if self._if_package_not_available(profile, version):
+            if os.path.isdir(dst_dir):
+                shutil.rmtree(dst_dir)
+            raise PackageManagerException(
+                f"{profile}@{version} package not available. Use 'list' command to see available packages.")
         if use_dist:
             if self._install_from_dist(profile, version):
                 print_console(f"[*] Binary package {profile}@{version} installed successfully")
             else:
+                if os.path.isdir(dst_dir):
+                    shutil.rmtree(dst_dir)
                 raise PackageManagerException(f"[x] Failed to install {profile}@{version}.")
         else:
             if self._build_from_source(profile, version, source_path, dst_dir, logfile, use_meson=use_meson):
@@ -93,11 +198,12 @@ class PackageManager:
             raise PackageManagerException(f"[x] Unable to find installed package {package_name}")
 
     def _get_pkgname(self, profile, version, cos):
-        for pkgv in self._packages[profile]["versions"]:
-            if pkgv["id"] == version:
-                for pkgd in pkgv["packages"].keys():
-                    if pkgd == cos:
-                        return pkgv["packages"][pkgd]
+        if profile in self._packages:
+            for pkgv in self._packages[profile]["versions"]:
+                if pkgv["id"] == version:
+                    for pkgd in pkgv["packages"].keys():
+                        if pkgd == cos:
+                            return pkgv["packages"][pkgd]
         return None
 
     def _get_disturl(self, profile, version, cos):
@@ -190,7 +296,7 @@ class PackageManager:
     @staticmethod
     def _build_using_acr(source_path, dst_path, logfile, r2env_path):
         """Only works in Unix systems"""
-        if host_distname() in "w64":
+        if host_distname() is not None and host_distname() in "w64":
             raise PackageManagerException("acr is not supported on Windows platform. Use meson instead.")
         exit_if_not_exists(['make'])
         print_console("[-] Building package using acr...")
@@ -222,12 +328,9 @@ class PackageManager:
         print_console(f"Executing {cmd}")
         return os.system(cmd) == 0
 
-    def _exit_if_package_not_available(self, profile, version):
-        pkg_found = False
-        for available_version in self._packages[profile]["versions"]:
-            if available_version['id'] == version:
-                pkg_found = True
-                break
-        if not pkg_found:
-            raise PackageManagerException(
-                f"{profile}@{version} package not available. Use 'list' command to see available packages.")
+    def _if_package_not_available(self, profile, version):
+        if profile in self._packages:
+            for available_version in self._packages[profile]["versions"]:
+                if available_version['id'] == version:
+                    return False
+        return True

r2env/repl.py

@@ -31,6 +31,7 @@ class REPL:
       list          - list all packages available to r2env.
       shell         - enter a new shell with PATH env var set.
       purge         - remove ~/.r2env
+      update        - update list of available profiles
 
     Environment
 
@@ -55,6 +56,7 @@ class REPL:
             "install": self._core.install,
             "installed": self._core.list_installed_packages,
             "uninstall": self._core.uninstall,
+            "update": self._core.update_version_profiles,
             "rm": self._core.uninstall,
             "use": self._core.use,
             "purge": self._core.purge,

r2env/tools.py

@@ -81,7 +81,12 @@ def git_fetch(url, version, source_path):
                                       f"Verify the source path is not dirty. Error: {err}") \
             from err
     if version != "git":
-        repo.git.checkout(version)
+        try:
+            repo.git.checkout(version)
+        except git.GitCommandError:
+            # try again after pulling new tags from origin
+            repo.git.fetch("origin", "--tags")
+            repo.git.checkout(version)
     sms = repo.submodules
     for sm in sms:
         if not sm.module_exists():

r2env/version.txt

@@ -1 +1 @@
-0.5.6
+0.5.9

requirements/build.txt

@@ -1,4 +1,5 @@
 dploy>=0.1.2
 colorama>=0.4.4
 wget
+requests
 GitPython

setup.cfg

@@ -35,6 +35,7 @@ install_requires=
     dploy >= 0.1.2
     colorama >= 0.4.4
     GitPython
+    requests
 setup_requires =
     setuptools_scm >= 1.15
 include_package_data = True

setup.py

@@ -6,7 +6,7 @@ setup(
     setup_requires=['pbr'],
     pbr=True,
     package_files=[
-                   ('config', ['r2env/config/config.json', 'r2env/config/profiles.json']),
+                   ('config', ['r2env/config/config.json']),
                    ('', ['version.txt'])
                    ]
 )

tox.ini

@@ -3,10 +3,11 @@ envlist = py{38,39,310}-{tests},{py38,py39,310}-codecheck-{pylint,flake8,bandit}
 skipsdist = True
 
 [testenv]
+allowlist_externals = bash 
 deps = -r {toxinidir}/requirements/test.txt
 passenv = *
 commands =
-    py{38,39,310}-tests: {toxinidir}/run_tests.sh unit
+    py{38,39,310}-tests: bash {toxinidir}/run_tests.sh unit
     py{38,39,310}-codecheck-pylint: pylint r2env
     py{38,39,310}-codecheck-flake8: flake8 r2env
     py{38,39,310}-codecheck-bandit: bandit -c .bandit.yml -r r2env