Merge pull request #2796 from peace-maker/stable_backports

Backport bugfixes from 5.0 to stable

.coveragerc

@@ -1,12 +0,0 @@
-[run]
-branch = True
-parallel = True
-concurrency = thread,multiprocessing
-omit =
-    */constants/*
-source =
-    pwn
-    pwnlib
-    ~/.cache/.pwntools-cache-2.7/
-    ~/.cache/.pwntools-cache-3.8/
-disable_warnings = module-not-imported

.github/ISSUE_TEMPLATE/bug_report.md

@@ -39,7 +39,7 @@ You should see `[DEBUG]` statements that show what's happening behind the scenes
 
 ## Verify on Ubuntu
 
-If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+If possible, please verify that your issue occurs on 64-bit Ubuntu 22.04.  We provide a Dockerfile based on Ubuntu 22.04 via `docker.io` to make this super simple, no VM required!
 
 ```sh
 # Download the Docker image

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/android.yml

@@ -3,27 +3,24 @@ name: Android Tests
 on: [push, pull_request]
 
 jobs:
-  test:
+  android-test:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Cache for pip
-      uses: actions/cache@v1
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Install Linux dependencies
       run: |
@@ -33,14 +30,27 @@ jobs:
           qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
-          libc6-dbg \
-          openjdk-8-jre-headless
+          libc6-dbg
+
+    - name: Cache for avd
+      uses: actions/cache@v4
+      id: cache-avd
+      with:
+        path: |
+          ~/.android
+          /usr/local/lib/android/sdk/emulator
+          /usr/local/lib/android/sdk/platform-tools
+          /usr/local/lib/android/sdk/system-images
+        key: ${{ matrix.os }}-cache-avd-${{ hashFiles('travis/setup_avd*.sh') }}
+        restore-keys: |
+          ${{ matrix.os }}-cache-avd-
 
     - name: Install Android AVD
       run: |
-        USER=travis source travis/setup_avd.sh
+        sudo usermod -aG kvm $USER
+        source travis/setup_avd_fast.sh
         sed -i 's/skip_android = True/skip_android = False/' docs/source/conf.py
-        set | egrep '^(ANDROID|PATH)' >.android.env
+        set | grep ^PATH >.android.env
 
     - name: Install dependencies
       run: |

.github/workflows/changelog.yml

@@ -1,5 +1,9 @@
 name: Changelog
-on: [pull_request]
+on:
+  pull_request:
+    paths:
+    - pwnlib/**
+    - pwn/**
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/ci.yml

@@ -5,31 +5,98 @@ jobs:
   test:
     strategy:
       matrix:
-        python-version: [2.7, 3.8]
+        python_version: ['3.10', '3.12', '3.13', '3.14', '3.15']
         os: [ubuntu-latest]
+        include:
+          - python_version: '2.7'
+            os: ubuntu-22.04
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
+    services:
+      libcdb-cache:
+        image: nginx
+        volumes:
+          - /home/runner/libcdb-cache:/var/cache/nginx
+        ports:
+          - 3000:3000  # https://debuginfod.elfutils.org proxy cache
+          - 3001:3001  # https://libc.rip/ proxy cache
+          - 3002:3002  # http://archive.ubuntu.com/ proxy cache
+          - 3003:3003  # https://gitlab.com/ proxy cache
+          - 3004:3004  # https://debuginfod.ubuntu.com/ proxy cache
+    env:
+      DEBUGINFOD_URLS: "http://localhost:3000/ http://localhost:3004/"
+      PWN_LIBCRIP_URL: http://localhost:3001/
+      PWN_UBUNTU_ARCHIVE_URL: http://localhost:3002/
+      PWN_GITLAB_LIBCDB_URL:  http://localhost:3003/
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
-        fetch-depth: 0
+        fetch-depth: 20
 
     - name: Git History
       run: |
         git fetch origin
         git log --oneline --graph -10
+    
+    - name: Fix libcdb-cache permissions
+      id: fix-perms
+      run: |
+        sudo chown -R runner:runner /home/runner/libcdb-cache
+        echo "date=$(/bin/date -u "+%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
 
+    - name: Cache for libcdb requests
+      uses: actions/cache@v4
+      with:
+        path: ~/libcdb-cache
+        key: libcdb-python${{ matrix.python_version }}-${{ steps.fix-perms.outputs.date }}
+        restore-keys: |
+          libcdb-python${{ matrix.python_version }}-
+          libcdb-
+
+    - name: Install libcdb-cache service config
+      run: |
+        sudo chown -R 101:101 /home/runner/libcdb-cache
+        container_id=$(docker ps --all --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker cp ./travis/libcdb_nginx_cache.conf $container_id:/etc/nginx/nginx.conf
+        docker restart $container_id
+
+    - name: Install RPyC for gdb
+      run: |
+        # The version packaged in python3-rpyc is too old on Ubuntu 24.04
+        # We use ^6.0 from pip.
+        sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
+        /usr/bin/python -m pip install --break-system-packages rpyc || /usr/bin/python -m pip install rpyc
+        gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
+    
     - name: Cache for pip
-      uses: actions/cache@v1
-      id: cache-pip
+      uses: actions/cache@v4
+      if: matrix.python_version == '2.7'
       with:
         path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+        key: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-${{ hashFiles('**/pyproject.toml', '**/requirements*.txt') }}
+        restore-keys: ${{ matrix.os }}-${{ matrix.python_version }}-cache-pip-
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+    - name: Set up Python ${{ matrix.python_version }}
+      if: matrix.python_version != '2.7'
+      uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python_version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
+
+    - name: Set up Python 2.7
+      if: matrix.python_version == '2.7'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          python2.7 python2.7-dev python2-pip-whl
+        sudo ln -sf python2.7 /usr/bin/python
+        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
+        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
+        sudo chown -R $USER /usr/local/lib/python2.7
+
 
     - name: Verify tag against version
       if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
@@ -49,30 +116,29 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          pandoc gdb gdbserver socat sshpass \
+          gdb gdbserver socat \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
+          binutils-mipsel-linux-gnu \
           binutils-msp430 \
           binutils-powerpc-linux-gnu \
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
+          binutils-riscv64-linux-gnu \
           gcc-multilib \
-          libc6-dbg
-
-    - name: Install RPyC for GDB
-      run: |
-        sudo apt-get install -y python3-pip
-        /usr/bin/python3 -m pip install rpyc
-
+          libc6-dbg \
+          elfutils \
+          patchelf
+      
     - name: Testing Corefiles
       run: |
         ulimit -a
         ulimit -c unlimited
         cat /proc/sys/kernel/core_pattern
         cat /proc/sys/kernel/core_uses_pid
-        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
+        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash /var/lib/apport/coredump/core*) || true
 
     - name: Set up SSH
       run: |
@@ -81,18 +147,21 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
+        pip install --upgrade wheel build
         pip install --upgrade flake8 appdirs
-        python setup.py egg_info
         pip install --upgrade --editable .
+        # Pin to known working version until 2.1.5 is out. https://github.com/unicorn-engine/unicorn/issues/2134
+        pip install 'unicorn==2.1.2'
 
     - name: Sanity checks
       run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
 
     - name: Install documentation dependencies
-      run:  pip install -r docs/requirements.txt
+      run: pip install -r docs/requirements.txt
 
     - name: Manually install non-broken Unicorn
-      run:  pip install unicorn==1.0.2rc3
+      if:   matrix.python_version == '2.7'
+      run:  pip install unicorn==2.0.0rc7
 
     - name: Disable yama ptrace_scope
       run: |
@@ -100,6 +169,10 @@ jobs:
 
     - name: Coverage doctests
       run: |
+        # Python version installed using setup-python interferes with gdb's python
+        # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules
+        # like _socket. This is a workaround.
+        unset LD_LIBRARY_PATH
         PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples
@@ -127,11 +200,18 @@ jobs:
         pwn shellcraft --list |tail
         pwn shellcraft -l --syscalls |tail
         pwn shellcraft -l execve
+        pwn shellcraft -l execve + exit
         pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft --show i386.linux.loader_append + i386.linux.sh
         pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f asm --color amd64.linux.setreuid + amd64.linux.cat /etc/passwd
+        pwn shellcraft -f asm --color amd64.linux.setreuid = amd64.linux.cat /key+secret --delim =
         pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f elf amd64.linux.cat /etc/passwd + amd64.linux.syscalls.exit 0 </dev/null |pwn hex
         pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd + amd64.linux.sh </dev/null
         pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f c   amd64.linux.cat /etc/passwd + amd64.linux.syscalls.exit 0 </dev/null
         pwn shellcraft -f str aarch64.linux.sh </dev/null
         pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
         pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
@@ -140,6 +220,11 @@ jobs:
         pwn disasm --color ff3424c3ebfe
         pwn asm -f hex nop
 
+        pwn hex ABCD
+        pwn hex ABCD --separator ' '
+        pwn hex ABCD --prefix '\x'
+        pwn hex ABCD -p '0x' -s ' '
+
         pwn hex abcd
         pwn unhex 4141 4141
 
@@ -147,6 +232,7 @@ jobs:
         pwn phd -l 0x3d --color=always /etc/os-release
 
         pwn checksec /bin/bash
+        (ulimit -v 500000 && pwn checksec /bin/bash)
 
         pwn errno 2
         pwn errno -1
@@ -155,28 +241,87 @@ jobs:
         pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
         pwn constgrep ^MAP_ 0
         pwn constgrep -e O_RDWR
+        pwn constgrep C
+
+        pwn libcdb file /lib/x86_64-linux-gnu/libc.so.6
+        pwn libcdb lookup puts 5f0 __libc_start_main_ret d0a
+        pwn libcdb hash b229d1da1e161f95e839cf90cded5f719e5de308
 
     - name: Build source and wheel distributions
+      if: matrix.python_version != '2.7'
       run: |
-        python setup.py sdist
-        python setup.py bdist_wheel --universal
+        python -m build
 
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
+      if: matrix.python_version == '3.10'
       with:
         name: packages
         path: dist/
+        include-hidden-files: true
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: coverage-${{ matrix.python_version }}
+        path: .coverage*
+        include-hidden-files: true
+    
+    - name: Fix libcdb-cache permissions
+      run: |
+        container_id=$(docker ps --filter volume=/home/runner/libcdb-cache --no-trunc --format "{{.ID}}")
+        docker stop $container_id
+        sudo chown -R runner:runner /home/runner/libcdb-cache
+
+  windows-test:
+    runs-on: windows-latest
+    timeout-minutes: 30
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run: |
+        python -bb -c 'from pwn import *'
+        python -bb examples/text.py
+
+  upload-coverage:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 20
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: coverage-*
+        merge-multiple: true
+
+    - name: Install coveralls
+      run: |
+        pip install --break-system-packages tomli coveralls || pip install tomli coveralls
 
     - name: Upload coverage to coveralls.io
       run: |
         coverage combine
         COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
 
+
   staging-merge:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
     needs: test
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 20
     - name: Push changes to protected branch
@@ -189,22 +334,22 @@ jobs:
         git push origin "$TARGET"
         git push origin --delete "$BRANCH"
 
+
   pypi:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    permissions:
+      id-token: write
     needs: test
     steps:
     - name: Download artifacts
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
         name: packages
         path: dist
 
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@v1.1.0
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+      uses: pypa/gh-action-pypi-publish@release/v1
 
     - if: failure()
       run: ls -R

.github/workflows/docker.yml

@@ -0,0 +1,64 @@
+name: Publish Docker images
+on:
+  push:
+    branches:
+      - dev
+      - beta
+      - stable
+  workflow_dispatch:
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled'
+    steps:
+      # Required for subdirectories in Git context
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v6
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          context: "{{defaultContext}}:extra/docker/base"
+          push: true
+          tags: pwntools/pwntools:base
+
+      - name: Build and push stable image
+        uses: docker/build-push-action@v6
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/stable')
+        with:
+          context: "{{defaultContext}}:extra/docker/stable"
+          push: true
+          tags: pwntools/pwntools:stable
+
+      - name: Build and push beta image
+        uses: docker/build-push-action@v6
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/beta')
+        with:
+          context: "{{defaultContext}}:extra/docker/beta"
+          push: true
+          tags: pwntools/pwntools:beta
+
+      - name: Build and push dev image
+        uses: docker/build-push-action@v6
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:extra/docker/dev"
+          push: true
+          tags: |
+            pwntools/pwntools:dev
+            pwntools/pwntools:latest
+
+      - name: Build and push ci image
+        uses: docker/build-push-action@v6
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:travis/docker"
+          push: true
+          tags: pwntools/pwntools:ci

.github/workflows/lint.yml

@@ -5,23 +5,21 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
-    - name: Cache for pip
-      uses: actions/cache@v1
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: Critical lint
       run: |

.github/workflows/merge-conflict.yml

@@ -1,5 +1,7 @@
 name: Check for merge markers
-on: [pull_request]
+on:
+  pull_request:
+    types: [synchronize]
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/pylint.yml

@@ -5,33 +5,32 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
-    - name: Cache for pip
-      uses: actions/cache@v1
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: ${{ matrix.os }}-cache-pip
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          **/pyproject.toml
+          **/requirements*.txt
 
     - name: PyLint
       run: |
         set -x
-        pip install pylint
-        pip install --upgrade -r requirements.txt
-        pylint --exit-zero --errors-only  pwnlib > current.txt
+        pip install 'pylint<4'
+        pip install --upgrade -e .
+        run_pylint() { pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- | sed 's/line [0-9]\+/line XXXX/g'; }
+        run_pylint > current.txt
         git fetch origin
         git checkout origin/"$GITHUB_BASE_REF"
-        pylint --exit-zero --errors-only  pwnlib > base.txt
+        run_pylint > base.txt
         if diff base.txt current.txt | grep '>'; then
           false
         fi

.gitignore

@@ -21,3 +21,4 @@ venv
 .idea
 __pycache__
 !.github
+.DS_Store

.readthedocs.yaml

@@ -0,0 +1,15 @@
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+sphinx:
+  configuration: docs/source/conf.py
+
+python:
+  install:
+  - requirements: docs/requirements.txt

.travis.yml

@@ -1,66 +0,0 @@
-language: python
-addons:
-  apt:
-    packages:
-    - lib32stdc++6
-    - ash
-    - bash
-    - dash
-    - ksh
-    - mksh
-    - zsh
-    - pandoc
-    - gdb
-    - socat
-    - sshpass
-    - binutils
-    - qemu-user-static
-    - binutils-multiarch
-    - binutils-aarch64-linux-gnu
-    - binutils-arm-linux-gnueabihf
-    - binutils-mips-linux-gnu
-    - binutils-msp430
-    - binutils-powerpc-linux-gnu
-    - binutils-s390x-linux-gnu
-    - binutils-sparc64-linux-gnu
-    - bash-static
-    - python3-pip
-cache:
-    - pip
-    - directories:
-        - usr
-        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.15/bin/
-        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
-        - /home/travis/virtualenv/python3.8.*/bin/
-        - docs/build/doctest/.doctrees
-python:
-  - "2.7"
-  - "3.8"
-before_install:
-  - echo $-
-  - source travis/install.sh
-  - source travis/ssh_setup.sh
-  - echo $-
-install:
-  - /usr/bin/python3 -m pip install rpyc # for GDB
-  - pip install --upgrade pip
-  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
-  - python setup.py egg_info      # pypa/pip#6275
-  - pip install --upgrade --editable .
-  - pip install --upgrade --requirement docs/requirements.txt
-before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-script:
-  # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-  - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
-  - coverage combine
-after_success:
-  - coveralls
-  - source travis/update_demo.sh

CHANGELOG.md

@@ -9,9 +9,21 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.8.0](#480)    | `dev`    |
-| [4.7.0](#470)    | `beta`   |
-| [4.6.0](#460)    | `stable` | Jul 12, 2021
+| [5.0.0](#500-dev)  | `dev`    |
+| [4.15.0](#4150-stable)  | `stable` | Oct 12, 2025
+| [4.14.1](#4141)  |          | Mar 24, 2025
+| [4.14.0](#4140)  |          | Jan 15, 2025
+| [4.13.1](#4131)  |          | Sep 29, 2024
+| [4.13.0](#4130)  |          | Aug 12, 2024
+| [4.12.0](#4120)  |          | Feb 22, 2024
+| [4.11.1](#4111)  |          | Nov 14, 2023
+| [4.11.0](#4110)  |          | Sep 15, 2023
+| [4.10.0](#4100)  |          | May 21, 2023
+| [4.9.0](#490)    |          | Dec 29, 2022
+| [4.8.0](#480)    |          | Apr 21, 2022
+| [4.7.1](#471)    |          | Apr 20, 2022
+| [4.7.0](#470)    |          | Nov 15, 2021
+| [4.6.0](#460)    |          | Jul 12, 2021
 | [4.5.1](#451)    |          | May 30, 2021
 | [4.5.0](#450)    |          | Apr 30, 2021
 | [4.4.0](#440)    |          | Mar 29, 2021
@@ -61,11 +73,435 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.8.0 (`dev`)
+## 5.0.0 (`dev`)
 
+- [#2419][2419] riscv: avoid compressed instructions (if you need compressed, use .option rvc)
+- [#2551][2551] Detect when kitty is being used as terminal
+- [#2519][2519] Drop Python 2.7 support / Require Python 3.10
+- [#2507][2507] Add `+LINUX` and `+WINDOWS` doctest options and start proper testing on Windows
+- [#2522][2522] Support starting a kitty debugging window with the 'kitten' command
+- [#2526][2526] Properly make use of extra arguments in `packing` utilities. `sign` parameter requires keyword syntax to specify it.
+- [#2517][2517] Allow to passthru kwargs on `ssh.__getattr__` convenience function to fix SSH motd problems
+- [#2529][2529] Add LoongArch64 support
+- [#2504][2504] doc: add example case for `tuple` (host, port pair) in `gdb.attach`
+- [#2546][2546] ssh: Allow passing disabled_algorithms keyword argument from ssh to paramiko
+- [#2538][2538] Add `ssh -L` / `ssh.connect_remote()` workaround when `AllowTcpForwarding` is disabled
+- [#2574][2574] Allow creating an ELF from in-memory bytes
+- [#2575][2575] Detect when Terminator is being used as terminal
+- [#2578][2578] Add gnome-terminal, Alacritty, Ttilix for run_in_new_terminal
+- [#2590][2590] Add support for finding corefiles under WSL2
+- [#2496][2496] Add linux ko file search support
+- [#2542][2542] Decode `_IO_*` flags in `FileStructure` member
+- [#2608][2608] Abort on `libcdb file libc.so --unstrip` if eu-unstrip is not installed
+- [#2611][2611] Cleanup `pwnlib.lexer` exports and imports
+- [#2615][2615] tube/process: Fix redirecting stderr to stdout on Windows
 
+[2419]: https://github.com/Gallopsled/pwntools/pull/2419
+[2551]: https://github.com/Gallopsled/pwntools/pull/2551
+[2519]: https://github.com/Gallopsled/pwntools/pull/2519
+[2507]: https://github.com/Gallopsled/pwntools/pull/2507
+[2522]: https://github.com/Gallopsled/pwntools/pull/2522
+[2526]: https://github.com/Gallopsled/pwntools/pull/2526
+[2517]: https://github.com/Gallopsled/pwntools/pull/2517
+[2529]: https://github.com/Gallopsled/pwntools/pull/2529
+[2504]: https://github.com/Gallopsled/pwntools/pull/2504
+[2546]: https://github.com/Gallopsled/pwntools/pull/2546
+[2538]: https://github.com/Gallopsled/pwntools/pull/2538
+[2574]: https://github.com/Gallopsled/pwntools/pull/2574
+[2575]: https://github.com/Gallopsled/pwntools/pull/2575
+[2578]: https://github.com/Gallopsled/pwntools/pull/2578
+[2590]: https://github.com/Gallopsled/pwntools/pull/2590
+[2496]: https://github.com/Gallopsled/pwntools/pull/2496
+[2542]: https://github.com/Gallopsled/pwntools/pull/2542
+[2608]: https://github.com/Gallopsled/pwntools/pull/2608
+[2611]: https://github.com/Gallopsled/pwntools/pull/2611
+[2615]: https://github.com/Gallopsled/pwntools/pull/2615
 
-## 4.7.0 (`beta`)
+## 4.15.1
+
+- [#2694][2694] fix: pad bytes fields to correct field size in FileStructure
+- [#2701][2701] Fix `adb._build_date()` crash on devices with non-standard locale date strings
+- [#2721][2721] Allow running with Unicorn 2.1.[34] but throw when emulating MIPS
+- [#2773][2773] Disable debuginfod doctests in libcdb.unstrip_libc
+- [#2772][2772] fix(packing): `dd()` no longer crashes on a `bytearray` destination
+- [#2787][2787] gdb.corefile: Run gdb directly to prevent detach race
+- [#2795][2795] regsort: Fix self-assignment detection through register aliases
+- [#2524][2524] Raise EOFError during `process.recv` when stdout closes on Windows
+- [#2530][2530] Do NOT error when passing directory arguments in `checksec` commandline tool.
+- [#2506][2506] ROP: fix `ROP(ELF(exe)).leave` is `None` in some ELF
+- [#2592][2592] pwnlib.config: Fix customization of `context.timeout`
+- [#2598][2598] aarch64: Fix ABI definition
+- [#2610][2610] Fix `log.progress` ignoring `context.log_console`
+- [#2627][2627] Remove `pwnlib.util.iters.lookahead` (broken anyway, fails on Python 3.14+)
+- [#2653][2653] Add aarch64_be qemu mapping
+- [#2642][2642] Remove `pip` as explicit dependency
+- [#2646][2646] fix(libcdb-cli): return early if no matched libc found
+- [#2669][2669] asm: try native binutils before fallback architectures
+- [#2677][2677] refactor: replace unsafe eval with safeeval.const in ROP cache loading
+- [#2699][2699] Fix `tty` and `raw` arguments in `ssh.process()`
+- [#2688][2688] Close SSH client connection when authentication failed
+- [#2704][2704] ssh: Fix distro lookup on Ubuntu 24.04+
+- [#2762][2762] fix(srop): correct amd64 SigreturnFrame `uc_sigmask` offset
+- [#2767][2767] Support paginated libc.rip API requests to include all search results
+- [#2739][2739] shellcraft: migrate lazy importer to find_spec for Python 3.12+
+- [#2792][2792] Fix deprecation warning for not using `RLIM_INFINITY` in `resource.setrlimit` (Python 3.15)
+
+[2694]: https://github.com/Gallopsled/pwntools/pull/2694
+[2701]: https://github.com/Gallopsled/pwntools/pull/2701
+[2721]: https://github.com/Gallopsled/pwntools/pull/2721
+[2773]: https://github.com/Gallopsled/pwntools/pull/2773
+[2772]: https://github.com/Gallopsled/pwntools/pull/2772
+[2787]: https://github.com/Gallopsled/pwntools/pull/2787
+[2795]: https://github.com/Gallopsled/pwntools/pull/2795
+[2524]: https://github.com/Gallopsled/pwntools/pull/2524
+[2530]: https://github.com/Gallopsled/pwntools/pull/2530
+[2506]: https://github.com/Gallopsled/pwntools/pull/2506
+[2592]: https://github.com/Gallopsled/pwntools/pull/2592
+[2598]: https://github.com/Gallopsled/pwntools/pull/2598
+[2610]: https://github.com/Gallopsled/pwntools/pull/2610
+[2627]: https://github.com/Gallopsled/pwntools/pull/2627
+[2653]: https://github.com/Gallopsled/pwntools/pull/2653
+[2642]: https://github.com/Gallopsled/pwntools/pull/2642
+[2646]: https://github.com/Gallopsled/pwntools/pull/2646
+[2669]: https://github.com/Gallopsled/pwntools/pull/2669
+[2677]: https://github.com/Gallopsled/pwntools/pull/2677
+[2699]: https://github.com/Gallopsled/pwntools/pull/2699
+[2688]: https://github.com/Gallopsled/pwntools/pull/2688
+[2704]: https://github.com/Gallopsled/pwntools/pull/2704
+[2762]: https://github.com/Gallopsled/pwntools/pull/2762
+[2767]: https://github.com/Gallopsled/pwntools/pull/2767
+[2739]: https://github.com/Gallopsled/pwntools/pull/2739
+[2792]: https://github.com/Gallopsled/pwntools/pull/2792
+
+## 4.15.0 (`stable`)
+
+- [#2508][2508] Ignore a warning when compiling with asm on nix
+- [#2471][2471] Properly close spawned kitty window
+- [#2358][2358] Cache output of `asm()`
+- [#2457][2457] Catch exception of non-ELF files in checksec.
+- [#2444][2444] Add `ELF.close()` to release resources
+- [#2413][2413] libcdb: improve the search speed of `search_by_symbol_offsets` in local libc-database
+- [#2470][2470] Fix waiting for gdb under WSL2
+- [#2479][2479] Support extracting libraries from Docker image in `pwn template`
+- [#2483][2483] Only print `checksec` output of `ELF.libc` when it was printed for the `ELF` already
+- [#2482][2482] Throw error when using `sni` and setting `server_hostname` manually in `remote`
+- [#2478][2478] libcdb-cli: add `--offline-only`, refactor unstrip and add fetch parser for download libc-database
+- [#2484][2484] Allow to disable caching
+- [#2291][2291] Fix attaching to a gdbserver with tuple `gdb.attach(('0.0.0.0',12345))`
+- [#2410][2410] Add `tube.upload_manually` to upload files in chunks
+- [#2502][2502] Fix loading ELF files without valid .dynamic section
+- [#2476][2476] Deprecate 'keepends' argument in favor of 'drop' in `tube.recvline*`
+- [#2364][2364] Deprecate direct commandline scripts invocation and exclude nonsense ones
+- [#2570][2570] Fix `pwn template` to skip Docker library extraction for statically linked binaries
+- [#2545][2545] SSH: fix download/upload with -1 exit status
+- [#2567][2567] Fix mistakenly parsing of ld-linux error messages.
+- [#2576][2576] regsort: respect register aliases
+- [#2595][2595] libcdb: Add Ubuntu and Debian debuginfod servers to default list
+- [#2593][2593] Use unicorn on macOS w/ SIP enabled
+- [#2587][2587] Support longer function names in Windows `getexport` shellcode
+- [#2596][2596] Ignore `colored_traceback` error when TERM envvar is unset
+- [#2579][2579] Fix poll error in `process.libs()` and clean up maps parsing
+- [#2602][2602] Allow setting debugger path via context.gdb_binary
+- [#2609][2609] Fix log level of child remotes of `server` tube
+- [#2612][2612] Fix lookup of binutils for `mipsel` architecture
+- [#2624][2624] Fix regression: gdbserver can't handle command-line argument containing whitespace
+
+[2508]: https://github.com/Gallopsled/pwntools/pull/2508
+[2471]: https://github.com/Gallopsled/pwntools/pull/2471
+[2358]: https://github.com/Gallopsled/pwntools/pull/2358
+[2457]: https://github.com/Gallopsled/pwntools/pull/2457
+[2444]: https://github.com/Gallopsled/pwntools/pull/2444
+[2413]: https://github.com/Gallopsled/pwntools/pull/2413
+[2470]: https://github.com/Gallopsled/pwntools/pull/2470
+[2479]: https://github.com/Gallopsled/pwntools/pull/2479
+[2483]: https://github.com/Gallopsled/pwntools/pull/2483
+[2482]: https://github.com/Gallopsled/pwntools/pull/2482
+[2478]: https://github.com/Gallopsled/pwntools/pull/2478
+[2484]: https://github.com/Gallopsled/pwntools/pull/2484
+[2291]: https://github.com/Gallopsled/pwntools/pull/2291
+[2410]: https://github.com/Gallopsled/pwntools/pull/2410
+[2502]: https://github.com/Gallopsled/pwntools/pull/2502
+[2476]: https://github.com/Gallopsled/pwntools/pull/2476
+[2364]: https://github.com/Gallopsled/pwntools/pull/2364
+[2570]: https://github.com/Gallopsled/pwntools/pull/2570
+[2545]: https://github.com/Gallopsled/pwntools/pull/2545
+[2567]: https://github.com/Gallopsled/pwntools/pull/2567
+[2576]: https://github.com/Gallopsled/pwntools/pull/2576
+[2595]: https://github.com/Gallopsled/pwntools/pull/2595
+[2593]: https://github.com/Gallopsled/pwntools/pull/2593
+[2587]: https://github.com/Gallopsled/pwntools/pull/2587
+[2596]: https://github.com/Gallopsled/pwntools/pull/2596
+[2579]: https://github.com/Gallopsled/pwntools/pull/2579
+[2602]: https://github.com/Gallopsled/pwntools/pull/2602
+[2609]: https://github.com/Gallopsled/pwntools/pull/2609
+[2612]: https://github.com/Gallopsled/pwntools/pull/2612
+[2624]: https://github.com/Gallopsled/pwntools/pull/2624
+
+## 4.14.1
+
+- [#2451][2451] Show symbols defined to value 0 (start of file)
+- [#2533][2533] Fix installation on Python 3.5 and lower
+- [#2518][2518] fix: update apport coredump path handling for CorefileFinder
+- [#2552][2552] Fix memcpy shellcraft template off-by-one
+- [#2559][2559] Fix parsing corefile with missing auxv
+- [#2562][2562] Fix syntax highlighting of multiline comments in ASM
+- [#2565][2565] Exclude broken Unicorn
+
+[2451]: https://github.com/Gallopsled/pwntools/pull/2451
+[2533]: https://github.com/Gallopsled/pwntools/pull/2533
+[2518]: https://github.com/Gallopsled/pwntools/pull/2518
+[2552]: https://github.com/Gallopsled/pwntools/pull/2552
+[2559]: https://github.com/Gallopsled/pwntools/pull/2559
+[2562]: https://github.com/Gallopsled/pwntools/pull/2562
+[2565]: https://github.com/Gallopsled/pwntools/pull/2565
+
+## 4.14.0
+
+- [#2356][2356] Add local libc database provider for libcdb
+- [#2360][2360] Add offline parameter for `search_by_hash` series function
+- [#2388][2388] libcdb: add `offline_only` to `search_by_symbol_offsets`
+- [#2374][2374] libcdb.unstrip_libc: debug symbols are fetched only if not present
+- [#2327][2327] Add basic support to debug processes on Windows
+- [#2437][2437] Support asm/disasm on Windows
+- [#2330][2330] Change `context.newline` when setting `context.os` to `"windows"`
+- [#2322][2322] Add basic RISCV64 shellcraft support
+- [#2376][2376] Return buffered data on first EOF in tube.readline()
+- [#2371][2371] Add functions for retrieving process mappings
+- [#2398][2398] Add support for generating multiple shellcodes at a time in shellcraft
+- [#2389][2389] Fix passing bytes to `context.log_file` and `crc.BitPolynom`
+- [#2391][2391] Fix error message when passing invalid kwargs to `xor`
+- [#2387][2387] Convert apport_corefile() output from bytes-like object to string
+- [#2415][2415] Add shellcraft template for IPv6 socket
+- [#2405][2405] Add "none" ssh authentication method
+- [#2427][2427] Document behaviour of remote()'s sni argument as string.
+- [#2382][2382] added optional port, gdb_args and gdbserver_args parameters to gdb.debug()
+- [#2435][2435] Speed up gdbserver handshake in gdb.debug()
+- [#2436][2436] Add resolution_addr parameter to Ret2dlresolvePayload
+- [#2497][2497] Fix remote.fromsocket() to handle AF_INET6 socket
+
+[2436]: https://github.com/Gallopsled/pwntools/pull/2436
+[2371]: https://github.com/Gallopsled/pwntools/pull/2371
+[2360]: https://github.com/Gallopsled/pwntools/pull/2360
+[2356]: https://github.com/Gallopsled/pwntools/pull/2356
+[2374]: https://github.com/Gallopsled/pwntools/pull/2374
+[2327]: https://github.com/Gallopsled/pwntools/pull/2327
+[2322]: https://github.com/Gallopsled/pwntools/pull/2322
+[2330]: https://github.com/Gallopsled/pwntools/pull/2330
+[2389]: https://github.com/Gallopsled/pwntools/pull/2389
+[2391]: https://github.com/Gallopsled/pwntools/pull/2391
+[2376]: https://github.com/Gallopsled/pwntools/pull/2376
+[2387]: https://github.com/Gallopsled/pwntools/pull/2387
+[2388]: https://github.com/Gallopsled/pwntools/pull/2388
+[2398]: https://github.com/Gallopsled/pwntools/pull/2398
+[2415]: https://github.com/Gallopsled/pwntools/pull/2415
+[2405]: https://github.com/Gallopsled/pwntools/pull/2405
+[2427]: https://github.com/Gallopsled/pwntools/pull/2405
+[2382]: https://github.com/Gallopsled/pwntools/pull/2382
+[2435]: https://github.com/Gallopsled/pwntools/pull/2435
+[2437]: https://github.com/Gallopsled/pwntools/pull/2437
+[2497]: https://github.com/Gallopsled/pwntools/pull/2497
+
+## 4.13.1
+
+- [#2445][2445] Fix parsing the PLT on Windows
+- [#2466][2466] Fix PLT emulation with Unicorn 2.1.0
+- [#2466][2466] Switch to PyPi Simple API for update checks
+- [#2467][2467] Fix loading at all on Windows
+- [#2469][2469] GDB 15+ rpyc detection
+
+[2445]: https://github.com/Gallopsled/pwntools/pull/2445
+[2466]: https://github.com/Gallopsled/pwntools/pull/2466
+[2467]: https://github.com/Gallopsled/pwntools/pull/2467
+[2469]: https://github.com/Gallopsled/pwntools/pull/2469
+
+## 4.13.0
+
+- [#2242][2242] Term module revamp: activating special handling of terminal only when necessary
+- [#2277][2277] elf: Resolve more relocations into GOT entries
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2293][2293] Add x86 CET status to checksec output
+- [#1763][1763] Allow to add to the existing environment in `process` instead of replacing it
+- [#2307][2307] Fix `pwn libcdb file` crashing if "/bin/sh" string was not found
+- [#2309][2309] Detect challenge binary and libc in `pwn template`
+- [#2308][2308] Fix WinExec shellcraft to make sure it's 16 byte aligned
+- [#2279][2279] Make `pwn template` always set context.binary
+- [#2310][2310] Add support to start a process on Windows
+- [#2335][2335] Add lookup optimizations in DynELF
+- [#2334][2334] Speed up disasm commandline tool with colored output
+- [#2328][2328] Lookup using $PATHEXT file extensions in `which` on Windows
+- [#2189][2189] Explicitly define p64/u64 functions for IDE support
+- [#2339][2339] Fix: Allow setting attributes on gdb Breakpoints
+- [#2323][2323] Retry failed lookups after one week in libcdb
+- [#2325][2325] Match against local system libc first in libcdb
+- [#2336][2336] Add `ELF.stripped` and `ELF.debuginfo` properties
+- [#2161][2161] Add basic support for darwin shellcraft/asm/disasm/run_shellcode/run_assembly
+- [#2161][2161] Fix freebsd amd64 SyscallABI
+- [#2160][2161] Fix invalid shellcraft.mov on arm64
+- [#2284][2161] Fix invalid shellcraft.pushstr_array on arm64
+- [#2345][2345] Fix pwn constgrep when it matches a non-constant type
+- [#2338][2338] Fix: follow symlink for libs on ssh connection
+- [#2341][2341] Launch GDB correctly in iTerm on Mac
+- [#2268][2268] Add a `flatten` argument to `ssh.libs`
+- [#2347][2347] Fix/workaround Unicorn Engine 1GB limit that calls exit()
+- [#2233][2233] Fix gdb.debug: exe parameter now respected, allow empty argv
+- [#2373][2373] Fix displaying bright color variation in terminal output
+- [#2378][2378] Don't go though a shell in `gdb.debug`
+
+[2242]: https://github.com/Gallopsled/pwntools/pull/2242
+[2277]: https://github.com/Gallopsled/pwntools/pull/2277
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2293]: https://github.com/Gallopsled/pwntools/pull/2293
+[1763]: https://github.com/Gallopsled/pwntools/pull/1763
+[2307]: https://github.com/Gallopsled/pwntools/pull/2307
+[2309]: https://github.com/Gallopsled/pwntools/pull/2309
+[2308]: https://github.com/Gallopsled/pwntools/pull/2308
+[2279]: https://github.com/Gallopsled/pwntools/pull/2279
+[2310]: https://github.com/Gallopsled/pwntools/pull/2310
+[2335]: https://github.com/Gallopsled/pwntools/pull/2335
+[2334]: https://github.com/Gallopsled/pwntools/pull/2334
+[2328]: https://github.com/Gallopsled/pwntools/pull/2328
+[2189]: https://github.com/Gallopsled/pwntools/pull/2189
+[2339]: https://github.com/Gallopsled/pwntools/pull/2339
+[2323]: https://github.com/Gallopsled/pwntools/pull/2323
+[2325]: https://github.com/Gallopsled/pwntools/pull/2325
+[2336]: https://github.com/Gallopsled/pwntools/pull/2336
+[2161]: https://github.com/Gallopsled/pwntools/pull/2161
+[2345]: https://github.com/Gallopsled/pwntools/pull/2345
+[2338]: https://github.com/Gallopsled/pwntools/pull/2338
+[2341]: https://github.com/Gallopsled/pwntools/pull/2341
+[2268]: https://github.com/Gallopsled/pwntools/pull/2268
+[2347]: https://github.com/Gallopsled/pwntools/pull/2347
+[2233]: https://github.com/Gallopsled/pwntools/pull/2233
+[2373]: https://github.com/Gallopsled/pwntools/pull/2373
+[2378]: https://github.com/Gallopsled/pwntools/pull/2378
+
+## 4.12.0
+
+- [#2202][2202] Fix `remote` and `listen` in sagemath
+- [#2117][2117] Add -p (--prefix) and -s (--separator) arguments to `hex` command
+- [#2221][2221] Add shellcraft.sleep template wrapping SYS_nanosleep
+- [#2219][2219] Fix passing arguments on the stack in shellcraft syscall template
+- [#2212][2212] Add `--libc libc.so` argument to `pwn template` command
+- [#2257][2257] Allow creation of custom templates for `pwn template` command
+- [#2225][2225] Allow empty argv in ssh.process()
+- [#2349][2349] Fix term.readline omitting a trailing \n
+- [#2352][2352] add `RETURN_CONST` as an allowed `_const_code` in safeeval
+
+[2202]: https://github.com/Gallopsled/pwntools/pull/2202
+[2117]: https://github.com/Gallopsled/pwntools/pull/2117
+[2221]: https://github.com/Gallopsled/pwntools/pull/2221
+[2219]: https://github.com/Gallopsled/pwntools/pull/2219
+[2212]: https://github.com/Gallopsled/pwntools/pull/2212
+[2257]: https://github.com/Gallopsled/pwntools/pull/2257
+[2225]: https://github.com/Gallopsled/pwntools/pull/2225
+[2349]: https://github.com/Gallopsled/pwntools/pull/2349
+[2352]: https://github.com/Gallopsled/pwntools/pull/2352
+
+## 4.11.1
+
+- [#2271][2271] FIX: Generated shebang with path to python invalid if path contains spaces
+- [#2272][2272] Fix `tube.clean_and_log` not logging buffered data
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2287][2287] Fix `_countdown_handler` not invoking `timeout_change`
+- [#2294][2294] Fix atexit SEGV in aarch64 loader
+
+[2271]: https://github.com/Gallopsled/pwntools/pull/2271
+[2272]: https://github.com/Gallopsled/pwntools/pull/2272
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2287]: https://github.com/Gallopsled/pwntools/pull/2287
+[2294]: https://github.com/Gallopsled/pwntools/pull/2294
+
+## 4.11.0
+
+- [#2185][2185] make fmtstr module able to create payload without $ notation
+- [#2103][2103] Add search for libc binary by leaked function addresses `libcdb.search_by_symbol_offsets()`
+- [#2177][2177] Support for RISC-V 64-bit architecture
+- [#2186][2186] Enhance `ELF.nx` and `ELF.execstack`
+- [#2129][2129] Handle `context.newline` correctly when typing in `tube.interactive()`
+- [#2214][2214] Fix bug at ssh.py:`download` and `download_file` with relative paths
+- [#2241][2241] Fix ssh.process not setting ssh_process.cwd attribute
+- [#2261][2261] Fix corefile module after pyelftools update
+
+[2185]: https://github.com/Gallopsled/pwntools/pull/2185
+[2103]: https://github.com/Gallopsled/pwntools/pull/2103
+[2177]: https://github.com/Gallopsled/pwntools/pull/2177
+[2186]: https://github.com/Gallopsled/pwntools/pull/2186
+[2129]: https://github.com/Gallopsled/pwntools/pull/2129
+[2214]: https://github.com/Gallopsled/pwntools/pull/2214
+[2241]: https://github.com/Gallopsled/pwntools/pull/2241
+[2261]: https://github.com/Gallopsled/pwntools/pull/2261
+
+## 4.10.0
+
+In memoriam — [Zach Riggle][zach] — long time contributor and maintainer of Pwntools.
+
+- [#2062][2062] make pwn cyclic -l work with entry larger than 4 bytes
+- [#2092][2092] shellcraft: dup() is now called dupio() consistently across all supported arches
+- [#2093][2093] setresuid() in shellcraft uses current euid by default
+- [#2125][2125] Allow tube.recvregex to return capture groups
+- [#2144][2144] Removes `p2align 2` `asm()` headers from `x86-32`, `x86-64` and `mips` architectures to avoid inconsistent instruction length when patching binaries
+
+[2062]: https://github.com/Gallopsled/pwntools/pull/2062
+[2092]: https://github.com/Gallopsled/pwntools/pull/2092
+[2093]: https://github.com/Gallopsled/pwntools/pull/2093
+[2125]: https://github.com/Gallopsled/pwntools/pull/2125
+[2144]: https://github.com/Gallopsled/pwntools/pull/2144
+[zach]: https://github.com/zachriggle
+
+## 4.9.0
+
+- [#1975][1975] Add libcdb commandline tool
+- [#1979][1979] Add `js_escape()` and `js_unescape()` to `util.fiddling`
+- [#2011][2011] Fix tube's debug output of same byte compression
+- [#2023][2023] Support KDE Konsole in run_in_new_terminal function
+- [#2027][2027] Fix ELF.libc_start_main_return with glibc 2.34
+- [#2033][2033] Quote file and core path in generated GDB script
+- [#2035][2035] Change Buffer's parent class to object
+- [#2037][2037] Allow SSH tunnel to be treated like a TCP socket (with 'raw=True')
+- [#2123][2123] Fix ROP without a writeable cache directory
+- [#2124][2124] Fix `tube.recvpred()` timeout argument
+
+[1975]: https://github.com/Gallopsled/pwntools/pull/1975
+[1979]: https://github.com/Gallopsled/pwntools/pull/1979
+[2011]: https://github.com/Gallopsled/pwntools/pull/2011
+[2023]: https://github.com/Gallopsled/pwntools/pull/2023
+[2027]: https://github.com/Gallopsled/pwntools/pull/2027
+[2033]: https://github.com/Gallopsled/pwntools/pull/2033
+[2035]: https://github.com/Gallopsled/pwntools/pull/2035
+[2037]: https://github.com/Gallopsled/pwntools/pull/2037
+[2123]: https://github.com/Gallopsled/pwntools/pull/2123
+[2124]: https://github.com/Gallopsled/pwntools/pull/2124
+
+## 4.8.0
+
+- [#1922][1922] Fix logic in `wait_for_debugger`
+- [#1828][1828] libcdb: Load debug info and unstrip libc binary
+- [#1939][1939] Fix error in validating log levels
+- [#1981][1981] Fix `cyclic_find()` to make it work with large int values
+
+[1922]: https://github.com/Gallopsled/pwntools/pull/1922
+[1828]: https://github.com/Gallopsled/pwntools/pull/1828
+[1939]: https://github.com/Gallopsled/pwntools/pull/1939
+[1981]: https://github.com/Gallopsled/pwntools/pull/1981
+
+## 4.7.1
+
+- [#1784][1784] Use temporary cache directory when persistent cache cannot be used
+- [#1973][1973] ELF symbols can be looked up by bytes values
+- several bugfixes ([#2012][2012], [#2031][2031], [#1912][1912], [#1961][1961],
+[#2007][2007], [#2040][2040], [#2051][2051])
+
+[1784]: https://github.com/Gallopsled/pwntools/pull/1784
+[1912]: https://github.com/Gallopsled/pwntools/pull/1912
+[1961]: https://github.com/Gallopsled/pwntools/pull/1961
+[1973]: https://github.com/Gallopsled/pwntools/pull/1973
+[2007]: https://github.com/Gallopsled/pwntools/pull/2007
+[2012]: https://github.com/Gallopsled/pwntools/pull/2012
+[2031]: https://github.com/Gallopsled/pwntools/pull/2031
+[2040]: https://github.com/Gallopsled/pwntools/pull/2040
+[2051]: https://github.com/Gallopsled/pwntools/pull/2051
+
+## 4.7.0
 
 - [#1733][1733] Update libc headers -> more syscalls available!
 - [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException
@@ -78,6 +514,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#1904][1904] Add bash completion script
 - [#1906][1906] Defer import of several modules to save on startup time
 - [#1921][1921] Add basic support for the bare-metal ARM specific toolchain
+- [#1995][1995] Add `shellcraft.*.linux.cat2`, which uses alloc+read+write instead of sendfile
 
 [1733]: https://github.com/Gallopsled/pwntools/pull/1733
 [1876]: https://github.com/Gallopsled/pwntools/pull/1876
@@ -90,8 +527,9 @@ The table below shows which release corresponds to each branch, and what date th
 [1904]: https://github.com/Gallopsled/pwntools/pull/1904
 [1906]: https://github.com/Gallopsled/pwntools/pull/1906
 [1921]: https://github.com/Gallopsled/pwntools/pull/1921
+[1995]: https://github.com/Gallopsled/pwntools/pull/1995
 
-## 4.6.0 (`stable`)
+## 4.6.0
 
 - [#1429][1429] Add a mechanism for ret2csu (originally #1138)
 - [#1566][1566] Add `ignore_config` argument to `pwnlib.tubes.ssh` and improve `allow_agent` implementation
@@ -250,7 +688,7 @@ The table below shows which release corresponds to each branch, and what date th
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.7 (`stable`)
+## 4.1.7
 
 - [#1615][1615] Fix aarch64 pushstr and pushstr_array
 

MANIFEST.in

@@ -1,7 +1,9 @@
+graft build
 graft examples
 graft extra
+graft travis
 include *.md *.txt *.sh *.yml MANIFEST.in
 recursive-include docs   *.rst *.png Makefile *.py *.txt
 recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh __doc__ *.mako
 recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
-recursive-exclude *.pyc
+global-exclude *.pyc

README.md

@@ -3,10 +3,10 @@
 
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![GitHub Workflow Status (dev)](https://img.shields.io/github/actions/workflow/status/Gallopsled/pwntools/ci.yml?branch=dev&logo=GitHub)](https://github.com/Gallopsled/pwntools/actions/workflows/ci.yml?query=branch%3Adev)
 [![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Packaging status](https://img.shields.io/repology/repositories/python:pwntools)](https://repology.org/project/python:pwntools/versions)
 [![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
 [![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
@@ -32,13 +32,13 @@ To get you started, we've provided some example solutions for past CTF challenge
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
+Pwntools is best supported on 64-bit Ubuntu LTS releases (18.04, 20.04, 22.04, and 24.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
 
 Python3 is suggested, but Pwntools still works with Python 2.7.  Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
 ```sh
-apt-get update
-apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+sudo apt-get update
+sudo apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
 python3 -m pip install --upgrade pwntools
 ```

SECURITY.md

@@ -0,0 +1,32 @@
+# Security Policy
+
+## Supported Versions
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| latest dev    | :white_check_mark: |
+| latest beta   | :white_check_mark: |
+| latest stable | :white_check_mark: |
+| anything else | :x: |
+
+## Reporting a Vulnerability
+
+The aim of pwntools is exploiting software vulnerabilities, which is an unusual position, but it nevertheless can have its own security issues.
+Especially that an attacker (=re-victim) is usually not prepared to be attacked back (by the re-attacker).
+
+The first question to ask yourself is: is this an actual vulnerability?
+- can it be triggered by a re-attacker (malicious honeypot pretending to be a vulnerable service)?
+- does it impact the attacker (=re-victim)?
+- is it serious?
+  * *availability: medium* means *at least* exhausting RAM or disk space of the attacker (=re-victim)
+  * *confidentiality: medium* means *at least* reading the filesystem of the attacker (=re-victim)
+  * *integrity: medium* means *at least* performing uncontrolled actions or data corruption on behalf of the attacker (=re-victim)
+  * if crucial for some sophisticated exploit chain, it is always serious
+  * `safe_eval` bypasses **are** serious.
+  * an example of what was **kind of** serious: [#1732](https://github.com/Gallopsled/pwntools/pull/1732)
+- can it be fixed without compromising on Pwntools' usability?
+
+If at least one of the answers is no, then this is NOT a vulnerability, so just file a bug report or feature request, without the weird confidential disclosure dance.
+
+Just e-mail the maintainers.  Arusekk is the one that is currently the most excited to fix vulnerabilities.
+Or create a CTF task!  Prove a point the good old hacker way!

docs/Makefile

@@ -6,7 +6,6 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
-DASHBUILD     = ./dashbuild.py
 TAR           = tar
 
 # Internal variables.
@@ -145,8 +144,8 @@ gettext:
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
 dash:
-	$(DASHBUILD) source $(BUILDDIR)/dash
-	$(TAR) --exclude='.DS_Store' -cvzf $(BUILDDIR)/dash/pwntools.tgz -C $(BUILDDIR)/dash pwntools.docset
+	$(SPHINXBUILD) -b html -d $(BUILDDIR)/doctrees -t dash source $(BUILDDIR)/html
+	doc2dash $(BUILDDIR)/html -d $(BUILDDIR)/dash -n pwntools -f -I index.html
 	@echo "Build finished. The Dash docset is in $(BUILDDIR)/dash."
 
 changes:

docs/dashbuild.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-
-# -*- coding: utf-8 -*-
-
-#
-# Helper to build Dash docset from sphinx source files.
-#
-# Dash docsets can be read by various applications:
-#
-# Dash         OS X, iOS         https://kapeli.com
-# Zeal         Linux, Windows    https://zealdocs.org
-# Velocity     Windows           http://velocity.silverlakesoftware.com
-# LovelyDocs   Android           http://lovelydocs.io
-# dasht        POSIX             https://github.com/sunaku/dasht
-# Helm Dash    emacs             https://github.com/areina/helm-dash
-#
-
-import argparse
-import doc2dash.__main__
-import os, os.path
-import sphinx
-import sqlite3
-import sys
-
-sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
-import version
-
-def main(args):
-	"""Generate a Dash docset from Sphinx source files."""
-
-	srcdir = args.srcdir
-	dstdir = args.dstdir
-	name = args.name
-
-	if not os.path.exists(dstdir):
-		os.makedirs(dstdir)
-
-	# Generate HTML without indices.
-	sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
-		"-t", "dash", srcdir, os.path.join(dstdir, "html") ])
-
-	# Convert to docset.
-	try:
-		doc2dash.__main__.main.main( \
-			[ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
-					"-f", "-I", "index.html"], "doc2dash", False)
-	except SystemExit as e:
-		pass
-
-	# Insert a link to the online version.
-	online = args.online
-	if online is not None and online != "":
-		url = online.replace("@VERSION", args.version)
-		with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
-			pl = f_info.read()
-			pl = pl.replace("</dict>", \
-				"\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
-			f_info.seek(0)
-			f_info.write(pl)
-			f_info.truncate()
-
-	# Modify the CSS to hide the menu included in the HTML.
-	with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
-		css = f_css.read()
-		css = css.replace( \
-			'@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
-			'@media screen {.wy-body-for-nav{background:#fcfcfc}' )
-		css = css.replace( \
-			'@media screen and (max-width: 480px)', \
-			'@media screen ')
-		f_css.seek(0)
-		f_css.write(css)
-		f_css.truncate()
-
-	# Modify the index
-	db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
-	try:
-		db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
-			'("1 Contents", "Guide", "index.html"), ' \
-			'("2 About pwntools", "Guide", "about.html"), ' \
-			'("3 Installation", "Guide", "install.html"), ' \
-			'("4 Getting Started", "Guide", "intro.html"), ' \
-			'("5 Globals (pwn)", "Guide", "globals.html"), ' \
-			'("6 Command Line Tools", "Guide", "commandline.html")')
-		db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
-		db_conn.commit()
-	finally:
-		db_conn.close()
-
-	return 0
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--name", help="docset name", default="pwntools")
-parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
-parser.add_argument("--version", help="pwntools version", default=version.__version__)
-parser.add_argument("srcdir", help="Source directory containing .rst files")
-parser.add_argument("dstdir", help="Destination and working directory")
-
-main(parser.parse_args())

docs/requirements.txt

@@ -1,17 +1,23 @@
 capstone
-coveralls
+coverage[toml]
+python-dateutil
 doc2dash
+docutils<0.18; python_version<'3'
+docutils>=0.18; python_version>='3'
+intervaltree
 isort
 mako>=1.0.0
 paramiko>=1.15.2
-pip>=6.0.8
-pyelftools>=0.2.3
+pyelftools>=0.29, <0.30; python_version<'3'
+pyelftools>=0.29; python_version>='3'
 pygments>=2.0
 pypandoc
 pyserial>=2.7
+pysocks
+psutil
 requests>=2.5.1
 ropgadget>=5.3
-sphinx<3.4
+sphinx==1.8.6; python_version<'3'
+sphinx>=8.1.3, <9; python_version>='3'
 sphinx_rtd_theme
-sphinxcontrib-napoleon
-sphinxcontrib-autoprogram<=0.1.5
+sphinxcontrib-autoprogram<=0.1.5

docs/source/conf.py

@@ -42,8 +42,8 @@ extensions = [
     'sphinx.ext.coverage',
     'sphinx.ext.todo',
     'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
     'sphinxcontrib.autoprogram',
-    'sphinxcontrib.napoleon'
 ]
 
 # Disable "info" logging directly to stdout by Sphinx
@@ -71,6 +71,7 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+import six
 import pwnlib.update
 import pwnlib.util.fiddling
 import logging
@@ -97,8 +98,8 @@ pwnlib.context.ContextType.defaults['log_console'] = stdout()
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
 local_doctest = os.environ.get('USER') == 'pwntools'
-branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 skip_android = True
+is_python2 = six.PY2
 '''
 
 autoclass_content = 'both'
@@ -169,7 +170,7 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -265,8 +266,8 @@ latex_documents = [
    u'2016, Gallopsled et al.', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None),
-                       'paramiko': ('https://paramiko-docs.readthedocs.org/en/2.1/', None)}
+intersphinx_mapping = {'python': ('https://docs.python.org/3/', None),
+                       'paramiko': ('https://docs.paramiko.org/en/2.1/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
@@ -359,6 +360,9 @@ def linkcode_resolve(domain, info):
     else:
         filename = info['module'].replace('.', '/') + '.py'
 
+        if isinstance(val, property):
+            val = val.fget
+
         if isinstance(val, (types.ModuleType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType) + six.class_types):
             try:
                 lines, first = inspect.getsourcelines(val)
@@ -377,11 +381,10 @@ if build_dash:
     on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
     if not on_rtd:  # only import and set the theme if we're building docs locally
-        import sphinx_rtd_theme
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-    # otherwise, readthedocs.org uses their theme by default, so no need to specify it
+        import alabaster
+        html_theme = 'alabaster'
+        html_theme_path = [alabaster.get_path()]
+        html_theme_options = { 'nosidebar' : True }
 
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------

docs/source/elf/elf.rst

@@ -3,6 +3,7 @@
    from pwn import *
    from glob import glob
    from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   import shutil
 
 :mod:`pwnlib.elf.elf` --- ELF Files
 ===========================================================

docs/source/index.rst

@@ -13,7 +13,7 @@ readthedocs_. It comes in three primary flavors:
 - Dev_
 
 .. _readthedocs: https://readthedocs.org
-.. _docs.pwntools.com: https://docs.pwntools.com
+.. _docs.pwntools.com: https://docs.pwntools.com/en/latest
 .. _Stable: https://docs.pwntools.com/en/stable
 .. _Beta: https://docs.pwntools.com/en/beta
 .. _Dev: https://docs.pwntools.com/en/dev
@@ -77,12 +77,12 @@ Each of the ``pwntools`` modules is documented here.
    update
    useragents
    util/*
+   windbg
 
 .. toctree::
    :hidden:
 
    testexample
-   rop/call
 
 .. only:: not dash
 

docs/source/install.rst

@@ -15,6 +15,9 @@ following system libraries installed.
 
    install/*
 
+
+Note: For Mac OS X you will need to have cmake ``brew install cmake`` and pkg-config ``brew install pkg-config`` installed.
+
 Released Version
 -----------------
 
@@ -25,8 +28,8 @@ Python3
 
 .. code-block:: bash
 
-    $ apt-get update
-    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ sudo apt-get update
+    $ sudo apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
     $ python3 -m pip install --upgrade pip
     $ python3 -m pip install --upgrade pwntools
 
@@ -40,8 +43,8 @@ Additionally, due to `pip` dropping support for Python2, a specfic version of `p
 
 .. code-block:: bash
 
-    $ apt-get update
-    $ apt-get install python python-pip python-dev git libssl-dev libffi-dev build-essential
+    $ sudo apt-get update
+    $ sudo apt-get install python python-pip python-dev git libssl-dev libffi-dev build-essential
     $ python2 -m pip install --upgrade pip==20.3.4
     $ python2 -m pip install --upgrade pwntools
 
@@ -51,13 +54,11 @@ Command-Line Tools
 
 When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
 
-However, if you run as an unprivileged user, you may see a warning message that looks like this:
+However, if you run as an unprivileged user, you may see a warning message that looks like this::
 
-.. code-block::
-
-      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
-      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
-      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+    WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+    elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+    unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
 
 Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
 

docs/source/install/binutils.rst

@@ -5,7 +5,7 @@ Assembly of foreign architectures (e.g. assembling Sparc shellcode on
 Mac OS X) requires cross-compiled versions of ``binutils`` to be
 installed. We've made this process as smooth as we can.
 
-In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, mips64, vax, etc.).
+In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, aarch64, mips64, vax, etc.).
 
 Building `binutils` from source takes about 60 seconds on a modern 8-core machine.
 
@@ -18,28 +18,35 @@ Ubuntu Xenial (16.04) has official packages for most architectures, and does not
 
 .. code-block:: bash
 
-    $ apt-get install software-properties-common
-    $ apt-add-repository ppa:pwntools/binutils
-    $ apt-get update
+    $ sudo apt-get install software-properties-common
+    $ sudo apt-add-repository ppa:pwntools/binutils
+    $ sudo apt-get update
 
 Then, install the binutils for your architecture.
 
 .. code-block:: bash
 
-    $ apt-get install binutils-$ARCH-linux-gnu
+    $ sudo apt-get install binutils-$ARCH-linux-gnu
 
 Mac OS X
 ^^^^^^^^^^^^^^^^
 
 Mac OS X is just as easy, but requires building binutils from source.
-However, we've made ``homebrew`` recipes to make this a single command.
-After installing `brew <http://brew.sh>`__, grab the appropriate
+However, we've made ``homebrew`` recipes to make this just two commands.
+After installing `brew <https://brew.sh>`__, grab the appropriate
 recipe from our `binutils
 repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
+    $ wget https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
+    $ brew install ./binutils-$ARCH.rb
+
+Windows
+^^^^^^^^^^^^^^^^
+
+Windows support is experimental. You can try installing a prebuilt version of binutils
+for your desired architecture from the `GNU Toolchains <https://gnutoolchains.com/>`__ project.
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^
@@ -51,10 +58,10 @@ OSes, ``binutils`` is simple to build by hand.
 
     #!/usr/bin/env bash
 
-    V=2.25   # Binutils Version
+    V=2.38   # Binutils Version
     ARCH=arm # Target architecture
 
-    cd /tmp
+    cd ${TMPDIR:-/tmp}
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz.sig
 
@@ -70,7 +77,7 @@ OSes, ``binutils`` is simple to build by hand.
     export AS=as
 
     ../binutils-$V/configure \
-        --prefix=/usr/local \
+        --prefix=${PREFIX:-/usr/local} \
         --target=$ARCH-unknown-linux-gnu \
         --disable-static \
         --disable-multilib \

docs/source/install/headers.rst

@@ -10,7 +10,7 @@ Ubuntu
 
 .. code-block:: bash
 
-    $ apt-get install python-dev
+    $ sudo apt-get install python-dev
 
 Mac OS X
 ^^^^^^^^^^^^^^^^

docs/source/intro.rst

@@ -34,6 +34,8 @@ This exposes a standard interface to talk to processes, sockets, serial ports,
 and all manner of things, along with some nifty helpers for common tasks.
 For example, remote connections via :mod:`pwnlib.tubes.remote`.
 
+::
+
     >>> conn = remote('ftp.ubuntu.com',21)
     >>> conn.recvline() # doctest: +ELLIPSIS
     b'220 ...'

docs/source/protocols.rst

@@ -1 +1,10 @@
+.. testsetup:: *
 
+   from pwn import *
+
+
+:mod:`pwnlib.protocols.adb` --- Protocol implementations
+========================================================
+
+.. automodule:: pwnlib.protocols.adb
+   :members:

docs/source/rop/rop.rst

@@ -15,6 +15,7 @@
    from pwnlib.tubes.process import process
    from pwnlib import shellcraft
    from pwnlib.util.misc import which
+   import pwnlib.data
 
    context.clear()
 

docs/source/shellcraft/amd64.rst

@@ -17,3 +17,9 @@
 
 .. automodule:: pwnlib.shellcraft.amd64.linux
    :members:
+
+:mod:`pwnlib.shellcraft.amd64.windows`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.amd64.windows
+   :members:

docs/source/shellcraft/riscv64.rst

@@ -0,0 +1,19 @@
+.. testsetup:: *
+
+   from pwn import *
+   context.clear(arch='riscv64')
+
+:mod:`pwnlib.shellcraft.riscv64` --- Shellcode for RISCV64
+==========================================================
+
+:mod:`pwnlib.shellcraft.riscv64`
+--------------------------------
+
+.. automodule:: pwnlib.shellcraft.riscv64
+   :members:
+
+:mod:`pwnlib.shellcraft.riscv64.linux`
+--------------------------------------
+
+.. automodule:: pwnlib.shellcraft.riscv64.linux
+   :members:

docs/source/windbg.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+    from pwn import *
+
+:mod:`pwnlib.windbg` --- Working with WinDbg
+============================================
+
+.. automodule:: pwnlib.windbg
+   :members:

examples/clean_and_log.py

@@ -11,18 +11,24 @@ Solution:
 """
 
 from pwn import *
+from multiprocessing import Process
 
-os.system('''((
-echo prefix sometext ;
-echo prefix someothertext ;
-echo here comes the flag ;
-echo LostInTheInterTubes
-) | nc -l 1337) &
-''')
+def submit_data():
+    with context.quiet:
+        with listen(1337) as io:
+            io.wait_for_connection()
+            io.sendline(b'prefix sometext')
+            io.sendline(b'prefix someothertext')
+            io.sendline(b'here comes the flag')
+            io.sendline(b'LostInTheInterTubes')
 
-r = remote('localhost', 1337)
-atexit.register(r.clean_and_log)
+if __name__ == '__main__':
+    p = Process(target=submit_data)
+    p.start()
 
-while True:
-    line = r.recvline()
-    print(re.findall(r'^prefix (\S+)$', line)[0])
+    r = remote('localhost', 1337)
+    atexit.register(r.clean_and_log)
+
+    while True:
+        line = r.recvline()
+        print(re.findall(br'^prefix (\S+)$', line)[0])

examples/options.py

@@ -4,5 +4,5 @@ Example showing `pwnlib.ui.options()`
 
 from pwn import *
 
-opts = [string.letters[x] for x in range(10)]
+opts = [string.ascii_letters[x] for x in range(12)]
 print('You choose "%s"' % opts[options('Pick one:', opts)])

extra/docker/Makefile

@@ -1,4 +1,4 @@
-subdirs=$(shell find . -type d -depth 1 | xargs basename)
+subdirs=$(shell find . -mindepth 1 -maxdepth 1 -type d | xargs -n1 basename)
 tags=base stable beta dev
 ROOT=$(shell git rev-parse --show-toplevel)
 CMD ?= zsh
@@ -7,7 +7,7 @@ ifneq ($(HISTFILE),)
 MOUNT_HISTFILE=--mount type=bind,source="$(HISTFILE)",target=/home/pwntools/.history
 endif
 
-all: $(subdirs) $(tags)
+all: $(tags) $(subdirs)
 
 tags: $(tags)
 
@@ -36,4 +36,4 @@ clean:
 
 FORCE:
 
-.PHONY: all $(subdirs) $(makefiles) $(tags)
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/base/Dockerfile

@@ -3,45 +3,51 @@
 # Based on Ubuntu
 ############################################################
 
-FROM ubuntu:bionic
+FROM ubuntu:jammy
 MAINTAINER Maintainer Gallopsled et al.
 
-env DEBIAN_FRONTEND=noninteractive
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update \
-    && apt-get install locales \
-    && locale-gen en_US.UTF-8 \
     && apt-get install -y \
+        sudo \
+        locales \
         build-essential \
+        elfutils \
         git \
         libssl-dev \
         libffi-dev \
-        python \
-        python-pip \
-        python-dev \
+        python2.7 \
+        python2.7-dev \
+        python2-pip-whl \
         python3 \
         python3-pip \
         python3-dev \
+        python-is-python3 \
         qemu-user-static \
         binutils-arm-linux-gnueabihf \
         binutils-aarch64-linux-gnu \
         binutils-mips-linux-gnu \
         binutils-mipsel-linux-gnu \
+        binutils-msp430 \
         binutils-powerpc-linux-gnu \
         binutils-powerpc64-linux-gnu \
+        binutils-s390x-linux-gnu \
         binutils-sparc64-linux-gnu \
+        binutils-riscv64-linux-gnu \
         tmux \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && pip3 install --upgrade pip \
-    && python3 -m pip install --upgrade pwntools \
+        patchelf \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --no-cache-dir --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --no-cache-dir --upgrade pwntools \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \
     && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
-WORKDIR /home/pwntools

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/buster/Dockerfile

@@ -5,4 +5,4 @@ RUN apt-get -y dist-upgrade
 RUN apt-get -y install python3 python3-pip
 RUN apt-get -y install git wget unzip
 
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@dev

extra/docker/dev/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -5,8 +5,8 @@ ENV HISTFILE=/home/pwntools/.history
 
 # Uninstall existing versions of pwntools
 USER root
-RUN python -m pip uninstall -q -y pwntools \
- && python3 -m pip uninstall -q -y pwntools
+RUN python2.7 -m pip uninstall -q -y pwntools \
+    && python3 -m pip uninstall -q -y pwntools
 
 # Switch back to the pwntools user from here forward
 USER pwntools
@@ -18,20 +18,23 @@ ENV PATH="/home/pwntools/.local/bin:${PATH}"
 
 # Install Pwntools to the home directory, make it an editable install
 RUN git clone https://github.com/Gallopsled/pwntools \
- && python  -m pip install --upgrade --editable pwntools \
+ && python2.7 -m pip install --upgrade --editable pwntools \
  && python3 -m pip install --upgrade --editable pwntools \
  && PWNLIB_NOTERM=1 pwn version
 
 # Requirements for running the tests
-RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
- && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+RUN python2.7 -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+    && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
 
 # Python niceties for debugging
-RUN python  -m pip install -U ipython ipdb \
- && python3 -m pip install -U ipython ipdb
+RUN python2.7 -m pip install -U ipython ipdb \
+    && python3 -m pip install -U ipython ipdb
 
 # Dependencies from .travis.yml addons -> apt -> packages
-RUN sudo apt-get install -y \
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ="UTC"
+RUN sudo apt-get update && sudo -E apt-get install -y \
+	tzdata \
 	ash \
 	bash \
 	bash-static \
@@ -41,7 +44,7 @@ RUN sudo apt-get install -y \
 	dash \
 	gcc \
 	gcc-multilib \
-	gdb \ 
+	gdb \
 	ksh \
 	lib32stdc++6 \
 	libc6-dev-i386 \
@@ -51,12 +54,9 @@ RUN sudo apt-get install -y \
 	socat \
 	sshpass \
 	vim \
-	zsh
-
+	zsh \
 # Misc useful things when developing
-RUN sudo apt-get install -y  \
 	curl \
-	ipython \
 	ipython3 \
 	lsb-release \
 	ssh \
@@ -64,7 +64,7 @@ RUN sudo apt-get install -y  \
 	wget
 
 # Use zsh by default
-RUN sudo chsh -s /bin/zsh pwntools
+RUN sudo -E chsh -s /bin/zsh pwntools
 
 # Get and install prezto
 RUN git clone --recursive https://github.com/sorin-ionescu/prezto.git .zprezto
@@ -83,4 +83,4 @@ ADD 10-import.py       /home/pwntools/.ipython/profile_default/startup
 ADD ipython_config.py  /home/pwntools/.ipython/profile_default
 
 # Do not require password for sudo
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --no-cache-dir --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --no-cache-dir --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwn/toplevel.py

@@ -16,7 +16,6 @@ import tempfile
 import threading
 import time
 
-import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -85,7 +84,20 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-colored_traceback.add_hook()
+try:
+    import colored_traceback
+except ImportError:
+    pass
+else:
+    try:
+        colored_traceback.add_hook()
+    except Exception:
+        # Exception: curses.error
+        # colored_traceback (curses.setupterm()) fails if TERM is unset.
+        # This is not critical, so we just ignore it.
+        # We cannot import `curses` for `curses.error` because it is not
+        # available on all platforms (e.g. Windows).
+        pass
 
 # Equivalence with the default behavior of "from import *"
 # __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/__init__.py

@@ -36,6 +36,7 @@ __all__ = [
     'util',
     'update',
     'version',
+    'windbg',
 ]
 
 from . import args

pwnlib/abi.py

@@ -48,6 +48,8 @@ class ABI(object):
         (32, 'mips', 'linux'):   linux_mips,
         (32, 'powerpc', 'linux'): linux_ppc,
         (64, 'powerpc', 'linux'): linux_ppc64,
+        (32, 'riscv32', 'linux'): linux_riscv32,
+        (64, 'riscv64', 'linux'): linux_riscv64,
         (32, 'i386', 'freebsd'):  freebsd_i386,
         (64, 'aarch64', 'freebsd'): freebsd_aarch64,
         (64, 'amd64', 'freebsd'): freebsd_amd64,
@@ -58,6 +60,8 @@ class ABI(object):
         (64, 'powerpc', 'freebsd'): freebsd_ppc64,
         (32, 'i386', 'windows'):  windows_i386,
         (64, 'amd64', 'windows'): windows_amd64,
+        (64, 'amd64', 'darwin'): darwin_amd64,
+        (64, 'aarch64', 'darwin'): darwin_aarch64,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -76,6 +80,8 @@ class ABI(object):
         (64, 'aarch64', 'linux'):   linux_aarch64_syscall,
         (32, 'powerpc', 'linux'): linux_ppc_syscall,
         (64, 'powerpc', 'linux'): linux_ppc64_syscall,
+        (32, 'riscv32', 'linux'): linux_riscv32_syscall,
+        (64, 'riscv64', 'linux'): linux_riscv64_syscall,
         (32, 'i386', 'freebsd'):  freebsd_i386_syscall,
         (64, 'amd64', 'freebsd'): freebsd_amd64_syscall,
         (64, 'aarch64', 'freebsd'): freebsd_aarch64_syscall,
@@ -85,6 +91,8 @@ class ABI(object):
         (64, 'aarch64', 'freebsd'):   freebsd_aarch64_syscall,
         (32, 'powerpc', 'freebsd'): freebsd_ppc_syscall,
         (64, 'powerpc', 'freebsd'): freebsd_ppc64_syscall,
+        (64, 'amd64', 'darwin'): darwin_amd64_syscall,
+        (64, 'aarch64', 'darwin'): darwin_aarch64_syscall,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -99,13 +107,18 @@ class ABI(object):
         (32, 'arm', 'linux'):   linux_arm_sigreturn,
         (32, 'thumb', 'linux'):   linux_arm_sigreturn,
         (64, 'aarch64', 'linux'):   linux_aarch64_sigreturn,
+        (32, 'riscv32', 'linux'):   linux_riscv32_sigreturn,
+        (64, 'riscv64', 'linux'):   linux_riscv64_sigreturn,
         (32, 'i386', 'freebsd'):  freebsd_i386_sigreturn,
         (64, 'amd64', 'freebsd'): freebsd_amd64_sigreturn,
         (32, 'arm', 'freebsd'):   freebsd_arm_sigreturn,
         (32, 'thumb', 'freebsd'):   freebsd_arm_sigreturn,
         (64, 'aarch64', 'freebsd'):   freebsd_aarch64_sigreturn,
+        (64, 'amd64', 'darwin'):   darwin_amd64_sigreturn,
+        (64, 'aarch64', 'darwin'):   darwin_aarch64_sigreturn,
         }[(context.bits, context.arch, context.os)]
 
+
 class SyscallABI(ABI):
     """
     The syscall ABI treats the syscall number as the zeroth argument,
@@ -115,6 +128,7 @@ class SyscallABI(ABI):
         super(SyscallABI, self).__init__(*a, **kw)
         self.syscall_register = self.register_arguments[0]
 
+
 class SigreturnABI(SyscallABI):
     """
     The sigreturn ABI is similar to the syscall ABI, except that
@@ -128,10 +142,12 @@ class SigreturnABI(SyscallABI):
 linux_i386   = ABI('esp', [], 4, 0)
 linux_amd64  = ABI('rsp', ['rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
 linux_arm    = ABI('sp', ['r0', 'r1', 'r2', 'r3'], 8, 0)
-linux_aarch64 = ABI('sp', ['x0', 'x1', 'x2', 'x3'], 16, 0)
+linux_aarch64 = ABI('sp', ['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7'], 16, 0)
 linux_mips  = ABI('$sp', ['$a0','$a1','$a2','$a3'], 4, 0)
 linux_ppc = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 4, 0)
 linux_ppc64 = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 8, 0)
+linux_riscv32 = ABI('sp', ['a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7'], 8, 0)
+linux_riscv64 = ABI('sp', ['a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7'], 8, 0)
 
 sysv_i386 = linux_i386
 sysv_amd64 = linux_amd64
@@ -140,24 +156,33 @@ sysv_aarch64 = linux_aarch64
 sysv_mips = linux_mips
 sysv_ppc = linux_ppc
 sysv_ppc64 = linux_ppc64
+sysv_riscv32 = linux_riscv32
+sysv_riscv64 = linux_riscv64
 
+# Docs: https://man7.org/linux/man-pages/man2/syscall.2.html
 linux_i386_syscall = SyscallABI('esp', ['eax', 'ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp'], 4, 0)
 linux_amd64_syscall = SyscallABI('rsp', ['rax', 'rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'],   8, 0)
-linux_arm_syscall   = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
-linux_aarch64_syscall   = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6'], 16, 0)
+linux_arm_syscall = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
+linux_aarch64_syscall = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5'], 16, 0)
 linux_mips_syscall  = SyscallABI('$sp', ['$v0','$a0','$a1','$a2','$a3'], 4, 0)
-linux_ppc_syscall = ABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 4, 0)
-linux_ppc64_syscall = ABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 8, 0)
+linux_ppc_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 4, 0)
+linux_ppc64_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8'], 8, 0)
+linux_riscv32_syscall = SyscallABI('sp', ['a7', 'a0', 'a1', 'a2', 'a3', 'a4', 'a5'], 4, 0)
+linux_riscv64_syscall = SyscallABI('sp', ['a7', 'a0', 'a1', 'a2', 'a3', 'a4', 'a5'], 8, 0)
 
 linux_i386_sigreturn = SigreturnABI('esp', ['eax'], 4, 0)
-linux_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 4, 0)
+linux_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 8, 0)
 linux_arm_sigreturn = SigreturnABI('sp', ['r7'], 4, 0)
 linux_aarch64_sigreturn = SigreturnABI('sp', ['x8'], 16, 0)
+linux_riscv32_sigreturn = SigreturnABI('sp', ['a7'], 4, 0)
+linux_riscv64_sigreturn = SigreturnABI('sp', ['a7'], 8, 0)
 
 sysv_i386_sigreturn = linux_i386_sigreturn
 sysv_amd64_sigreturn = linux_amd64_sigreturn
 sysv_arm_sigreturn = linux_arm_sigreturn
 sysv_aarch64_sigreturn = linux_aarch64_sigreturn
+sysv_riscv32_sigreturn = linux_riscv32_sigreturn
+sysv_riscv64_sigreturn = linux_riscv64_sigreturn
 
 freebsd_i386 = sysv_i386
 freebsd_amd64 = sysv_amd64
@@ -168,7 +193,7 @@ freebsd_ppc = sysv_ppc
 freebsd_ppc64 = sysv_ppc64
 
 freebsd_i386_syscall   = SyscallABI('esp', ['eax'], 4, 0)
-freebsd_amd64_syscall  = SyscallABI('rsp', ['rax','rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
+freebsd_amd64_syscall  = SyscallABI('rsp', ['rax','rdi','rsi','rdx','r10','r8','r9'], 8, 0)
 freebsd_arm_syscall    = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3'], 8, 0)
 freebsd_aarch64_syscall = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3'], 16, 0)
 freebsd_mips_syscall  = SyscallABI('$sp', ['$v0','$a0','$a1','$a2','$a3'], 4, 0)
@@ -182,3 +207,11 @@ freebsd_aarch64_sigreturn = sysv_aarch64_sigreturn
 
 windows_i386  = ABI('esp', [], 4, 0)
 windows_amd64 = ABI('rsp', ['rcx','rdx','r8','r9'], 32, 32)
+
+darwin_aarch64 = sysv_aarch64
+darwin_aarch64_syscall = SyscallABI('sp', ['x16', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5'], 16, 0)
+darwin_aarch64_sigreturn = SigreturnABI('sp', ['x16'], 16, 0)
+
+darwin_amd64 = sysv_amd64
+darwin_amd64_syscall = SyscallABI('rsp', ['rax', 'rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'], 8, 0)
+darwin_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 8, 0)

pwnlib/adb/adb.py

@@ -66,6 +66,7 @@ from pwnlib import tubes
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.device import Device
+from pwnlib.exception import PwnlibException
 from pwnlib.log import getLogger
 from pwnlib.protocols.adb import AdbClient
 from pwnlib.util.packing import _decode
@@ -122,7 +123,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device  # doctest: +ELLIPSIS
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_armv7', model='sdk ...phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone..._...', model='...', device='...')
         >>> device.port
         'emulator'
     """
@@ -252,13 +253,13 @@ class AdbDevice(Device):
 
         >>> device = adb.wait_for_device()
         >>> device.arch
-        'arm'
+        'amd64'
         >>> device.bits
-        32
+        64
         >>> device.os
         'android'
         >>> device.product  # doctest: +ELLIPSIS
-        'sdk_...phone_armv7'
+        'sdk_...phone..._...'
         >>> device.serial
         'emulator-5554'
     """
@@ -384,6 +385,9 @@ class AdbDevice(Device):
             >>> adb.getprop(property) == device.getprop(property)
             True
         """
+        if name.startswith('_'):
+            raise AttributeError(name)
+
         with context.local(device=self):
             g = globals()
 
@@ -876,7 +880,7 @@ def which(name, all = False, *a, **kw):
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
-        ['/system/bin/sh']
+        ['/system/bin/sh', '/vendor/bin/sh']
 
         >>> adb.which('foobar') is None
         True
@@ -984,7 +988,7 @@ def proc_exe(pid):
        :skipif: skip_android
 
         >>> adb.proc_exe(1)
-        b'/init'
+        b'/system/bin/init'
     """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
@@ -1271,8 +1275,28 @@ properties = Property()
 
 def _build_date():
     """Returns the build date in the form YYYY-MM-DD as a string"""
+    import datetime
+
+    # Use ro.build.date.utc (integer epoch seconds) which is set by the
+    # AOSP build system and available on all standard Android devices.
+    # This avoids ro.build.date which is locale-dependent and can contain
+    # non-ASCII characters that dateutil cannot parse.  See #2513.
+    utc = getprop('ro.build.date.utc')
+    if utc and utc.strip().isdigit():
+        try:
+            as_datetime = datetime.datetime.fromtimestamp(int(utc.strip()), dateutil.tz.UTC)
+            return as_datetime.strftime('%Y-%b-%d')
+        except (OSError, OverflowError, ValueError):
+            pass
+
+    # Fallback for non-standard builds missing ro.build.date.utc.
     as_string = getprop('ro.build.date')
-    as_datetime =  dateutil.parser.parse(as_string)
+    if not as_string:
+        return ''
+    try:
+        as_datetime = dateutil.parser.parse(as_string)
+    except (ValueError, OverflowError):
+        return as_string
     return as_datetime.strftime('%Y-%b-%d')
 
 def find_ndk_project_root(source):
@@ -1361,14 +1385,18 @@ def compile(source):
         >>> filename = adb.compile(temp)
         >>> sent = adb.push(filename, "/data/local/tmp")
         >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
-        b'... /system/bin/linker\n...'
+        b'... /system/lib64/libc++.so\n...'
     """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
         # Ensure that we can find the NDK.
-        ndk = os.environ.get('NDK', None)
-        if ndk is None:
+        for envvar in ('NDK', 'ANDROID_NDK', 'ANDROID_NDK_ROOT',
+                       'ANDROID_NDK_HOME', 'ANDROID_NDK_LATEST_HOME'):
+            ndk = os.environ.get(envvar)
+            if ndk is not None:
+                break
+        else:
             log.error('$NDK must be set to the Android NDK directory')
         ndk_build = os.path.join(ndk, 'ndk-build')
 
@@ -1483,8 +1511,9 @@ class Partitions(object):
     @context.quietfunc
     def by_name_dir(self):
         try:
-            return next(find('/dev/block/platform','by-name'))
-        except StopIteration:
+            with context.local(log_level=logging.FATAL):
+                return next(find('/dev/block/platform','by-name'))
+        except (StopIteration, PwnlibException):
             return '/dev/block'
 
     @context.quietfunc
@@ -1513,7 +1542,7 @@ class Partitions(object):
         return iter(names)
 
     def __getattr__(self, attr):
-        if attr.startswith("_"):
+        if attr.startswith('_'):
             raise AttributeError(attr)
 
         for name in self:
@@ -1584,7 +1613,10 @@ def uninstall(package, *arguments):
 @context.quietfunc
 def packages():
     """Returns a list of packages installed on the system"""
-    packages = process(['pm', 'list', 'packages']).recvall()
+    # Decodes the received bytes as UTF-8 per:
+    # https://developer.android.com/reference/java/nio/charset/Charset#defaultCharset()
+    # where it is specified that UTF-8 is the default charset for Android.
+    packages = process(['pm', 'list', 'packages']).recvall().decode('utf-8')
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
 
 @context.quietfunc

pwnlib/adb/bootimg.py

@@ -46,7 +46,7 @@ class BootImage(object):
         self.kernel = self.data[PAGE:PAGE+self.kernel_size]
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)

pwnlib/adb/bootloader.py

@@ -114,10 +114,11 @@ class BootloaderImage(object):
         return '\n'.join(rv)
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)
+
 
 if __name__ == '__main__':
     # Easy sanity checking

pwnlib/args.py

@@ -62,6 +62,8 @@ from pwnlib.context import context
 
 class PwnlibArgs(collections.defaultdict):
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         return self[attr]
 
 args = PwnlibArgs(str)
@@ -157,6 +159,11 @@ def STDERR(v):
     """Sends logging to ``stderr`` by default, instead of ``stdout``"""
     context.log_console = sys.stderr
 
+def LOCAL_LIBCDB(v):
+    """Sets path to local libc-database via ``context.local_libcdb``, e.g. 
+    ``LOCAL_LIBCDB='/path/to/libc-databse'``"""
+    context.local_libcdb = v
+
 hooks = {
     'LOG_LEVEL': LOG_LEVEL,
     'LOG_FILE': LOG_FILE,
@@ -168,6 +175,7 @@ hooks = {
     'NOASLR': NOASLR,
     'NOPTRACE': NOPTRACE,
     'STDERR': STDERR,
+    'LOCAL_LIBCDB': LOCAL_LIBCDB,
 }
 
 def initialize():

pwnlib/asm.py

@@ -59,6 +59,9 @@ from pwnlib import shellcraft
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
+from pwnlib.util.hashes import sha1sumhex
+from pwnlib.util.packing import _encode
+from pwnlib.version import __version__
 
 log = getLogger(__name__)
 
@@ -136,8 +139,23 @@ Try installing binutils for this architecture:
 %(instructions)s
 """.strip() % locals())
 
+
+def check_binutils_version(util):
+    if util_versions[util]:
+        return util_versions[util]
+    result = subprocess.check_output([util, '--version','/dev/null'],
+                                     stderr=subprocess.STDOUT, universal_newlines=True)
+    if 'clang' in result:
+        log.warn_once('Your binutils is clang-based and may not work!\n'
+            'Try installing with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', result.strip())
+    version = re.search(r' (\d+\.\d+)', result).group(1)
+    util_versions[util] = version = tuple(map(int, version.split('.')))
+    return version
+
+
 @LocalContext
-def which_binutils(util):
+def which_binutils(util, check_version=False):
     """
     Finds a binutils in the PATH somewhere.
     Expects that the utility is prefixed with the architecture name.
@@ -152,6 +170,8 @@ def which_binutils(util):
         '.../bin/arm-...-as'
         >>> which_binutils('as', arch='powerpc') #doctest: +ELLIPSIS
         '.../bin/powerpc...-as'
+        >>> which_binutils('as', arch='mips', endianness='little') #doctest: +ELLIPSIS
+        '.../bin/mipsel...-as'
         >>> which_binutils('as', arch='msp430') #doctest: +SKIP
         ...
         Traceback (most recent call last):
@@ -160,26 +180,44 @@ def which_binutils(util):
     """
     arch = context.arch
 
+    # Handle endianness-specific naming for mips/mips64
+    arch = {
+        ('mips', 'little'): 'mipsel',
+        ('mips64', 'little'): 'mips64el',
+    }.get((arch, context.endianness), arch)
+
     # Fix up pwntools vs Debian triplet naming, and account
     # for 'thumb' being its own pwntools architecture.
-    arches = [arch] + {
+    aliases = {
         'thumb':  ['arm',    'aarch64'],
-        'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
-        'amd64':  ['x86_64', 'i386'],
-        'mips64': ['mips'],
-        'powerpc64': ['powerpc'],
-        'sparc64': ['sparc'],
+        'amd64':  ['x86_64'],
     }.get(arch, [])
 
+    # Some binutils can support multiple architectures. Try them as fallbacks.
+    fallback_arches = {
+        'i386':   ['x86_64', 'amd64'],
+        'amd64':  ['i386'],
+        'arm':  ['aarch64'],
+        'mips': ['mipsel'],
+        'mipsel': ['mips'],
+        'mips64': ['mips64el', 'mips', 'mipsel'],
+        'mips64el': ['mips64', 'mipsel', 'mips'],
+        'powerpc64': ['powerpc'],
+        'sparc64': ['sparc'],
+        'riscv32': ['riscv64', 'riscv'],
+        'riscv64': ['riscv32', 'riscv'],
+    }.get(arch, [])
+    arches = [arch] + aliases + fallback_arches
+
     # If one of the candidate architectures matches the native
-    # architecture, use that as a last resort.
+    # architecture, use that as a last resort before trying fallbacks.
     machine = platform.machine()
     machine = 'i386' if machine == 'i686' else machine
     try:
         with context.local(arch = machine):
             if context.arch in arches:
-                arches.append(None)
+                arches = [arch] + aliases + [None] + fallback_arches
     except AttributeError:
         log.warn_once("Your local binutils won't be used because architecture %r is not supported." % machine)
 
@@ -189,6 +227,11 @@ def which_binutils(util):
     if platform.system() == 'Darwin':
         utils = ['g'+util, util]
 
+    if platform.system() == 'Windows':
+        utils = [util + '.exe']
+
+    # Try the explicit tools for the target architecture first,
+    # then the native one optionally, then fallbacks.
     for arch in arches:
         for gutil in utils:
             # e.g. objdump
@@ -203,18 +246,24 @@ def which_binutils(util):
                             '%s-%s' % (arch, gutil)]
 
             for pattern in patterns:
-                for dir in environ['PATH'].split(':'):
-                    res = sorted(glob(path.join(dir, pattern)))
-                    if res:
-                        return res[0]
+                for dir in environ['PATH'].split(os.pathsep):
+                    for res in sorted(glob(path.join(dir, pattern))):
+                        if check_version:
+                            ver = check_binutils_version(res)
+                            return res, ver
+                        return res
 
     # No dice!
     print_binutils_instructions(util, context)
 
-checked_assembler_version = defaultdict(lambda: False)
+util_versions = defaultdict(tuple)
 
 def _assembler():
-    gas = which_binutils('as')
+    gas, version = which_binutils('as', check_version=True)
+    if version < (2, 19):
+        log.warn_once('Your binutils version is too old and may not work!\n'
+            'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', version)
 
     E = {
         'big':    '-EB',
@@ -241,30 +290,19 @@ def _assembler():
         'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
 
         # ia64 only accepts -mbe or -mle
-        'ia64':    [gas, '-m%ce' % context.endianness[0]]
+        'ia64':    [gas, '-m%ce' % context.endianness[0]],
+
+        # riscv64-unknown-elf-as supports riscv32 as well as riscv64
+        'riscv32': [gas, '-march=rv32gc', '-mabi=ilp32'],
+        'riscv64': [gas, '-march=rv64gc', '-mabi=lp64'],
     }
 
     assembler = assemblers.get(context.arch, [gas])
 
-    if not checked_assembler_version[gas]:
-        checked_assembler_version[gas] = True
-        result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT, universal_newlines=True)
-        version = re.search(r' (\d+\.\d+)', result).group(1)
-        if 'clang' in result:
-            log.warn_once('Your binutils is clang version and may not work!\n'
-                'Try install with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-        elif version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-
-
     return assembler
 
 def _linker():
-    ld  = [which_binutils('ld')]
+    ld, _ = which_binutils('ld', check_version=True)
     bfd = ['--oformat=' + _bfdname()]
 
     E = {
@@ -276,7 +314,16 @@ def _linker():
         'i386': ['-m', 'elf_i386'],
     }.get(context.arch, [])
 
-    return ld + bfd + [E] + arguments
+    return [ld] + bfd + [E] + arguments
+
+
+def _execstack(linker):
+    ldflags = ['-z', 'execstack']
+    version = util_versions[linker[0]]
+    if version >= (2, 39):
+        return ldflags + ['--no-warn-execstack', '--no-warn-rwx-segments']
+    return ldflags
+
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -305,20 +352,23 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
-                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
-        'i386'  :  ['.intel_syntax noprefix'],
-        'amd64' :  ['.intel_syntax noprefix'],
+        'i386'  :  ['.intel_syntax noprefix', '.p2align 0'],
+        'amd64' :  ['.intel_syntax noprefix', '.p2align 0'],
         'arm'   : ['.syntax unified',
                    '.arch armv7-a',
-                   '.arm'],
+                   '.arm',
+                   '.p2align 2'],
         'thumb' : ['.syntax unified',
                    '.arch armv7-a',
-                   '.thumb'],
+                   '.thumb',
+                   '.p2align 2'
+                   ],
         'mips'  : ['.set mips2',
                    '.set noreorder',
+                   '.p2align 2'
                    ],
     }
 
@@ -344,7 +394,8 @@ def _bfdname():
         'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
-        'riscv'   : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv32' : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv64' : 'elf%d-%sriscv' % (context.bits, E),
         'vax'     : 'elf32-vax',
         's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
@@ -367,6 +418,8 @@ def _bfdarch():
         'powerpc64': 'powerpc',
         'sparc64':   'sparc',
         'thumb':     'arm',
+        'riscv32':   'riscv',
+        'riscv64':   'riscv',
     }
 
     if arch in convert:
@@ -430,18 +483,24 @@ def cpp(shellcode):
         >>> cpp("SYS_setresuid", os = "freebsd")
         '311\n'
     """
+    if platform.system() == 'Windows':
+        cpp = which_binutils('cpp')
+    else:
+        cpp = 'cpp'
+
     code = _include_header() + shellcode
     cmd  = [
-        'cpp',
+        cpp,
+        '-Wno-unused-command-line-argument',
         '-C',
         '-nostdinc',
         '-undef',
         '-P',
         '-I' + _incdir,
-        '/dev/stdin'
     ]
     return _run(cmd, code).strip('\n').rstrip() + '\n'
 
+
 @LocalContext
 def make_elf_from_assembly(assembly,
                            vma=None,
@@ -595,7 +654,7 @@ def make_elf(data,
 
         _run(assembler + ['-o', step2, step1])
 
-        linker_options = ['-z', 'execstack']
+        linker_options = _execstack(linker)
         if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
@@ -624,6 +683,68 @@ def make_elf(data,
 
     return retval
 
+
+@LocalContext
+def make_macho_from_assembly(shellcode):
+    return make_macho(shellcode, is_shellcode=True)
+
+
+@LocalContext
+def make_macho(data, is_shellcode=False):
+    prefix = []
+    if context.arch == 'amd64':
+        prefix = [
+            '.intel_syntax noprefix',
+        ]
+    prefix.extend([
+        '.text',
+        '.global _start',
+        '_start:',
+        '.p2align 2',
+    ])
+    code = ''
+    code += '\n'.join(prefix) + '\n'
+    if is_shellcode:
+        code += cpp(data)
+    else:
+        code += '.string "%s"' % ''.join('\\x%02x' % c for c in bytearray(data))
+
+    log.debug('Assembling\n%s' % code)
+
+    tmpdir = tempfile.mkdtemp(prefix = 'pwn-asm-')
+    step1 = path.join(tmpdir, 'step1')
+    step2 = path.join(tmpdir, 'step2')
+    step3 = path.join(tmpdir, 'step3')
+
+    with open(step1, 'w') as fd:
+        fd.write(code)
+
+    assembler = [
+        '/usr/bin/as',
+    ]
+    asflags = [
+        '-mmacosx-version-min=11.0',
+        '-o', step2, step1,
+    ]
+    _run(assembler + asflags)
+
+    linker = [
+        '/usr/bin/ld',
+    ]
+    ldflags = [
+        '-macos_version_min', '11.0',
+        '-l', 'System',
+        '-e', '_start',
+        '-L', '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib',
+        '-o', step3, step2,
+    ]
+    _run(linker + ldflags)
+
+    os.chmod(step3, 0o755)
+
+    return step3
+
+
 @LocalContext
 def asm(shellcode, vma = 0, extract = True, shared = False):
     r"""asm(code, vma = 0, extract = True, shared = False, ...) -> str
@@ -661,8 +782,23 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'0@*\x00'
         >>> asm("la %r0, 42", arch = 's390', bits=64)
         b'A\x00\x00*'
+
+        The output is cached:
+
+        >>> start = time.time()
+        >>> asm("lea rax, [rip+0]", arch = 'amd64', cache_dir = None) # force uncached time
+        b'H\x8d\x05\x00\x00\x00\x00'
+        >>> uncached_time = time.time() - start
+        >>> asm("lea rax, [rip+0]", arch = 'amd64') # cache it
+        b'H\x8d\x05\x00\x00\x00\x00'
+        >>> start = time.time()
+        >>> asm("lea rax, [rip+0]", arch = 'amd64')
+        b'H\x8d\x05\x00\x00\x00\x00'
+        >>> cached_time = time.time() - start
+        >>> uncached_time > cached_time
+        True
     """
-    result = ''
+    result = b''
 
     assembler = _assembler()
     linker    = _linker()
@@ -673,6 +809,30 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
 
     log.debug('Assembling\n%s' % code)
 
+    cache_file = None
+    if context.cache_dir:
+        cache_dir = os.path.join(context.cache_dir, 'asm-cache')
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir)
+
+        # Include the context in the hash in addition to the shellcode
+        hash_params = '{}_{}_{}_{}'.format(vma, extract, shared, __version__)
+        fingerprint_params = _encode(code) + _encode(hash_params) + _encode(' '.join(assembler)) + _encode(' '.join(linker)) + _encode(' '.join(objcopy))
+        asm_hash = sha1sumhex(fingerprint_params)
+        cache_file = os.path.join(cache_dir, asm_hash)
+        if os.path.exists(cache_file):
+            log.debug('Using cached assembly output from %r', cache_file)
+            if extract:
+                with open(cache_file, 'rb') as f:
+                    return f.read()
+
+            # Create a temporary copy of the cached file to avoid modification.
+            tmpdir = tempfile.mkdtemp(prefix = 'pwn-asm-')
+            atexit.register(shutil.rmtree, tmpdir)
+            step3 = os.path.join(tmpdir, 'step3')
+            shutil.copy(cache_file, step3)
+            return step3
+
     tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
     step1     = path.join(tmpdir, 'step1')
     step2     = path.join(tmpdir, 'step2')
@@ -689,7 +849,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if vma or not extract:
-            ldflags = ['-z', 'execstack', '-o', step3, step2]
+            ldflags = _execstack(linker) + ['-o', step3, step2]
             if vma:
                 ldflags += ['--section-start=.shellcode=%#x' % vma,
                             '--entry=%#x' % vma]
@@ -720,6 +880,8 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if not extract:
+            if cache_file is not None:
+                shutil.copy(step3, cache_file)
             return step3
 
         _run(objcopy + [step3, step4])
@@ -733,6 +895,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
     else:
         atexit.register(lambda: shutil.rmtree(tmpdir))
 
+    if cache_file is not None and result != b'':
+        with open(cache_file, 'wb') as f:
+            f.write(result)
+
     return result
 
 @LocalContext
@@ -765,13 +931,13 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   b8 17 00 00 00          mov    eax, 0x17
         >>> print(disasm(unhex('48c7c017000000'), arch = 'amd64'))
            0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
-        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))
-           0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
+        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))  # doctest: +ELLIPSIS
+           0:   e51f0004        ldr     r0, [pc, #-4]   ...
            4:   00900052        addseq  r0, r0, r2, asr r0
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
-           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           0:   gs data16 fs rex nop WORD PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
         >>> print(disasm(unhex('01000000'), arch='sparc64'))
            0:   01 00 00 00     nop
@@ -779,6 +945,11 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   60 00 00 00     nop
         >>> print(disasm(unhex('00000000'), arch='mips64'))
            0:   00000000        nop
+        >>> print(disasm(unhex('48b84141414141414100c3'), arch='amd64'))
+           0:   48 b8 41 41 41 41 41 41 41 00   movabs rax, 0x41414141414141
+           a:   c3                      ret
+        >>> print(disasm(unhex('00000000'), vma=0x80000000, arch='mips'))
+        80000000:       00000000        nop
     """
     result = ''
 
@@ -790,7 +961,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     bfdarch = _bfdarch()
     bfdname = _bfdname()
-    objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
+    objdump = _objdump() + ['-w', '-d', '--adjust-vma', str(vma), '-b', bfdname]
     objcopy = _objcopy() + [
         '-I', 'binary',
         '-O', bfdname,
@@ -815,7 +986,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         _run(objcopy + [step1, step2])
 
         output0 = _run(objdump + [step2])
-        output1 = output0.split('<.text>:\n')
+        output1 = re.split(r'<\.text(?:\+0x0)?>:\n', output0, flags=re.MULTILINE)
 
         if len(output1) != 2:
             log.error('Could not find .text in objdump output:\n%s' % output0)
@@ -828,6 +999,8 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
 
     lines = []
+
+    # Note: those patterns are also used in pwnlib/commandline/disasm.py
     pattern = '^( *[0-9a-f]+: *)', '((?:[0-9a-f]+ )+ *)', '(.*)'
     if not byte:
         pattern = pattern[::2]

pwnlib/commandline/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 __all__ = [
     'asm',
     'checksec',

pwnlib/commandline/asm.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -138,4 +137,4 @@ def main(args):
         args.output.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/checksec.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -16,7 +15,6 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     'elf',
     nargs='*',
-    type=argparse.FileType('rb'),
     help='Files to check'
 )
 parser.add_argument(
@@ -24,19 +22,21 @@ parser.add_argument(
     nargs='*',
     dest='elf2',
     metavar='elf',
-    type=argparse.FileType('rb'),
     help='File to check (for compatibility with checksec.sh)'
 )
 
 def main(args):
-    files  = args.elf or args.elf2 or []
+    files = args.elf or args.elf2 or []
 
     if not files:
         parser.print_usage()
         return
 
     for f in files:
-        e = ELF(f.name)
+        try:
+            e = ELF(f)
+        except Exception as e:
+            print("{name}: {error}".format(name=f, error=e))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/common.py

@@ -25,8 +25,25 @@ parser = argparse.ArgumentParser(description='Pwntools Command-line Interface',
                                  prog='pwn')
 parser_commands = parser.add_subparsers(dest='command')
 
-def main(file=sys.argv[0]):
-    import pwnlib.commandline.main
+def main(file=sys.argv[0], command_main=None):
     name = os.path.splitext(os.path.basename(file))[0]
+    if command_main is None:
+        import importlib
+        command_main = importlib.import_module('pwnlib.commandline.%s' % name).main
     sys.argv.insert(1, name)
-    pwnlib.commandline.main.main()
+    entrypoint({name: command_main})
+
+def deprecated_main():
+    file=sys.argv[0]
+    name = os.path.splitext(os.path.basename(file))[0]
+    import warnings
+    warnings.warn("The '%s' command is deprecated and will be removed in a future version. Please use 'pwn %s' instead." % (name, name), DeprecationWarning, stacklevel=2)
+    main(file)
+
+def entrypoint(commands):
+    if len(sys.argv) < 2:
+        parser.print_usage()
+        sys.exit()
+    args = parser.parse_args()
+    with context.local(log_console = sys.stderr):
+        commands[args.command](args)

pwnlib/commandline/constgrep.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -92,9 +91,13 @@ def main(args):
             if not matcher.search(k):
                 continue
 
+            # Check if the value has proper type
+            val = getattr(mod, k)
+            if not isinstance(val, pwnlib.constants.constant.Constant):
+                continue
+
             # Check the constant
             if constant is not None:
-                val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
                         continue
@@ -103,7 +106,7 @@ def main(args):
                         continue
 
             # Append it
-            out.append((getattr(mod, k), k))
+            out.append((val, k))
             maxlen = max(len(k), maxlen)
 
         # Output all matching constants
@@ -130,4 +133,4 @@ def main(args):
                 print('(%s) == %s' % (' | '.join(k for v, k in good), args.constant))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/cyclic.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -72,13 +71,16 @@ def main(args):
 
         try:
             pat = int(pat, 0)
+            pat = pack(pat, 'all')
         except ValueError:
             pass
         pat = flat(pat, bytes=args.length)
 
-        if len(pat) != subsize:
-            log.critical('Subpattern must be %d bytes' % subsize)
+        if len(pat) < subsize:
+            log.critical('Subpattern must be at least %d bytes' % subsize)
             sys.exit(1)
+        else:
+            pat = pat[:subsize]
 
         if not all(c in alphabet for c in pat):
             log.critical('Pattern contains characters not present in the alphabet')
@@ -105,4 +107,4 @@ def main(args):
             out.write(b'\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/debug.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -103,4 +102,4 @@ def main(args):
         gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive()
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disablenx.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -25,4 +24,4 @@ def main(args):
         ELF(e.path)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/disasm.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import argparse
+import re
 import string
 import sys
 
@@ -77,21 +77,37 @@ def main(args):
         from pygments.formatters import TerminalFormatter
         from pwnlib.lexer import PwntoolsLexer
 
-        offsets = disasm(dat, vma=safeeval.const(args.address), instructions=False, byte=False)
-        bytes   = disasm(dat, vma=safeeval.const(args.address), instructions=False, offset=False)
-        instrs  = disasm(dat, vma=safeeval.const(args.address), byte=False, offset=False)
-        # instrs  = highlight(instrs, PwntoolsLexer(), TerminalFormatter())
+        dis = disasm(dat, vma=safeeval.const(args.address))
 
-        for o,b,i in zip(*map(str.splitlines, (offsets, bytes, instrs))):
-            b = b.replace('00', text.red('00'))
-            b = b.replace('0a', text.red('0a'))
-            i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
-            i = i.replace(',',', ')
+        # Note: those patterns are copied from disasm function
+        pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
+        lines = []
+        for line in dis.splitlines():
+            match = re.search(pattern, line)
+            if not match:
+                # Append as one element tuple
+                lines.append((line,))
+                continue
 
-            print(o,b,i)
+            groups = match.groups()
+            o, b, i = groups
+
+            lines.append((o, b, i))
+
+
+        highlight_bytes = lambda t: ''.join(map(lambda x: x.replace('00', text.red('00')).replace('0a', text.red('0a')), group(2, t)))
+        for line in lines:
+            if len(line) == 3:
+                o, b, i = line
+                b = ' '.join(highlight_bytes(bb) for bb in b.split(' '))
+                i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
+                i = i.replace(',',', ')
+                print(o,b,i)
+            else:
+                print(line[0])
         return
 
     print(disasm(dat, vma=safeeval.const(args.address)))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfdiff.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -60,4 +59,4 @@ def main(a):
     print(diff(x, y))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/elfpatch.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -35,4 +34,4 @@ def main(a):
     getattr(sys.stdout, 'buffer', sys.stdout).write(elf.get_data())
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/errno.py

@@ -46,4 +46,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/hex.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -7,6 +6,7 @@ import sys
 
 from pwnlib.commandline import common
 from pwnlib.util.fiddling import enhex
+from pwnlib.util.lists import group
 
 parser = common.parser_commands.add_parser(
     'hex',
@@ -16,14 +16,38 @@ parser = common.parser_commands.add_parser(
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
+parser.add_argument(
+    '-p', '--prefix',
+    metavar = 'prefix',
+    type = str,
+    default = '',
+    help = 'Insert a prefix before each byte',
+)
+
+parser.add_argument(
+    '-s', '--separator',
+    metavar = 'separator',
+    type = str,
+    default = '',
+    help = 'Add a separator between each byte',
+)
+
+def format_hex(hex_string, prefix, separator):
+    return separator.join([prefix + x for x in group(2, hex_string)])
+
 def main(args):
     if not args.data:
-        print(enhex(getattr(sys.stdin, 'buffer', sys.stdin).read()))
+        encoded = enhex(getattr(sys.stdin, 'buffer', sys.stdin).read())
     else:
         data = ' '.join(args.data)
         if not hasattr(data, 'decode'):
             data = data.encode('utf-8', 'surrogateescape')
-        print(enhex(data))
+        encoded = enhex(data)
+
+    if args.prefix or args.separator:
+        encoded = format_hex(encoded, args.prefix, args.separator)
+
+    print(encoded)
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/libcdb.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import shutil
+import sys
+
+import pwnlib.args
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'libcdb',
+    help = 'Print various information about a libc binary',
+    description = 'Print various information about a libc binary'
+)
+
+libc_commands = parser.add_subparsers(
+    dest = 'libc_command'
+)
+
+lookup_parser = libc_commands.add_parser(
+    'lookup',
+    help = 'Lookup a libc version by function offsets',
+    description = 'Lookup a libc version by function offsets'
+)
+
+lookup_parser.add_argument(
+    'symbol_offset_pairs',
+    metavar = 'symbol_offset_pairs',
+    nargs = '+',
+    help = 'Symbol and offset pairs to lookup matching libc version. Can be any number of pairs to narrow the search. Example: "read 3e0 write 520"'
+)
+
+lookup_parser.add_argument(
+    '-d', '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+lookup_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+lookup_parser.add_argument(
+    '--offline-only',
+    action = 'store_true',
+    default = False,
+    dest = 'offline_only',
+    help = 'Attempt to searching with offline only mode'
+)
+
+hash_parser = libc_commands.add_parser(
+    'hash',
+    help = 'Display information of a libc version given an unique hash',
+    description = 'Display information of a libc version given an unique hash'
+)
+
+hash_parser.add_argument(
+    'hash_value',
+    metavar = 'hash_value',
+    nargs = '+',
+    help = 'Hex encoded hash value'
+)
+
+hash_parser.add_argument(
+    '-t', '--hash_type',
+    nargs = '?',
+    type = str,
+    choices = ['id', 'buildid', 'md5', 'sha1', 'sha256'],
+    default = 'buildid',
+    help = 'The type of the provided hash value. Supported hashtypes: id, buildid, md5, sha1, sha256'
+)
+
+hash_parser.add_argument(
+    '-d', '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+hash_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+hash_parser.add_argument(
+    '--offline-only',
+    action = 'store_true',
+    default = False,
+    dest = 'offline_only',
+    help = 'Attempt to searching with offline only mode'
+)
+
+file_parser = libc_commands.add_parser(
+    'file',
+    help = 'Dump information about a libc binary',
+    description = 'Dump information about a libc binary'
+)
+
+file_parser.add_argument(
+    'files',
+    metavar = 'files',
+    nargs = '+',
+    help = 'Libc binary to dump'
+)
+
+file_parser.add_argument(
+    '-s', '--symbols',
+    metavar = 'symbols',
+    nargs = '*',
+    help = 'List of symbol offsets to dump in addition to the common ones'
+)
+
+file_parser.add_argument(
+    '-o', '--offset',
+    metavar = 'offset',
+    type = str,
+    help = 'Display all offsets relative to this symbol'
+)
+
+file_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    dest = 'unstrip',
+    help = 'Attempt to unstrip the libc binary inplace with debug symbols from a debuginfod server'
+)
+
+fetch_parser = libc_commands.add_parser(
+    'fetch',
+    help = 'Fetch libc database',
+    description = 'Fetch libc database. If no argument passed, it will init and upgrade libc-database repository',
+)
+
+fetch_parser.add_argument(
+    'path',
+    nargs = '?',
+    default = context.local_libcdb,
+    help = 'Set libc-database path, If it is empty, the default path will be `context.local_libcdb` (%s)' % context.local_libcdb
+)
+
+fetch_parser.add_argument(
+    '-u', '--update',
+    metavar = 'update',
+    nargs = '+',
+    choices = ['all', 'ubuntu', 'debian', 'rpm', 'centos', 'arch', 'alpine', 'kali', 'parrotsec', 'launchpad'],
+    help = 'Fetch the desired libc categories'
+)
+
+common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
+
+def print_libc_info(libc):
+    log.info('%s', text.red(libc['id']))
+    log.indented('\t%-20s %s', text.green('BuildID:'), libc['buildid'])
+    log.indented('\t%-20s %s', text.green('MD5:'), libc['md5'])
+    log.indented('\t%-20s %s', text.green('SHA1:'), libc['sha1'])
+    log.indented('\t%-20s %s', text.green('SHA256:'), libc['sha256'])
+    log.indented('\t%s', text.green('Symbols:'))
+    for symbol in libc['symbols'].items():
+        log.indented('\t%25s = %s', symbol[0], symbol[1])
+
+def print_libc_elf(exe):
+    from hashlib import md5, sha1, sha256
+
+    log.info('%s', text.red(os.path.basename(exe.path)))
+
+    libc_version = get_libc_version(exe)
+    if libc_version:
+        log.indented('%-20s %s', text.green('Version:'), libc_version)
+
+    if exe.buildid:
+        log.indented('%-20s %s', text.green('BuildID:'), enhex(exe.buildid))
+
+    log.indented('%-20s %s', text.green('MD5:'), md5(exe.data).hexdigest())
+    log.indented('%-20s %s', text.green('SHA1:'), sha1(exe.data).hexdigest())
+    log.indented('%-20s %s', text.green('SHA256:'), sha256(exe.data).hexdigest())
+
+    # Always dump the basic list of common symbols
+    log.indented('%s', text.green('Symbols:'))
+    synthetic_symbols = collect_synthetic_symbols(exe)
+
+    symbols = common_symbols + (args.symbols or []) + synthetic_symbols
+    symbols.sort()
+    for symbol in symbols:
+        if symbol not in exe.symbols:
+            log.indented('%25s = %s', symbol, text.red('not found'))
+        else:
+            log.indented('%25s = %#x', symbol, translate_offset(exe.symbols[symbol], args, exe))
+
+def get_libc_version(exe):
+    res = re.search(br'libc[ -](\d+\.\d+)', exe.data)
+    if res:
+        return res.group(1).decode()
+    return None
+
+def translate_offset(offs, args, exe):
+    if args.offset:
+        if args.offset not in exe.symbols:
+            log.info_once('offset symbol %s not found. ignoring.', args.offset)
+            return offs
+        return offs - exe.symbols[args.offset]
+    return offs
+
+def collect_synthetic_symbols(exe):
+    available_symbols = []
+    try:
+        exe.symbols['str_bin_sh'] = next(exe.search(b'/bin/sh\x00'))
+        available_symbols.append('str_bin_sh')
+    except StopIteration:
+        pass
+
+    libc_start_main_return = exe.libc_start_main_return
+    if libc_start_main_return > 0:
+        exe.symbols['__libc_start_main_ret'] = libc_start_main_return
+        available_symbols.append('__libc_start_main_ret')
+
+    return available_symbols
+
+def main(args):
+    if len(sys.argv) < 3:
+        parser.print_usage()
+        sys.exit()
+
+    if args.libc_command == 'lookup':
+        pairs = args.symbol_offset_pairs
+        if len(pairs) % 2 != 0:
+            log.failure('Uneven number of arguments. Please provide "symbol offset" pairs')
+            return
+
+        symbols = {pairs[i]:pairs[i+1] for i in range(0, len(pairs), 2)}
+        matched_libcs = libcdb.search_by_symbol_offsets(symbols, offline_only=args.offline_only, return_raw=True)
+
+        if not matched_libcs:
+            return
+
+        for libc in matched_libcs:
+            print_libc_info(libc)
+            if args.download_libc:
+                path = libcdb.search_by_build_id(libc['buildid'], args.unstrip)
+                if path:
+                    shutil.copy(path, './{}.so'.format(libc['id']))
+
+    elif args.libc_command == 'hash':
+        inverted_map = {v: k for k, v in libcdb.MAP_TYPES.items()}
+        hash_type = inverted_map.get(args.hash_type, args.hash_type)
+
+        for hash_value in args.hash_value:
+            path = libcdb.search_by_hash(hash_value, hash_type, unstrip=args.unstrip, offline_only=args.offline_only)
+            exe = ELF(path, checksec=False)
+            print_libc_elf(exe)
+
+            if args.download_libc:
+                # if we cannot get actual libc version then copy with cache name
+                shutil.copy(path, './libc-{}.so'.format(get_libc_version(exe) or Path(path).stem))
+
+    elif args.libc_command == 'file':
+        for file in args.files:
+            if not os.path.exists(file) or not os.path.isfile(file):
+                log.failure('File does not exist %s', args.file)
+                continue
+
+            if args.unstrip:
+                libcdb.unstrip_libc(file)
+
+            print_libc_elf(ELF(file, checksec=False))
+
+    elif args.libc_command == 'fetch':
+
+        if args.update:
+            subprocess.check_call(['./get'] + args.update, cwd=args.path)
+
+        else:
+            if not Path(args.path).exists():
+                if yesno("Would you like to initialize the libc-database repository? "
+                         "If the path already exists, this prompt will not display, and automatically upgrade repository."):
+                    log.waitfor("init libc-database repository")
+                    subprocess.check_call(['git', 'clone', 'https://github.com/niklasb/libc-database/', args.path])
+            else:
+                log.waitfor("upgrade libc-database repository")
+                subprocess.check_call(['git', 'pull'], cwd=args.path)
+
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/main.py

@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-import sys
-
 from pwnlib.commandline import asm
 from pwnlib.commandline import checksec
 from pwnlib.commandline import common
@@ -14,6 +12,7 @@ from pwnlib.commandline import elfdiff
 from pwnlib.commandline import elfpatch
 from pwnlib.commandline import errno
 from pwnlib.commandline import hex
+from pwnlib.commandline import libcdb
 from pwnlib.commandline import phd
 from pwnlib.commandline import pwnstrip
 from pwnlib.commandline import scramble
@@ -22,8 +21,7 @@ from pwnlib.commandline import template
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
 from pwnlib.commandline import version
-from pwnlib.commandline.common import parser
-from pwnlib.context import context
+from pwnlib.commandline.common import parser as parser
 
 commands = {
     'asm': asm.main,
@@ -37,6 +35,7 @@ commands = {
     'elfpatch': elfpatch.main,
     'errno': errno.main,
     'hex': hex.main,
+    'libcdb': libcdb.main,
     'phd': phd.main,
     'pwnstrip': pwnstrip.main,
     'scramble': scramble.main,
@@ -48,12 +47,7 @@ commands = {
 }
 
 def main():
-    if len(sys.argv) < 2:
-        parser.print_usage()
-        sys.exit()
-    args = parser.parse_args()
-    with context.local(log_console = sys.stderr):
-        commands[args.command](args)
+    common.entrypoint(commands)
 
 if __name__ == '__main__':
     main()

pwnlib/commandline/phd.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -110,4 +109,4 @@ def main(args):
         pass
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/pwnstrip.py

@@ -53,4 +53,4 @@ def main(args):
     args.output.write(result)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/scramble.py

@@ -110,4 +110,4 @@ def main(args):
 
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/shellcraft.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -72,17 +71,9 @@ p.add_argument(
 
 p.add_argument(
     'shellcode',
-    nargs = '?',
-    help = 'The shellcode you want',
-    type = str
-)
-
-p.add_argument(
-    'args',
     nargs = '*',
-    metavar = 'arg',
-    default = (),
-    help = 'Argument to the chosen shellcode',
+    help = 'The shellcodes you want.  shellcode [args ...] [+ shellcode [args ...]]',
+    type = str
 )
 
 p.add_argument(
@@ -92,6 +83,12 @@ p.add_argument(
     action='store_true'
 )
 
+p.add_argument(
+    '--delim',
+    help='Set the delimiter between multilple shellcodes',
+    default='+'
+)
+
 p.add_argument(
     '-b',
     '--before',
@@ -173,24 +170,50 @@ p.add_argument(
     help='Generated ELF is a shared library'
 )
 
-def get_template(name):
-    func = shellcraft
-    for attr in name.split('.'):
-        func = getattr(func, attr)
-    return func
+def get_template(shellcodes):
+    funcs = []
+    for shellcode in shellcodes:
+        func = shellcraft
+        cur_name = shellcode[0]
+        args = []
+        if len(shellcode) > 1:
+            args = shellcode[1:]
+        for attr in cur_name.split('.'):
+            func = getattr(func, attr)
+        funcs.append((cur_name, func, args))
+    return funcs
 
 def is_not_a_syscall_template(name):
     template_src = shellcraft._get_source(name)
     return '/syscalls' not in template_src
 
 def main(args):
+    delim = '+'
+    if args.delim:
+        delim = args.delim.strip()
+
+    shellcodes = []
+    if args.shellcode:
+        current = []
+        for s in args.shellcode:
+            if s.strip() == delim:
+                shellcodes.append(current)
+                current = []
+            else:
+                current.append(s)
+        if len(current) > 0:
+            shellcodes.append(current)
+
     if args.list:
         templates = shellcraft.templates
 
         if args.shellcode:
-            templates = filter(lambda a: args.shellcode in a, templates)
+            template_array = []
+            for s in shellcodes:
+                template_array.extend(list(filter(lambda a: s[0] in a, templates)))
+            templates = template_array
         elif not args.syscalls:
-            templates = filter(is_not_a_syscall_template, templates)
+            templates = list(filter(is_not_a_syscall_template, templates))
 
         print('\n'.join(templates))
         exit()
@@ -200,84 +223,92 @@ def main(args):
         exit()
 
     try:
-        func = get_template(args.shellcode)
+        funcs = get_template(shellcodes)
     except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
     if args.show:
-        # remove doctests
-        doc = []
-        in_doctest = False
-        block_indent = None
-        caption = None
-        lines = func.__doc__.splitlines()
-        i = 0
-        while i < len(lines):
-            line = lines[i]
-            if line.lstrip().startswith('>>>'):
-                # this line starts a doctest
-                in_doctest = True
-                block_indent = None
-                if caption:
-                    # delete back up to the caption
-                    doc = doc[:caption - i]
-                    caption = None
-            elif line == '':
-                # skip blank lines
-                pass
-            elif in_doctest:
-                # indentation marks the end of a doctest
-                indent = len(line) - len(line.lstrip())
-                if block_indent is None:
-                    if not line.lstrip().startswith('...'):
-                        block_indent = indent
-                elif indent < block_indent:
-                    in_doctest = False
+        for (name, func, _args) in funcs:
+            # remove doctests
+            doc = []
+            in_doctest = False
+            block_indent = None
+            caption = None
+            lines = func.__doc__.splitlines()
+            i = 0
+            if len(funcs) > 1:
+                print('%s:' % name)
+            while i < len(lines):
+                line = lines[i]
+                if line.lstrip().startswith('>>>'):
+                    # this line starts a doctest
+                    in_doctest = True
                     block_indent = None
-                    # re-evalutate this line
-                    continue
-            elif line.endswith(':'):
-                # save index of caption
-                caption = i
-            else:
-                # this is not blank space and we're not in a doctest, so the
-                # previous caption (if any) was not for a doctest
-                caption = None
+                    if caption:
+                        # delete back up to the caption
+                        doc = doc[:caption - i]
+                        caption = None
+                elif line == '':
+                    # skip blank lines
+                    pass
+                elif in_doctest:
+                    # indentation marks the end of a doctest
+                    indent = len(line) - len(line.lstrip())
+                    if block_indent is None:
+                        if not line.lstrip().startswith('...'):
+                            block_indent = indent
+                    elif indent < block_indent:
+                        in_doctest = False
+                        block_indent = None
+                        # re-evalutate this line
+                        continue
+                elif line.endswith(':'):
+                    # save index of caption
+                    caption = i
+                else:
+                    # this is not blank space and we're not in a doctest, so the
+                    # previous caption (if any) was not for a doctest
+                    caption = None
 
-            if not in_doctest:
-                doc.append(line)
-            i += 1
-        print('\n'.join(doc).rstrip())
+                if not in_doctest:
+                    doc.append(line)
+                i += 1
+            print('\n'.join(doc).rstrip())
+            if len(funcs) > 1:
+                print('')
         exit()
 
-    defargs = len(six.get_function_defaults(func) or ())
-    reqargs = six.get_function_code(func).co_argcount - defargs
-    if len(args.args) < reqargs:
-        if defargs > 0:
-            log.critical('%s takes at least %d arguments' % (args.shellcode, reqargs))
-            sys.exit(1)
-        else:
-            log.critical('%s takes exactly %d arguments' % (args.shellcode, reqargs))
-            sys.exit(1)
+    code_array = []
+    for (name, func, func_args) in funcs:
+        defargs = len(six.get_function_defaults(func) or ())
+        reqargs = six.get_function_code(func).co_argcount - defargs
 
-    # Captain uglyness saves the day!
-    for i, val in enumerate(args.args):
-        try:
-            args.args[i] = util.safeeval.expr(val)
-        except ValueError:
-            pass
+        if len(func_args) < reqargs:
+            if defargs > 0:
+                log.critical('%s takes at least %d arguments' % (name, reqargs))
+                sys.exit(1)
+            else:
+                log.critical('%s takes exactly %d arguments' % (name, reqargs))
+                sys.exit(1)
 
-    # And he strikes again!
-    list(map(common.context_arg, args.shellcode.split('.')))
-    code = func(*args.args)
+        # Captain uglyness saves the day!
+        for i, val in enumerate(func_args):
+            try:
+                func_args[i] = util.safeeval.expr(val)
+            except ValueError:
+                pass
 
+        # And he strikes again!
+        list(map(common.context_arg, name.split('.')))
+        code_array.append(func(*func_args))
+
+    code = "".join(code_array)
 
     if args.before:
         code = shellcraft.trap() + code
     if args.after:
         code = code + shellcraft.trap()
 
-
     if args.format in ['a', 'asm', 'assembly']:
         if args.color:
             from pygments import highlight
@@ -320,7 +351,7 @@ def main(args):
         else:
             args.format = 'raw'
 
-    arch = args.shellcode.split('.')[0]
+    arch = name.split('.')[0]
 
     if args.debug:
         if not args.avoid:
@@ -353,4 +384,4 @@ def main(args):
     args.out.write(code)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/template.py

@@ -1,30 +1,184 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
+from __future__ import print_function
 
 from pwn import *
 from pwnlib.commandline import common
+from pwnlib.util.misc import which, parse_ldd_output, write
 
-from mako.lookup import TemplateLookup
+from sys import stderr
+from mako.lookup import TemplateLookup, Template
 
 parser = common.parser_commands.add_parser(
     'template',
     help = 'Generate an exploit template',
-    description = 'Generate an exploit template'
+    description = 'Generate an exploit template. If no arguments are given, '
+                    'the current directory is searched for an executable binary and ' 
+                    'libc. If only one binary is found, it is assumed to be the '
+                    'challenge binary.'
 )
 
-parser.add_argument('exe', nargs='?', help='Target binary')
+# change path to hardcoded one when building the documentation
+printable_data_path = "pwnlib/data" if 'sphinx' in sys.modules else pwnlib.data.path
+
+parser.add_argument('exe', nargs='?', help='Target binary. If not given, the current directory is searched for an executable binary.')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
 parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
+parser.add_argument('--libc', help='Path to libc binary to use. If not given, the current directory is searched for a libc binary.')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
 parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
+parser.add_argument('--template', help='Path to a custom template. Tries to use \'~/.config/pwntools/templates/pwnup.mako\', if it exists. '
+                                   'Check \'%s\' for the default template shipped with pwntools.' % 
+                                        os.path.join(printable_data_path, "templates", "pwnup.mako"))
+parser.add_argument('--no-auto', help='Do not automatically detect missing binaries', action='store_false', dest='auto')
+
+def get_docker_image_libraries():
+    """Tries to retrieve challenge libraries from a Docker image built from the Dockerfile in the current working directory.
+    
+    The libraries are retrieved by parsing the output of running ldd on /bin/sh.
+    Supports regular Docker images as well as jail images.
+    """
+    with log.progress("Extracting challenge libraries from Docker image") as progress:
+        if not which("docker"):
+            progress.failure("docker command not found")
+            return None, None
+        # maps jail image name to the root directory of the child image
+        jail_image_to_chroot_dir = {
+            "pwn.red/jail": "/srv",
+        }
+        dockerfile = open("Dockerfile", "r").read()
+        jail = None
+        chroot_dir = "/"
+        for jail_image in jail_image_to_chroot_dir:
+            if re.search(r"^FROM %s" % jail_image, dockerfile, re.MULTILINE):
+                jail = jail_image
+                chroot_dir = jail_image_to_chroot_dir[jail_image]
+                break
+        try:
+            progress.status("Building image")
+            image_sha = subprocess.check_output(["docker", "build", "-q", "."], stderr=subprocess.PIPE, shell=False).decode().strip()
+
+            progress.status("Retrieving library paths")
+            ldd_command = ["-c", "chroot %s /bin/sh -c 'ldd /bin/sh'" % chroot_dir]
+            ldd_output = subprocess.check_output([
+                    "docker",
+                    "run",
+                    "--rm",
+                    "--entrypoint",
+                    "/bin/sh",
+                    ] + (["--privileged"] if jail else []) + [
+                        image_sha,
+                    ] + ldd_command,
+                stderr=subprocess.PIPE, 
+                shell=False
+            ).decode()
+            
+            libc, ld = None, None
+            libc_basename, ld_basename = None, None
+            for lib_path in parse_ldd_output(ldd_output):
+                if "libc." in lib_path:
+                    libc = lib_path
+                    libc_basename = os.path.basename(lib_path)
+                if "ld-" in lib_path:
+                    ld = lib_path
+                    ld_basename = os.path.basename(lib_path)
+
+            if not (libc and ld):
+                progress.failure("Could not find libraries")
+                return None, None
+
+            progress.status("Copying libraries to current directory")
+            for filename, basename in zip((libc, ld), (libc_basename, ld_basename)):
+                cat_command = ["-c", "chroot %s /bin/sh -c '/bin/cat %s'" % (chroot_dir, filename)]
+                contents = subprocess.check_output([
+                        "docker",
+                        "run",
+                        "--rm",
+                        "--entrypoint",
+                        "/bin/sh",
+                        ] + (["--privileged"] if jail else []) + [
+                            image_sha
+                        ] + cat_command,
+                    stderr=subprocess.PIPE, 
+                    shell=False
+                )
+                write(basename, contents)
+
+        except subprocess.CalledProcessError as e:
+            print(e.stderr.decode(), file=sys.stderr)
+            progress.failure("docker failed with status: %d" % e.returncode)
+            return None, None
+
+        progress.success("Retrieved libraries from Docker image")
+    return libc_basename, ld_basename
+
+def detect_missing_binaries(args):
+    """Automatically detects challenge binaries and libraries in the current directory.
+    
+    This function scans the current directory for executable files, libc, and ld libraries.
+    If a Dockerfile is present and no libraries are found, it attempts to extract them from
+    the Docker image, but only if the binary is not statically linked.
+    
+    Args:
+        args: Argument namespace containing exe and libc attributes.
+        
+    Returns:
+        tuple: A pair of (executable_path, libc_path) where either may be None if not found.
+    """
+    log.info("Automatically detecting challenge binaries...")
+    # look for challenge binary, libc, and ld in current directory
+    exe, libc, ld = args.exe, args.libc, None
+    has_dockerfile = False
+    other_files = []
+    for filename in os.listdir("."):
+        if not os.path.isfile(filename):
+            continue
+        if not libc and ('libc-' in filename or 'libc.' in filename):
+            libc = filename
+        elif not ld and 'ld-' in filename:
+            ld = filename
+        elif filename == "Dockerfile":
+            has_dockerfile = True
+        else:
+            if os.access(filename, os.X_OK):
+                other_files.append(filename)
+    if not exe:
+        if len(other_files) == 1:
+            exe = other_files[0]
+        elif len(other_files) > 1:
+            log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
+    
+    # Check if the binary is statically linked before trying to extract libraries from Docker
+    is_statically_linked = False
+    if exe:
+        try:
+            binary = ELF(exe, checksec=False)
+            is_statically_linked = binary.statically_linked
+            if is_statically_linked:
+                log.info("Binary is statically linked, no need for external libraries")
+        except Exception as e:
+            log.warning("Could not check if binary is statically linked: %s", e)
+    
+    # Only extract libraries from Docker if the binary is not statically linked
+    if has_dockerfile and exe and not (libc or ld) and not is_statically_linked: 
+        libc, ld = get_docker_image_libraries()
+
+    if exe != args.exe:
+        log.success("Found challenge binary %r", exe)
+    if libc != args.libc:
+        log.success("Found libc binary %r", libc)
+    return exe, libc
 
 def main(args):
+
     lookup = TemplateLookup(
-        directories      = [os.path.join(pwnlib.data.path, 'templates')],
+        directories      = [
+            os.path.expanduser('~/.config/pwntools/templates/'),
+            os.path.join(pwnlib.data.path, 'templates')
+        ],
         module_directory = None
     )
 
@@ -34,25 +188,32 @@ def main(args):
         if not (args.path or args.exe):
             log.error("Must specify --path or a exe")
 
-        s = ssh(args.user, args.host, args.port or 22, args.password or None)
-
-        try:
-            remote_file = args.path or args.exe
-            s.download(remote_file)
-        except Exception:
-            log.warning("Could not download file %r, opening a shell", remote_file)
-            s.interactive()
-            return
+        with ssh(args.user, args.host, args.port or 22, args.password or None) as s:
+            try:
+                remote_file = args.path or args.exe
+                s.download(remote_file)
+            except Exception:
+                log.warning("Could not download file %r, opening a shell", remote_file)
+                s.interactive()
+                return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)
 
-    template = lookup.get_template('pwnup.mako')
+    if args.auto and (args.exe is None or args.libc is None):
+        args.exe, args.libc = detect_missing_binaries(args)
+    
+    if args.template:
+        template = Template(filename=args.template) # Failing on invalid file is ok
+    else:
+        template = lookup.get_template('pwnup.mako')
+    
     output = template.render(args.exe,
                              args.host,
                              args.port,
                              args.user,
                              args.password,
+                             args.libc,
                              args.path,
                              args.quiet)
 
@@ -74,4 +235,5 @@ def main(args):
         except OSError: pass
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)
+    

pwnlib/commandline/unhex.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -28,6 +27,7 @@ def main(args):
             o.write(unhex(''.join(args.hex)))
     except TypeError as e:
         sys.stderr.write(str(e) + '\n')
+        raise
 
 if __name__ == '__main__':
-    common.main(__file__)
+    common.main(__file__, main)

pwnlib/commandline/update.py

@@ -30,4 +30,4 @@ def main(a):
         subprocess.check_call(result, shell=False)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/commandline/version.py

@@ -29,4 +29,4 @@ def main(a):
 	log.info("Pwntools v%s" % version)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    pwnlib.commandline.common.main(__file__, main)

pwnlib/config.py

@@ -55,8 +55,8 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
-    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
-                       os.path.expanduser("~/.config"))
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME',
+                                     os.path.expanduser("~/.config"))
 
     c = configparser.ConfigParser()
     c.read(['/etc/pwn.conf',
@@ -65,7 +65,7 @@ def initialize():
 
     for section in c.sections():
         if section not in registered_configs:
-            log.warn("Unknown configuration section %r" % section)
+            log.warn("Unknown configuration section %r", section)
             continue
         settings = dict(c.items(section))
         registered_configs[section](settings)

pwnlib/context/__init__.py

@@ -6,17 +6,20 @@ contexts work properly and as expected.
 from __future__ import absolute_import
 from __future__ import division
 
+import atexit
 import collections
+import errno
 import functools
 import logging
 import os
+import os.path
 import platform
+import shutil
 import six
 import socket
-import stat
 import string
-import subprocess
 import sys
+import tempfile
 import threading
 import time
 
@@ -299,7 +302,7 @@ class ContextType(object):
         >>> context.os == 'linux'
         True
         >>> context.arch = 'arm'
-        >>> vars(context) == {'arch': 'arm', 'bits': 32, 'endian': 'little', 'os': 'linux'}
+        >>> vars(context) == {'arch': 'arm', 'bits': 32, 'endian': 'little', 'os': 'linux', 'newline': b'\n'}
         True
         >>> context.endian
         'little'
@@ -345,6 +348,10 @@ class ContextType(object):
         'binary': None,
         'bits': 32,
         'buffer_size': 4096,
+        'cache_dir_base': os.environ.get(
+            'XDG_CACHE_HOME',
+            os.path.join(os.path.expanduser('~'), '.cache')
+        ),
         'cyclic_alphabet': string.ascii_lowercase.encode(),
         'cyclic_size': 4,
         'delete_corefiles': False,
@@ -352,13 +359,16 @@ class ContextType(object):
         'encoding': 'auto',
         'endian': 'little',
         'gdbinit': "",
+        'gdb_binary': "",
         'kernel': None,
+        'local_libcdb': "/var/lib/libc-database",
         'log_level': logging.INFO,
         'log_file': _devnull(),
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
         'newline': b'\n',
+        'throw_eof_on_incomplete_line': None,
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
@@ -368,8 +378,19 @@ class ContextType(object):
         'timeout': Timeout.maximum,
     }
 
-    #: Valid values for :meth:`pwnlib.context.ContextType.os`
-    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal'))
+    unix_like    = {'newline': b'\n'}
+    windows_like = {'newline': b'\r\n'}
+
+    #: Keys are valid values for :meth:`pwnlib.context.ContextType.os`
+    oses = _longest({
+        'linux':     unix_like,
+        'freebsd':   unix_like,
+        'windows':   windows_like,
+        'cgc':       unix_like,
+        'android':   unix_like,
+        'baremetal': unix_like,
+        'darwin':    unix_like,
+    })
 
     big_32    = {'endian': 'big', 'bits': 32}
     big_64    = {'endian': 'big', 'bits': 64}
@@ -397,7 +418,8 @@ class ContextType(object):
         'msp430':    little_16,
         'powerpc':   big_32,
         'powerpc64': big_64,
-        'riscv':     little_32,
+        'riscv32':   little_32,
+        'riscv64':   little_64,
         's390':      big_32,
         'sparc':     big_32,
         'sparc64':   big_64,
@@ -437,14 +459,14 @@ class ContextType(object):
 
 
     def copy(self):
-        """copy() -> dict
+        r"""copy() -> dict
         Returns a copy of the current context as a dictionary.
 
         Examples:
 
             >>> context.clear()
             >>> context.os   = 'linux'
-            >>> vars(context) == {'os': 'linux'}
+            >>> vars(context) == {'os': 'linux', 'newline': b'\n'}
             True
         """
         return self._tls.copy()
@@ -573,9 +595,9 @@ class ContextType(object):
             ...         log.debug("DEBUG")
             ...         log.info("INFO")
             ...         log.warn("WARN")
-            [DEBUG] DEBUG
-            [*] INFO
-            [!] WARN
+            [...] DEBUG
+            [...] INFO
+            [...] WARN
         """
         level = 'error'
         if context.log_level <= logging.DEBUG:
@@ -643,7 +665,7 @@ class ContextType(object):
             information is printed.
 
             >>> with context.verbose: func()
-            [DEBUG] Hello
+            [...] Hello
 
         """
         return self.local(log_level='debug')
@@ -762,12 +784,15 @@ class ContextType(object):
         # We have to make sure that x86_64 appears before x86 for this to work correctly.
         transform = [('ppc64', 'powerpc64'),
                      ('ppc', 'powerpc'),
+                     ('x86-64', 'amd64'),
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
                      ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
-                     ('arm64', 'aarch64')]
+                     ('arm64', 'aarch64'),
+                     ('rv32', 'riscv32'),
+                     ('rv64', 'riscv64')]
         for k, v in transform:
             if arch.startswith(k):
                 arch = v
@@ -776,7 +801,7 @@ class ContextType(object):
         try:
             defaults = self.architectures[arch]
         except KeyError:
-            raise AttributeError('AttributeError: arch must be one of %r' % sorted(self.architectures))
+            raise AttributeError('AttributeError: arch (%r) must be one of %r' % (arch, sorted(self.architectures)))
 
         for k,v in defaults.items():
             if k not in self._tls:
@@ -819,6 +844,7 @@ class ContextType(object):
         The default value is ``32``, but changes according to :attr:`arch`.
 
         Examples:
+
             >>> context.clear()
             >>> context.bits == 32
             True
@@ -972,7 +998,10 @@ class ContextType(object):
         except AttributeError:  pass
 
         # Otherwise, fail
-        level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
+        try:
+            level_names = logging._levelToName.values()
+        except AttributeError:
+            level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
         permitted = sorted(level_names)
         raise AttributeError('log_level must be an integer or one of %r' % permitted)
 
@@ -1006,6 +1035,8 @@ class ContextType(object):
         """
         if isinstance(value, (bytes, six.text_type)):
             # check if mode was specified as "[value],[mode]"
+            from pwnlib.util.packing import _need_text
+            value = _need_text(value)
             if ',' not in value:
                 value += ',a'
             filename, mode = value.rsplit(',', 1)
@@ -1056,31 +1087,105 @@ class ContextType(object):
             stream = open(stream, 'wt')
         return stream
 
+    @_validator
+    def local_libcdb(self, path):
+        """ 
+        Sets path to local libc-database, get more information for libc-database:
+        https://github.com/niklasb/libc-database
+
+        Works in :attr:`pwnlib.libcdb` when searching by local database provider.
+
+        The default value is ``/var/lib/libc-database``.
+
+        Sets `context.local_libcdb` to empty string or `None` will turn off local libc-database integration.
+
+        Examples:
+
+            >>> context.local_libcdb = pwnlib.data.elf.path
+            >>> context.local_libcdb = 'foobar'
+            Traceback (most recent call last):
+            ...
+            AttributeError: 'foobar' does not exist, please download libc-database first
+        """
+
+        if not os.path.isdir(path):
+            raise AttributeError("'%s' does not exist, please download libc-database first" % path)
+
+        return path
+
     @property
     def mask(self):
         return (1 << self.bits) - 1
 
     @_validator
     def os(self, os):
-        """
+        r"""
         Operating system of the target machine.
 
         The default value is ``linux``.
 
         Allowed values are listed in :attr:`pwnlib.context.ContextType.oses`.
 
+        Side Effects:
+
+            If an os is specified some attributes will be set on the context
+            if a user has not already set a value.
+
+            The following property may be modified:
+
+            - :attr:`newline`
+
+        Raises:
+            AttributeError: An invalid os was specified
+
         Examples:
 
-            >>> context.os = 'linux'
+            >>> context.clear()
+            >>> context.os == 'linux' # Default os
+            True
+
+            >>> context.os = 'freebsd'
+            >>> context.os == 'freebsd'
+            True
+
             >>> context.os = 'foobar' #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
             AttributeError: os must be one of ['android', 'baremetal', 'cgc', 'freebsd', 'linux', 'windows']
+
+            >>> context.clear()
+            >>> context.newline == b'\n' # Default value
+            True
+            >>> context.os = 'windows'
+            >>> context.newline == b'\r\n' # New value
+            True
+
+            Note that expressly setting :attr:`newline` means that we use
+            that value instead of the default
+
+            >>> context.clear()
+            >>> context.newline = b'\n'
+            >>> context.os = 'windows'
+            >>> context.newline == b'\n'
+            True
+
+            Setting the os can override the default for :attr:`newline`
+
+            >>> context.clear()
+            >>> context.os = 'windows'
+            >>> vars(context) == {'os': 'windows', 'newline': b'\r\n'}
+            True
         """
         os = os.lower()
 
-        if os not in self.oses:
-            raise AttributeError("os must be one of %r" % self.oses)
+        try:
+            defaults = self.oses[os]
+        except KeyError:
+            raise AttributeError("os must be one of %r" % sorted(self.oses))
+
+        for k,v in defaults.items():
+            if k not in self._tls:
+                self._tls[k] = v
 
         return os
 
@@ -1269,12 +1374,30 @@ class ContextType(object):
         """
         return int(size)
 
+    @_validator
+    def cache_dir_base(self, new_base):
+        """Base directory to use for caching content.
+
+        Changing this to a different value will clear the :attr:`cache_dir` path
+        stored in TLS since a new path will need to be generated to respect the
+        new :attr:`cache_dir_base` value.
+        """
+
+        if new_base != self.cache_dir_base:
+            del self._tls["cache_dir"]
+        if os.access(new_base, os.F_OK) and not os.access(new_base, os.W_OK):
+            raise OSError(errno.EPERM, "Cache base dir is not writable")
+        return new_base
+
     @property
     def cache_dir(self):
         """Directory used for caching data.
 
         Note:
             May be either a path string, or :const:`None`.
+            Set to :const:`None` to disable caching.
+            Set to :const:`True` to generate the default cache directory path
+            based on :attr:`cache_dir_base` again.
 
         Example:
 
@@ -1282,32 +1405,67 @@ class ContextType(object):
             >>> cache_dir is not None
             True
             >>> os.chmod(cache_dir, 0o000)
+            >>> context.cache_dir = True
             >>> context.cache_dir is None
             True
             >>> os.chmod(cache_dir, 0o755)
             >>> cache_dir == context.cache_dir
             True
+            >>> context.cache_dir = None
+            >>> context.cache_dir is None
+            True
+            >>> context.cache_dir = True
+            >>> context.cache_dir is not None
+            True
         """
-        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
-                         os.path.join(os.path.expanduser('~'), '.cache')
+        try:
+            # If the TLS already has a cache directory path, we return it
+            # without any futher checks since it must have been valid when it
+            # was set and if that has changed, hiding the TOCTOU here would be
+            # potentially confusing
+            return self._tls["cache_dir"]
+        except KeyError:
+            pass
 
-        if not os.access(xdg_cache_home, os.W_OK):
+        # Attempt to create a Python version specific cache dir and its parents
+        cache_dirname = '.pwntools-cache-%d.%d' % sys.version_info[:2]
+        cache_dirpath = os.path.join(self.cache_dir_base, cache_dirname)
+        try:
+            os.makedirs(cache_dirpath)
+        except OSError as exc:
+            # If we failed for any reason other than the cache directory
+            # already existing then we'll fall back to a temporary directory
+            # object which doesn't respect the `cache_dir_base`
+            if exc.errno != errno.EEXIST:
+                try:
+                    cache_dirpath = tempfile.mkdtemp(prefix=".pwntools-tmp")
+                except IOError:
+                    # This implies no good candidates for temporary files so we
+                    # have to return `None`
+                    return None
+                else:
+                    # Ensure the temporary cache dir is cleaned up on exit. A
+                    # `TemporaryDirectory` would do this better upon garbage
+                    # collection but this is necessary for Python 2 support.
+                    atexit.register(shutil.rmtree, cache_dirpath)
+        # By this time we have a cache directory which exists but we don't know
+        # if it is actually writable. Some wargames e.g. pwnable.kr have
+        # created dummy directories which cannot be modified by the user
+        # account (owned by root).
+        if os.access(cache_dirpath, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = cache_dirpath
+            return cache_dirpath
+        else:
             return None
 
-        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
-
-        if not os.path.exists(cache):
-            try:
-                os.mkdir(cache)
-            except OSError:
-                return None
-
-        # Some wargames e.g. pwnable.kr have created dummy directories
-        # which cannot be modified by the user account (owned by root).
-        if not os.access(cache, os.W_OK):
-            return None
-
-        return cache
+    @cache_dir.setter
+    def cache_dir(self, v):
+        if v is True:
+            del self._tls["cache_dir"]
+        elif v is None or os.access(v, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = v
 
     @_validator
     def delete_corefiles(self, v):
@@ -1345,6 +1503,25 @@ class ContextType(object):
         # circular imports
         from pwnlib.util.packing import _need_bytes
         return _need_bytes(v)
+    
+    @_validator
+    def throw_eof_on_incomplete_line(self, v):
+        """Whether to raise an :class:`EOFError` if an EOF is received before a newline in ``tube.recvline``.
+
+        Controls if an :class:`EOFError` is treated as newline in ``tube.recvline`` and similar functions
+        and whether a warning should be logged about it.
+
+        Possible values are:
+
+        - ``True``: Raise an :class:`EOFError` if an EOF is received before a newline.
+        - ``False``: Return the data received so far if an EOF is received
+          before a newline without logging a warning.
+        - ``None``: Return the data received so far if an EOF is received
+          before a newline and log a warning.
+
+        Default value is ``None``.
+        """
+        return v if v is None else bool(v)
 
 
     @_validator
@@ -1363,6 +1540,20 @@ class ContextType(object):
         """
         return str(value)
 
+    @_validator
+    def gdb_binary(self, value):
+        """Path to the binary that is used when running GDB locally.
+
+        This is useful when you have multiple versions of gdb installed or the gdb binary is
+        called something different.
+
+        If set to an empty string, pwntools will try to search for a reasonable gdb binary from 
+        the path.
+
+        Default value is ``""``.
+        """
+        return str(value)
+
     @_validator
     def cyclic_alphabet(self, alphabet):
         """Cyclic alphabet.
@@ -1394,7 +1585,7 @@ class ContextType(object):
         from pwnlib.tubes.ssh import ssh
 
         if not isinstance(shell, ssh):
-            raise AttributeError("context.ssh_session must be an ssh tube") 
+            raise AttributeError("context.ssh_session must be an ssh tube")
 
         return shell
 
@@ -1557,7 +1748,7 @@ def update_context_defaults(section):
 
         default = ContextType.defaults[key]
 
-        if isinstance(default, six.string_types + six.integer_types + (tuple, list, dict)):
+        if isinstance(default, six.string_types + six.integer_types + (float, tuple, list, dict)):
             value = safeeval.expr(value)
         else:
             log.warn("Unsupported configuration option %r in section %r" % (key, 'context'))

pwnlib/data/crcsums.txt

@@ -1,58 +1,59 @@
 # This file was generated using the command:
-# curl -s 'http://reveng.sourceforge.net/crc-catalogue/all.htm#crc.legend' | sed -nr -e 's@<H3><A NAME="(crc\.cat[^"]*).*@\1@p' -e 's@.*<CODE>(width.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /' > crcsums.txt
+# curl -s 'https://reveng.sourceforge.io/crc-catalogue/all.htm' | sed -nr -e 's@.*?<A NAME="(crc\.cat[^"]*)">C.*@\1@p' -e 's@.*<CODE>(width=.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /'
 #
 # At the time of download, that page was available under the GPLv3 license, which can found here:
 # https://www.gnu.org/licenses/gpl.html
 #
+crc.cat.crc-3-gsm width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
 crc.cat.crc-3-rohc width=3  poly=0x3  init=0x7  refin=true  refout=true  xorout=0x0  check=0x6  residue=0x0  name="CRC-3/ROHC"
-crc.cat-bits.3 width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
-crc.cat-bits.4 width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
-crc.cat.crc-4-itu width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/ITU"
-crc.cat-bits.5 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC"
-crc.cat.crc-5-itu width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/ITU"
+crc.cat.crc-4-g-704 width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/G-704"
+crc.cat.crc-4-interlaken width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
+crc.cat.crc-5-epc-c1g2 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC-C1G2"
+crc.cat.crc-5-g-704 width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/G-704"
 crc.cat.crc-5-usb width=5  poly=0x05  init=0x1f  refin=true  refout=true  xorout=0x1f  check=0x19  residue=0x06  name="CRC-5/USB"
-crc.cat-bits.6 width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
+crc.cat.crc-6-cdma2000-a width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
 crc.cat.crc-6-cdma2000-b width=6  poly=0x07  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x3b  residue=0x00  name="CRC-6/CDMA2000-B"
 crc.cat.crc-6-darc width=6  poly=0x19  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-6/DARC"
+crc.cat.crc-6-g-704 width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/G-704"
 crc.cat.crc-6-gsm width=6  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x3f  check=0x13  residue=0x3a  name="CRC-6/GSM"
-crc.cat.crc-6-itu width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/ITU"
-crc.cat-bits.7 width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7"
+crc.cat.crc-7-mmc width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7/MMC"
 crc.cat.crc-7-rohc width=7  poly=0x4f  init=0x7f  refin=true  refout=true  xorout=0x00  check=0x53  residue=0x00  name="CRC-7/ROHC"
 crc.cat.crc-7-umts width=7  poly=0x45  init=0x00  refin=false  refout=false  xorout=0x00  check=0x61  residue=0x00  name="CRC-7/UMTS"
-crc.cat-bits.8 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8"
 crc.cat.crc-8-autosar width=8  poly=0x2f  init=0xff  refin=false  refout=false  xorout=0xff  check=0xdf  residue=0x42  name="CRC-8/AUTOSAR"
+crc.cat.crc-8-bluetooth width=8  poly=0xa7  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-8/BLUETOOTH"
 crc.cat.crc-8-cdma2000 width=8  poly=0x9b  init=0xff  refin=false  refout=false  xorout=0x00  check=0xda  residue=0x00  name="CRC-8/CDMA2000"
 crc.cat.crc-8-darc width=8  poly=0x39  init=0x00  refin=true  refout=true  xorout=0x00  check=0x15  residue=0x00  name="CRC-8/DARC"
 crc.cat.crc-8-dvb-s2 width=8  poly=0xd5  init=0x00  refin=false  refout=false  xorout=0x00  check=0xbc  residue=0x00  name="CRC-8/DVB-S2"
-crc.cat.crc-8-ebu width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/EBU"
 crc.cat.crc-8-gsm-a width=8  poly=0x1d  init=0x00  refin=false  refout=false  xorout=0x00  check=0x37  residue=0x00  name="CRC-8/GSM-A"
 crc.cat.crc-8-gsm-b width=8  poly=0x49  init=0x00  refin=false  refout=false  xorout=0xff  check=0x94  residue=0x53  name="CRC-8/GSM-B"
+crc.cat.crc-8-hitag width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0x00  check=0xb4  residue=0x00  name="CRC-8/HITAG"
+crc.cat.crc-8-i-432-1 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/I-432-1"
 crc.cat.crc-8-i-code width=8  poly=0x1d  init=0xfd  refin=false  refout=false  xorout=0x00  check=0x7e  residue=0x00  name="CRC-8/I-CODE"
-crc.cat.crc-8-itu width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/ITU"
 crc.cat.crc-8-lte width=8  poly=0x9b  init=0x00  refin=false  refout=false  xorout=0x00  check=0xea  residue=0x00  name="CRC-8/LTE"
-crc.cat.crc-8-maxim width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM"
+crc.cat.crc-8-maxim-dow width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM-DOW"
+crc.cat.crc-8-mifare-mad width=8  poly=0x1d  init=0xc7  refin=false  refout=false  xorout=0x00  check=0x99  residue=0x00  name="CRC-8/MIFARE-MAD"
+crc.cat.crc-8-nrsc-5 width=8  poly=0x31  init=0xff  refin=false  refout=false  xorout=0x00  check=0xf7  residue=0x00  name="CRC-8/NRSC-5"
 crc.cat.crc-8-opensafety width=8  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x00  check=0x3e  residue=0x00  name="CRC-8/OPENSAFETY"
 crc.cat.crc-8-rohc width=8  poly=0x07  init=0xff  refin=true  refout=true  xorout=0x00  check=0xd0  residue=0x00  name="CRC-8/ROHC"
 crc.cat.crc-8-sae-j1850 width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0xff  check=0x4b  residue=0xc4  name="CRC-8/SAE-J1850"
-crc.cat.crc-8-wdcma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
-crc.cat-bits.10 width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10"
+crc.cat.crc-8-smbus width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8/SMBUS"
+crc.cat.crc-8-tech-3250 width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/TECH-3250"
+crc.cat.crc-8-wcdma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
+crc.cat.crc-10-atm width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10/ATM"
 crc.cat.crc-10-cdma2000 width=10  poly=0x3d9  init=0x3ff  refin=false  refout=false  xorout=0x000  check=0x233  residue=0x000  name="CRC-10/CDMA2000"
 crc.cat.crc-10-gsm width=10  poly=0x175  init=0x000  refin=false  refout=false  xorout=0x3ff  check=0x12a  residue=0x0c6  name="CRC-10/GSM"
-crc.cat-bits.11 width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11"
+crc.cat.crc-11-flexray width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11/FLEXRAY"
 crc.cat.crc-11-umts width=11  poly=0x307  init=0x000  refin=false  refout=false  xorout=0x000  check=0x061  residue=0x000  name="CRC-11/UMTS"
-crc.cat-bits.12 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
+crc.cat.crc-12-cdma2000 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
 crc.cat.crc-12-dect width=12  poly=0x80f  init=0x000  refin=false  refout=false  xorout=0x000  check=0xf5b  residue=0x000  name="CRC-12/DECT"
 crc.cat.crc-12-gsm width=12  poly=0xd31  init=0x000  refin=false  refout=false  xorout=0xfff  check=0xb34  residue=0x178  name="CRC-12/GSM"
 crc.cat.crc-12-umts width=12  poly=0x80f  init=0x000  refin=false  refout=true  xorout=0x000  check=0xdaf  residue=0x000  name="CRC-12/UMTS"
-crc.cat-bits.13 width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
-crc.cat-bits.14 width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
+crc.cat.crc-13-bbc width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
+crc.cat.crc-14-darc width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
 crc.cat.crc-14-gsm width=14  poly=0x202d  init=0x0000  refin=false  refout=false  xorout=0x3fff  check=0x30ae  residue=0x031e  name="CRC-14/GSM"
-crc.cat-bits.15 width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15"
+crc.cat.crc-15-can width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15/CAN"
 crc.cat.crc-15-mpt1327 width=15  poly=0x6815  init=0x0000  refin=false  refout=false  xorout=0x0001  check=0x2566  residue=0x6815  name="CRC-15/MPT1327"
-crc.cat-bits.16 width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="ARC"
-crc.cat.crc-16-aug-ccitt width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/AUG-CCITT"
-crc.cat.crc-16-buypass width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/BUYPASS"
-crc.cat.crc-16-ccitt-false width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/CCITT-FALSE"
+crc.cat.crc-16-arc width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="CRC-16/ARC"
 crc.cat.crc-16-cdma2000 width=16  poly=0xc867  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x4c06  residue=0x0000  name="CRC-16/CDMA2000"
 crc.cat.crc-16-cms width=16  poly=0x8005  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0xaee7  residue=0x0000  name="CRC-16/CMS"
 crc.cat.crc-16-dds-110 width=16  poly=0x8005  init=0x800d  refin=false  refout=false  xorout=0x0000  check=0x9ecf  residue=0x0000  name="CRC-16/DDS-110"
@@ -62,44 +63,55 @@ crc.cat.crc-16-dnp width=16  poly=0x3d65  init=0x0000  refin=true  refout=true
 crc.cat.crc-16-en-13757 width=16  poly=0x3d65  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xc2b7  residue=0xa366  name="CRC-16/EN-13757"
 crc.cat.crc-16-genibus width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xd64e  residue=0x1d0f  name="CRC-16/GENIBUS"
 crc.cat.crc-16-gsm width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xce3c  residue=0x1d0f  name="CRC-16/GSM"
+crc.cat.crc-16-ibm-3740 width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/IBM-3740"
+crc.cat.crc-16-ibm-sdlc width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="CRC-16/IBM-SDLC"
+crc.cat.crc-16-iso-iec-14443-3-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-16/ISO-IEC-14443-3-A"
+crc.cat.crc-16-kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="CRC-16/KERMIT"
 crc.cat.crc-16-lj1200 width=16  poly=0x6f63  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xbdf4  residue=0x0000  name="CRC-16/LJ1200"
-crc.cat.crc-16-maxim width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM"
+crc.cat.crc-16-m17 width=16  poly=0x5935  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x772b  residue=0x0000  name="CRC-16/M17"
+crc.cat.crc-16-maxim-dow width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM-DOW"
 crc.cat.crc-16-mcrf4xx width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x6f91  residue=0x0000  name="CRC-16/MCRF4XX"
+crc.cat.crc-16-modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="CRC-16/MODBUS"
+crc.cat.crc-16-nrsc-5 width=16  poly=0x080b  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0xa066  residue=0x0000  name="CRC-16/NRSC-5"
 crc.cat.crc-16-opensafety-a width=16  poly=0x5935  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x5d38  residue=0x0000  name="CRC-16/OPENSAFETY-A"
-crc.cat.crc-16-opensafety-a width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
+crc.cat.crc-16-opensafety-b width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
 crc.cat.crc-16-profibus width=16  poly=0x1dcf  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xa819  residue=0xe394  name="CRC-16/PROFIBUS"
 crc.cat.crc-16-riello width=16  poly=0x1021  init=0xb2aa  refin=true  refout=true  xorout=0x0000  check=0x63d0  residue=0x0000  name="CRC-16/RIELLO"
+crc.cat.crc-16-spi-fujitsu width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/SPI-FUJITSU"
 crc.cat.crc-16-t10-dif width=16  poly=0x8bb7  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xd0db  residue=0x0000  name="CRC-16/T10-DIF"
 crc.cat.crc-16-teledisk width=16  poly=0xa097  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x0fb3  residue=0x0000  name="CRC-16/TELEDISK"
 crc.cat.crc-16-tms37157 width=16  poly=0x1021  init=0x89ec  refin=true  refout=true  xorout=0x0000  check=0x26b1  residue=0x0000  name="CRC-16/TMS37157"
+crc.cat.crc-16-umts width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/UMTS"
 crc.cat.crc-16-usb width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0xb4c8  residue=0xb001  name="CRC-16/USB"
-crc.cat.crc-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-A"
-crc.cat.kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="KERMIT"
-crc.cat.modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="MODBUS"
-crc.cat.x-25 width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="X-25"
-crc.cat.xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="XMODEM"
-crc.cat-bits.24 width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24"
+crc.cat.crc-16-xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="CRC-16/XMODEM"
+crc.cat.crc-17-can-fd width=17  poly=0x1685b  init=0x00000  refin=false  refout=false  xorout=0x00000  check=0x04f03  residue=0x00000  name="CRC-17/CAN-FD"
+crc.cat.crc-21-can-fd width=21  poly=0x102899  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x0ed841  residue=0x000000  name="CRC-21/CAN-FD"
 crc.cat.crc-24-ble width=24  poly=0x00065b  init=0x555555  refin=true  refout=true  xorout=0x000000  check=0xc25a56  residue=0x000000  name="CRC-24/BLE"
 crc.cat.crc-24-flexray-a width=24  poly=0x5d6dcb  init=0xfedcba  refin=false  refout=false  xorout=0x000000  check=0x7979bd  residue=0x000000  name="CRC-24/FLEXRAY-A"
 crc.cat.crc-24-flexray-b width=24  poly=0x5d6dcb  init=0xabcdef  refin=false  refout=false  xorout=0x000000  check=0x1f23b8  residue=0x000000  name="CRC-24/FLEXRAY-B"
 crc.cat.crc-24-interlaken width=24  poly=0x328b63  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0xb4f3e6  residue=0x144e63  name="CRC-24/INTERLAKEN"
 crc.cat.crc-24-lte-a width=24  poly=0x864cfb  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0xcde703  residue=0x000000  name="CRC-24/LTE-A"
 crc.cat.crc-24-lte-b width=24  poly=0x800063  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x23ef52  residue=0x000000  name="CRC-24/LTE-B"
-crc.cat-bits.30 width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
-crc.cat-bits.31 width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
-crc.cat-bits.32 width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32"
+crc.cat.crc-24-openpgp width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24/OPENPGP"
+crc.cat.crc-24-os-9 width=24  poly=0x800063  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0x200fa5  residue=0x800fe3  name="CRC-24/OS-9"
+crc.cat.crc-30-cdma width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
+crc.cat.crc-31-philips width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
+crc.cat.crc-32-aixm width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32/AIXM"
 crc.cat.crc-32-autosar width=32  poly=0xf4acfb13  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x1697d06a  residue=0x904cddbf  name="CRC-32/AUTOSAR"
+crc.cat.crc-32-base91-d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32/BASE91-D"
 crc.cat.crc-32-bzip2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0xffffffff  check=0xfc891918  residue=0xc704dd7b  name="CRC-32/BZIP2"
-crc.cat.crc-32c width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32C"
-crc.cat.crc-32d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32D"
+crc.cat.crc-32-cd-rom-edc width=32  poly=0x8001801b  init=0x00000000  refin=true  refout=true  xorout=0x00000000  check=0x6ec2edc4  residue=0x00000000  name="CRC-32/CD-ROM-EDC"
+crc.cat.crc-32-cksum width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/CKSUM"
+crc.cat.crc-32-iscsi width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32/ISCSI"
+crc.cat.crc-32-iso-hdlc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32/ISO-HDLC"
+crc.cat.crc-32-jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="CRC-32/JAMCRC"
+crc.cat.crc-32-mef width=32  poly=0x741b8cd7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0xd2c22f51  residue=0x00000000  name="CRC-32/MEF"
 crc.cat.crc-32-mpeg-2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0x00000000  check=0x0376e6e7  residue=0x00000000  name="CRC-32/MPEG-2"
-crc.cat.crc-32-posix width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/POSIX"
-crc.cat.crc-32q width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32Q"
-crc.cat.jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="JAMCRC"
-crc.cat.xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="XFER"
-crc.cat-bits.40 width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
-crc.cat-bits.64 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64"
+crc.cat.crc-32-xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="CRC-32/XFER"
+crc.cat.crc-40-gsm width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
+crc.cat.crc-64-ecma-182 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64/ECMA-182"
 crc.cat.crc-64-go-iso width=64  poly=0x000000000000001b  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0xb90956c775a41001  residue=0x5300000000000000  name="CRC-64/GO-ISO"
+crc.cat.crc-64-ms width=64  poly=0x259c84cba6426349  init=0xffffffffffffffff  refin=true  refout=true  xorout=0x0000000000000000  check=0x75d4b74f024eceea  residue=0x0000000000000000  name="CRC-64/MS"
 crc.cat.crc-64-we width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=false  refout=false  xorout=0xffffffffffffffff  check=0x62ec59e3f1a4f00a  residue=0xfcacbebd5931a992  name="CRC-64/WE"
 crc.cat.crc-64-xz width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0x995dc9bbdf1939fa  residue=0x49958c9abd7d353f  name="CRC-64/XZ"
-crc.cat-bits.82 width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"
+crc.cat.crc-82-darc width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"

pwnlib/data/includes/generator/linux/diet/riscv64/syscalls.h

@@ -0,0 +1,304 @@
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_newfstatat 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range 84
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_arch_specific_syscall 244
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at 264
+#define __NR_open_by_handle_at 265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_sched_setattr 274
+#define __NR_sched_getattr 275
+#define __NR_renameat2 276
+#define __NR_seccomp 277
+#define __NR_getrandom 278
+#define __NR_memfd_create 279
+#define __NR_bpf 280
+#define __NR_execveat 281
+#define __NR_userfaultfd 282
+#define __NR_membarrier 283
+#define __NR_mlock2 284
+#define __NR_copy_file_range 285
+#define __NR_preadv2 286
+#define __NR_pwritev2 287
+#define __NR_pkey_mprotect 288
+#define __NR_pkey_alloc 289
+#define __NR_pkey_free 290
+#define __NR_statx 291
+#define __NR_io_pgetevents 292
+#define __NR_rseq 293
+#define __NR_kexec_file_load 294
+#define __NR_pidfd_send_signal 424
+#define __NR_io_uring_setup 425
+#define __NR_io_uring_enter 426
+#define __NR_io_uring_register 427
+#define __NR_open_tree		428
+#define __NR_move_mount		429
+#define __NR_fsopen		430
+#define __NR_fsconfig		431
+#define __NR_fsmount		432
+#define __NR_fspick		433
+#define __NR_pidfd_open		434
+#define __NR_clone3		435
+#define __NR_close_range	436
+#define __NR_openat2		437
+#define __NR_pidfd_getfd	438
+#define __NR_faccessat2		439
+#define __NR_process_madvise	440
+#define __NR_epoll_pwait2	441
+#define __NR_mount_setattr	442
+#define __NR_landlock_create_ruleset	444
+#define __NR_landlock_add_rule	445
+#define __NR_landlock_restrict_self	446
+
+#define __NR_sysriscv __NR_arch_specific_syscall
+#define __NR_riscv_flush_icache (__NR_sysriscv + 15)

pwnlib/data/includes/generator/linux/riscv64.h

@@ -0,0 +1,4 @@
+// https://git.musl-libc.org/cgit/musl/plain/arch/riscv64/bits/syscall.h.in
+#define __riscv64__
+#include <riscv64/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/syscall_map.h

@@ -44,6 +44,7 @@
 #define SYS_clone2 __NR_clone2
 #define SYS_clone3 __NR_clone3
 #define SYS_close __NR_close
+#define SYS_close_range __NR_close_range
 #define SYS_connect __NR_connect
 #define SYS_copy_file_range __NR_copy_file_range
 #define SYS_creat __NR_creat
@@ -58,6 +59,7 @@
 #define SYS_epoll_ctl __NR_epoll_ctl
 #define SYS_epoll_ctl_old __NR_epoll_ctl_old
 #define SYS_epoll_pwait __NR_epoll_pwait
+#define SYS_epoll_pwait2 __NR_epoll_pwait2
 #define SYS_epoll_wait __NR_epoll_wait
 #define SYS_epoll_wait_old __NR_epoll_wait_old
 #define SYS_eventfd __NR_eventfd
@@ -69,6 +71,7 @@
 #define SYS_exit __NR_exit
 #define SYS_exit_group __NR_exit_group
 #define SYS_faccessat __NR_faccessat
+#define SYS_faccessat2 __NR_faccessat2
 #define SYS_fadvise64 __NR_fadvise64
 #define SYS_fadvise64_64 __NR_fadvise64_64
 #define SYS_fallocate __NR_fallocate
@@ -191,6 +194,9 @@
 #define SYS_kexec_load __NR_kexec_load
 #define SYS_keyctl __NR_keyctl
 #define SYS_kill __NR_kill
+#define SYS_landlock_add_rule __NR_landlock_add_rule
+#define SYS_landlock_create_ruleset __NR_landlock_create_ruleset
+#define SYS_landlock_restrict_self __NR_landlock_restrict_self
 #define SYS_lchown __NR_lchown
 #define SYS_lchown32 __NR_lchown32
 #define SYS_lgetxattr __NR_lgetxattr
@@ -228,6 +234,7 @@
 #define SYS_mmap2 __NR_mmap2
 #define SYS_modify_ldt __NR_modify_ldt
 #define SYS_mount __NR_mount
+#define SYS_mount_setattr __NR_mount_setattr
 #define SYS_move_mount __NR_move_mount
 #define SYS_move_pages __NR_move_pages
 #define SYS_mprotect __NR_mprotect
@@ -402,6 +409,7 @@
 #define SYS_preadv __NR_preadv
 #define SYS_preadv2 __NR_preadv2
 #define SYS_prlimit64 __NR_prlimit64
+#define SYS_process_madvise __NR_process_madvise
 #define SYS_process_vm_readv __NR_process_vm_readv
 #define SYS_process_vm_writev __NR_process_vm_writev
 #define SYS_prof __NR_prof
@@ -440,6 +448,7 @@
 #define SYS_reserved221 __NR_reserved221
 #define SYS_reserved82 __NR_reserved82
 #define SYS_restart_syscall __NR_restart_syscall
+#define SYS_riscv_flush_icache __NR_riscv_flush_icache
 #define SYS_rmdir __NR_rmdir
 #define SYS_rseq __NR_rseq
 #define SYS_rtas __NR_rtas
@@ -572,6 +581,7 @@
 #define SYS_sys_kexec_load __NR_sys_kexec_load
 #define SYS_syslog __NR_syslog
 #define SYS_sysmips __NR_sysmips
+#define SYS_sysriscv __NR_sysriscv
 #define SYS_sys_setaltroot __NR_sys_setaltroot
 #define SYS_tee __NR_tee
 #define SYS_tgkill __NR_tgkill

pwnlib/data/syscalls/Makefile

@@ -7,4 +7,7 @@ all: generate.py functions.py
 functions.py:
 	wget https://raw.githubusercontent.com/zachriggle/functions/master/functions.py
 
+generate_darwin:
+	python3 generate_darwin.py "$(ROOT)"
+
 .phony: all

pwnlib/data/syscalls/generate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 from __future__ import division
 import argparse
 import keyword
@@ -72,7 +72,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[index] = arg
+                stack_arguments[name] = arg
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
@@ -100,7 +100,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[target] = arg
+                stack_arguments[name] = arg
 
     # Some syscalls have different names on various architectures.
     # Determine which syscall number to use for the current architecture.
@@ -108,7 +108,7 @@ CALL = """
         if hasattr(pwnlib.constants, syscall):
             break
     else:
-        raise Exception("Could not locate any syscalls: %r" % syscalls)
+        raise Exception("Could not locate any syscalls: %r" % {syscalls!r})
 %>
     /* {name}(${{', '.join(syscall_repr)}}) */
 %for name, arg in string_arguments.items():
@@ -137,6 +137,7 @@ def can_be_string(arg):
     if arg.type == 'void' and arg.derefcnt == 1:
         return True
 
+
 def can_be_array(arg):
     if arg.type == 'char' and arg.derefcnt == 2:
         return True
@@ -160,11 +161,13 @@ def fix_bad_arg_names(func, arg):
 def get_arg_default(arg):
     return 0
 
+
 def fix_rt_syscall_name(name):
     if name.startswith('rt_'):
         return name[3:]
     return name
 
+
 def fix_syscall_names(name):
     # Do not use old_mmap
     if name == 'SYS_mmap':
@@ -180,6 +183,7 @@ def main(target):
         with context.local(arch=arch):
             generate_one(target)
 
+
 def generate_one(target):
     SYSCALL_NAMES = [c for c in dir(constants) if c.startswith('SYS_')]
 
@@ -214,46 +218,50 @@ def generate_one(target):
 
         # Set up the argument string for Mako
         argument_names = []
+        argument_names_ = []
         argument_defaults = []
 
-        #
-
-        for arg in function.args:
-            argname = fix_bad_arg_names(function, arg)
-            default = get_arg_default(arg)
-
-            # Mako is unable to use *vararg and *kwarg, so we just stub in
-            # a whole bunch of additional arguments.
-            if argname == 'vararg':
-                for j in range(5):
-                    argname = 'vararg_%i' % j
-                    argument_names.append(argname)
-                    argument_defaults.append('%s=%s' % (argname, None))
-                break
-
-            argument_names.append(argname)
-            argument_defaults.append('%s=%s' % (argname, default))
-
-        arguments_default_values = ', '.join(argument_defaults)
-        arguments_comma_separated = ', '.join(argument_names)
-
         string_arguments = []
         array_arguments = []
         arg_docs = []
 
+        #
+
         for arg in function.args:
+            argname_ = fix_bad_arg_names(function, arg)
+            argname = argname_.rstrip('_')
+            default = get_arg_default(arg)
 
             if can_be_array(arg):
-                array_arguments.append(arg.name)
+                array_arguments.append(argname)
 
             if can_be_string(arg):
-                string_arguments.append(arg.name)
+                string_arguments.append(argname)
 
-            argname = arg.name
             argtype = str(arg.type) + ('*' * arg.derefcnt)
             arg_docs.append(
-                '    {argname}({argtype}): {argname}'.format(argname=argname,
-                                                             argtype=argtype))
+                '    {argname_}({argtype}): {argname}'.format(
+                    argname_=argname_,
+                    argname=argname,
+                    argtype=argtype,
+                ))
+
+            # Mako is unable to use *vararg and *kwarg, so we just stub in
+            # a whole bunch of additional arguments.
+            if argname == 'vararg':
+                for j in range(6):
+                    argname = 'vararg_%i' % j
+                    argument_names.append(argname)
+                    argument_names_.append(argname)
+                    argument_defaults.append('%s=%s' % (argname, None))
+                break
+
+            argument_names.append(argname)
+            argument_names_.append(argname_)
+            argument_defaults.append('%s=%s' % (argname_, default))
+
+        arguments_default_values = ', '.join(argument_defaults)
+        arguments_comma_separated = ', '.join(argument_names_)
 
         return_type = str(function.type) + ('*' * function.derefcnt)
         arg_docs = '\n'.join(arg_docs)
@@ -282,6 +290,7 @@ def generate_one(target):
         with open(os.path.join(target, name + '.asm'), 'wt') as f:
             f.write('\n'.join(map(str.strip, lines)) + '\n')
 
+
 if __name__ == '__main__':
     p = argparse.ArgumentParser()
     p.add_argument('target_directory')

pwnlib/data/syscalls/generate_darwin.py

@@ -0,0 +1,73 @@
+# ./pwnlib/data/includes/darwin/aarch64.h
+# ./pwnlib/constants/darwin/aarch64.py
+
+# https://github.com/nullgemm/instant_macos_sdk (old sdk here, please use real macos device)
+# https://github.com/joseluisq/macosx-sdks (old sdk here, please use real macos device)
+# /Library/Developer/CommandLineTools/SDKs/MacOSX14.sdk/System/Library/Frameworks/Kernel.framework/Versions/A/Headers/sys/
+
+from pathlib import Path
+import re
+import sys
+
+# In the future, you should change the version of `MacOSX15.sdk`
+sdk_path = Path('/Library/Developer/CommandLineTools/SDKs/MacOSX15.sdk/System/Library/Frameworks/Kernel.framework/Versions/A/Headers/sys/')
+if not sdk_path.exists():
+    print('missing MacOSX sdk')
+    exit(1)
+
+project_path = Path(sys.argv[1])
+
+regex = re.compile(r'^#define\s+([a-zA-Z0-9_-]+)\s+([0-9]+|0x[0-9a-fA-F]+)(?:\s|$)', re.DOTALL)
+
+out_data = {}
+for file in sdk_path.iterdir():
+    if not file.is_file():
+        continue
+
+    print(file.name)
+    for line in file.read_text(errors='ignore').split('\n'):
+        matched = regex.search(line)
+        if not matched:
+            continue
+
+        key, value = matched.groups()
+        if value.startswith('0') and not value.startswith('0x') and len(value) > 1:
+            value = '0o'+value
+
+        print(key, value)
+        out_data[key] = value
+
+
+outbuf1_aarch64 = ''
+outbuf1_aarch64 += "from pwnlib.constants.constant import Constant\n"
+
+outbuf1_amd64 = ''
+outbuf1_amd64 += "from pwnlib.constants.constant import Constant\n"
+
+outbuf2_aarch64 = ''
+outbuf2_amd64 = ''
+
+# https://www.idryman.org/blog/2014/12/02/writing-64-bit-assembly-on-mac-os-x/
+# on amd64 syscall offsets from 0x2000000 + syscall number
+
+for key, value in out_data.items():
+    value_octal = value
+    if value_octal.startswith('0o'):
+        value_octal = value_octal.replace('0o', '0')
+
+    outbuf1_aarch64 += "{} = Constant('{}',{})\n".format(key, key, value)
+    outbuf2_aarch64 += "#define {} {}\n".format(key, value_octal)
+
+    if key.startswith('SYS_'):
+        value = f'{value} + 0x2000000'
+        value_octal = f'{value_octal} + 0x2000000'
+
+    outbuf1_amd64 += "{} = Constant('{}',{})\n".format(key, key, value)
+    outbuf2_amd64 += "#define {} {}\n".format(key, value_octal)
+
+pp = project_path
+(pp / Path('./pwnlib/constants/darwin/aarch64.py')).write_bytes(outbuf1_aarch64.encode())
+(pp / Path('./pwnlib/data/includes/darwin/aarch64.h')).write_bytes(outbuf2_aarch64.encode())
+
+(pp / Path('./pwnlib/constants/darwin/amd64.py')).write_bytes(outbuf1_amd64.encode())
+(pp / Path('./pwnlib/data/includes/darwin/amd64.h')).write_bytes(outbuf2_amd64.encode())

pwnlib/data/templates/pwnup.mako

@@ -1,4 +1,4 @@
-<%page args="binary, host=None, port=None, user=None, password=None, remote_path=None, quiet=False"/>\
+<%page args="binary, host=None, port=None, user=None, password=None, libc=None, remote_path=None, quiet=False"/>\
 <%
 import os
 import sys
@@ -31,6 +31,7 @@ elif host and not port:
 remote_path = remote_path or exe
 password = password or 'secret1234'
 binary_repr = repr(binary)
+libc_repr = repr(libc)
 %>\
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
@@ -43,8 +44,8 @@ from pwn import *
 %if not quiet:
 # Set up pwntools for the correct architecture
 %endif
-%if ctx.binary:
-exe = context.binary = ELF(${binary_repr})
+%if ctx.binary or not host:
+exe = context.binary = ELF(args.EXE or ${binary_repr})
 <% binary_repr = 'exe.path' %>
 %else:
 context.update(arch='i386')
@@ -58,7 +59,7 @@ exe = ${binary_repr}
 # for all created processes...
 # ./exploit.py DEBUG NOASLR
 %if host or port or user:
-# ./exploit.py GDB HOST=example.com PORT=4141
+# ./exploit.py GDB HOST=example.com PORT=4141 EXE=/tmp/executable
 %endif
 %endif
 %if host:
@@ -83,6 +84,31 @@ if not args.LOCAL:
     shell.set_working_directory(symlink=True)
 %endif
 
+%if libc:
+%if not quiet:
+# Use the specified remote libc version unless explicitly told to use the
+# local system version with the `LOCAL_LIBC` argument.
+# ./exploit.py LOCAL LOCAL_LIBC
+%endif
+if args.LOCAL_LIBC:
+    libc = exe.libc
+%if host:
+elif args.LOCAL:
+%else:
+else:
+%endif
+    library_path = libcdb.download_libraries(${libc_repr})
+    if library_path:
+        exe = context.binary = ELF.patch_custom_libraries(${binary_repr}, library_path)
+        libc = exe.libc
+    else:
+        libc = ELF(${libc_repr})
+%if host:
+else:
+    libc = ELF(${libc_repr})
+%endif
+%endif
+
 %if host:
 def start_local(argv=[], *a, **kw):
     '''Execute the target binary locally'''

pwnlib/data/useragents/download-useragents.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 """Script for downloading lists of user agent strings."""
 from __future__ import division
 import os
@@ -43,7 +43,7 @@ with log.waitfor('Fetching from from http://www.user-agents.org') as l:
     l.success()
 
 with log.waitfor('Parsing list') as l:
-    for item in soup.__getattr__('user-agents'):
+    for item in getattr(soup, 'user-agents'):
         if item.name == 'user-agent':
             ua = item.select('string')[0].string.strip()
             try:

pwnlib/dynelf.py

@@ -15,7 +15,7 @@ Example
     # leaks at least one byte at that address.
     def leak(address):
         data = p.read(address, 4)
-        log.debug("%#x => %s" % (address, enhex(data or '')))
+        log.debug("%#x => %s", address, enhex(data or ''))
         return data
 
     # For the sake of this example, let's say that we
@@ -62,8 +62,7 @@ from pwnlib.elf import constants
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
 from pwnlib.util.fiddling import enhex
-from pwnlib.util.packing import unpack, _need_bytes
-from pwnlib.util.web import wget
+from pwnlib.util.packing import _need_bytes
 
 log    = getLogger(__name__)
 sizeof = ctypes.sizeof
@@ -141,10 +140,10 @@ class DynELF(object):
     .. _.got.plt:  https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic/specialsections.html
     .. _DYNAMIC:   http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#dynamic_section
     .. _SYSV:      https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#hash
-    .. _GNU:       https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+    .. _GNU:       https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
     .. _DT_DEBUG:  https://reverseengineering.stackexchange.com/questions/6525/elf-link-map-when-linked-as-relro
     .. _link map:  https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=eaca8028e45a859ac280301a6e955a14eed1b887;hb=HEAD#l84
-    .. _DT_PLTGOT: http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
+    .. _DT_PLTGOT: https://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
     '''
 
     def __init__(self, leak, pointer=None, elf=None, libcdb=True):
@@ -166,6 +165,18 @@ class DynELF(object):
         self._waitfor  = None
         self._bases    = {}
         self._dynamic  = None
+        self.elf = None
+
+        if elf:
+            path = elf
+            if isinstance(elf, ELF):
+                path = elf.path
+
+            # Load a fresh copy of the ELF
+            with context.local(log_level='error'):
+                w = self.waitfor("Loading from %r" % path)
+                self.elf = ELF(path)
+                w.success("[LOADED]")
 
         if not (pointer or (elf and elf.address)):
             log.error("Must specify either a pointer into a module and/or an ELF file with a valid base address")
@@ -178,12 +189,15 @@ class DynELF(object):
         if not elf:
             log.warn_once("No ELF provided.  Leaking is much faster if you have a copy of the ELF being leaked.")
 
-        self.elf     = elf
         self.leak    = leak
         self.libbase = self._find_base(pointer or elf.address)
 
         if elf:
-            self._find_linkmap_assisted(elf)
+            self._elftype = self.elf.elftype
+            self._elfclass = self.elf.elfclass
+            self.elf.address = self.libbase
+            self._dynamic = self.elf.get_section_by_name('.dynamic').header.sh_addr
+            self._dynamic = self._make_absolute_ptr(self._dynamic) 
 
     @classmethod
     def for_one_lib_only(cls, leak, ptr):
@@ -242,49 +256,6 @@ class DynELF(object):
             self._dynamic = self._find_dynamic_phdr()
         return self._dynamic
 
-    def _find_linkmap_assisted(self, path):
-        """Uses an ELF file to assist in finding the link_map.
-        """
-        if isinstance(path, ELF):
-            path = path.path
-
-        # Load a fresh copy of the ELF
-        with context.local(log_level='error'):
-            elf = ELF(path)
-        elf.address = self.libbase
-
-        w = self.waitfor("Loading from %r" % elf.path)
-
-        # Save our real leaker
-        real_leak = self.leak
-
-        # Create a fake leaker which just leaks out of the 'loaded' ELF
-        # However, we may load things which are outside of the ELF (e.g.
-        # the linkmap or GOT) so we need to fall back on the real leak.
-        @MemLeak
-        def fake_leak(address):
-            try:
-                return elf.read(address, 4)
-            except ValueError:
-                return real_leak.b(address)
-
-        # Save off our real leaker, use the fake leaker
-        self.leak = fake_leak
-
-        # Get useful pointers for resolving the linkmap faster
-        w.status("Searching for DT_PLTGOT")
-        pltgot = self._find_dt(constants.DT_PLTGOT)
-
-        w.status("Searching for DT_DEBUG")
-        debug  = self._find_dt(constants.DT_DEBUG)
-
-        # Restore the real leaker
-        self.leak = real_leak
-
-        # Find the linkmap using the helper pointers
-        self._find_linkmap(pltgot, debug)
-        self.success('Done')
-
     def _find_base(self, ptr):
         page_size = 0x1000
         page_mask = ~(page_size - 1)
@@ -381,6 +352,27 @@ class DynELF(object):
 
         return dynamic
 
+    def _find_dt_optimized(self, name):
+        """
+        Find an entry in the DYNAMIC array through an ELF
+
+        Arguments:
+            name(str): Name of the tag to find ('DT_DEBUG', 'DT_PLTGOT', ...)
+
+        Returns:
+            Pointer to the data described by the specified entry.
+        """
+        if not self.elf:
+            return None
+
+        ptr = self.elf.dynamic_value_by_tag(name)
+        if ptr:
+            ptr = self._make_absolute_ptr(ptr)
+            self.success("Found %s at %#x" % (name, ptr))
+            return ptr
+        return None
+
+
     def _find_dt(self, tag):
         """
         Find an entry in the DYNAMIC array.
@@ -391,11 +383,16 @@ class DynELF(object):
         Returns:
             Pointer to the data described by the specified entry.
         """
-        leak    = self.leak
         base    = self.libbase
         dynamic = self.dynamic
+        leak    = self.leak
         name    = next(k for k,v in ENUM_D_TAG.items() if v == tag)
 
+        # Read directly from the ELF if possible
+        ptr = self._find_dt_optimized(name)
+        if ptr:
+            return ptr
+
         Dyn = {32: elf.Elf32_Dyn,    64: elf.Elf64_Dyn}     [self.elfclass]
 
         # Found the _DYNAMIC program header, now find PLTGOT entry in it
@@ -408,10 +405,10 @@ class DynELF(object):
             self.failure("Could not find tag %s" % name)
             return None
 
-        self.status("Found %s at %#x" % (name, dynamic))
         ptr = leak.field(dynamic, Dyn.d_ptr)
 
         ptr = self._make_absolute_ptr(ptr)
+        self.status("Found %s at %#x" % (name, ptr))
 
         return ptr
 
@@ -562,7 +559,7 @@ class DynELF(object):
         else:   dynlib = self
 
         if dynlib is None:
-            log.failure("Could not find %r" % lib)
+            log.failure("Could not find %r", lib)
             return None
 
         #
@@ -573,7 +570,7 @@ class DynELF(object):
             self.status("Trying lookup based on Build ID")
             build_id = dynlib._lookup_build_id(lib=lib)
             if build_id:
-                log.info("Trying lookup based on Build ID: %s" % build_id)
+                log.info("Trying lookup based on Build ID: %s", build_id)
                 path = libcdb.search_by_build_id(build_id)
                 if path:
                     with context.local(log_level='error'):
@@ -600,6 +597,10 @@ class DynELF(object):
         Return a dictionary mapping library path to its base address.
         '''
         if not self._bases:
+            if self.link_map is None:
+                self.failure("Cannot determine bases without linkmap")
+                return {}
+                
             leak    = self.leak
             LinkMap = {32: elf.Elf32_Link_Map, 64: elf.Elf64_Link_Map}[self.elfclass]
 
@@ -615,7 +616,7 @@ class DynELF(object):
                 addr   = leak.field(cur, LinkMap.l_addr)
                 cur    = leak.field(cur, LinkMap.l_next)
 
-                log.debug('Found %r @ %#x' % (name, addr))
+                log.debug('Found %r @ %#x', name, addr)
 
                 self._bases[name] = addr
 
@@ -651,7 +652,7 @@ class DynELF(object):
                 break
 
             if name:
-                self.status('Skipping %s' % name)
+                self.status('Skipping %r' % name)
 
             cur = leak.field(cur, LinkMap.l_next)
         else:
@@ -667,6 +668,62 @@ class DynELF(object):
         lib._waitfor = self._waitfor
         return lib
 
+    def _rel_lookup(self, symb, strtab=None, symtab=None, jmprel=None):
+        """Performs slower symbol lookup using DT_JMPREL(.rela.plt)"""
+        leak = self.leak
+        elf_obj = self.elf
+        symb_name = symb.decode()
+
+        # If elf is available look for the symbol in it
+        if elf_obj and symb_name in elf_obj.symbols:
+            self.success("Symbol '%s' found in ELF!" % symb_name)
+            return elf_obj.symbols[symb_name]
+
+        log.warning("Looking up symbol through DT_JMPREL. This might be slower...")
+
+
+        strtab  = strtab or self._find_dt(constants.DT_STRTAB)
+        symtab  = symtab or self._find_dt(constants.DT_SYMTAB)
+        jmprel  = jmprel or self._find_dt(constants.DT_JMPREL) # .rela.plt
+
+        strtab = self._make_absolute_ptr(strtab)
+        symtab = self._make_absolute_ptr(symtab)
+        jmprel = self._make_absolute_ptr(jmprel)
+
+        w = self.waitfor("Looking for %s in .rel.plt" % symb)
+        # We look for the symbol by iterating through each Elf64_Rel entry.
+        # For each Elf64_Rel, get the Elf64_Sym for that entry
+        # Then compare the Elf64_Sym.st_name with the symbol name
+       
+        Rel = {32: elf.Elf32_Rel, 64: elf.Elf64_Rel}[self.elfclass]
+        Sym = {32: elf.Elf32_Sym, 64: elf.Elf64_Sym}[self.elfclass]
+
+        rel_addr = jmprel
+        rel_entry = None
+        while True:
+            rel_entry = leak.struct(rel_addr, Rel)
+
+            # We ran out of entries in DT_JMPREL 
+            if rel_entry.r_offset == 0:
+                return None
+
+            sym_idx = rel_entry.r_info >> 32 # might be different for 32-bit
+            sym_entry_address = symtab + ( sym_idx * sizeof(Sym) )
+            sym_str_off = leak.field(sym_entry_address, Sym.st_name)
+            symb_str = leak.s(strtab+sym_str_off)
+
+            if symb_str == symb:
+                w.success("Found matching Elf64_Rel entry!")
+                break
+
+            rel_addr += sizeof(Rel)
+
+        symbol_address = self._make_absolute_ptr(rel_entry.r_offset)
+
+        return symbol_address
+
+
+
     def _lookup(self, symb):
         """Performs the actual symbol lookup within one ELF file."""
         leak = self.leak
@@ -699,9 +756,39 @@ class DynELF(object):
         #
         # Perform the hash lookup
         #
+
+        # Save off our real leaker in case we use the fake leaker
+        real_leak = self.leak
+        if self.elf:
+
+            # Create a fake leaker which just leaks out of the 'loaded' ELF
+            # However, we may load things which are outside of the ELF (e.g.
+            # the linkmap or GOT) so we need to fall back on the real leak.
+            @MemLeak
+            def fake_leak(address):
+                try:
+                    return self.elf.read(address, 4)
+                except ValueError:
+                    return real_leak.b(address)
+            # Use fake leaker since ELF is available
+            self.leak = fake_leak
+
         routine = {'sysv': self._resolve_symbol_sysv,
                    'gnu':  self._resolve_symbol_gnu}[hshtype]
-        return routine(self.libbase, symb, hshtab, strtab, symtab)
+        resolved_addr = routine(self.libbase, symb, hshtab, strtab, symtab)
+
+        if resolved_addr:
+            # Restore the original leaker
+            self.leak = real_leak
+            return resolved_addr
+
+        # if symbol not found in GNU_Hash, try looking in JMPREL
+        resolved_addr = self._rel_lookup(symb, strtab, symtab)
+
+        # Restore the original leaker
+        self.leak = real_leak
+
+        return resolved_addr
 
     def _resolve_symbol_sysv(self, libbase, symb, hshtab, strtab, symtab):
         """
@@ -770,7 +857,7 @@ class DynELF(object):
             structure.
 
             Again, Oracle has good documentation.
-            https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+            https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
 
             You can force an ELF to use this type of symbol table by compiling
             with 'gcc -Wl,--hash-style=gnu'

pwnlib/elf/corefile.py

@@ -76,7 +76,6 @@ import tempfile
 from io import BytesIO, StringIO
 
 import elftools
-from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
@@ -106,43 +105,11 @@ prstatus_types = {
     'aarch64': elf_prstatus_aarch64
 }
 
-prpsinfo_types = {
-    32: elf_prpsinfo_32,
-    64: elf_prpsinfo_64,
-}
-
 siginfo_types = {
     32: elf_siginfo_32,
     64: elf_siginfo_64
 }
 
-# Slightly modified copy of the pyelftools version of the same function,
-# until they fix this issue:
-# https://github.com/eliben/pyelftools/issues/93
-def iter_notes(self):
-    """ Iterates the list of notes in the segment.
-    """
-    offset = self['p_offset']
-    end = self['p_offset'] + self['p_filesz']
-    while offset < end:
-        note = struct_parse(
-            self.elffile.structs.Elf_Nhdr,
-            self.stream,
-            stream_pos=offset)
-        note['n_offset'] = offset
-        offset += self.elffile.structs.Elf_Nhdr.sizeof()
-        self.stream.seek(offset)
-        # n_namesz is 4-byte aligned.
-        disk_namesz = roundup(note['n_namesz'], 2)
-        note['n_name'] = bytes2str(
-            CString('').parse(self.stream.read(disk_namesz)))
-        offset += disk_namesz
-
-        desc_data = bytes2str(self.stream.read(note['n_descsz']))
-        note['n_desc'] = desc_data
-        offset += roundup(note['n_descsz'], 2)
-        note['n_size'] = offset - note['n_offset']
-        yield note
 
 class Mapping(object):
     """Encapsulates information about a memory mapping in a :class:`Corefile`.
@@ -271,8 +238,8 @@ class Corefile(ELF):
     Registers can be accessed directly, e.g. via ``core_obj.eax`` and enumerated
     via :data:`Corefile.registers`.
 
-    Memory can be accessed directly via :meth:`.read` or :meth:`.write`, and also
-    via :meth:`.pack` or :meth:`.unpack` or even :meth:`.string`.
+    Memory can be accessed directly via :meth:`pwnlib.elf.elf.ELF.read` or :meth:`pwnlib.elf.elf.ELF.write`, and also
+    via :meth:`pwnlib.elf.elf.ELF.pack` or :meth:`pwnlib.elf.elf.ELF.unpack` or even :meth:`.string`.
 
     Arguments:
         core: Path to the core file.  Alternately, may be a :class:`.process` instance,
@@ -400,6 +367,7 @@ class Corefile(ELF):
         automatically invokes GDB to attach and dump a corefile.
 
         >>> core = io.corefile
+        >>> io.close()
 
         The corefile can be inspected and read from, and even exposes various mappings
 
@@ -408,8 +376,8 @@ class Corefile(ELF):
         >>> core.exe.data[0:4]
         b'\x7fELF'
 
-        It also supports all of the features of :class:`ELF`, so you can :meth:`.read`
-        or :meth:`.write` or even the helpers like :meth:`.pack` or :meth:`.unpack`.
+        It also supports all of the features of :class:`ELF`, so you can :meth:`pwnlib.elf.elf.ELF.read`
+        or :meth:`pwnlib.elf.elf.ELF.write` or even the helpers like :meth:`pwnlib.elf.elf.ELF.pack` or :meth:`pwnlib.elf.elf.ELF.unpack`.
 
         Don't forget to call :meth:`.ELF.save` to save the changes to disk.
 
@@ -444,7 +412,7 @@ class Corefile(ELF):
         will be loaded.
 
         >>> process('bash').corefile.libc # doctest: +ELLIPSIS
-        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+        Mapping('.../libc...so...', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
 
         The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
@@ -529,7 +497,6 @@ class Corefile(ELF):
         >>> io = elf.process()
         >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -588,6 +555,9 @@ class Corefile(ELF):
         # Pointer to the entry point
         self.at_entry = 0
 
+        # Pointer to the vdso
+        self.at_sysinfo_ehdr = None
+
         try:
             super(Corefile, self).__init__(*a, **kw)
         except IOError:
@@ -604,7 +574,6 @@ class Corefile(ELF):
             log.warn_once("%s does not use a supported corefile architecture, registers are unavailable" % self.file.name)
 
         prstatus_type = prstatus_types.get(self.arch)
-        prpsinfo_type = prpsinfo_types.get(self.bits)
         siginfo_type = siginfo_types.get(self.bits)
 
         with log.waitfor("Parsing corefile...") as w:
@@ -615,45 +584,38 @@ class Corefile(ELF):
                     continue
 
 
-                # Note that older versions of pyelftools (<=0.24) are missing enum values
-                # for NT_PRSTATUS, NT_PRPSINFO, NT_AUXV, etc.
-                # For this reason, we have to check if note.n_type is any of several values.
-                for note in iter_notes(segment):
-                    if not isinstance(note.n_desc, bytes):
-                        note['n_desc'] = note.n_desc.encode('latin1')
+                for note in segment.iter_notes():
                     # Try to find NT_PRSTATUS.
-                    if prstatus_type and \
-                       note.n_descsz == ctypes.sizeof(prstatus_type) and \
-                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRSTATUS'):
+                    if note.n_type == 'NT_PRSTATUS':
                         self.NT_PRSTATUS = note
                         self.prstatus = prstatus_type.from_buffer_copy(note.n_desc)
 
                     # Try to find NT_PRPSINFO
-                    if prpsinfo_type and \
-                       note.n_descsz == ctypes.sizeof(prpsinfo_type) and \
-                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRPSINFO'):
+                    if note.n_type == 'NT_PRPSINFO':
                         self.NT_PRPSINFO = note
-                        self.prpsinfo = prpsinfo_type.from_buffer_copy(note.n_desc)
+                        self.prpsinfo = note.n_desc
 
                     # Try to find NT_SIGINFO so we can see the fault
-                    if note.n_type in (0x53494749, 'NT_SIGINFO'):
+                    if note.n_type == 'NT_SIGINFO':
                         self.NT_SIGINFO = note
                         self.siginfo = siginfo_type.from_buffer_copy(note.n_desc)
 
                     # Try to find the list of mapped files
-                    if note.n_type in (constants.NT_FILE, 'NT_FILE'):
+                    if note.n_type == 'NT_FILE':
                         with context.local(bytes=self.bytes):
                             self._parse_nt_file(note)
 
                     # Try to find the auxiliary vector, which will tell us
                     # where the top of the stack is.
-                    if note.n_type in (constants.NT_AUXV, 'NT_AUXV'):
+                    if note.n_type == 'NT_AUXV':
                         self.NT_AUXV = note
                         with context.local(bytes=self.bytes):
                             self._parse_auxv(note)
 
             if not self.stack and self.mappings:
                 self.stack = self.mappings[-1].stop
+                if self.mappings[-1].start == 0xffffffffff600000 and len(self.mappings) > 1:
+                    self.stack = self.mappings[-2].stop
 
             if self.stack and self.mappings:
                 for mapping in self.mappings:
@@ -665,7 +627,7 @@ class Corefile(ELF):
                     log.warn('Could not find the stack!')
                     self.stack = None
 
-            with context.local(bytes=self.bytes, log_level='warn'):
+            with context.local(bytes=self.bytes):
                 try:
                     self._parse_stack()
                 except ValueError:
@@ -683,31 +645,16 @@ class Corefile(ELF):
             self._describe_core()
 
     def _parse_nt_file(self, note):
-        t = tube()
-        t.unrecv(note.n_desc)
-
-        count = t.unpack()
-        page_size = t.unpack()
-
         starts = []
         addresses = {}
 
-        for i in range(count):
-            start = t.unpack()
-            end = t.unpack()
-            offset = t.unpack()
-            starts.append((start, offset))
-
-        for i in range(count):
-            filename = t.recvuntil(b'\x00', drop=True)
+        for vma, filename in zip(note.n_desc.Elf_Nt_File_Entry, note.n_desc.filename):
             if not isinstance(filename, str):
-                filename = filename.decode('utf-8')
-            (start, offset) = starts[i]
-
+                filename = filename.decode('utf-8', 'surrogateescape')
             for mapping in self.mappings:
-                if mapping.start == start:
+                if mapping.start == vma.vm_start:
                     mapping.name = filename
-                    mapping.page_offset = offset
+                    mapping.page_offset = vma.page_offset
 
         self.mappings = sorted(self.mappings, key=lambda m: m.start)
 
@@ -758,7 +705,7 @@ class Corefile(ELF):
     @property
     def libc(self):
         """:class:`Mapping`: First mapping for ``libc.so``"""
-        expr = r'libc\b.*so$'
+        expr = r'^libc\b.*so(?:\.6)?$'
 
         for m in self.mappings:
             if not m.name:
@@ -1183,14 +1130,13 @@ class Corefile(ELF):
         pwnlib.gdb.attach(self, exe=self.exe.path)
 
     def __getattr__(self, attribute):
-        if self.prstatus:
-            if hasattr(self.prstatus, attribute):
-                return getattr(self.prstatus, attribute)
+        if attribute.startswith('_') or not self.prstatus:
+            raise AttributeError(attribute)
 
-            if hasattr(self.prstatus.pr_reg, attribute):
-                return getattr(self.prstatus.pr_reg, attribute)
+        if hasattr(self.prstatus, attribute):
+            return getattr(self.prstatus, attribute)
 
-        return super(Corefile, self).__getattribute__(attribute)
+        return getattr(self.prstatus.pr_reg, attribute)
 
     # Override routines which don't make sense for Corefiles
     def _populate_got(*a): pass
@@ -1388,6 +1334,10 @@ class CorefileFinder(object):
         except Exception:
             pass
 
+        # Convert bytes-like object to string
+        if isinstance(data, bytes):
+            data = data.decode('utf-8')
+
         return data
 
     def systemd_coredump_corefile(self):
@@ -1406,9 +1356,8 @@ class CorefileFinder(object):
                     "coredumpctl",
                     "dump",
                     "--output=%s" % filename,
-                    # Filter coredump by pid and filename
+                    # Filter coredump by pid
                     str(self.pid),
-                    self.basename,
                 ],
                 stdout=open(os.devnull, 'w'),
                 stderr=subprocess.STDOUT,
@@ -1456,6 +1405,10 @@ class CorefileFinder(object):
                     f.write(apport_core)
                 return filename
 
+            filename = self.apport_coredump()
+            if filename:
+                return filename
+
             # Pretend core_pattern was just 'core', and see if we come up with anything
             self.kernel_core_pattern = 'core'
             return self.native_corefile_pattern()
@@ -1545,6 +1498,53 @@ class CorefileFinder(object):
         for corefile in sorted(glob.glob(corefile_path), reverse=True):
             return corefile
 
+    def apport_coredump(self):
+        """Find new-style apport coredump of executables not belonging
+        to a system package
+        """
+        # Now Ubuntu, which is the most silly distro of all, doesn't follow
+        # anybody else's rules either...
+        # ...and it uses apport FROM SOME OTHER REPO THAN THE DOCS SAY
+        # Hey, thanks for making our lives easier, Canonical :----)
+        # Seriously, why is Ubuntu even considered to be the default distro
+        # on GH Actions?
+        #
+        #     core.<_path_to_target_binary>.<uid>.<boot_id>.<pid>.<timestamp>
+        #
+        # Note that we don't give any fucks about the timestamp, since the PID
+        # should be unique enough that we can just glob.
+
+        boot_id = read('/proc/sys/kernel/random/boot_id').strip().decode()
+
+        # Use the absolute path of the executable
+        # Apport uses the executable's path to determine the core dump filename
+        #
+        # Reference source:
+        # https://github.com/canonical/apport/blob/4bbb179b8f92989bf7c1ee3692074f35d70ef3e8/data/apport#L110
+        # https://github.com/canonical/apport/blob/4bbb179b8f92989bf7c1ee3692074f35d70ef3e8/apport/fileutils.py#L599
+        #
+        # Apport calls `get_core_path` with `options.executable_path`, which corresponds to
+        # the executable's pathname, as specified by the `%E` placeholder
+        # in the core pattern (see `man core` and `apport --help`).
+        path = os.path.abspath(self.exe).replace('/', '_').replace('.', '_')
+
+        # Format the name
+        corefile_name = 'core.{path}.{uid}.{boot_id}.{pid}.*'.format(
+            path=path,
+            uid=self.uid,
+            boot_id=boot_id,
+            pid=self.pid,
+        )
+
+        # Get the full path
+        corefile_path = os.path.join('/var/lib/apport/coredump', corefile_name)
+
+        log.debug("Trying corefile_path: %r" % corefile_path)
+
+        # Glob all of them, return the *most recent* based on numeric sort order.
+        for corefile in sorted(glob.glob(corefile_path), reverse=True):
+            return corefile
+
     def binfmt_lookup(self):
         """Parses /proc/sys/fs/binfmt_misc to find the interpreter for a file"""