Inform GDB of file when using QEMU (#1630)

* Inform GDB of file when using QEMU

Fixes #1625

* Sync CHANGELOG from dev + version bump

Co-authored-by: Heap Crash <[email protected]>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: backport-required, bug
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!
+
+## Update Pwntools First
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+## Debug Output
+
+Having the extra debug output really helps us, and might help you diagnose the problem yourself.
+
+When submitting an issue that has output from Pwntools, make sure to run your script as shown below, to enable the extra debugging data.
+
+```sh
+$ python exploit.py DEBUG LOG_FILE=log.txt
+```
+
+You should see `[DEBUG]` statements that show what's happening behind the scenes:
+
+```
+[+] Starting local process '/bin/sh' argv=['sh'] : pid 16823
+[DEBUG] Sent 0x7 bytes:
+    'whoami\n'
+[DEBUG] Sent 0x5 bytes:
+    'exit\n'
+[+] Receiving all data: Done (11B)
+[DEBUG] Received 0xb bytes:
+    'crashheap\n'
+[*] Process '/bin/sh' stopped with exit code 0 (pid 16823)
+```
+
+## Verify on Ubuntu
+
+If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+
+```sh
+# Download the Docker image
+$ docker pull pwntools/pwntools:stable
+
+# Boot the image
+$ docker run -it pwntools/pwntools:stable
+
+pwntools@7dc3ef409476:~$ python
+>>> from pwn import *
+>>> # Test your code here
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!   Ideas from the community help make Pwntools an amazing tool for everybody.
+
+If you've got an idea for a new feature, please provide information about:
+
+* What the feature does
+* Why the feature should exit
+* What tests should be included
+
+If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask us a question about Pwntools
+title: ''
+labels: question
+assignees: ''
+
+---
+
+Thanks for using Pwntools!  If you have a question, here are some tips that will make it easier for us to help you.
+
+## Check the Documentation
+
+Pwntools has REALLY extensive documentation.  Please check there first!  https://docs.pwntools.com
+
+## Debug Output
+
+If you're having an issue with an exploit, run your script with full debugging output, i.e. `python exploit.py DEBUG`.  This may help you to see what's happening under the hood, and the debug output will help to diagnose any issues you're encountering.

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,20 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md

.github/workflows/ci.yml

@@ -8,8 +8,15 @@ jobs:
         python-version: [2.7, 3.8]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Git History 2
+      run: git log --oneline --graph -10
+
     - name: Cache for pip
       uses: actions/cache@v1
       id: cache-pip
@@ -17,60 +24,69 @@ jobs:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
-    - name: Cache for dependencies
-      uses: actions/cache@v1
-      id: cache-deps
-      with:
-        path: android-?dk
-        key: ${{ matrix.os }}-cache-deps
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Lint
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
       run: |
-        pip install flake8
-        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
+
+    - name: Detect whether java is needed
+      id: java-needed
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
+      run: |
+        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
+          echo "Found release or important branch ($GITHUB_REF), forcing tests."
+        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
+          echo 'Incorrect commit range, forcing android tests.'
+        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
+          echo 'Found Android-related commits, forcing tests.'
+        else
+          # clear files that cause Android doctests
+          : > docs/source/adb.rst > docs/source/protocols/adb.rst
+          exit 0
+        fi
+        echo ::set-output name=need::openjdk-8-jre-headless
 
     - name: Install Linux dependencies
+      env:
+        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
       run: |
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          pandoc gdb socat sshpass \
+          pandoc gdb gdbserver socat sshpass \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
           binutils-powerpc-linux-gnu \
           gcc-multilib \
-          openjdk-8-jre-headless
-        sudo apt-get install -y -o Acquire::Retries=3 \
-          gcc-aarch64-linux-gnu \
-          gcc-arm-linux-gnueabihf
+          libc6-dbg \
+          $ANDROID_JRE
 
     - name: Install android avd
-      if:  steps.cache-deps.outputs.cache-hit != 'true'
+      if:  steps.java-needed.outputs.need
       run: |
         USER=travis source travis/install.sh
-        adb emu kill
-        set | egrep '^(ANDROID|PATH)' >android-sdk/.android.env
+        set | egrep '^(ANDROID|PATH)' >.android.env
 
     - name: Set up SSH
       run: |
-        chmod og-rw ~ # see https://stackoverflow.com/a/60367309/3869724
-        ssh-keygen -t ed25519 -f ~/.ssh/pwntools-ci -N ''
-        cat > ~/.ssh/config <<EOF
-        Host example.pwnme
-            User $USER
-            HostName 127.0.0.1
-            IdentityFile ~/.ssh/pwntools-ci
-        EOF
-        echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/pwntools-ci.pub > ~/.ssh/authorized_keys
-        ssh -o 'StrictHostKeyChecking no' example.pwnme id
+        travis/ssh_setup.sh
 
     - name: Install dependencies
       run: |
@@ -85,34 +101,121 @@ jobs:
     - name: Install documentation dependencies
       run:  pip install -r docs/requirements.txt
 
+    - name: Disable broken tests
+      run: |
+        rm -f docs/source/elf/corefile.rst
+
+    - name: Manually install non-broken Unicorn
+      run:  pip install unicorn==1.0.2rc3
+
     - name: Coverage doctests
       run: |
-        source android-sdk/.android.env
-        android-sdk/emulator/emulator -avd android-$ANDROID_ABI -no-window -no-boot-anim -read-only -no-audio -no-window -no-snapshot &
-        adb wait-for-device
+        source .android.env || :
         echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
         PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples
       run: |
         export TERM=linux
+        set -x
         python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
-        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
         coverage run examples/asm.py
         coverage run examples/asm.py
         coverage run examples/text.py
         # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
 
+    - name: Coverage running commandline tools
+      run: |
+        export TERM=linux
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        set -x
+
+        pwn cyclic 32
+        pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
+
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
+
+        pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
+
+        pwn hex abcd
+        pwn unhex 4141 4141
+
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
+
+        pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
+
+        pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
+        pwn constgrep ^MAP_ 0
+        pwn constgrep -e O_RDWR
+
     - name: Build source and wheel distributions
       run: |
         python setup.py sdist
         python setup.py bdist_wheel --universal
 
-    - uses: actions/upload-artifact@v2-preview
+    - uses: actions/upload-artifact@v2
       with:
-        path: dist/*
+        name: packages
+        path: dist/
 
     - name: Upload coverage to coveralls.io
       run: |
         coverage combine
         COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
+
+  staging-merge:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
+    - name: Push changes to protected branch
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        BRANCH=${GITHUB_REF#refs/heads/}
+        TARGET=${BRANCH%-staging}
+        git branch -f "$TARGET"
+        git push origin "$TARGET"
+
+  pypi:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: build
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v2
+      with:
+        name: packages
+        path: dist
+
+    - name: Publish package
+      uses: pypa/[email protected]
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}
+
+    - if: failure()
+      run: ls -R

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__

.github/workflows/merge-conflict.yml

@@ -0,0 +1,28 @@
+name: Check for merge markers
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  mergecheck:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check for merge conflict markers commited
+      run: |
+        if git show ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+\+(>>>>>|=====|<<<<<)'; then
+            echo 'WARNING: Merge marker commits found in history.'
+            if git diff ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+(>>>>>|=====|<<<<<)'; then
+                echo 'ERROR: Merge markers still present! Do something!'
+                exit 1
+            fi
+        else
+            echo 'Congrats! All the merges you did were clean or resolved correctly :)'
+        fi

.travis.yml

@@ -2,7 +2,6 @@ language: python
 addons:
   apt:
     packages:
-    - gcc-multilib
     - lib32stdc++6
     - ash
     - bash
@@ -21,6 +20,7 @@ addons:
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
     - binutils-powerpc-linux-gnu
+    - bash-static
 cache:
     - pip
     - directories:
@@ -54,7 +54,6 @@ script:
   # stop the build if there are Python syntax errors or undefined names
   - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
   # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-  - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
   - coverage combine
 after_success:

CHANGELOG.md

@@ -9,12 +9,20 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.3.0](#430)    | `dev`    | Jun 30, 2020 (planned)
-| [4.2.0](#420)    | `beta`   | May 30, 2020 (planned)
-| [4.1.0](#410)    | `stable` | May 8, 2020
+| [4.4.0](#440)    | `dev`    | Sep 1, 2020 (planned)
+| [4.3.0](#430)    | `beta`   | Aug 1, 2020 (planned)
+| [4.2.0](#420)    | `stable` | Jul 3, 2020 (planned)
+| [4.1.7](#417)    |          | Jun 30, 2020
+| [4.1.5](#415)    |          | Jun 27, 2020
+| [4.1.4](#414)    |          | Jun 26, 2020
+| [4.1.3](#413)    |          | Jun 23, 2020
+| [4.1.2](#412)    |          | Jun 5, 2020
+| [4.1.1](#411)    |          | Jun 3, 2020
+| [4.1.0](#410)    |          | May 8, 2020
 | [4.0.1](#401)    |          | Jan 22, 2020
 | [4.0.0](#400)    |          | Jan 09, 2020
 | [3.13.0](#3130)  |          | Nov 5, 2019
+| [3.12.2](#3122)  |          | Jan 8, 2019
 | [3.12.1](#3121)  |          | Sept 17, 2018
 | [3.12.0](#3120)  |          | Feb 22, 2018
 | [3.11.0](#3110)  |          | Jan 3, 2018
@@ -47,23 +55,92 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.3.0 (`dev`)
+## 4.4.0 (`dev`)
 
-To be released on Jun 30, 2020.
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
 
-## 4.2.0 (`beta`)
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
 
-To be released on May 30, 2020.
+## 4.3.0 (`beta`)
 
-- #1436 Add ret2dlresolve automation
+- [#1576][1576] Add `executable=` argument to `ELF.search`
+- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
+- [#1592][1592] Fix over-verbose logging of process() environment
+- [#1593][1593] Colorize output of `pwn template`
+- [#1601][1601] Add `pwn version` command line tool
+- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
+- [#1613][1613] Permit `--password` for `pwn template`
+- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
+- [#1621][1621] Permit negative values in flat() and fit()
+
+[1576]: https://github.com/Gallopsled/pwntools/pull/1576
+[1584]: https://github.com/Gallopsled/pwntools/pull/1584
+[1592]: https://github.com/Gallopsled/pwntools/pull/1592
+[1593]: https://github.com/Gallopsled/pwntools/pull/1593
+[1601]: https://github.com/Gallopsled/pwntools/pull/1601
+[1605]: https://github.com/Gallopsled/pwntools/pull/1605
+[1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1564]: https://github.com/Gallopsled/pwntools/pull/1564
+[1621]: https://github.com/Gallopsled/pwntools/pull/1621
+
+## 4.2.1 (`stable`)
+
+- [#1625][1625] GDB now properly loads executables with QEMU
+
+[1625]: https://github.com/Gallopsled/pwntools/pull/1625
+
+## 4.2.0
+
+- [#1436][1436] Add ret2dlresolve automation
 - [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
 - Miscellanous improvements to DynElf and fmtstr leaker (see examples/fmtstr/exploit2.py)
-- #1454 Support for windows console colors
+- [#1454][1454] Support for windows console colors
 
-[fecf9f]: http://github.com/Gallopsled/pwntols/commit/fecf9f
+[1436]: https://github.com/Gallopsled/pwntools/pull/1436
+[fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
+[1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
+## 4.1.7 (`stable`)
 
-## 4.1.0 (`stable`)
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
+
+- Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
+- Fix wrong ELF/context unpack handling (c4c11a37)
+- Fix updating of ELF.functions addresses after changing ELF.address #1512 (#1513)
+- Update Corefile warnings and replace asserts with normal checks (#1526)
+- several py2-py3 issues (#1451)
+- Fix cyclic command
+
+## 4.1.0
 
 - [#1316][1316] Fix connect shellcraft in python 3
 - [#1323][1323] Fix issues related with debugging
@@ -72,15 +149,19 @@ To be released on May 30, 2020.
 - [#1241][1241] Launch QEMU with sysroot if specified
 - [#1218][1218] Support for FileStructure exploitation
 
-[1241]: https://github.com/Gallopsled/pwntools/pulls/1218
-[1218]: https://github.com/Gallopsled/pwntools/pulls/1218
+[1316]: https://github.com/Gallopsled/pwntools/pull/1316
+[1323]: https://github.com/Gallopsled/pwntools/pull/1323
+[1001]: https://github.com/Gallopsled/pwntools/pull/1001
+[1389]: https://github.com/Gallopsled/pwntools/pull/1389
+[1241]: https://github.com/Gallopsled/pwntools/pull/1241
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
 
 ## 4.0.1
 
 - [#1412][1412] `recvline_pred()` and similar do not reorder data
 - Bypass unicorn-engine/unicorn#1100 and unicorn-engine/unicorn#1170 requiring unstable package
 
-[1412]: https://github.com/Gallopsled/pwntools/pulls/1412
+[1412]: https://github.com/Gallopsled/pwntools/pull/1412
 
 ## 4.0.0
 
@@ -94,12 +175,13 @@ To be released on May 30, 2020.
 - [#1202][1202] Docker: Kill 14 layers in pwntools base images
 - [#1182][1182] shellcraft.dupio() for mips
 
-[1391]: https://github.com/Gallopsled/pwntools/pulls/1391
-[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
-[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
-[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
-[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
-[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[1402]: https://github.com/Gallopsled/pwntools/pull/1402
+[1391]: https://github.com/Gallopsled/pwntools/pull/1391
+[1317]: https://github.com/Gallopsled/pwntools/pull/1317
+[1285]: https://github.com/Gallopsled/pwntools/pull/1285
+[1216]: https://github.com/Gallopsled/pwntools/pull/1216
+[1202]: https://github.com/Gallopsled/pwntools/pull/1202
+[1182]: https://github.com/Gallopsled/pwntools/pull/1182
 [76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
 
 ## 3.13.0
@@ -110,32 +192,16 @@ To be released on May 30, 2020.
 - [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
 - [#1292][1292] Provide correct arch name to gdb for sparc64
 
-[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
-[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
-[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
-[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
-[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
-
-## 3.12.1
-
-- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
-- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
-- [5fdc08][5fdc08] Work around broken `pidof` on Android
-- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
-- [#1094][1094] Make hexdump output alignment more consistent
-- [#1096][1096] `flat()` and `fit()` are now the same function
-
-[1104]: https://github.com/Gallopsled/pwntools/pull/1104
-[1101]: https://github.com/Gallopsled/pwntools/pull/1101
-[1094]: https://github.com/Gallopsled/pwntools/pull/1094
-[1096]: https://github.com/Gallopsled/pwntools/pull/1096
-[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
-[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+[1175]: https://github.com/Gallopsled/pwntools/pull/1175
+[1204]: https://github.com/Gallopsled/pwntools/pull/1204
+[1277]: https://github.com/Gallopsled/pwntools/pull/1277
+[1292]: https://github.com/Gallopsled/pwntools/pull/1292
+[1355]: https://github.com/Gallopsled/pwntools/pull/1355
 
 ## 3.12.2
 
-- [1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
-- [1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
+- [#1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
+- [#1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
 
 [1242]: https://github.com/Gallopsled/pwntools/pull/1242
 [1243]: https://github.com/Gallopsled/pwntools/pull/1243
@@ -153,7 +219,15 @@ To be released on May 30, 2020.
 - [#1131][1131] Add "libc-" to libc prefixes in `process` tubes
 - [#1125][1125] Fix a typo
 - [#1121][1121] Fix tests which were broken by an upstream Sphinx change
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
 
+[1198]: https://github.com/Gallopsled/pwntools/pull/1198
+[1191]: https://github.com/Gallopsled/pwntools/pull/1191
 [1159]: https://github.com/Gallopsled/pwntools/pull/1159
 [1162]: https://github.com/Gallopsled/pwntools/pull/1162
 [1150]: https://github.com/Gallopsled/pwntools/pull/1150
@@ -163,6 +237,12 @@ To be released on May 30, 2020.
 [1131]: https://github.com/Gallopsled/pwntools/pull/1131
 [1125]: https://github.com/Gallopsled/pwntools/pull/1125
 [1121]: https://github.com/Gallopsled/pwntools/pull/1121
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
 
 ## 3.12.0
 

docs/source/about.rst

@@ -47,7 +47,7 @@ This module is our "clean" python-code. As a rule, we do not think that
 importing :mod:`pwnlib` or any of the submodules should have any significant
 side-effects (besides e.g. caching).
 
-For the most part, you will also only get the bits you import. You for instance
+For the most part, you will also only get the bits you import. You for instance would
 not get access to :mod:`pwnlib.util.packing` simply by doing ``import
 pwnlib.util``.
 

docs/source/conf.py

@@ -88,6 +88,9 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
 '''
 
 autoclass_content = 'both'

docs/source/install.rst

@@ -25,9 +25,9 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get update
-    $ apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-    $ pip install --upgrade pip
-    $ pip install --upgrade pwntools
+    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ python3 -m pip install --upgrade pip
+    $ python3 -m pip install --upgrade pwntools
 
 Development
 --------------

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

pwnlib/commandline/asm.py

@@ -5,6 +5,9 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 

pwnlib/commandline/cyclic.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from __future__ import division
 
 import argparse
+import six
 import string
 import sys
 
@@ -21,7 +22,7 @@ parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
     default = string.ascii_lowercase.encode(),
-    type = bytes,
+    type = six.ensure_binary,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -42,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -52,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -65,9 +67,10 @@ def main(args):
         pat = args.lookup
 
         try:
-            pat = packing.pack(int(pat, 0), subsize*8)
+            pat = int(pat, 0)
         except ValueError:
             pass
+        pat = flat(pat)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -88,7 +91,7 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
         out = getattr(sys.stdout, 'buffer', sys.stdout)

pwnlib/commandline/errno.py

@@ -45,4 +45,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/phd.py

@@ -85,12 +85,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/unhex.py

@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/data/elf/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
 
 import os

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/fmtstr/vuln.c

@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11111000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11111110)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

pwnlib/elf/corefile.py

@@ -945,16 +945,11 @@ class Corefile(ELF):
 
         # Sanity check!
         try:
-            assert stack[address] == b'\x00'
-        except AssertionError:
-            # Something weird is happening.  Just don't touch it.
-            log.debug("Something is weird")
-            return
+            if stack[address] != b'\x00':
+                log.warning("Error parsing corefile stack: Could not find end of environment")
+                return
         except ValueError:
-            # If the stack is not actually present in the coredump, we can't
-            # read from the stack.  This will fail as:
-            # ValueError: 'seek out of range'
-            log.debug("ValueError")
+            log.warning("Error parsing corefile stack: Address out of bounds")
             return
 
         # address is currently set to the NULL terminator of the last
@@ -968,12 +963,14 @@ class Corefile(ELF):
         p_last_env_addr = stack.find(pack(last_env_addr), None, last_env_addr)
         if p_last_env_addr < 0:
             # Something weird is happening.  Just don't touch it.
-            log.warn_once("Found bad environment at %#x", last_env_addr)
+            log.warn_once("Error parsing corefile stack: Found bad environment at %#x", last_env_addr)
             return
 
         # Sanity check that we did correctly find the envp NULL terminator.
         envp_nullterm = p_last_env_addr+context.bytes
-        assert self.unpack(envp_nullterm) == 0
+        if self.unpack(envp_nullterm) != 0:
+            log.warning("Error parsing corefile stack: Could not find end of environment variables")
+            return
 
         # We've successfully located the end of the envp[] array.
         #
@@ -1535,4 +1532,3 @@ class CorefileFinder(object):
                 return keys['interpreter']
 
         return ''
-

pwnlib/elf/elf.py

@@ -592,6 +592,8 @@ class ELF(ELFFile):
         self.symbols = dotdict({k:update(v) for k,v in self.symbols.items()})
         self.plt     = dotdict({k:update(v) for k,v in self.plt.items()})
         self.got     = dotdict({k:update(v) for k,v in self.got.items()})
+        for f in self.functions.values():
+            f.address += delta
 
         # Update our view of memory
         memory = intervaltree.IntervalTree()
@@ -1927,7 +1929,7 @@ class ELF(ELFFile):
     def unpack(self, address, *a, **kw):
         """Unpacks an integer from the specified ``address``."""
         self._update_args(kw)
-        return packing.unpack(self.read(address, context.bytes), *a, **kw)
+        return packing.unpack(self.read(address, self.bytes), *a, **kw)
 
     def string(self, address):
         """string(address) -> str

pwnlib/fmtstr.py

@@ -1,39 +1,39 @@
 r"""
 Provide some tools to exploit format string bug
 
-Examples:
+Let's use this program as an example:
 
-    >>> program = tempfile.mktemp()
-    >>> source  = program + ".c"
-    >>> write(source, '''
-    ... #include <stdio.h>
-    ... #include <stdlib.h>
-    ... #include <unistd.h>
-    ... #include <sys/mman.h>
-    ... #define MEMORY_ADDRESS ((void*)0x11111000)
-    ... #define MEMORY_SIZE 1024
-    ... #define TARGET ((int *) 0x11111110)
-    ... int main(int argc, char const *argv[])
-    ... {
-    ...        char buff[1024];
-    ...        void *ptr = NULL;
-    ...        int *my_var = TARGET;
-    ...        ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
-    ...        if(ptr != MEMORY_ADDRESS)
-    ...        {
-    ...                perror("mmap");
-    ...                return EXIT_FAILURE;
-    ...        }
-    ...        *my_var = 0x41414141;
-    ...        write(1, &my_var, sizeof(int *));
-    ...        scanf("%s", buff);
-    ...        dprintf(2, buff);
-    ...        write(1, my_var, sizeof(int));
-    ...        return 0;
-    ... }''')
-    >>> gcc = process(["gcc", source, "-Wno-format-security", "-m32", "-o", program])
-    >>> gcc.poll(True) and gcc.recvall()
-    0
+::
+
+    #include <stdio.h>
+    #include <stdlib.h>
+    #include <unistd.h>
+    #include <sys/mman.h>
+    #define MEMORY_ADDRESS ((void*)0x11111000)
+    #define MEMORY_SIZE 1024
+    #define TARGET ((int *) 0x11111110)
+    int main(int argc, char const *argv[])
+    {
+           char buff[1024];
+           void *ptr = NULL;
+           int *my_var = TARGET;
+           ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+           if(ptr != MEMORY_ADDRESS)
+           {
+                   perror("mmap");
+                   return EXIT_FAILURE;
+           }
+           *my_var = 0x41414141;
+           write(1, &my_var, sizeof(int *));
+           scanf("%s", buff);
+           dprintf(2, buff);
+           write(1, my_var, sizeof(int));
+           return 0;
+    }
+
+We can automate the exploitation of the process like so:
+
+    >>> program = pwnlib.data.elf.fmtstr.get('i386')
     >>> def exec_fmt(payload):
     ...     p = process(program)
     ...     p.sendline(payload)

pwnlib/gdb.py

@@ -110,6 +110,7 @@ from pwnlib.asm import make_elf_from_assembly
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
+from pwnlib.timeout import Timeout
 from pwnlib.util import misc
 from pwnlib.util import proc
 
@@ -384,8 +385,8 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
     by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.
 
     >>> # Connect to the SSH server
-    >>> shell = ssh('runner', 'example.pwnme', password='demopass')
     >>> # Start a process on the server
+    >>> shell = ssh('travis', 'example.pwnme', password='demopass')
     >>> io = gdb.debug(['bash'],
     ...                 ssh = shell,
     ...                 gdbscript = '''
@@ -544,59 +545,62 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
     Examples:
 
-    >>> # Attach directly to pid 1234
-    >>> gdb.attach(1234) # doctest: +SKIP
+        Attach to a process by PID
 
+        >>> pid = gdb.attach(1234) # doctest: +SKIP
 
-    >>> # Attach to the youngest "bash" process
-    >>> gdb.attach('bash') # doctest: +SKIP
+        Attach to the youngest process by name
 
+        >>> pid = gdb.attach('bash') # doctest: +SKIP
 
-    >>> # Start a process
-    >>> bash = process('bash')
-    >>> # Attach the debugger
-    >>> pid = gdb.attach(bash, '''
-    ... set follow-fork-mode child
-    ... break execve
-    ... continue
-    ... ''')
-    >>> # Interact with the process
-    >>> bash.sendline("whoami")
-    >>> bash.recvline()
-    b'runner\n'
-    >>> bash.close()
+        Attach a debugger to a :class:`.process` tube and automate interaction
 
-    >>> # Start a forking server
-    >>> server = process(['socat', 'tcp-listen:12345,fork,reuseaddr', 'exec:/bin/bash,nofork'])
-    >>> sleep(1)
-    >>> # Connect to the server
-    >>> io = remote('127.0.0.1', 12345)
-    >>> # Connect the debugger to the server-spawned process
-    >>> pid = gdb.attach(io, '''
-    ... break exit
-    ... continue
-    ... ''', exe = '/bin/bash')
-    >>> # Talk to the spawned 'bash'
-    >>> io.sendline("echo hello")
-    >>> io.recvline()
-    b'hello\n'
-    >>> io.sendline("exit")
-    >>> io.close()
+        >>> io = process('bash')
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from process debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from process debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
 
-    >>> # Connect to the SSH server
-    >>> shell = ssh('runner', 'example.pwnme', password='demopass')
-    >>> # Start a process on the server
-    >>> cat = shell.process(['cat'])
-    >>> # Attach a debugger to it
-    >>> gdb.attach(cat, '''
-    ... break exit
-    ... continue
-    ... ''')
-    >>> cat.sendline("hello")
-    >>> cat.recvline()
-    b'hello\n'
-    >>> # Cause `cat` to exit
-    >>> cat.close()
+        Attach to the remote process from a :class:`.remote` or :class:`.listen` tube,
+        as long as it is running on the same machine.
+
+        >>> server = process(['socat', 'tcp-listen:12345,reuseaddr,fork', 'exec:/bin/bash,nofork'])
+        >>> sleep(1) # Wait for socat to start
+        >>> io = remote('127.0.0.1', 12345)
+        >>> sleep(1) # Wait for process to fork
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from remote debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from remote debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
+
+        Attach to processes running on a remote machine via an SSH :class:`.ssh` process
+        
+        >>> shell = ssh('travis', 'example.pwnme', password='demopass')
+        >>> io = shell.process(['cat'])
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call sleep(5)
+        ... call puts("Hello from ssh debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline(timeout=5)  # doctest: +SKIP
+        b'Hello from ssh debugger!\n'
+        >>> io.sendline('This will be echoed back')
+        >>> io.recvline()
+        b'This will be echoed back\n'
+        >>> io.close()
     """
     if context.noptrace:
         log.warn_once("Skipping debug attach since context.noptrace==True")
@@ -627,6 +631,9 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         if context.os == 'android':
             pre += 'set gnutarget ' + _bfdname() + '\n'
 
+        if exe:
+            pre += 'file %s\n' % exe
+
     # let's see if we can find a pid to attach to
     pid = None
     if   isinstance(target, six.integer_types):
@@ -675,18 +682,24 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
     elif isinstance(target, tubes.sock.sock):
         pids = proc.pidof(target)
         if not pids:
-            log.error('could not find remote process (%s:%d) on this machine' %
+            log.error('Could not find remote process (%s:%d) on this machine' %
                       target.sock.getpeername())
-        waiting = 100
-        if exe:
-            while waiting:
-                waiting -= 1
-                for pid in pids:
-                    if proc.exe(pid) == exe:
-                        waiting = False
-                        break
-                else:
-                    time.sleep(0.01)
+        pid = pids[0]
+
+        # Specifically check for socat, since it has an intermediary process
+        # if you do not specify "nofork" to the EXEC: argument
+        # python(2640)───socat(2642)───socat(2643)───bash(2644)
+        if proc.exe(pid).endswith('/socat') and time.sleep(0.1) and proc.children(pid):
+            pid = proc.children(pid)[0]
+
+        # We may attach to the remote process after the fork but before it performs an exec.  
+        # If an exe is provided, wait until the process is actually running the expected exe
+        # before we attach the debugger.
+        t = Timeout()
+        with t.countdown(2):
+            while exe and os.realpath(proc.exe(pid)) != os.realpath(exe) and t.timeout:
+                time.sleep(0.1)
+
     elif isinstance(target, tubes.process.process):
         pid = proc.pidof(target)[0]
         exe = exe or target.executable
@@ -855,7 +868,7 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
     Example:
 
     >>> with context.local(log_level=9999):
-    ...     shell =  ssh(host='example.pwnme', user='runner', password='demopass')
+    ...     shell =  ssh(host='example.pwnme', user='travis', password='demopass')
     ...     bash_libs = gdb.find_module_addresses('/bin/bash', shell)
     >>> os.path.basename(bash_libs[0].path)
     'libc.so.6'
@@ -934,13 +947,23 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
     return rv
 
 def corefile(process):
-    r"""Drops a core file for the process.
+    r"""Drops a core file for a running local process.
+
+    Note:
+        You should use :meth:`.process.corefile` instead of using this method directly.
 
     Arguments:
         process: Process to dump
 
     Returns:
         :class:`.Core`: The generated core file
+
+    Example:
+
+        >>> io = process('bash')
+        >>> core = gdb.corefile(io)
+        >>> core.exe.name
+        '/bin/bash'
     """
 
     if context.noptrace:

pwnlib/gdb_faketerminal.py

@@ -3,7 +3,10 @@ from pwnlib.tubes.process import process
 from time import sleep
 from sys import argv
 sleep(1)
-sh = process(argv[1], shell=True)
+if len(argv) == 2:
+	sh = process(argv[1], shell=True)
+else:
+	sh = process(argv[1:])
 sh.sendline('set prompt (gdb)')
 res = sh.sendlineafter('(gdb)', 'c')
 while b'The program is not being run.' not in res:

pwnlib/rop/rop.py

@@ -977,7 +977,10 @@ class ROP(object):
                 SYS_sigreturn  = constants.SYS_rt_sigreturn
 
             for register, value in zip(frame.arguments, arguments):
-                frame[register] = value
+                if not isinstance(value, six.integer_types + (Unresolved,)):
+                    frame[register] = AppendedArgument(value)
+                else:
+                    frame[register] = value
 
         # Set up a call frame which will set EAX and invoke the syscall
         call = Call('SYS_sigreturn',

pwnlib/shellcraft/templates/aarch64/linux/loader_append.asm

@@ -17,18 +17,11 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    >>> gcc = process(['aarch64-linux-gnu-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "world");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-    >>> run_assembly(sc).recvline()
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
     b'Hello, world!\n'
 
 </%docstring>
@@ -47,7 +40,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/aarch64/pushstr.asm

@@ -15,64 +15,19 @@ Args:
 
 Examples:
 
-    >>> print(shellcraft.pushstr("Hello!").rstrip())
-        /* push b'Hello!\x00' */
-        /* Set x14 = 36762444129608 = 0x216f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #8559, lsl #0x20
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world!").rstrip())
-        /* push b'Hello, world!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 143418749551 = 0x21646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #33, lsl #0x20
-        stp x14, x15, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue").rstrip())
-        /* push b'Hello, world, bienvenue\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 28558089656888933 = 0x65756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #101, lsl #0x30
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue!").rstrip())
-        /* push b'Hello, world, bienvenue!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 2406458692908510821 = 0x2165756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #8549, lsl #0x30
-        mov  x15, xzr
-        stp x14, x15, [sp, #-16]!
+    >>> string = "Hello, world!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world!'
+
+    >>> string = "Hello, world! This is a long string! Wow!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world! This is a long string! Wow!'
 </%docstring>
 <%
 if isinstance(string, six.text_type):
@@ -91,6 +46,8 @@ while len(string) % 8:
 words = packing.unpack_many(string)
 pairs = lists.group(2, words)
 
+pairs = pairs[::-1]
+
 # The stack must be 16-byte aligned
 total = len(pairs) * 16
 

pwnlib/shellcraft/templates/aarch64/pushstr_array.asm

@@ -2,7 +2,7 @@
     from pwnlib import shellcraft
     from pwnlib.context import context as ctx
     from pwnlib.util.iters import group
-    from six import text_type, binary_type
+    from six import text_type, binary_type, ensure_binary
 %>
 <%docstring>
 Pushes an array/envp-style array of pointers onto the stack.
@@ -14,66 +14,44 @@ Arguments:
         Single argument or list of arguments to push.
         NULL termination is normalized so that each argument
         ends with exactly one NULL byte.
+
+Example:
+    >>> assembly = shellcraft.execve("/bin/sh", ["sh", "-c", "echo Hello string $WORLD"], {"WORLD": "World!"})
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello string World!\n'
 </%docstring>
 <%page args="reg, array, register1='x14', register2='x15'"/>
 <%
 if isinstance(array, (binary_type, text_type)):
     array = [array]
 
-# Normalize all of the arguments' endings
+# Convert all items to strings
+array = [ensure_binary(x) for x in array]
+
+# Normalize line endings for each item
 array = [arg.rstrip(b'\x00') + b'\x00' for arg in array]
 
+# Join everything in the string-to-be-pushed
+string = b''.join(array)
+
 # Maximum amount that we can adjust SP by at once is 4095,
 # which seems like a safe maximum.
 if len(array) * 8 > 4095:
     raise Exception("Array size is too large (%i), max=4095" % len(array))
-
-# Join them into one big string that can be pushed
-array_str = b''.join(array)
-
-# Create a listing of offsets from what will be the "top" of the stack.
-num_pointers = len(array)
-
-# Account for the NULL terminator
-num_pointers += 1
-
-while num_pointers % 2 != 0:
-    num_pointers += 1
-
-# Offset from the 'top' of the stack, to the data pointed at
-sp_to_data = num_pointers * ctx.bytes
-
-# List of amounts to subtract from $SP
-offsets = {}
-for i,arg in enumerate(reversed(array)):
-    offsets[i] = sp_to_data + len(array_str) - len(arg)
-
-# If the array length is ODD we can sneak in our null terminator at the end
-if len(array) % 2 == 1:
-    offsets[len(offsets)] = 'sp'
-
-sorted_offsets = []
-for key in sorted(offsets):
-    sorted_offsets.append(offsets[key])
-
-# Load the offsets into pairs
-pairwise_offsets = group(2, sorted_offsets)
 %>\
     /* push argument array ${repr(array)} */
-    ${shellcraft.pushstr(array_str, register1=register1, register2=register2)}
+    ${shellcraft.pushstr(string, register1=register1, register2=register2)}
 
-    /* adjust the stack pointer to account for the array of pointers */
-    sub sp, sp, ${num_pointers * ctx.bytes}
+    /* push null terminator */
+    ${shellcraft.mov(register1, 0)}
+    str ${register1}, [sp, #-8]!
 
-    /* push pointers onto the stack in pairs */
-%for i, (a, b) in enumerate(pairwise_offsets):
-    ${shellcraft.mov(register1, a)}
-    ${shellcraft.mov(register2, b)}
-    sub  ${register1}, sp, ${register1}
-    sub  ${register2}, sp, ${register2}
-    stp  ${register1}, ${register2}, [sp], ${i * 16}
+    /* push pointers onto the stack */
+%for i, value in enumerate(reversed(array)):
+   ${shellcraft.mov(register1, (i+1)*8 + string.index(value))}
+   add ${register1}, sp, ${register1}
+   str ${register1}, [sp, #-8]! /* ${array[-i]} */
 %endfor
-%if len(array[-1] != 'sp')
 
     /* set ${reg} to the current top of the stack */
     ${shellcraft.mov(reg,'sp')}

pwnlib/shellcraft/templates/amd64/linux/loader_append.asm

@@ -14,23 +14,15 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m64','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "amd64");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'amd64'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    >>> run_assembly(sc).recvline() == b'Hello, amd64!\n'
-    True
 
 </%docstring>
 <%page args="data = None"/>
@@ -48,7 +40,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/i386/linux/loader_append.asm

@@ -14,23 +14,14 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m32','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "i386");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    >>> run_assembly(sc).recvline()
-    b'Hello, i386!\n'
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'i386'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
 </%docstring>
 <%page args="data = None"/>
@@ -48,7 +39,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/thumb/linux/loader_append.asm

@@ -17,20 +17,16 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    >>> gcc = process(['arm-linux-gnueabihf-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "world");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-    >>> run_assembly(sc).recvline()
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'arm'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
     b'Hello, world!\n'
 
+
 </%docstring>
 <%page args="data = None"/>
 <%
@@ -45,7 +41,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/tubes/ssh.py

@@ -560,12 +560,15 @@ class ssh(Timeout, Logger):
 
         Example proxying:
 
+        .. doctest::
+           :skipif: github_actions
+
             >>> s1 = ssh(host='example.pwnme',
-            ...          user='runner',
+            ...          user='travis',
             ...          password='demopass')
             >>> r1 = s1.remote('localhost', 22)
             >>> s2 = ssh(host='example.pwnme',
-            ...          user='runner',
+            ...          user='travis',
             ...          password='demopass',
             ...          proxy_sock=r1.sock)
             >>> r2 = s2.remote('localhost', 22) # and so on...
@@ -631,7 +634,7 @@ class ssh(Timeout, Logger):
             if os.path.exists(known_hosts):
                 self.client.load_host_keys(known_hosts)
 
-            has_proxy = (proxy_sock or proxy_command) and True
+            has_proxy = bool(proxy_sock or proxy_command)
             if has_proxy:
                 if 'ProxyCommand' not in dir(paramiko):
                     self.error('This version of paramiko does not support proxies.')
@@ -702,7 +705,7 @@ class ssh(Timeout, Logger):
 
         Examples:
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> sh = s.shell('/bin/sh')
             >>> sh.sendline(b'echo Hello; exit')
@@ -781,7 +784,7 @@ class ssh(Timeout, Logger):
 
         Examples:
             >>> s = ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> sh = s.process('/bin/sh', env={'PS1':''})
             >>> sh.sendline(b'echo Hello; exit')
@@ -828,10 +831,14 @@ class ssh(Timeout, Logger):
             >>> print(s.process('false', preexec_fn=uses_globals).recvall().strip().decode()) # doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            NameError: name 'bar' is not defined
+            NameError: ... name 'bar' is not defined
 
             >>> s.process('echo hello', shell=True).recvall()
             b'hello\n'
+
+            >>> io = s.process(['cat'], timeout=5)
+            >>> io.recvline()
+            b''
         """
         if not argv and not executable:
             self.error("Must specify argv or executable")
@@ -1049,11 +1056,14 @@ os.execve(exe, argv, env)
 
         msg = 'Starting remote process %s on %s' % (execve_repr, self.host)
 
+        if timeout == Timeout.default:
+            timeout = self.timeout
+
         with self.progress(msg) as h:
 
-            script = 'for py in python3 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2' % sh_string(script)
-            with context.local(log_level='error'):
-                python = ssh_process(self, script, tty=True, raw=True, level=self.level, timeout=self.timeout)
+            script = 'for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2' % sh_string(script)
+            with context.quiet:
+                python = ssh_process(self, script, tty=True, raw=True, level=self.level, timeout=timeout)
 
             try:
                 result = safeeval.const(python.recvline())
@@ -1132,7 +1142,7 @@ os.execve(exe, argv, env)
 
         Examples:
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> py = s.run('python -i')
             >>> _ = py.recvuntil(b'>>> ')
@@ -1193,7 +1203,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         Examples:
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> print(s.run_to_end('echo Hello; exit 17'))
             (b'Hello\n', 17)
@@ -1218,7 +1228,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             >>> from pwn import *
             >>> l = listen()
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> a = s.connect_remote(s.host, l.lport)
             >>> a=a; b = l.wait_for_connection()  # a=a; prevents hangs
@@ -1243,7 +1253,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
             >>> from pwn import *
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> l = s.listen_remote()
             >>> a = remote(s.host, l.port)
@@ -1263,7 +1273,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         Examples:
 
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> print(repr(s['echo hello']))
             b'hello'
@@ -1276,7 +1286,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         Examples:
 
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> print(repr(s('echo hello')))
             b'hello'
@@ -1289,12 +1299,12 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         Examples:
 
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> s.echo('hello')
             b'hello'
             >>> s.whoami()
-            b'runner'
+            b'travis'
             >>> s.echo(['huh','yay','args'])
             b'huh yay args'
         """
@@ -1322,7 +1332,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         Example:
 
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> s.connected()
             True
@@ -1480,7 +1490,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             >>> with open('/tmp/bar','w+') as f:
             ...     _ = f.write('Hello, world')
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass',
             ...         cache=False)
             >>> s.download_data('/tmp/bar')
@@ -1570,7 +1580,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         Example:
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> s.upload_data(b'Hello, world', '/tmp/upload_foo')
             >>> print(open('/tmp/upload_foo').read())
@@ -1801,7 +1811,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         Examples:
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> cwd = s.set_working_directory()
             >>> s.ls()
@@ -1810,7 +1820,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             True
 
             >>> s =  ssh(host='example.pwnme',
-            ...         user='runner',
+            ...         user='travis',
             ...         password='demopass')
             >>> homedir = s.pwd()
             >>> _=s.touch('foo')
@@ -1980,7 +1990,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         Example:
 
-            >>> s = ssh("runner", "example.pwnme")
+            >>> s = ssh("travis", "example.pwnme")
             >>> s.aslr
             True
         """

pwnlib/util/misc.py

@@ -5,6 +5,7 @@ import errno
 import os
 import platform
 import re
+import six
 import socket
 import stat
 import string
@@ -237,7 +238,7 @@ def run_in_new_terminal(command, terminal = None, args = None):
 
     argv = [which(terminal)] + args
 
-    if isinstance(command, str):
+    if isinstance(command, six.string_types):
         if ';' in command:
             log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += [command]

pwnlib/util/packing.py

@@ -33,6 +33,7 @@ Examples:
 from __future__ import absolute_import
 from __future__ import division
 
+import collections
 import six
 import struct
 import sys
@@ -491,9 +492,11 @@ def _fit(pieces, preprocessor, packer, filler):
     pad = bytearray()
     def fill(key):
         key = bytearray(key)
-        while len(pad) < len(key) or pad[-len(key):] != key:
+        offset = pad.find(key)
+        while offset == -1:
             pad.append(next(filler))
-        return len(pad) - len(key)
+            offset = pad.find(key, -len(key))
+        return offset
 
     # Key conversion:
     # - convert str/unicode keys to offsets
@@ -614,26 +617,103 @@ def flat(*args, **kwargs):
       sign (str): Signedness of the converted integer (False/True)
 
     Examples:
-      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
-      b'\x01\x00testABABABABABAB'
-      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
-      b'234'
-      >>> flat({12: 0x41414141,
-      ...       24: 'Hello',
-      ...      })
-      b'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> flat({'caaa': ''})
-      b'aaaabaaa'
-      >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
-      b'ABABABABABABXXXXABAB'
-      >>> flat({ 8: [0x41414141, 0x42424242],
-      ...       20: 'CCCC'})
-      b'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> flat({ 0x61616162: 'X'})
-      b'aaaaX'
-      >>> flat({4: {0: 'X', 4: 'Y'}})
-      b'aaaaXaaaY'
 
+        (Test setup, please ignore)
+    
+        >>> context.clear()
+
+        Basic usage of :meth:`flat` works similar to the pack() routines.
+
+        >>> flat(4)
+        b'\x04\x00\x00\x00'
+
+        :meth:`flat` works with strings, bytes, lists, and dictionaries.
+
+        >>> flat(b'X')
+        b'X'
+        >>> flat([1,2,3])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
+        >>> flat({4:'X'})
+        b'aaaaX'
+
+        :meth:`.flat` flattens all of the values provided, and allows nested lists
+        and dictionaries.
+
+        >>> flat([{4:'X'}] * 2)
+        b'aaaaXaaacX'
+        >>> flat([[[[[[[[[1]]]], 2]]]]])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00'
+
+        You can also provide additional arguments like endianness, word-size, and
+        whether the values are treated as signed or not.
+
+        >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
+        b'\x01\x00testABABABABABAB'
+
+        A preprocessor function can be provided in order to modify the values in-flight.
+        This example converts increments each value by 1, then converts to a string.
+
+        >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
+        b'234'
+
+        Using dictionaries is a fast way to get specific values at specific offsets,
+        without having to do ``data += "foo"`` repeatedly.
+
+        >>> flat({12: 0x41414141,
+        ...       24: 'Hello',
+        ...      })
+        b'aaaabaaacaaaAAAAeaaafaaaHello'
+
+        Dictionary usage permits directly using values derived from :func:`.cyclic`.
+        See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
+        for more options.  
+
+        The cyclic pattern can be provided as either the text or hexadecimal offset.
+
+        >>> flat({ 0x61616162: 'X'})
+        b'aaaaX'
+        >>> flat({'baaa': 'X'})
+        b'aaaaX'
+
+        Fields do not have to be in linear order, and can be freely mixed.
+        This also works with cyclic offsets.
+
+        >>> flat({2: 'A', 0:'B'})
+        b'BaA'
+        >>> flat({0x61616161:'x', 0x61616162:'y'})
+        b'xaaay'
+        >>> flat({0x61616162:'y', 0x61616161:'x'})
+        b'xaaay'
+
+        Fields do not have to be in order, and can be freely mixed.
+
+        >>> flat({'caaa': 'XXXX', 16: '\x41', 20: 0xdeadbeef})
+        b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
+        >>> flat({ 8: [0x41414141, 0x42424242], 20: 'CCCC'})
+        b'aaaabaaaAAAABBBBeaaaCCCC'
+        >>> fit({
+        ...     0x61616161: 'a',
+        ...     1: 'b',
+        ...     0x61616161+2: 'c',
+        ...     3: 'd',
+        ... })
+        b'abadbaaac'
+
+        By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
+        You can customize this by providing an iterable or method for the ``filler``
+        argument.
+
+        >>> flat({12: 'XXXX'}, filler = b'_', length = 20)
+        b'____________XXXX____'
+        >>> flat({12: 'XXXX'}, filler = b'AB', length = 20)
+        b'ABABABABABABXXXXABAB'
+
+        Nested dictionaries also work as expected.
+
+        >>> flat({4: {0: 'X', 4: 'Y'}})
+        b'aaaaXaaaY'
+        >>> fit({4: {4: 'XXXX'}})
+        b'aaaabaaaXXXX'
     """
     # HACK: To avoid circular imports we need to delay the import of `cyclic`
     from pwnlib.util import cyclic
@@ -642,6 +722,9 @@ def flat(*args, **kwargs):
     filler       = kwargs.pop('filler', cyclic.de_bruijn())
     length       = kwargs.pop('length', None)
 
+    if isinstance(filler, str):
+        filler = bytearray(six.ensure_binary(filler))
+
     if kwargs != {}:
         raise TypeError("flat() does not support argument %r" % kwargs.popitem()[0])
 
@@ -656,7 +739,7 @@ def flat(*args, **kwargs):
     return out
 
 def fit(*args, **kwargs):
-    """Legacy alias for `:func:flat`"""
+    """Legacy alias for :func:`flat`"""
     return flat(*args, **kwargs)
 
 """

pwnlib/util/proc.py

@@ -9,6 +9,7 @@ import psutil
 
 from pwnlib import tubes
 from pwnlib.log import getLogger
+from pwnlib.timeout import Timeout
 
 log = getLogger(__name__)
 
@@ -337,17 +338,23 @@ def wait_for_debugger(pid, debugger_pid=None):
     Returns:
         None
     """
-    with log.waitfor('Waiting for debugger') as l:
-        if debugger_pid:
-            debugger = psutil.Process(debugger_pid)
-            while tracer(pid) is None:
-                try:
-                    debugger.wait(0.01)
-                except psutil.TimeoutExpired:
-                    pass
-                else:
-                    l.failure("debugger exited! (maybe check /proc/sys/kernel/yama/ptrace_scope)")
+    t = Timeout()
+    with t.countdown(timeout=15):
+        with log.waitfor('Waiting for debugger') as l:
+            if debugger_pid:
+                debugger = psutil.Process(debugger_pid)
+                while t.timeout and tracer(pid) is None:
+                    try:
+                        debugger.wait(0.01)
+                    except psutil.TimeoutExpired:
+                        pass
+                    else:
+                        l.failure("debugger exited! (maybe check /proc/sys/kernel/yama/ptrace_scope)")
+            else:
+                while t.timeout and tracer(pid) is None:
+                    time.sleep(0.01)
+
+        if tracer(pid):
+            l.success()
         else:
-            while tracer(pid) is None:
-                time.sleep(0.01)
-        l.success()
+            l.failure('Debugger did not attach to pid %d within 15 seconds', pid)

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.2.0beta0'
+__version__ = '4.2.1'

setup.py

@@ -60,7 +60,8 @@ install_requires     = ['paramiko>=1.15.2',
                         'psutil>=3.3.0',
                         'intervaltree>=3.0',
                         'sortedcontainers',
-                        'unicorn>=1.0.2rc1', # see unicorn-engine/unicorn#1100, unicorn-engine/unicorn#1170
+                        'unicorn>=1.0.2rc1,<1.0.2rc4', # see unicorn-engine/unicorn#1100, unicorn-engine/unicorn#1170, Gallopsled/pwntools#1538
+                        'six>=1.12.0',
 ]
 
 # Check that the user has installed the Python development headers
@@ -82,7 +83,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.2.0beta0',
+    version              = '4.2.1',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],

travis/install.sh

@@ -64,6 +64,11 @@ setup_travis()
     rm -rf usr/share
 }
 
+setup_ipv6()
+{
+    echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
+}
+
 setup_gdbserver()
 {
     # https://docs.improbable.io/reference/14.3/shared/debug-cloud-workers#common-issues
@@ -186,6 +191,7 @@ setup_osx()
 
 if [[ "$USER" == "travis" ]]; then
 #   setup_travis
+    setup_ipv6
     setup_gdbserver
     setup_android_emulator
 elif [[ "$USER" == "shippable" ]]; then

travis/ssh_setup.sh

@@ -32,6 +32,7 @@ fi
 
 # Generate a new key so that we can log into it
 ssh-keygen -t rsa -f ~/.ssh/$U -N ''
+chmod og-rw ~/.ssh
 
 # Load the public key into a memory for below
 pubkey=$(cat ~/.ssh/$U.pub)
@@ -42,6 +43,7 @@ USUDO mkdir $H/.ssh || true
 USUDO tee -a $H/.ssh/authorized_keys <<EOF
 from="127.0.0.1" $pubkey
 EOF
+USUDO chmod 700 $H $H/.ssh $H/.ssh/authorized_keys
 
 # In the pwntools examples, we ssh to 'example.pwnme'
 # Set up an SSH config entry to make this actually work
@@ -51,8 +53,13 @@ Host example.pwnme
     User $U
     HostName 127.0.0.1
     IdentityFile ~/.ssh/$U
+    StrictHostKeyChecking no
 EOF
+chmod 700 ~ ~/.ssh
 
-ssh -o "StrictHostKeyChecking no" -vvvv [email protected] id
+ls -la ~/.ssh
+USUDO ls -la $H/.ssh
+
+ssh -v [email protected] id
 
 set +ex