Use Github Actions for CI (#3713)

switch to github actions for CI

.appveyor.yml

@@ -1,38 +0,0 @@
-version: '{build}'
-build: off  # Not a C# project
-
-branches:
-  except:
-    - requires-io-master
-
-environment:
-  CI_DEPS: codecov>=2.0.5
-  CI_COMMANDS: codecov
-  matrix:
-    - PYTHON: "C:\\Python36"
-      TOXENV: "py36"
-      PYINSTALLER: "1"
-      WININSTALLER: "1"
-    - PYTHON: "C:\\Python37"
-      TOXENV: "py37"
-
-install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "python -m pip install --disable-pip-version-check -U pip"
-  - "pip install -U tox"
-
-test_script:
-  - ps: |
-      if ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[notest]")) {
-        echo "!!!! Skipping tests."
-      } else {
-        tox -- --verbose --cov-report=term
-      }
-  - ps: tox -e cibuild -- build
-
-deploy_script:
-  ps: tox -e cibuild -- upload
-
-cache:
-  - C:\projects\mitmproxy\release\installbuilder\setup -> .appveyor.yml
-  - C:\Users\appveyor\AppData\Local\pip\cache

.gitattributes

@@ -1,2 +1,3 @@
 mitmproxy/tools/web/static/**/* -diff linguist-vendored
 web/src/js/filt/filt.js -diff
+*.bin binary

.github/workflows/main.yml

@@ -0,0 +1,210 @@
+name: CI
+
+on: [push, pull_request]
+
+# We currently use Python 3.7 for most things:
+# - zstandard currently doesn't have 3.8 wheels,
+# - we need to upgrade cryptography from version 2.4, which also doesn't have wheels
+
+env:
+  # Codecov
+  CODECOV_TOKEN: "0409bdfd-57a4-477d-a8af-f6172ef431d3"
+
+jobs:
+  lint-pr:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: TrueBrain/actions-flake8@v1.2
+  lint-local:
+    # do not use external action when secrets are exposed.
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+      - run: pip install flake8
+      - run: flake8 mitmproxy pathod examples test release
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+      - run: pip install mypy
+      - run: mypy .
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - run: printenv
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - run: pip install tox
+      - run: tox -e py37
+      # codecov's GitHub action only supports Linux. https://github.com/codecov/codecov-action/issues/7
+      # codecov's Python uploader has no github actions support yet. https://github.com/codecov/codecov-python/pull/214
+      - name: Extract branch name  # https://stackoverflow.com/a/58035262/934719
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - run: pip install codecov
+      - run: >
+          codecov -f coverage.xml
+          --name python-${{ matrix.os }}
+          --commit ${{ github.sha }}
+          --slug ${{ github.repository }}
+          --branch ${{ steps.extract_branch.outputs.branch }}
+  test-py35:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.5'
+      - run: pip install tox
+      - run: tox -e py35
+  build-wheel:
+    runs-on: ubuntu-latest
+    env:
+      CI_BUILD_WHEEL: 1
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - run: pip install tox
+      - run: tox -e cibuild -- build
+      - uses: actions/upload-artifact@master
+        with:
+          name: wheel
+          path: release/dist
+  build-binaries:
+    strategy:
+      fail-fast: false
+      matrix:
+        # Old Ubuntu version for old glibc
+        os: [macos-latest, windows-latest, ubuntu-16.04]
+    runs-on: ${{ matrix.os }}
+    env:
+      CI_BUILD_PYINSTALLER: 1
+      CI_BUILD_WININSTALLER: ${{ matrix.os == 'windows-latest' }}
+      CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }}
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - if: matrix.os == 'windows-latest'
+        uses: actions/cache@v1
+        with:
+          path: release/installbuilder/setup
+          key: installbuilder
+      - run: pip install tox
+      - run: tox -e cibuild -- build
+      # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24
+      - uses: actions/upload-artifact@master
+        with:
+          name: binaries.${{ matrix.os }}
+          path: release/dist
+
+  test-web-ui:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - run: git rev-parse --abbrev-ref HEAD
+      - uses: actions/setup-node@v1
+      - id: yarn-cache
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - uses: actions/cache@v1
+        with:
+          path: ${{ steps.yarn-cache.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - working-directory: ./web
+        run: yarn
+      - working-directory: ./web
+        run: npm test
+      - run: bash <(curl -s https://codecov.io/bash)
+
+  docs:
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - run: pip install tox
+      - run: |
+          wget https://github.com/gohugoio/hugo/releases/download/v0.59.1/hugo_0.59.1_Linux-64bit.deb
+          sudo dpkg -i hugo*.deb
+      - run: tox -e docs
+
+  # Separate from everything else because slow.
+  build-and-deploy-docker:
+    if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
+    needs: [test, test-web-ui, build-wheel]
+    runs-on: ubuntu-latest
+    env:
+      CI_BUILD_DOCKER: 1
+      DOCKER_USERNAME: mitmbot
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - run: pip install tox
+      - uses: actions/download-artifact@master
+        with:
+          name: wheel
+          path: release/dist
+      - run: tox -e cibuild -- build
+      - run: tox -e cibuild -- upload
+
+  deploy:
+    if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [test, test-web-ui, build-wheel, build-binaries]
+    env:
+      CI_BUILD_WHEEL: 1
+      CI_BUILD_PYINSTALLER: 1
+      CI_BUILD_WININSTALLER: 1
+      TWINE_USERNAME: mitmproxy
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      # artifacts must be downloaded individually, see https://github.com/actions/download-artifact/issues/6
+      - uses: actions/download-artifact@master
+        with:
+          name: wheel
+          path: release/dist
+      - uses: actions/download-artifact@master
+        with:
+          name: binaries.windows-latest
+          path: release/dist
+      - uses: actions/download-artifact@master
+        with:
+          name: binaries.macos-latest
+          path: release/dist
+      - uses: actions/download-artifact@master
+        with:
+          name: binaries.ubuntu-16.04
+          path: release/dist
+      - run: ls release/dist
+      - run: pip install tox
+      - run: tox -e cibuild -- upload

.travis.yml

@@ -1,103 +0,0 @@
-language: python
-
-branches:
-  except:
-    - requires-io-master
-
-env:
-  global:
-    - CI_DEPS=codecov>=2.0.5
-    - CI_COMMANDS=codecov
-
-git:
-  depth: 10000
-
-matrix:
-  fast_finish: true
-  include:
-    - python: 3.5
-      env: TOXENV=py35 # this just makes sure that our version detection shows an appropriate error message
-    - python: 3.6
-      env: TOXENV=lint
-    - os: osx
-      osx_image: xcode7.3
-      language: generic
-      env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1
-    - python: 3.6
-      env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 WHEEL=1 DOCKER=1
-      sudo: required
-      services:
-        - docker
-    - python: 3.6
-      env: TOXENV=individual_coverage
-    - python: 3.7
-      env: TOXENV=py37
-      dist: xenial
-    - language: node_js
-      node_js: "node"
-      before_install:
-          - curl -o- -L https://yarnpkg.com/install.sh | bash
-          - export PATH=$HOME/.yarn/bin:$PATH
-      install:
-          - cd web && yarn
-          - yarn global add codecov
-      script: npm test && codecov
-      cache:
-          yarn: true
-          directories:
-              - web/node_modules
-    - python: 3.6
-      env: NAME=docs TOXENV=docs
-      install:
-        - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb
-        - sudo dpkg -i hugo*.deb
-        - pip install -U tox virtualenv setuptools
-      script:
-        - tox
-      after_success:
-        - echo done
-
-install:
-  - |
-    if [[ $TRAVIS_OS_NAME == "osx" ]]
-    then
-      brew update || brew update
-      brew outdated pyenv || brew upgrade pyenv
-      eval "$(pyenv init -)"
-      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9
-      pyenv global 3.6.9
-      pyenv shell 3.6.9
-    fi
-  - pip install -U tox virtualenv setuptools
-
-script:
-  # All these steps MUST succeed for the job to be successful!
-  # Using the after_success block DOES NOT capture errors.
-  # Pull requests might break our build - we need to check this.
-  # Pull requests are not allowed to upload build artifacts - enforced by cibuild script.
-  - |
-    if [[ $TRAVIS_COMMIT_MESSAGE = *"[notest]"* ]]; then
-      echo "!!!! Skipping tests."
-    else
-      tox -- --verbose --cov-report=term
-    fi
-
-  - |
-    if [[ $CIBUILD == "1" ]]
-    then
-        git fetch --unshallow --tags
-        tox -e cibuild -- build && tox -e cibuild -- upload
-    fi
-
-notifications:
-  slack:
-      -rooms:
-          mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu
-      on_success: change
-      on_failure: change
-      on_start: never
-
-cache:
-  directories:
-    - $HOME/.pyenv
-    - $HOME/.cache/pip

docs/ci

@@ -7,7 +7,7 @@ set -e
 
 # Only upload if we have defined credentials - we only have these defined for
 # trusted commits (i.e. not PRs).
-if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
+if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $GITHUB_REF == "refs/heads/master" ]]; then
     aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
     aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
         --paths "/master/*"

release/cibuild.py

@@ -14,9 +14,10 @@ import urllib.request
 import zipfile
 
 import click
-import cryptography.fernet
 import parver
 
+import cryptography.fernet
+
 
 @contextlib.contextmanager
 def chdir(path: str):  # pragma: no cover
@@ -30,6 +31,14 @@ class BuildError(Exception):
     pass
 
 
+def bool_from_env(envvar: str) -> bool:
+    val = os.environ.get(envvar, "")
+    if not val or val.lower() in ("0", "false"):
+        return False
+    else:
+        return True
+
+
 class BuildEnviron:
     PLATFORM_TAGS = {
         "Darwin": "osx",
@@ -38,25 +47,27 @@ class BuildEnviron:
     }
 
     def __init__(
-        self,
-        *,
-        system="",
-        root_dir="",
-        travis_tag="",
-        travis_branch="",
-        travis_pull_request="",
-        appveyor_repo_tag_name="",
-        appveyor_repo_branch="",
-        appveyor_pull_request_number="",
-        should_build_wheel=False,
-        should_build_docker=False,
-        should_build_pyinstaller=False,
-        should_build_wininstaller=False,
-        has_aws_creds=False,
-        has_twine_creds=False,
-        docker_username="",
-        docker_password="",
-        rtool_key="",
+            self,
+            *,
+            system="",
+            root_dir="",
+            travis_tag="",
+            travis_branch="",
+            travis_pull_request="",
+            appveyor_repo_tag_name="",
+            appveyor_repo_branch="",
+            appveyor_pull_request_number="",
+            github_ref="",
+            github_event_name="",
+            should_build_wheel=False,
+            should_build_docker=False,
+            should_build_pyinstaller=False,
+            should_build_wininstaller=False,
+            has_aws_creds=False,
+            has_twine_creds=False,
+            docker_username="",
+            docker_password="",
+            build_key="",
     ):
         self.system = system
         self.root_dir = root_dir
@@ -80,11 +91,14 @@ class BuildEnviron:
         self.appveyor_repo_branch = appveyor_repo_branch
         self.appveyor_pull_request_number = appveyor_pull_request_number
 
+        self.github_ref = github_ref
+        self.github_event_name = github_event_name
+
         self.has_aws_creds = has_aws_creds
         self.has_twine_creds = has_twine_creds
         self.docker_username = docker_username
         self.docker_password = docker_password
-        self.rtool_key = rtool_key
+        self.build_key = build_key
 
     @classmethod
     def from_env(cls):
@@ -96,19 +110,18 @@ class BuildEnviron:
             travis_pull_request=os.environ.get("TRAVIS_PULL_REQUEST"),
             appveyor_repo_tag_name=os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
             appveyor_repo_branch=os.environ.get("APPVEYOR_REPO_BRANCH", ""),
-            appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
-            should_build_wheel="WHEEL" in os.environ,
-            should_build_pyinstaller="PYINSTALLER" in os.environ,
-            should_build_wininstaller="WININSTALLER" in os.environ,
-            should_build_docker="DOCKER" in os.environ,
-            has_aws_creds="AWS_ACCESS_KEY_ID" in os.environ,
-            has_twine_creds=(
-                "TWINE_USERNAME" in os.environ and
-                "TWINE_PASSWORD" in os.environ
-            ),
-            docker_username=os.environ.get("DOCKER_USERNAME"),
-            docker_password=os.environ.get("DOCKER_PASSWORD"),
-            rtool_key=os.environ.get("RTOOL_KEY"),
+            appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER", ""),
+            github_ref=os.environ.get("GITHUB_REF", ""),
+            github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""),
+            should_build_wheel=bool_from_env("CI_BUILD_WHEEL"),
+            should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"),
+            should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"),
+            should_build_docker=bool_from_env("CI_BUILD_DOCKER"),
+            has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"),
+            has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"),
+            docker_username=os.environ.get("DOCKER_USERNAME", ""),
+            docker_password=os.environ.get("DOCKER_PASSWORD", ""),
+            build_key=os.environ.get("CI_BUILD_KEY", ""),
         )
 
     def archive(self, path):
@@ -143,26 +156,34 @@ class BuildEnviron:
         return ret
 
     @property
-    def branch(self):
-        return self.travis_branch or self.appveyor_repo_branch
+    def branch(self) -> str:
+        if self.travis_branch:
+            return self.travis_branch
+        if self.appveyor_repo_branch:
+            return self.appveyor_repo_branch
+        if self.github_ref and self.github_ref.startswith("refs/heads/"):
+            return self.github_ref.replace("refs/heads/", "")
+        if self.github_ref and self.github_ref.startswith("refs/pull/"):
+            return "pr-" + self.github_ref.split("/")[2]
+        return ""
 
     @property
-    def build_dir(self):
+    def build_dir(self) -> str:
         return os.path.join(self.release_dir, "build")
 
     @property
-    def dist_dir(self):
+    def dist_dir(self) -> str:
         return os.path.join(self.release_dir, "dist")
 
     @property
-    def docker_tag(self):
+    def docker_tag(self) -> str:
         if self.branch == "master":
             t = "dev"
         else:
             t = self.version
         return "mitmproxy/mitmproxy:{}".format(t)
 
-    def dump_info(self, fp=sys.stdout):
+    def dump_info(self, fp=sys.stdout) -> None:
         lst = [
             "version",
             "tag",
@@ -176,7 +197,9 @@ class BuildEnviron:
             "upload_dir",
             "should_build_wheel",
             "should_build_pyinstaller",
+            "should_build_wininstaller",
             "should_build_docker",
+            "should_upload_aws",
             "should_upload_docker",
             "should_upload_pypi",
         ]
@@ -232,6 +255,8 @@ class BuildEnviron:
 
     @property
     def is_pull_request(self) -> bool:
+        if self.github_event_name == "pull_request":
+            return True
         if self.appveyor_pull_request_number:
             return True
         if self.travis_pull_request and self.travis_pull_request != "false":
@@ -239,13 +264,13 @@ class BuildEnviron:
         return False
 
     @property
-    def platform_tag(self):
+    def platform_tag(self) -> str:
         if self.system in self.PLATFORM_TAGS:
             return self.PLATFORM_TAGS[self.system]
         raise BuildError("Unsupported platform: %s" % self.system)
 
     @property
-    def release_dir(self):
+    def release_dir(self) -> str:
         return os.path.join(self.root_dir, "release")
 
     @property
@@ -256,6 +281,13 @@ class BuildEnviron:
             self.has_docker_creds,
         ])
 
+    @property
+    def should_upload_aws(self) -> bool:
+        return all([
+            self.has_aws_creds,
+            (self.should_build_wheel or self.should_build_pyinstaller or self.should_build_wininstaller),
+        ])
+
     @property
     def should_upload_pypi(self) -> bool:
         return all([
@@ -265,18 +297,24 @@ class BuildEnviron:
         ])
 
     @property
-    def tag(self):
-        return self.travis_tag or self.appveyor_repo_tag_name
+    def tag(self) -> str:
+        if self.travis_tag:
+            return self.travis_tag
+        if self.appveyor_repo_tag_name:
+            return self.appveyor_repo_tag_name
+        if self.github_ref and self.github_ref.startswith("refs/tags/"):
+            return self.github_ref.replace("refs/tags/", "")
+        return ""
 
     @property
-    def upload_dir(self):
+    def upload_dir(self) -> str:
         if self.tag:
             return self.version
         else:
             return "branches/%s" % self.version
 
     @property
-    def version(self):
+    def version(self) -> str:
         if self.tag:
             if self.tag.startswith("v"):
                 try:
@@ -300,13 +338,14 @@ def build_wheel(be: BuildEnviron):  # pragma: no cover
         "bdist_wheel",
         "--dist-dir", be.dist_dir,
     ])
-    whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
+    whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
     click.echo("Found wheel package: {}".format(whl))
     subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
     return whl
 
 
-def build_docker_image(be: BuildEnviron, whl: str):  # pragma: no cover
+def build_docker_image(be: BuildEnviron):  # pragma: no cover
+    whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
     click.echo("Building Docker images...")
     subprocess.check_call([
         "docker",
@@ -410,22 +449,25 @@ def build_pyinstaller(be: BuildEnviron):  # pragma: no cover
 
 
 def build_wininstaller(be: BuildEnviron):  # pragma: no cover
+    if not be.build_key:
+        click.echo("Cannot build windows installer without secret key.")
+        return
     click.echo("Building wininstaller package...")
 
-    IB_VERSION = "18.8.0"
+    IB_VERSION = "19.10.0"
     IB_DIR = pathlib.Path(be.release_dir) / "installbuilder"
     IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
     IB_CLI = fr"C:\Program Files (x86)\BitRock InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe"
     IB_LICENSE = IB_DIR / "license.xml"
 
-    if True or not os.path.isfile(IB_CLI):
+    if not os.path.isfile(IB_CLI):
         if not os.path.isfile(IB_SETUP):
             click.echo("Downloading InstallBuilder...")
 
             def report(block, blocksize, total):
                 done = block * blocksize
                 if round(100 * done / total) != round(100 * (done - blocksize) / total):
-                    click.secho(f"Downloading... {round(100*done/total)}%")
+                    click.secho(f"Downloading... {round(100 * done / total)}%")
 
             urllib.request.urlretrieve(
                 f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe",
@@ -435,14 +477,13 @@ def build_wininstaller(be: BuildEnviron):  # pragma: no cover
             shutil.move(IB_SETUP.with_suffix(".tmp"), IB_SETUP)
 
         click.echo("Install InstallBuilder...")
-        subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"],
-                       check=True)
+        subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
         assert os.path.isfile(IB_CLI)
 
     click.echo("Decrypt InstallBuilder license...")
-    f = cryptography.fernet.Fernet(be.rtool_key.encode())
-    with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, open(IB_LICENSE,
-                                                                        "wb") as outfile:
+    f = cryptography.fernet.Fernet(be.build_key.encode())
+    with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
+            open(IB_LICENSE, "wb") as outfile:
         outfile.write(f.decrypt(infile.read()))
 
     click.echo("Run InstallBuilder...")
@@ -479,13 +520,12 @@ def build():  # pragma: no cover
     os.makedirs(be.dist_dir, exist_ok=True)
 
     if be.should_build_wheel:
-        whl = build_wheel(be)
-        # Docker image requires wheels
-        if be.should_build_docker:
-            build_docker_image(be, whl)
+        build_wheel(be)
+    if be.should_build_docker:
+        build_docker_image(be)
     if be.should_build_pyinstaller:
         build_pyinstaller(be)
-    if be.should_build_wininstaller and be.rtool_key:
+    if be.should_build_wininstaller:
         build_wininstaller(be)
 
 
@@ -499,12 +539,15 @@ def upload():  # pragma: no cover
         Pushes the Docker image to Docker Hub.
     """
     be = BuildEnviron.from_env()
+    be.dump_info()
 
     if be.is_pull_request:
         click.echo("Refusing to upload artifacts from a pull request!")
         return
 
-    if be.has_aws_creds:
+    if be.should_upload_aws:
+        num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)])
+        click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...")
         subprocess.check_call([
             "aws", "s3", "cp",
             "--acl", "public-read",

test/mitmproxy/contentviews/test_protobuf.py

@@ -8,7 +8,7 @@ datadir = "mitmproxy/contentviews/test_protobuf_data/"
 
 def test_view_protobuf_request(tdata):
     v = full_eval(protobuf.ViewProtobuf())
-    p = tdata.path(datadir + "protobuf01")
+    p = tdata.path(datadir + "protobuf01.bin")
 
     with open(p, "rb") as f:
         raw = f.read()
@@ -19,12 +19,12 @@ def test_view_protobuf_request(tdata):
         v(b'foobar')
 
 
-@pytest.mark.parametrize("filename", ["protobuf02", "protobuf03"])
+@pytest.mark.parametrize("filename", ["protobuf02.bin", "protobuf03.bin"])
 def test_format_pbuf(filename, tdata):
     path = tdata.path(datadir + filename)
     with open(path, "rb") as f:
         input = f.read()
-    with open(path + "-decoded") as f:
+    with open(path.replace(".bin", "-decoded.bin")) as f:
         expected = f.read()
 
     assert protobuf.format_pbuf(input) == expected

test/mitmproxy/io/test_compat.py

@@ -5,7 +5,7 @@ from mitmproxy import exceptions
 
 
 def test_load(tdata):
-    with open(tdata.path("mitmproxy/data/dumpfile-011"), "rb") as f:
+    with open(tdata.path("mitmproxy/data/dumpfile-011.bin"), "rb") as f:
         flow_reader = io.FlowReader(f)
         flows = list(flow_reader.stream())
         assert len(flows) == 1
@@ -13,7 +13,7 @@ def test_load(tdata):
 
 
 def test_load_018(tdata):
-    with open(tdata.path("mitmproxy/data/dumpfile-018"), "rb") as f:
+    with open(tdata.path("mitmproxy/data/dumpfile-018.bin"), "rb") as f:
         flow_reader = io.FlowReader(f)
         flows = list(flow_reader.stream())
         assert len(flows) == 1
@@ -21,7 +21,7 @@ def test_load_018(tdata):
 
 
 def test_cannot_convert(tdata):
-    with open(tdata.path("mitmproxy/data/dumpfile-010"), "rb") as f:
+    with open(tdata.path("mitmproxy/data/dumpfile-010.bin"), "rb") as f:
         flow_reader = io.FlowReader(f)
         with pytest.raises(exceptions.FlowReadException):
             list(flow_reader.stream())

test/mitmproxy/proxy/test_config.py

@@ -14,7 +14,7 @@ class TestProxyConfig:
 
     def test_invalid_certificate(self, tdata):
         opts = options.Options()
-        opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")]
+        opts.certs = [tdata.path("mitmproxy/data/dumpfile-011.bin")]
         with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"):
             ProxyConfig(opts)
 

test/mitmproxy/test_proxy.py

@@ -1,20 +1,19 @@
 import argparse
 import platform
 from unittest import mock
+
 import pytest
 
-from mitmproxy.tools import cmdline
-from mitmproxy.tools import main
 from mitmproxy import options
 from mitmproxy.proxy import ProxyConfig
-from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
 from mitmproxy.proxy import config
-
+from mitmproxy.proxy.server import ConnectionHandler, DummyServer, ProxyServer
+from mitmproxy.tools import cmdline
+from mitmproxy.tools import main
 from ..conftest import skip_windows
 
 
 class MockParser(argparse.ArgumentParser):
-
     """
     argparse.ArgumentParser sys.exits() by default.
     Make it more testable by throwing an exception instead.
@@ -53,11 +52,9 @@ class TestProcessProxyOptions:
 class TestProxyServer:
 
     @skip_windows
-    @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'],
-                        reason='Skipping due to macOS Mojave')
+    @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-only")
     def test_err(self):
-        # binding to 0.0.0.0:1 works without special permissions on Windows and
-        # macOS Mojave
+        # binding to 0.0.0.0:1 works without special permissions on Windows and macOS Mojave+
         conf = ProxyConfig(options.Options(listen_port=1))
         with pytest.raises(Exception, match="Error starting proxy server"):
             ProxyServer(conf)

test/release/test_cibuild.py

@@ -58,31 +58,60 @@ def test_buildenviron_pr():
     )
     assert be.is_pull_request
 
-    # Mini test for appveyor
-    be = cibuild.BuildEnviron(
-        appveyor_pull_request_number="xxxx",
+
+def test_ci_systems():
+    appveyor = cibuild.BuildEnviron(
+        appveyor_pull_request_number="1234",
+        appveyor_repo_branch="foo",
+        appveyor_repo_tag_name="qux",
     )
-    assert be.is_pull_request
-    assert not be.is_prod_release
-    assert not be.is_maintenance_branch
+    assert appveyor.is_pull_request
+    assert appveyor.branch == "foo"
+    assert appveyor.tag == "qux"
+
+    travis = cibuild.BuildEnviron(
+        travis_pull_request="1234",
+        travis_branch="foo",
+        travis_tag="foo",
+    )
+    assert travis.is_pull_request
+    assert travis.branch == "foo"
+    assert travis.tag == "foo"
+
+    github = cibuild.BuildEnviron(
+        github_event_name="pull_request",
+        github_ref="refs/heads/master"
+    )
+    assert github.is_pull_request
+    assert github.branch == "master"
+    assert github.tag == ""
+
+    github2 = cibuild.BuildEnviron(
+        github_event_name="pull_request",
+        github_ref="refs/tags/qux"
+    )
+    assert github2.is_pull_request
+    assert github2.branch == ""
+    assert github2.tag == "qux"
 
 
 def test_buildenviron_commit():
     # Simulates an ordinary commit on the master branch.
     be = cibuild.BuildEnviron(
-        travis_tag="",
-        travis_branch="master",
-        travis_pull_request="false",
+        github_ref="refs/heads/master",
+        github_event_name="push",
         should_build_wheel=True,
         should_build_pyinstaller=True,
         should_build_docker=True,
         docker_username="foo",
         docker_password="bar",
+        has_aws_creds=True,
     )
     assert be.docker_tag == "mitmproxy/mitmproxy:dev"
     assert be.should_upload_docker
     assert not be.should_upload_pypi
     assert be.should_upload_docker
+    assert be.should_upload_aws
     assert not be.is_prod_release
     assert not be.is_maintenance_branch
 
@@ -244,3 +273,20 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir):
     else:
         with pytest.raises(ValueError):
             be.check_version()
+
+
+def test_bool_from_env(monkeypatch):
+    monkeypatch.setenv("FOO", "1")
+    assert cibuild.bool_from_env("FOO")
+
+    monkeypatch.setenv("FOO", "0")
+    assert not cibuild.bool_from_env("FOO")
+
+    monkeypatch.setenv("FOO", "false")
+    assert not cibuild.bool_from_env("FOO")
+
+    monkeypatch.setenv("FOO", "")
+    assert not cibuild.bool_from_env("FOO")
+
+    monkeypatch.delenv("FOO")
+    assert not cibuild.bool_from_env("FOO")

tox.ini

@@ -5,17 +5,14 @@ toxworkdir={env:TOX_WORK_DIR:.tox}
 
 [testenv]
 deps =
-  {env:CI_DEPS:}
   -rrequirements.txt
-passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL RTOOL_*
 setenv = HOME = {envtmpdir}
 commands =
   mitmdump --version
-  pytest --timeout 60 --cov-report='' \
+  pytest --timeout 60 --cov-report xml \
     --cov=mitmproxy --cov=pathod --cov=release \
     --full-cov=mitmproxy/ --full-cov=pathod/ \
     {posargs}
-  {env:CI_COMMANDS:python -c ""}
 
 [testenv:py35]
 whitelist_externals =
@@ -40,7 +37,7 @@ commands =
   python ./test/individual_coverage.py
 
 [testenv:cibuild]
-passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER
+passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_*
 deps =
   -rrequirements.txt
   pyinstaller==3.5
@@ -62,7 +59,7 @@ commands =
   pathoc --version
 
 [testenv:docs]
-passenv = TRAVIS_* APPVEYOR_* AWS_*
+passenv = GITHUB_* AWS_*
 deps =
   -rrequirements.txt
   awscli