fix pyinstaller

amongst other things, fix #1746

.appveyor.yml

@@ -24,17 +24,20 @@ install:
 
 test_script:
   - ps: "tox -- --cov mitmproxy --cov pathod -v"
+  - ps: |
+      $Env:VERSION = $(python mitmproxy/version.py)
+      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
+      tox -e wheel -- https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl
+      tox -e rtool -- bdist
 
 deploy_script:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   ps: |
     if(
       ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-      pip install -U virtualenv
-      .\dev.ps1
-      cmd /c "python -u .\release\rtool.py bdist 2>&1"
-      python -u .\release\rtool.py upload-snapshot --bdist --wheel
+      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+    ) {
+      tox -e rtool -- upload-snapshot --bdist --wheel
     }
 
 cache:

.travis.yml

@@ -51,18 +51,21 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov mitmproxy --cov pathod -v
+script:
+  - |
+    tox -- --cov mitmproxy --cov pathod -v
+    if [[ $BDIST == "1" ]]
+    then
+        git fetch --unshallow --tags
+        tox -e rtool -- bdist
+    fi
 
 after_success:
+  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
-        git fetch --unshallow
-        ./dev.sh 3.5
-        source venv3.5/bin/activate
-        pip install -e ./release
-        python -u ./release/rtool.py bdist
-        python -u ./release/rtool.py upload-snapshot --bdist
+        tox -e rtool -- upload-snapshot --bdist
     fi
 
 notifications:

dev.ps1

@@ -1,7 +1,7 @@
 $ErrorActionPreference = "Stop"
 $VENV = ".\venv"
 
-virtualenv $VENV --always-copy
+python3 -m venv $VENV --copies
 & $VENV\Scripts\activate.ps1
 
 python -m pip install --disable-pip-version-check -U pip

mitmproxy/tools/__init__.py

@@ -1,5 +0,0 @@
-from mitmproxy.tools import web
-from mitmproxy.tools import console
-from mitmproxy.tools import dump
-
-__all__ = ["web", "console", "dump"]

mitmproxy/utils/data.py

@@ -15,7 +15,7 @@ class Data:
         """
             Change the data object to a path relative to the module.
         """
-        dirname = os.path.join(self.dirname, subpath)
+        dirname = os.path.normpath(os.path.join(self.dirname, subpath))
         ret = Data(self.name)
         ret.dirname = dirname
         return ret
@@ -27,7 +27,7 @@ class Data:
 
             This function will raise ValueError if the path does not exist.
         """
-        fullpath = os.path.join(self.dirname, path)
+        fullpath = os.path.normpath(os.path.join(self.dirname, path))
         if not os.path.exists(fullpath):
             raise ValueError("dataPath: %s does not exist." % fullpath)
         return fullpath

mitmproxy/version.py

@@ -2,3 +2,6 @@ IVERSION = (0, 19)
 VERSION = ".".join(str(i) for i in IVERSION)
 PATHOD = "pathod " + VERSION
 MITMPROXY = "mitmproxy " + VERSION
+
+if __name__ == "__main__":
+    print(VERSION)

release/hooks/hook-cryptography.py

@@ -0,0 +1,43 @@
+# Taken from the latest pyinstaller master on 2016-11-27 (0729a2b).
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2005-2016, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License with exception
+# for distributing bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+
+"""
+Hook for cryptography module from the Python Cryptography Authority.
+"""
+
+import os.path
+import glob
+
+from PyInstaller.compat import EXTENSION_SUFFIXES
+from PyInstaller.utils.hooks import collect_submodules, get_module_file_attribute
+from PyInstaller.utils.hooks import copy_metadata
+
+# get the package data so we can load the backends
+datas = copy_metadata('cryptography')
+
+# Add the backends as hidden imports
+hiddenimports = collect_submodules('cryptography.hazmat.backends')
+
+# Add the OpenSSL FFI binding modules as hidden imports
+hiddenimports += collect_submodules('cryptography.hazmat.bindings.openssl') + ['_cffi_backend']
+
+
+# Include the cffi extensions as binaries in a subfolder named like the package.
+# The cffi verifier expects to find them inside the package directory for
+# the main module. We cannot use hiddenimports because that would add the modules
+# outside the package.
+binaries = []
+cryptography_dir = os.path.dirname(get_module_file_attribute('cryptography'))
+for ext in EXTENSION_SUFFIXES:
+    ffimods = glob.glob(os.path.join(cryptography_dir, '*_cffi_*%s*' % ext))
+    for f in ffimods:
+        binaries.append((f, 'cryptography'))

release/hooks/hook-mitmproxy.addons.onboardingapp.py

@@ -1,3 +1,3 @@
 from PyInstaller.utils.hooks import collect_data_files
 
-datas = collect_data_files('mitmproxy.addons.onboardingapp')
+datas = collect_data_files('mitmproxy.addons.onboardingapp')

release/hooks/hook-mitmproxy.tools.web.py

@@ -0,0 +1,3 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = collect_data_files('mitmproxy.tools.web')

release/rtool.py

@@ -8,10 +8,9 @@ import runpy
 import shlex
 import shutil
 import subprocess
-import sys
 import tarfile
 import zipfile
-from os.path import join, abspath, normpath, dirname, exists, basename
+from os.path import join, abspath, dirname, exists, basename
 
 import click
 import pysftp
@@ -20,8 +19,13 @@ import pysftp
 # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
 if platform.system() == "Windows":
     VENV_BIN = "Scripts"
+    PYINSTALLER_ARGS = [
+        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+    ]
 else:
     VENV_BIN = "bin"
+    PYINSTALLER_ARGS = []
 
 # ZipFile and tarfile have slightly different APIs. Fix that.
 if platform.system() == "Windows":
@@ -46,13 +50,9 @@ PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
 PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
 
 VENV_DIR = join(BUILD_DIR, "venv")
-VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
-VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
 
 # Project Configuration
 VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-PROJECT_NAME = "mitmproxy"
-PYTHON_VERSION = "py2.py3"
 BDISTS = {
     "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
     "pathod": ["pathoc", "pathod"]
@@ -62,20 +62,20 @@ if platform.system() == "Windows":
 
 TOOLS = [
     tool
-    for tools in BDISTS.values()
+    for tools in sorted(BDISTS.values())
     for tool in tools
 ]
 
 
-def get_version() -> str:
-    return runpy.run_path(VERSION_FILE)["VERSION"]
-
-
 def git(args: str) -> str:
     with chdir(ROOT_DIR):
         return subprocess.check_output(["git"] + shlex.split(args)).decode()
 
 
+def get_version() -> str:
+    return runpy.run_path(VERSION_FILE)["VERSION"]
+
+
 def get_snapshot_version() -> str:
     last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
     tag_dist = int(tag_dist)
@@ -109,25 +109,11 @@ def archive_name(bdist: str) -> str:
 
 
 def wheel_name() -> str:
-    return "{project}-{version}-{py_version}-none-any.whl".format(
-        project=PROJECT_NAME,
+    return "mitmproxy-{version}-py3-none-any.whl".format(
         version=get_version(),
-        py_version=PYTHON_VERSION
     )
 
 
-@contextlib.contextmanager
-def empty_pythonpath():
-    """
-    Make sure that the regular python installation is not on the python path,
-    which would give us access to modules installed outside of our virtualenv.
-    """
-    pythonpath = os.environ.get("PYTHONPATH", "")
-    os.environ["PYTHONPATH"] = ""
-    yield
-    os.environ["PYTHONPATH"] = pythonpath
-
-
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -156,58 +142,8 @@ def contributors():
             f.write(contributors_data.encode())
 
 
-@cli.command("wheel")
-def make_wheel():
-    """
-    Build wheel
-    """
-    with empty_pythonpath():
-        if exists(DIST_DIR):
-            shutil.rmtree(DIST_DIR)
-
-        print("Creating wheel...")
-        subprocess.check_call(
-            [
-                "python3", "./setup.py", "-q",
-                "bdist_wheel", "--dist-dir", DIST_DIR, "--universal"
-            ],
-            cwd=ROOT_DIR
-        )
-
-        print("Creating virtualenv for test install...")
-        if exists(VENV_DIR):
-            shutil.rmtree(VENV_DIR)
-        subprocess.check_call(["python3", "-m", "virtualenv", "-q", VENV_DIR])
-
-        with chdir(DIST_DIR):
-            print("Install wheel into virtualenv...")
-            # lxml...
-            if platform.system() == "Windows" and sys.version_info[0] == 3:
-                subprocess.check_call(
-                    [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]
-                )
-            subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()])
-
-            print("Running tools...")
-            for tool in TOOLS:
-                tool = join(VENV_DIR, VENV_BIN, tool)
-                print("> %s --version" % tool)
-                print(subprocess.check_output([tool, "--version"]).decode())
-
-            print("Virtualenv available for further testing:")
-            print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate")))
-
-
 @cli.command("bdist")
-@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False)
-@click.argument(
-    "pyinstaller_version",
-    envvar="PYINSTALLER_VERSION",
-    default="git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c"
-)
-@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1")
-@click.pass_context
-def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version):
+def make_bdist():
     """
     Build a binary distribution
     """
@@ -216,37 +152,40 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    if not use_existing_wheel:
-        ctx.invoke(make_wheel)
+    os.makedirs(DIST_DIR, exist_ok=True)
 
-    print("Installing PyInstaller and setuptools...")
-    subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version])
-    print(subprocess.check_output([VENV_PIP, "freeze"]).decode())
-
-    for bdist, tools in BDISTS.items():
+    for bdist, tools in sorted(BDISTS.items()):
         with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
             for tool in tools:
+                # We can't have a folder and a file with the same name.
+                if tool == "mitmproxy":
+                    tool = "mitmproxy_main"
                 # This is PyInstaller, so it messes up paths.
                 # We need to make sure that we are in the spec folder.
                 with chdir(PYINSTALLER_SPEC):
                     print("Building %s binary..." % tool)
+                    excludes = []
+                    if tool != "mitmweb":
+                        excludes.append("mitmproxy.tools.web")
+                    if tool != "mitmproxy_main":
+                        excludes.append("mitmproxy.tools.console")
                     subprocess.check_call(
                         [
-                            VENV_PYINSTALLER,
+                            "pyinstaller",
                             "--clean",
                             "--workpath", PYINSTALLER_TEMP,
                             "--distpath", PYINSTALLER_DIST,
                             "--additional-hooks-dir", PYINSTALLER_HOOKS,
-                            # PyInstaller 3.2 does not handle Python 3.5's ucrt correctly.
-                            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
                             "--onefile",
                             "--console",
                             "--icon", "icon.ico",
                             # This is PyInstaller, so setting a
                             # different log level obviously breaks it :-)
                             # "--log-level", "WARN",
-                            tool
                         ]
+                        + [x for e in excludes for x in ["--exclude-module", e]]
+                        + PYINSTALLER_ARGS
+                        + [tool]
                     )
                     # Delete the spec file - we're good without.
                     os.remove("{}.spec".format(tool))
@@ -255,6 +194,15 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
                 executable = join(PYINSTALLER_DIST, tool)
                 if platform.system() == "Windows":
                     executable += ".exe"
+
+                # Remove _main suffix from mitmproxy executable
+                if executable.startswith("mitmproxy_main"):
+                    shutil.move(
+                        executable,
+                        executable.replace("_main", "")
+                    )
+                    executable = executable.replace("_main", "")
+
                 print("> %s --version" % executable)
                 print(subprocess.check_output([executable, "--version"]).decode())
 
@@ -306,7 +254,7 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel,
             if wheel:
                 files.append(wheel_name())
             if bdist:
-                for bdist in BDISTS.keys():
+                for bdist in sorted(BDISTS.keys()):
                     files.append(archive_name(bdist))
 
             for f in files:

release/setup.py

@@ -7,8 +7,6 @@ setup(
     install_requires=[
         "click>=6.2, <7.0",
         "twine>=1.6.5, <1.9",
-        "virtualenv>=14.0.5, <15.2",
-        "wheel>=0.29.0, <0.30",
         "pysftp==0.2.8",
     ],
     entry_points={

requirements.txt

@@ -1,3 +1,2 @@
 https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5'
 -e .[dev,examples,contentviews]
--e ./release

setup.py

@@ -84,6 +84,9 @@ setup(
         "watchdog>=0.8.3, <0.9",
         "brotlipy>=0.5.1, <0.7",
         "sortedcontainers>=1.5.4, <1.6",
+        # transitive from cryptography, we just blacklist here.
+        # https://github.com/pypa/setuptools/issues/861
+        "setuptools>=11.3, !=29.0.0",
     ],
     extras_require={
         ':sys_platform == "win32"': [

test/mitmproxy/addons/test_script.py

@@ -156,8 +156,8 @@ class TestScript:
             sc.request(f)
             assert tctx.master.event_log[0][0] == "error"
             assert len(tctx.master.event_log[0][1].splitlines()) == 6
-            assert re.search('addonscripts/error.py", line \d+, in request', tctx.master.event_log[0][1])
-            assert re.search('addonscripts/error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1])
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
             assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n")
 
     def test_addon(self):

tox.ini

@@ -4,10 +4,11 @@ skipsdist = True
 toxworkdir={env:TOX_WORK_DIR:.tox}
 
 [testenv]
+basepython = python3.5
 deps =
   {env:CI_DEPS:}
   -rrequirements.txt
-passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_*
+passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_*
 setenv = HOME = {envtmpdir}
 commands =
   mitmdump --sysinfo
@@ -29,3 +30,27 @@ commands =
     mitmproxy/proxy/protocol/ \
     mitmproxy/log.py \
     mitmproxy/tools/dump.py mitmproxy/tools/web
+
+[testenv:wheel]
+recreate = True
+deps =
+commands =
+  python setup.py -q bdist_wheel --dist-dir release/dist
+  pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl
+  # skip `mitmproxy --version` if SKIP_MITMPROXY is defined.
+  {env:SKIP_MITMPROXY:mitmproxy --version}
+  mitmdump --version
+  mitmweb --version
+  pathod --version
+  pathoc --version
+
+[testenv:rtool]
+deps =
+  -rrequirements.txt
+  -e./release
+  # The 3.2 release is broken 🎉
+  # the next commit after this updates the bootloaders, which then segfault! 🎉
+  # https://github.com/pyinstaller/pyinstaller/issues/2232
+  git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c
+commands =
+  rtool {posargs}