diff --git a/.github/node-version.txt b/.github/node-version.txt new file mode 100644 index 000000000..8351c1939 --- /dev/null +++ b/.github/node-version.txt @@ -0,0 +1 @@ +14 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fc76e5fe9..4ed8a1cc3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: CI -on: [ push, pull_request ] +on: [ push, pull_request, workflow_dispatch ] permissions: contents: read @@ -152,7 +152,7 @@ jobs: - run: git rev-parse --abbrev-ref HEAD - uses: actions/setup-node@v3 with: - node-version: '14' + node-version-file: .github/node-version.txt - name: Cache Node.js modules uses: actions/cache@v3 with: @@ -191,6 +191,16 @@ jobs: with: name: docs path: docs/public + # For releases, also build the archive version of the docs. + - if: startsWith(github.ref, 'refs/tags/') + run: ./docs/build.py + env: + DOCS_ARCHIVE: true + - if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v3 + with: + name: docs-archive + path: docs/public # Separate from everything else because slow. build-and-deploy-docker: @@ -263,6 +273,11 @@ jobs: with: name: docs path: docs/public + - if: startsWith(github.ref, 'refs/tags/') + uses: actions/download-artifact@v3 + with: + name: docs-archive + path: docs/archive - uses: actions/download-artifact@v3 with: name: binaries.windows diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..d99bf2f4f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: 'The version to release (major.minor.patch)' + required: true + type: string + +jobs: + # this job is just here as a fail-safe to make sure that the invoking user has the necessary permissions + # before we start the release process. This way we hopefully don't have to clean up incomplete release processes. + permission-check: + runs-on: ubuntu-latest + environment: deploy-release + steps: + - run: echo "ok" + + release: + needs: permission-check + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version-file: .github/node-version.txt + - uses: actions/setup-python@v4 + with: + python-version-file: .github/python-version.txt + - run: ./release/release.py ${{ inputs.version }} diff --git a/release/README.md b/release/README.md index 5b6bafa17..3f5b37035 100644 --- a/release/README.md +++ b/release/README.md @@ -1,51 +1,48 @@ # Release Checklist - -These steps assume you are on the correct branch and have a git remote called `origin` that points to the -`mitmproxy/mitmproxy` repo. If necessary, create a major version branch starting off the release tag -(e.g. `git checkout -b v4.x 4.0.0`) first. - -- Update CHANGELOG. -- Verify that the compiled mitmweb assets are up-to-date (`npm start prod`). -- Verify that all CI tests pass. -- Verify that `mitmproxy/version.py` is correct. Remove `.dev` suffix if it exists. -- Tag the release and push to GitHub. - - `git tag 4.0.0` - - `git push origin 4.0.0` -- Wait for tag CI to complete. + +1. Make sure that `CHANGELOG.md` is up-to-date with all entries in the "Unreleased" section. +2. Invoke the [release workflow](https://github.com/mitmproxy/mitmproxy/actions/workflows/release.yml) from the GitHub UI. +3. The spawned workflow runs will require manual confirmation on GitHub which you need to approve twice: + https://github.com/mitmproxy/mitmproxy/actions +4. Once everything has been deployed, update the website. ### GitHub Releases -- Create release notice on GitHub - [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already - auto-created by the tag. -- We DO NOT upload release artifacts to GitHub anymore. Simply add the - following snippet to the notice: - `You can find the latest release packages at https://mitmproxy.org/downloads/.` +- CI will automatically create a GitHub release: + https://github.com/mitmproxy/mitmproxy/releases ### PyPi -- The created wheel is uploaded to PyPi automatically. -- Please verify that https://pypi.python.org/pypi/mitmproxy has the latest version. +- CI will automatically push a wheel to GitHub: + https://pypi.python.org/pypi/mitmproxy + +### Docker + +- CI will automatically push images to Docker Hub: + https://hub.docker.com/r/mitmproxy/mitmproxy/tags/ + +### Docs + +- CI will automatically update the stable docs and create an archive version: + `https://docs.mitmproxy.org/archive/vMAJOR/` + +### Download Server + +- CI will automatically push binaries to our download S3 bucket: + https://mitmproxy.org/downloads/ + +### Microsoft Store + +- CI will automatically update the Microsoft Store version: + https://apps.microsoft.com/store/detail/mitmproxy/9NWNDLQMNZD7 +- There is a review process, binaries may take a day to show up. ### Homebrew - The Homebrew maintainers are typically very fast and detect our new relese within a day. - If you feel the need, you can run this from a macOS machine: - `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v.tar.gz mitmproxy` - -### Docker - -- The docker image is built by our CI workers and pushed to Docker Hub automatically. -- Please verify that https://hub.docker.com/r/mitmproxy/mitmproxy/tags/ has the latest version. -- Please verify that the latest tag points to the most recent image (same digest / hash). - -### Docs - -- `./build.py`. If everything looks alright, continue with -- `./upload-stable.sh`, -- `DOCS_ARCHIVE=true ./build.py`, and -- `./upload-archive.sh v4`. Doing this now already saves you from switching back to an old state on the next release. + `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/.tar.gz mitmproxy` ### Website diff --git a/release/deploy.py b/release/deploy.py index 0037e0bab..f0774edec 100755 --- a/release/deploy.py +++ b/release/deploy.py @@ -29,14 +29,14 @@ if __name__ == "__main__": [ "aws", "s3", - "cp", + "sync", + "--delete", "--acl", "public-read", "--exclude", "*.msix", root / "release/dist", - f"s3://snapshots.mitmproxy.org/{upload_dir}/", - "--recursive", + f"s3://snapshots.mitmproxy.org/{upload_dir}", ] ) @@ -46,9 +46,8 @@ if __name__ == "__main__": (whl,) = root.glob("release/dist/mitmproxy-*-py3-none-any.whl") subprocess.check_call(["twine", "upload", whl]) - # Upload dev docs - if branch == "main": - print(f"Uploading dev docs...") + # Upload docs + def upload_docs(path: str, src: Path = root / "docs/public"): subprocess.check_call(["aws", "configure", "set", "preview.cloudfront", "true"]) subprocess.check_call( [ @@ -58,8 +57,8 @@ if __name__ == "__main__": "--delete", "--acl", "public-read", - root / "docs/public", - "s3://docs.mitmproxy.org/dev", + src, + f"s3://docs.mitmproxy.org{path}", ] ) subprocess.check_call( @@ -70,6 +69,14 @@ if __name__ == "__main__": "--distribution-id", "E1TH3USJHFQZ5Q", "--paths", - "/dev/*", + f"{path}/*", ] ) + + if branch == "main": + print(f"Uploading dev docs...") + upload_docs("/dev") + if tag: + print(f"Uploading release docs...") + upload_docs("/stable") + upload_docs(f"/archive/v{tag.split('.')[0]}", src=root / "docs/archive") diff --git a/release/github-release-notes.txt b/release/github-release-notes.txt new file mode 100644 index 000000000..49c12179a --- /dev/null +++ b/release/github-release-notes.txt @@ -0,0 +1,3 @@ +Changes: See [CHANGELOG.md](https://github.com/mitmproxy/mitmproxy/blob/main/CHANGELOG.md). + +You can find the latest release packages at https://mitmproxy.org/downloads/. diff --git a/release/release.py b/release/release.py new file mode 100755 index 000000000..b3c7b7a4c --- /dev/null +++ b/release/release.py @@ -0,0 +1,151 @@ +#!/usr/bin/env -S python3 -u +import datetime +import http.client +import json +import re +import subprocess +import sys +import time +from pathlib import Path + +# Security: No third-party dependencies here! + +root = Path(__file__).absolute().parent.parent + + +def get(url: str) -> http.client.HTTPResponse: + assert url.startswith("https://") + host, path = re.split(r"(?=/)", url.removeprefix("https://"), maxsplit=1) + conn = http.client.HTTPSConnection(host) + conn.request("GET", path, headers={"User-Agent": "mitmproxy/release-bot"}) + resp = conn.getresponse() + print(f"HTTP {resp.status} {resp.reason}") + return resp + + +def get_json(url: str) -> dict: + resp = get(url) + body = resp.read() + try: + return json.loads(body) + except Exception as e: + raise RuntimeError(f"{resp.status=} {body=}") from e + + +if __name__ == "__main__": + version = sys.argv[1] + assert re.match(r"^\d+\.\d+\.\d+$", version) + major_version = int(version.split(".")[0]) + + branch = subprocess.run( + ["git", "branch", "--show-current"], + cwd=root, check=True, capture_output=True, text=True + ).stdout.strip() + + print("➡️ Working dir clean?") + assert not subprocess.run(["git", "status", "--porcelain"]).stdout + + print(f"➡️ CI is passing for {branch}?") + assert get_json(f"https://api.github.com/repos/mitmproxy/mitmproxy/commits/{branch}/status")["state"] == "success" + + print("➡️ Updating CHANGELOG.md...") + changelog = root / "CHANGELOG.md" + date = datetime.date.today().strftime("%d %B %Y") + title = f"## {date}: mitmproxy {version}" + cl = changelog.read_text("utf8") + assert title not in cl + cl, ok = re.subn(r"(?<=## Unreleased: mitmproxy next)", f"\n\n\n\n{title}", cl) + assert ok == 1 + changelog.write_text(cl, "utf8") + + print("➡️ Updating web assets...") + subprocess.run(["npm", "ci"], cwd=root / "web", check=True, capture_output=True) + subprocess.run(["npm", "start", "prod"], cwd=root / "web", check=True, capture_output=True) + + print("➡️ Updating version...") + version_py = root / "mitmproxy" / "version.py" + ver = version_py.read_text("utf8") + ver, ok = re.subn(r'(?<=VERSION = ")[^"]+', version, ver) + assert ok == 1 + version_py.write_text(ver, "utf8") + + print("➡️ Do release commit...") + subprocess.run(["git", "config", "user.email", "noreply@mitmproxy.org"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "mitmproxy release bot"], cwd=root, check=True) + subprocess.run(["git", "commit", "-a", "-m", f"mitmproxy {version}"], cwd=root, check=True) + subprocess.run(["git", "tag", version], cwd=root, check=True) + + if branch == "main": + print("➡️ Bump version...") + next_dev_version = f"{major_version + 1}.0.0.dev" + ver, ok = re.subn(r'(?<=VERSION = ")[^"]+', next_dev_version, ver) + assert ok == 1 + version_py.write_text(ver, "utf8") + + print("➡️ Reopen main for development...") + subprocess.run(["git", "commit", "-a", "-m", f"reopen main for development"], cwd=root, check=True) + + print("➡️ Pushing...") + subprocess.run(["git", "push", "--atomic", "origin", branch, version], cwd=root, check=True) + + print("➡️ Creating release on GitHub...") + subprocess.run(["gh", "release", "create", version, + "--title", f"mitmproxy {version}", + "--notes-file", "release/github-release-notes.txt"], cwd=root, check=True) + + print("➡️ Dispatching release workflow...") + subprocess.run(["gh", "workflow", "run", "main.yml", "--ref", version], cwd=root, check=True) + + print("") + print("✅ CI is running now. Make sure to approve the deploy step: https://github.com/mitmproxy/mitmproxy/actions") + + for _ in range(60): + time.sleep(3) + print(".", end="") + print("") + + print("➡️ Checking GitHub Releases...") + resp = get(f"https://api.github.com/repos/mitmproxy/mitmproxy/releases/tags/{version}") + assert resp.status == 200 + + while True: + print("➡️ Checking PyPI...") + pypi_data = get_json("https://pypi.org/pypi/mitmproxy/json") + if version in pypi_data["releases"]: + print(f"{version} is on PyPI.") + break + else: + print(f"{version} not yet on PyPI.") + time.sleep(10) + + while True: + print("➡️ Checking docs archive...") + resp = get(f"https://docs.mitmproxy.org/archive/v{major_version}/") + if resp.status == 200: + break + else: + time.sleep(10) + + while True: + print(f"➡️ Checking Docker ({version} tag)...") + resp = get(f"https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/{version}") + if resp.status == 200: + break + else: + time.sleep(10) + + if branch == "main": + while True: + print("➡️ Checking Docker (latest tag)...") + docker_latest_data = get_json("https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/latest") + docker_last_updated = datetime.datetime.fromisoformat( + docker_latest_data["last_updated"].replace("Z", "+00:00")) + print(f"Last update: {docker_last_updated.isoformat(timespec='minutes')}") + if docker_last_updated > datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=2): + break + else: + time.sleep(10) + + print("") + print("✅ All done. 🥳") + print("")