mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-01-26 23:23:33 +00:00
Bug 1701608 - remove perma-busted python dependency generator. r=releng-reviewers,jmaher
I have manually pinned the requirements, and partials are working. We'll lose a way to auto-update our dependencies, but a) it's perma-busted, so realistically we're just losing a busted task, and b) mhentges is working on a way to auto-update pinned dependencies in-tree. Differential Revision: https://phabricator.services.mozilla.com/D110148
This commit is contained in:
Aki Sasaki
parent
44a2bdc31d
commit
fa4988e42e
13
.cron.yml
13
.cron.yml
@ -239,19 +239,6 @@ jobs:
|
||||
- {weekday: 'Monday', hour: 10, minute: 0}
|
||||
- {weekday: 'Thursday', hour: 10, minute: 0}
|
||||
|
||||
- name: python-dependency-update
|
||||
job:
|
||||
type: decision-task
|
||||
treeherder-symbol: Nfile
|
||||
target-tasks-method: python_dependency_update
|
||||
run-on-projects:
|
||||
- mozilla-central
|
||||
when:
|
||||
by-project:
|
||||
# No default branch
|
||||
mozilla-central:
|
||||
- {weekday: 'Monday', hour: 10, minute: 0}
|
||||
|
||||
- name: raptor-tp6m
|
||||
job:
|
||||
type: decision-task
|
||||
|
||||
@ -146,8 +146,6 @@ jobs:
|
||||
parent: push-to-try
|
||||
periodic-updates:
|
||||
symbol: I(file)
|
||||
python-dependency-update:
|
||||
symbol: I(pip)
|
||||
firefox-snap:
|
||||
symbol: I(snap)
|
||||
firefox-flatpak:
|
||||
|
||||
@ -1,45 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.python_update:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
|
||||
job-defaults:
|
||||
name: python_dependency_update
|
||||
description: Update requirements.txt
|
||||
run-on-projects: []
|
||||
treeherder:
|
||||
kind: build
|
||||
platform: linux64/opt
|
||||
tier: 1
|
||||
worker-type: b-linux
|
||||
worker:
|
||||
implementation: docker-worker
|
||||
os: linux
|
||||
docker-image: {in-tree: python-dependency-update}
|
||||
taskcluster-proxy: true
|
||||
max-run-time: 2400
|
||||
command:
|
||||
- /runme.sh
|
||||
artifacts:
|
||||
- name: 'public/build/requirements.txt.diff'
|
||||
path: '/home/worker/artifacts/requirements.txt.diff'
|
||||
type: file
|
||||
scopes:
|
||||
- secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token
|
||||
|
||||
|
||||
jobs:
|
||||
funsize-update-generator:
|
||||
treeherder:
|
||||
symbol: pydep(fug)
|
||||
worker:
|
||||
env:
|
||||
REQUIREMENTS_FILE: "taskcluster/docker/funsize-update-generator/requirements.in"
|
||||
PYTHON3: "1"
|
||||
REVIEWERS: "sfraser, bhearsum"
|
||||
@ -1,34 +0,0 @@
|
||||
FROM ubuntu:bionic
|
||||
MAINTAINER Ben Hearsum <bhearsum@mozilla.com>
|
||||
|
||||
# Required software
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
# %include python/mozbuild/mozbuild/action/tooltool.py
|
||||
ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
|
||||
|
||||
# %include testing/mozharness/external_tools/robustcheckout.py
|
||||
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
|
||||
|
||||
# %include taskcluster/docker/recipes/hgrc
|
||||
COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
|
||||
|
||||
# %include taskcluster/docker/recipes/install-mercurial.sh
|
||||
ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
|
||||
|
||||
ADD setup.sh /setup/setup.sh
|
||||
ADD requirements.txt /setup/requirements.txt
|
||||
|
||||
RUN cd /setup && ./setup.sh
|
||||
|
||||
COPY runme.sh /
|
||||
COPY scripts/* /home/worker/scripts/
|
||||
|
||||
ENV HOME /home/worker
|
||||
ENV SHELL /bin/bash
|
||||
ENV USER worker
|
||||
ENV LOGNAME worker
|
||||
ENV LC_ALL C.UTF-8
|
||||
ENV LANG C.UTF-8
|
||||
|
||||
CMD ["/runme.sh"]
|
||||
@ -1,15 +0,0 @@
|
||||
|
||||
==Python Dependency Updates==
|
||||
|
||||
This docker image contains the necessary dependencies and scripts to update
|
||||
in-tree requirement.txt produced by `pip-compile --generate-hashes`, produce a
|
||||
diff, and submit it to Phabricator.
|
||||
|
||||
|
||||
==Quick Start==
|
||||
|
||||
```sh
|
||||
docker build -t python-dependency-update --no-cache --rm .
|
||||
|
||||
docker run -e PYTHON3="1" -e BRANCH="mozilla-central" -e REQUIREMENTS_FILE="taskcluster/docker/funsize-update-generator/requirements.in" python-dependency-update
|
||||
```
|
||||
@ -1,3 +0,0 @@
|
||||
# bug 1698453 - pip-tools 6.0.0 breaks pip-compile due to missing
|
||||
# importlib.metadata plus no --index or --no-index
|
||||
pip-tools<6
|
||||
@ -1,14 +0,0 @@
|
||||
# SHA1:e0576f98505bf01e96b0dec209728de4a2bcfb65
|
||||
#
|
||||
# This file is autogenerated by pip-compile-multi
|
||||
# To update, run:
|
||||
#
|
||||
# pip-compile-multi
|
||||
#
|
||||
click==7.1.2
|
||||
# via pip-tools
|
||||
pip-tools==5.5.0
|
||||
# via -r ./requirements.in
|
||||
|
||||
# The following packages are considered to be unsafe in a requirements file:
|
||||
# pip
|
||||
@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
# Things to be set by task definition.
|
||||
# -b branch
|
||||
# -f requirements_file
|
||||
# -3 use python3
|
||||
|
||||
|
||||
test "${BRANCH}"
|
||||
test "${REQUIREMENTS_FILE}"
|
||||
|
||||
PIP_ARG="-2"
|
||||
if [ -n "${PYTHON3}" ]; then
|
||||
PIP_ARG="-3"
|
||||
fi
|
||||
|
||||
export ARTIFACTS_DIR="/home/worker/artifacts"
|
||||
mkdir -p "$ARTIFACTS_DIR"
|
||||
|
||||
# duplicate the functionality of taskcluster-lib-urls, but in bash..
|
||||
queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1"
|
||||
|
||||
# Get Arcanist API token
|
||||
|
||||
if [ -n "${TASK_ID}" ]
|
||||
then
|
||||
curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID"
|
||||
ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}')
|
||||
fi
|
||||
if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster
|
||||
then
|
||||
set +x # Don't echo these
|
||||
secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}"
|
||||
SECRET=$(curl "${secrets_url}")
|
||||
TOKEN=$(echo "${SECRET}" | jq -r '.secret.token')
|
||||
elif [ -n "${ARC_TOKEN}" ] # Allow for local testing.
|
||||
then
|
||||
TOKEN="${ARC_TOKEN}"
|
||||
fi
|
||||
|
||||
if [ -n "${TOKEN}" ]
|
||||
then
|
||||
cat >"${HOME}/.arcrc" <<END
|
||||
{
|
||||
"hosts": {
|
||||
"https://phabricator.services.mozilla.com/api/": {
|
||||
"token": "${TOKEN}"
|
||||
}
|
||||
}
|
||||
}
|
||||
END
|
||||
set -x
|
||||
chmod 600 "${HOME}/.arcrc"
|
||||
fi
|
||||
|
||||
export HGPLAIN=1
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
/home/worker/scripts/update_pipfiles.sh -b "${BRANCH}" -f "${REQUIREMENTS_FILE}" ${PIP_ARG}
|
||||
@ -1,134 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
function usage {
|
||||
cat <<EOF
|
||||
|
||||
Usage: $(basename "$0") -h # Displays this usage/help text
|
||||
Usage: $(basename "$0") -x # lists exit codes
|
||||
Usage: $(basename "$0") -b branch -r REQUIREMENTS_FILE [-2] [-3]
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
BRANCH=""
|
||||
PIP=""
|
||||
COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>'
|
||||
REPODIR=''
|
||||
HGHOST="hg.mozilla.org"
|
||||
BASEDIR="${HOME}"
|
||||
REQUIREMENTS_FILE=""
|
||||
|
||||
HG="$(command -v hg)"
|
||||
|
||||
# Clones an hg repo
|
||||
function clone_repo {
|
||||
cd "${BASEDIR}"
|
||||
if [ ! -d "${REPODIR}" ]; then
|
||||
CLONE_CMD="${HG} clone ${HGREPO} ${REPODIR}"
|
||||
${CLONE_CMD}
|
||||
fi
|
||||
|
||||
${HG} -R "${REPODIR}" pull
|
||||
${HG} -R "${REPODIR}" update -C default
|
||||
}
|
||||
|
||||
# Push all pending commits to Phabricator
|
||||
function push_repo {
|
||||
cd "${REPODIR}"
|
||||
if [ ! -r "${HOME}/.arcrc" ]
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
if ! ARC=$(command -v arc)
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
if [ -z "${REVIEWERS}" ]
|
||||
then
|
||||
return 1
|
||||
fi
|
||||
# Clean up older review requests
|
||||
# Turn Needs Review D624: No bug, Automated HSTS ...
|
||||
# into D624
|
||||
for diff in $($ARC list | grep "Needs Review" | grep "${REQUIREMENTS_FILE} pip-update" | awk 'match($0, /D[0-9]+[^: ]/) { print substr($0, RSTART, RLENGTH) }')
|
||||
do
|
||||
echo "Removing old request $diff"
|
||||
# There is no 'arc abandon', see bug 1452082
|
||||
echo '{"transactions": [{"type":"abandon"}], "objectIdentifier": "'"${diff}"'"}' | arc call-conduit differential.revision.edit
|
||||
done
|
||||
|
||||
$ARC diff --verbatim --reviewers "${REVIEWERS}"
|
||||
}
|
||||
|
||||
function update_requirements {
|
||||
pushd "${REPODIR}/${1}"
|
||||
pip-compile --generate-hashes "${2}"
|
||||
popd
|
||||
}
|
||||
|
||||
# Main
|
||||
|
||||
# Parse our command-line options.
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-h) usage; exit 0 ;;
|
||||
-b) BRANCH="$2"; shift ;;
|
||||
-r) REPODIR="$2"; shift ;;
|
||||
-2) PIP="pip" ;;
|
||||
-3) PIP="pip3" ;;
|
||||
-f) REQUIREMENTS_FILE="$2"; shift ;;
|
||||
-*) usage
|
||||
exit 11 ;;
|
||||
*) break ;; # terminate while loop
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Must supply a code branch to work with.
|
||||
if [ "${PIP}" == "" ]; then
|
||||
echo "Error: You must specify a python version with -2 or -3" >&2
|
||||
usage
|
||||
exit 12
|
||||
fi
|
||||
|
||||
# Must supply a code branch to work with.
|
||||
if [ "${BRANCH}" == "" ]; then
|
||||
echo "Error: You must specify a branch with -b branchname." >&2
|
||||
usage
|
||||
exit 13
|
||||
fi
|
||||
|
||||
if [ "${REPODIR}" == "" ]; then
|
||||
REPODIR="${BASEDIR}/$(basename "${BRANCH}")"
|
||||
fi
|
||||
|
||||
if [ "${BRANCH}" == "mozilla-central" ]; then
|
||||
HGREPO="https://${HGHOST}/${BRANCH}"
|
||||
elif [[ "${BRANCH}" == mozilla-* ]]; then
|
||||
HGREPO="https://${HGHOST}/releases/${BRANCH}"
|
||||
else
|
||||
HGREPO="https://${HGHOST}/projects/${BRANCH}"
|
||||
fi
|
||||
|
||||
clone_repo
|
||||
|
||||
requirements_basefile="$(basename "${REQUIREMENTS_FILE}")"
|
||||
requirements_dir="$(dirname "${REQUIREMENTS_FILE}")"
|
||||
update_requirements "${requirements_dir}" "${requirements_basefile}"
|
||||
requirements_newfile="${requirements_basefile%%.in}.txt"
|
||||
DIFF_ARTIFACT="${ARTIFACTS_DIR}/${requirements_newfile}.diff"
|
||||
|
||||
echo "INFO: diffing old/new ${requirements_newfile} into ${DIFF_ARTIFACT}"
|
||||
${HG} -R "${REPODIR}" diff "${BASEDIR}/${BRANCH}/${requirements_dir}/${requirements_newfile}" | tee "${DIFF_ARTIFACT}"
|
||||
|
||||
COMMIT_MESSAGE="No Bug, ${requirements_dir}/${requirements_newfile} pip-update."
|
||||
|
||||
if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}"
|
||||
then
|
||||
${HG} -R "${REPODIR}" out
|
||||
push_repo
|
||||
fi
|
||||
|
||||
echo "All done"
|
||||
@ -1,38 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
set -ve
|
||||
|
||||
tooltool_fetch() {
|
||||
cat >manifest.tt
|
||||
python2.7 /setup/tooltool.py fetch
|
||||
rm manifest.tt
|
||||
}
|
||||
|
||||
useradd -d /home/worker -s /bin/bash -m worker
|
||||
|
||||
apt-get update -q
|
||||
apt-get install -y --no-install-recommends \
|
||||
arcanist \
|
||||
curl \
|
||||
gcc \
|
||||
jq \
|
||||
libdpkg-perl \
|
||||
liblzma-dev \
|
||||
python \
|
||||
python-dev \
|
||||
python-pip \
|
||||
python3.8 \
|
||||
python3.8-dev \
|
||||
python3-setuptools \
|
||||
python3-wheel \
|
||||
python3-pip
|
||||
|
||||
apt-get clean
|
||||
|
||||
. install-mercurial.sh
|
||||
pip install -r requirements.txt
|
||||
|
||||
rm -rf /setup
|
||||
@ -580,11 +580,6 @@ repo-update
|
||||
Repo-Update tasks are tasks that perform some action on the project repo itself,
|
||||
in order to update its state in some way.
|
||||
|
||||
python-dependency-update
|
||||
------------------------
|
||||
Python-dependency-update runs `pip-compile --generate-hashes` against the specified `requirements.in` and
|
||||
submits patches to Phabricator.
|
||||
|
||||
partials
|
||||
--------
|
||||
Partials takes the complete.mar files produced in previous tasks and generates partial
|
||||
|
||||
@ -1054,17 +1054,6 @@ def target_tasks_chromium_update(full_task_graph, parameters, graph_config):
|
||||
]
|
||||
|
||||
|
||||
@_target_task("python_dependency_update")
|
||||
def target_tasks_python_update(full_task_graph, parameters, graph_config):
|
||||
"""Select the set of tasks required to perform nightly in-tree pipfile updates"""
|
||||
|
||||
def filter(task):
|
||||
# For now any task in the repo-update kind is ok
|
||||
return task.kind in ["python-dependency-update"]
|
||||
|
||||
return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)]
|
||||
|
||||
|
||||
@_target_task("file_update")
|
||||
def target_tasks_file_update(full_task_graph, parameters, graph_config):
|
||||
"""Select the set of tasks required to perform nightly in-tree file updates"""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user