mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-02-18 23:15:38 +00:00
Bug 1825830 - Remove updatebot from ESR102. r=gabriel,tjr a=NPOTB DONTBUILD
Differential Revision: https://phabricator.services.mozilla.com/D174335
This commit is contained in:
Ryan VanderMeulen
parent
778ca40c62
commit
c2a684f2b3
13
.cron.yml
13
.cron.yml
@ -149,19 +149,6 @@ jobs:
|
||||
when:
|
||||
- {hour: 2, minute: 0}
|
||||
|
||||
- name: updatebot-cron-job
|
||||
job:
|
||||
type: decision-task
|
||||
treeherder-symbol: updatebot
|
||||
target-tasks-method: updatebot_cron
|
||||
run-on-projects:
|
||||
- mozilla-central
|
||||
when:
|
||||
- {hour: 0, minute: 0}
|
||||
- {hour: 6, minute: 0}
|
||||
- {hour: 12, minute: 0}
|
||||
- {hour: 18, minute: 0}
|
||||
|
||||
- name: customv8-update
|
||||
job:
|
||||
type: decision-task
|
||||
|
||||
@ -194,9 +194,6 @@ jobs:
|
||||
symbol: I(PR)
|
||||
parent: debian10-base
|
||||
definition: partner-repack
|
||||
updatebot:
|
||||
symbol: I(3rdp)
|
||||
parent: push-to-try
|
||||
periodic-updates:
|
||||
symbol: I(file)
|
||||
parent: ubuntu1804-base
|
||||
|
||||
@ -17,4 +17,3 @@ jobs-from:
|
||||
- resource-monitor.yml
|
||||
- toolchain-clang-tidy-external.yml
|
||||
- toolchains.yml
|
||||
- updatebot.yml
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
cloud-sql-proxy-1.25.0:
|
||||
description: Windows executable for Google's Cloud SQL Proxy
|
||||
fetch:
|
||||
type: static-url
|
||||
url: https://storage.googleapis.com/cloudsql-proxy/v1.25.0/cloud_sql_proxy_x64.exe
|
||||
sha256: b922a8d3924499b829bfe53a25382f9b340ea804a738e2a7d8b34957a329a2dd
|
||||
size: 16195072
|
||||
artifact-name: cloud_sql_proxy.exe
|
||||
php-windows-7.4.24:
|
||||
description: VC15 x64 Non Thread Safe (2021-Sep-21 16:58:34)
|
||||
fetch:
|
||||
type: static-url
|
||||
url: https://windows.php.net/downloads/releases/archives/php-7.4.24-nts-Win32-vc15-x64.zip
|
||||
sha256: 1e93fb390ab2761a47d27c97225a518759731c632c0dd87436cbf25f19582436
|
||||
size: 26067094
|
||||
artifact-name: php.tar.zst
|
||||
add-prefix: php-win/
|
||||
arcanist:
|
||||
description: checkout of arcanist for running on Windows
|
||||
fetch:
|
||||
type: git
|
||||
repo: https://github.com/phacility/arcanist.git
|
||||
revision: a028291f8e5e79b6446ad67ed8b0be9c5ec8f029
|
||||
depot_tools:
|
||||
description: Google's depot_tools package
|
||||
fetch:
|
||||
type: git
|
||||
repo: https://chromium.googlesource.com/chromium/tools/depot_tools.git
|
||||
revision: 7b3351443198f4e7d068999b169385b55a43679a
|
||||
@ -1,91 +0,0 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: gecko_taskgraph.loader.transform:loader
|
||||
|
||||
kind-dependencies:
|
||||
- fetch
|
||||
- toolchain
|
||||
|
||||
transforms:
|
||||
- gecko_taskgraph.transforms.release_notifications:transforms
|
||||
- gecko_taskgraph.transforms.job:transforms
|
||||
- gecko_taskgraph.transforms.task:transforms
|
||||
|
||||
jobs:
|
||||
cron:
|
||||
description: Cron Job for Updatebot
|
||||
run-on-projects: []
|
||||
treeherder:
|
||||
kind: other
|
||||
platform: updatebot/all
|
||||
symbol: cron
|
||||
tier: 1
|
||||
worker-type: b-linux
|
||||
worker:
|
||||
docker-image: {in-tree: updatebot}
|
||||
max-run-time: 3600
|
||||
taskcluster-proxy: true
|
||||
scopes:
|
||||
- secrets:get:project/updatebot/{level}/try-sshkey
|
||||
- secrets:get:project/updatebot/{level}/phabricator-token
|
||||
- secrets:get:project/updatebot/{level}/bugzilla-api-key
|
||||
- secrets:get:project/updatebot/{level}/database-password
|
||||
- secrets:get:project/updatebot/{level}/sentry-url
|
||||
- secrets:get:project/updatebot/{level}/sql-proxy-config
|
||||
- hooks:trigger-hook:project-gecko/in-tree-action-1-generic/*
|
||||
run:
|
||||
using: run-task
|
||||
command: /builds/worker/run.py /builds/worker/checkouts/gecko /builds/worker/updatebot
|
||||
notifications:
|
||||
subject: 'updatebot cron failed'
|
||||
message: 'updatebot cron failed'
|
||||
status-types:
|
||||
- on-failed
|
||||
- on-exception
|
||||
emails: ["tom@mozilla.com", "jewilde@mozilla.com"]
|
||||
fetches:
|
||||
toolchain:
|
||||
- linux64-rust
|
||||
|
||||
cron-win:
|
||||
description: Cron Job for Updatebot (Windows)
|
||||
run-on-projects: []
|
||||
treeherder:
|
||||
kind: other
|
||||
platform: updatebot/all
|
||||
symbol: w-cron
|
||||
tier: 1
|
||||
worker-type: b-win2012
|
||||
worker:
|
||||
max-run-time: 3600
|
||||
taskcluster-proxy: true
|
||||
# begin _GENERATE_DEPOT_TOOLS_BINARIES_
|
||||
# artifacts:
|
||||
# - type: directory
|
||||
# name: public/depot_tools.zip
|
||||
# path: build\src\obj-build\depot_tools
|
||||
# end _GENERATE_DEPOT_TOOLS_BINARIES_
|
||||
env:
|
||||
TOOLTOOL_MANIFEST: "taskcluster/ci/updatebot/tooltool-manifests/updatebot.manifest"
|
||||
MSYSTEM: "MINGW64"
|
||||
run:
|
||||
using: run-task
|
||||
cwd: '{checkout}'
|
||||
command: ./taskcluster/docker/updatebot/windows-setup.sh
|
||||
tooltool-downloads: internal
|
||||
scopes:
|
||||
- secrets:get:project/updatebot/{level}/try-sshkey
|
||||
- secrets:get:project/updatebot/{level}/phabricator-token
|
||||
- secrets:get:project/updatebot/{level}/bugzilla-api-key
|
||||
- secrets:get:project/updatebot/{level}/database-password
|
||||
- secrets:get:project/updatebot/{level}/sentry-url
|
||||
- secrets:get:project/updatebot/{level}/sql-proxy-config
|
||||
- hooks:trigger-hook:project-gecko/in-tree-action-1-generic/*
|
||||
fetches:
|
||||
fetch:
|
||||
- cloud-sql-proxy-1.25.0
|
||||
- php-windows-7.4.24
|
||||
- arcanist
|
||||
- depot_tools
|
||||
@ -1,29 +0,0 @@
|
||||
[
|
||||
{
|
||||
"version": "vcruntime140.dll from Microsoft Visual C++ Redistributable for Visual Studio 2019 ver 14.29.30133.0",
|
||||
"size": 97160,
|
||||
"visibility": "internal",
|
||||
"digest": "aa6afd6bea27f554e3646152d8c4f96f7bcaaa4933f8b7c04346e410f93f23cfa6d29362fd5d51ccbb8b6223e094cd89e351f072ad0517553703f5bf9de28778",
|
||||
"algorithm": "sha512",
|
||||
"filename": "vcruntime140.dll",
|
||||
"unpack": false
|
||||
},
|
||||
{
|
||||
"version": "Visual Studio Community 2019, Windows SDK 10.0.19041.0, and a few system dlls Lobotmized for Angle Update",
|
||||
"filename": "vs-sdk-and-windir-shim.zip",
|
||||
"size": 463564917,
|
||||
"algorithm": "sha512",
|
||||
"digest": "20d93b0014ca44a666ca2b5cc570127f24378454889ff0a036df72213cd18140185d2c2e61df40bf1e4c0d9d2dc83a7796af9d2a13252de3c85faad068291945",
|
||||
"visibility": "internal",
|
||||
"unpack": true
|
||||
},
|
||||
{
|
||||
"version": "Windows binaries for depot_tools retrieved with depot_tools revision 7b3351443198f4e7d068999b169385b55a43679a on 2021-12-18",
|
||||
"filename": "depot_tools-preloaded-binaries-7b3351443198f4e7d068999b169385b55a43679a-2021-12-18.zip",
|
||||
"size": 315038144,
|
||||
"algorithm": "sha512",
|
||||
"digest": "250f3f3fcf631c53de1f505c53eb5ba782864fa92111eef0d710a1e7d0ad3da66db2fb8908d7c6d83e149674e976984d679f9a641eb9bea0b5c3d7b4f236920d",
|
||||
"visibility": "internal",
|
||||
"unpack": true
|
||||
}
|
||||
]
|
||||
@ -1,21 +0,0 @@
|
||||
FROM $DOCKER_IMAGE_PARENT
|
||||
MAINTAINER Tom Ritter <tom@mozilla.com>
|
||||
|
||||
VOLUME /builds/worker/checkouts
|
||||
|
||||
COPY privileged-setup.sh /setup/privileged-setup.sh
|
||||
COPY updatebot-version.sh /setup/updatebot-version.sh
|
||||
COPY setup.sh /builds/worker/setup.sh
|
||||
COPY run.py /builds/worker/run.py
|
||||
COPY hgrc /etc/mercurial/hgrc.d/updatebot.rc
|
||||
|
||||
RUN cd /setup && ./privileged-setup.sh
|
||||
|
||||
ENV HOME /builds/worker
|
||||
ENV SHELL /bin/bash
|
||||
ENV USER worker
|
||||
ENV LOGNAME worker
|
||||
ENV PYTHONUNBUFFERED 1
|
||||
ENV PATH "/builds/worker/go/bin:/builds/worker/fetches/rustc/bin:${PATH}"
|
||||
|
||||
RUN cd /builds/worker && ./setup.sh
|
||||
@ -1 +0,0 @@
|
||||
1
|
||||
@ -1,13 +0,0 @@
|
||||
diff --git a/src/workflow/ArcanistDiffWorkflow.php b/src/workflow/ArcanistDiffWorkflow.php
|
||||
index 38aa4b62..c9b7e215 100644
|
||||
--- a/src/workflow/ArcanistDiffWorkflow.php
|
||||
+++ b/src/workflow/ArcanistDiffWorkflow.php
|
||||
@@ -899,7 +899,7 @@ EOTEXT
|
||||
throw new Exception(pht('Repository API is not supported.'));
|
||||
}
|
||||
|
||||
- $limit = 1024 * 1024 * 4;
|
||||
+ $limit = 1024 * 1024 * 12;
|
||||
foreach ($changes as $change) {
|
||||
$size = 0;
|
||||
foreach ($change->getHunks() as $hunk) {
|
||||
@ -1,19 +0,0 @@
|
||||
diff --git a/src/channel/PhutilSocketChannel.php b/src/channel/PhutilSocketChannel.php
|
||||
index 4bd2a47a..2ad3fd85 100644
|
||||
--- a/src/channel/PhutilSocketChannel.php
|
||||
+++ b/src/channel/PhutilSocketChannel.php
|
||||
@@ -46,9 +46,11 @@ final class PhutilSocketChannel extends PhutilChannel {
|
||||
if (!$socket) {
|
||||
continue;
|
||||
}
|
||||
- $ok = stream_set_blocking($socket, false);
|
||||
- if (!$ok) {
|
||||
- throw new Exception(pht('Failed to set socket nonblocking!'));
|
||||
+ if (strtoupper(substr(PHP_OS, 0, 3)) !== 'WIN') {
|
||||
+ $ok = stream_set_blocking($socket, false);
|
||||
+ if (!$ok) {
|
||||
+ throw new Exception(pht('Failed to set socket nonblocking!'));
|
||||
+ }
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
[ui]
|
||||
ssh = ssh -i $HOME/id_rsa -l updatebot@mozilla.com -o UserKnownHostsFile=$HOME/ssh_known_hosts
|
||||
username = Updatebot <updatebot@mozilla.com>
|
||||
|
||||
[extensions]
|
||||
strip =
|
||||
push-to-try = $HOME/.mozbuild/version-control-tools/hgext/push-to-try
|
||||
@ -1,80 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
set -vex
|
||||
|
||||
. ./updatebot-version.sh # Get UPDATEBOT_REVISION
|
||||
|
||||
# If you edit this, be sure to edit fetch/updatebot.yml
|
||||
export SQLPROXY_REVISION=fb1939ab92846761595833361c6b0b0ecd543861
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Update apt-get lists
|
||||
apt-get update -y
|
||||
|
||||
# Install dependencies
|
||||
apt-get install -y --no-install-recommends \
|
||||
arcanist \
|
||||
bzr \
|
||||
ca-certificates \
|
||||
curl \
|
||||
ed \
|
||||
golang-go \
|
||||
gcc \
|
||||
libc6-dev \
|
||||
python-requests \
|
||||
python-requests-unixsocket \
|
||||
python3.5 \
|
||||
python3-minimal \
|
||||
python3-wheel \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
python3-requests \
|
||||
python3-requests-unixsocket \
|
||||
python3-setuptools \
|
||||
nodejs \
|
||||
npm \
|
||||
openssh-client \
|
||||
rsync \
|
||||
wget
|
||||
|
||||
mkdir -p /builds/worker/.mozbuild
|
||||
chown -R worker:worker /builds/worker/
|
||||
export GOPATH=/builds/worker/go
|
||||
|
||||
# pdf.js setup
|
||||
# We want to aviod downloading a ton of packages all the time, so
|
||||
# we will preload the pdf.js repo (and packages) in the Docker image
|
||||
# and only update it at runtime. This means that the `./mach vendor`
|
||||
# behavior for pdf.js will also be kind of custom
|
||||
npm install -g gulp-cli
|
||||
cd /builds/worker/
|
||||
git clone https://github.com/mozilla/pdf.js.git
|
||||
cd /builds/worker/pdf.js
|
||||
npm ci
|
||||
|
||||
# Build Google's Cloud SQL Proxy from source
|
||||
cd /builds/worker/
|
||||
mkdir cloud_sql_proxy
|
||||
cd cloud_sql_proxy
|
||||
go mod init .
|
||||
go get github.com/GoogleCloudPlatform/cloudsql-proxy/cmd/cloud_sql_proxy@$SQLPROXY_REVISION
|
||||
|
||||
# Check out source code
|
||||
cd /builds/worker/
|
||||
git clone https://github.com/mozilla-services/updatebot.git
|
||||
cd updatebot
|
||||
git checkout "$UPDATEBOT_REVISION"
|
||||
|
||||
# Set up dependencies
|
||||
cd /builds/worker/
|
||||
chown -R worker:worker .
|
||||
chown -R worker:worker .*
|
||||
|
||||
python3 -m pip install -U pip
|
||||
python3 -m pip install poetry
|
||||
|
||||
rm -rf /setup
|
||||
@ -1,223 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.append("/builds/worker/checkouts/gecko/third_party/python")
|
||||
sys.path.append(".")
|
||||
|
||||
import os
|
||||
import stat
|
||||
import base64
|
||||
import signal
|
||||
import platform
|
||||
import requests
|
||||
import subprocess
|
||||
import taskcluster
|
||||
|
||||
# Bump this number when you need to cause a commit for the job to re-run: 21
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage:", sys.argv[0], "gecko-dev-path updatebot-path [moz-fetches-dir]")
|
||||
sys.exit(1)
|
||||
|
||||
GECKO_DEV_PATH = sys.argv[1].replace("/", os.path.sep)
|
||||
UPDATEBOT_PATH = sys.argv[2].replace("/", os.path.sep)
|
||||
|
||||
# Only needed on Windows
|
||||
if len(sys.argv) > 3:
|
||||
FETCHES_PATH = sys.argv[3].replace("/", os.path.sep)
|
||||
else:
|
||||
FETCHES_PATH = None
|
||||
|
||||
HOME_PATH = os.path.expanduser("~")
|
||||
|
||||
OPERATING_MODE = (
|
||||
"prod"
|
||||
if os.environ.get("GECKO_HEAD_REPOSITORY", "")
|
||||
== "https://hg.mozilla.org/mozilla-central"
|
||||
else "dev"
|
||||
)
|
||||
|
||||
DEV_PHAB_URL = "https://phabricator-dev.allizom.org/"
|
||||
PROD_PHAB_URL = "https://phabricator.services.mozilla.com/"
|
||||
|
||||
phabricator_url = DEV_PHAB_URL if OPERATING_MODE == "dev" else PROD_PHAB_URL
|
||||
|
||||
|
||||
def log(*args):
|
||||
print(*args)
|
||||
|
||||
|
||||
def get_secret(name):
|
||||
secret = None
|
||||
if "TASK_ID" in os.environ:
|
||||
secrets_url = (
|
||||
"http://taskcluster/secrets/v1/secret/project/updatebot/"
|
||||
+ ("3" if OPERATING_MODE == "prod" else "2")
|
||||
+ "/"
|
||||
+ name
|
||||
)
|
||||
res = requests.get(secrets_url)
|
||||
res.raise_for_status()
|
||||
secret = res.json()
|
||||
else:
|
||||
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
|
||||
secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" + name)
|
||||
secret = secret["secret"] if "secret" in secret else None
|
||||
secret = secret["value"] if "value" in secret else None
|
||||
return secret
|
||||
|
||||
|
||||
# Get TC Secrets =======================================
|
||||
log("Operating mode is ", OPERATING_MODE)
|
||||
log("Getting secrets...")
|
||||
bugzilla_api_key = get_secret("bugzilla-api-key")
|
||||
phabricator_token = get_secret("phabricator-token")
|
||||
try_sshkey = get_secret("try-sshkey")
|
||||
database_config = get_secret("database-password")
|
||||
sentry_url = get_secret("sentry-url")
|
||||
sql_proxy_config = get_secret("sql-proxy-config")
|
||||
|
||||
# Update Updatebot =======================================
|
||||
if OPERATING_MODE == "dev":
|
||||
"""
|
||||
If we are in development mode, we will update from github.
|
||||
(This command will probably only work if we checkout a branch FWIW.)
|
||||
|
||||
This allows us to iterate faster by committing to github and
|
||||
re-running the cron job on Taskcluster, without rebuilding the
|
||||
Docker image.
|
||||
|
||||
However, this mechanism is bypassing the security feature we
|
||||
have in-tree, where upstream out-of-tree code is fixed at a known
|
||||
revision and cannot be changed without a commit to m-c.
|
||||
|
||||
Therefore, we only do this in dev mode when running on try.
|
||||
"""
|
||||
|
||||
os.chdir(UPDATEBOT_PATH)
|
||||
log("Performing git repo update...")
|
||||
command = ["git", "symbolic-ref", "-q", "HEAD"]
|
||||
|
||||
r = subprocess.run(command)
|
||||
if r.returncode == 0:
|
||||
# This indicates we are on a branch, and not a specific revision
|
||||
subprocess.check_call(["git", "pull", "origin"])
|
||||
|
||||
# Set Up SSH & Phabricator ==============================
|
||||
os.chdir(HOME_PATH)
|
||||
log("Setting up ssh and phab keys...")
|
||||
with open("id_rsa", "w") as sshkey:
|
||||
sshkey.write(try_sshkey)
|
||||
os.chmod("id_rsa", stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
arc_filename = ".arcrc"
|
||||
if platform.system() == "Windows":
|
||||
arc_path = os.path.join(FETCHES_PATH, "..", "AppData", "Roaming")
|
||||
os.makedirs(arc_path, exist_ok=True)
|
||||
os.chdir(arc_path)
|
||||
log("Writing %s to %s" % (arc_filename, arc_path))
|
||||
else:
|
||||
os.chdir(HOME_PATH)
|
||||
|
||||
arcrc = open(arc_filename, "w")
|
||||
towrite = """
|
||||
{
|
||||
"hosts": {
|
||||
"PHAB_URL_HERE": {
|
||||
"token": "TOKENHERE"
|
||||
}
|
||||
}
|
||||
}
|
||||
""".replace(
|
||||
"TOKENHERE", phabricator_token
|
||||
).replace(
|
||||
"PHAB_URL_HERE", phabricator_url + "api/"
|
||||
)
|
||||
arcrc.write(towrite)
|
||||
arcrc.close()
|
||||
os.chmod(arc_filename, stat.S_IRUSR | stat.S_IWUSR)
|
||||
|
||||
# Set up the Cloud SQL Proxy =============================
|
||||
os.chdir(HOME_PATH)
|
||||
log("Setting up cloud_sql_proxy...")
|
||||
with open("sql-proxy-key", "w") as proxy_key_file:
|
||||
proxy_key_file.write(
|
||||
base64.b64decode(sql_proxy_config["key-value"]).decode("utf-8")
|
||||
)
|
||||
|
||||
instance_name = sql_proxy_config["instance-name"]
|
||||
if platform.system() == "Linux":
|
||||
sql_proxy_command = "/builds/worker/go/bin/cloud_sql_proxy"
|
||||
else:
|
||||
sql_proxy_command = os.path.join(UPDATEBOT_PATH, "..", "cloud_sql_proxy.exe")
|
||||
|
||||
sql_proxy_command += (
|
||||
" -instances=" + instance_name + "=tcp:3306 -credential_file=sql-proxy-key"
|
||||
)
|
||||
sql_proxy_args = {
|
||||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.PIPE,
|
||||
"shell": True,
|
||||
"start_new_session": True,
|
||||
}
|
||||
|
||||
if platform.system() == "Windows":
|
||||
si = subprocess.STARTUPINFO()
|
||||
si.dwFlags = subprocess.CREATE_NEW_PROCESS_GROUP
|
||||
|
||||
sql_proxy_args["startupinfo"] = si
|
||||
|
||||
sql_proxy = subprocess.Popen((sql_proxy_command), **sql_proxy_args)
|
||||
|
||||
try:
|
||||
(stdout, stderr) = sql_proxy.communicate(input=None, timeout=2)
|
||||
log("sql proxy stdout:", stdout.decode("utf-8"))
|
||||
log("sql proxy stderr:", stderr.decode("utf-8"))
|
||||
except subprocess.TimeoutExpired:
|
||||
log("no sqlproxy output in 2 seconds, this means it probably didn't error.")
|
||||
log("sqlproxy pid:", sql_proxy.pid)
|
||||
|
||||
database_config["host"] = "127.0.0.1"
|
||||
|
||||
# Vendor =================================================
|
||||
log("Getting Updatebot ready...")
|
||||
os.chdir(UPDATEBOT_PATH)
|
||||
localconfig = {
|
||||
"General": {
|
||||
"env": OPERATING_MODE,
|
||||
"gecko-path": GECKO_DEV_PATH,
|
||||
},
|
||||
"Logging": {
|
||||
"local": True,
|
||||
"sentry": True,
|
||||
"sentry_config": {"url": sentry_url, "debug": False},
|
||||
},
|
||||
"Database": database_config,
|
||||
"Bugzilla": {
|
||||
"apikey": bugzilla_api_key,
|
||||
},
|
||||
"Taskcluster": {
|
||||
"url_treeherder": "https://treeherder.mozilla.org/",
|
||||
"url_taskcluster": "http://taskcluster/",
|
||||
},
|
||||
}
|
||||
|
||||
log("Writing local config file")
|
||||
config = open("localconfig.py", "w")
|
||||
config.write("localconfig = " + str(localconfig))
|
||||
config.close()
|
||||
|
||||
log("Running updatebot")
|
||||
# On Windows, Updatebot is run by windows-setup.sh
|
||||
if platform.system() == "Linux":
|
||||
subprocess.check_call(["python3", "-m", "poetry", "run", "./automation.py"])
|
||||
|
||||
# Clean up ===============================================
|
||||
log("Killing cloud_sql_proxy")
|
||||
os.kill(sql_proxy.pid, signal.SIGTERM)
|
||||
@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
set -vex
|
||||
|
||||
# Copy the system known_hosts to the home directory so we have uniformity with Windows
|
||||
# and the ssh command will find them in the same place.
|
||||
cp /etc/ssh/ssh_known_hosts "$HOME/ssh_known_hosts"
|
||||
|
||||
# If poetry is not run as worker, then it won't work when run as user later.
|
||||
cd /builds/worker/updatebot
|
||||
/usr/local/bin/poetry install
|
||||
@ -1 +0,0 @@
|
||||
export UPDATEBOT_REVISION=5ea77bfef02c065d77aa670ca1b0819295f11136
|
||||
@ -1,130 +0,0 @@
|
||||
[PHP]
|
||||
|
||||
engine = On
|
||||
short_open_tag = Off
|
||||
precision = 14
|
||||
output_buffering = 4096
|
||||
zlib.output_compression = Off
|
||||
implicit_flush = Off
|
||||
unserialize_callback_func =
|
||||
serialize_precision = -1
|
||||
disable_functions =
|
||||
disable_classes =
|
||||
zend.enable_gc = On
|
||||
zend.exception_ignore_args = Off
|
||||
expose_php = On
|
||||
max_execution_time = 30
|
||||
max_input_time = 60
|
||||
memory_limit = 128M
|
||||
error_reporting = E_ALL
|
||||
display_errors = On
|
||||
display_startup_errors = On
|
||||
log_errors = On
|
||||
log_errors_max_len = 1024
|
||||
ignore_repeated_errors = Off
|
||||
ignore_repeated_source = Off
|
||||
report_memleaks = On
|
||||
variables_order = "GPCS"
|
||||
request_order = "GP"
|
||||
register_argc_argv = Off
|
||||
auto_globals_jit = On
|
||||
post_max_size = 8M
|
||||
auto_prepend_file =
|
||||
auto_append_file =
|
||||
default_mimetype = "text/html"
|
||||
default_charset = "UTF-8"
|
||||
doc_root =
|
||||
user_dir =
|
||||
extension_dir = "ext"
|
||||
enable_dl = Off
|
||||
file_uploads = On
|
||||
upload_max_filesize = 2M
|
||||
max_file_uploads = 20
|
||||
allow_url_fopen = On
|
||||
allow_url_include = Off
|
||||
default_socket_timeout = 60
|
||||
extension=curl
|
||||
|
||||
[CLI Server]
|
||||
cli_server.color = On
|
||||
|
||||
[Pdo_mysql]
|
||||
pdo_mysql.default_socket=
|
||||
|
||||
[mail function]
|
||||
SMTP = localhost
|
||||
smtp_port = 25
|
||||
mail.add_x_header = Off
|
||||
|
||||
[ODBC]
|
||||
odbc.allow_persistent = On
|
||||
odbc.check_persistent = On
|
||||
odbc.max_persistent = -1
|
||||
odbc.max_links = -1
|
||||
odbc.defaultlrl = 4096
|
||||
odbc.defaultbinmode = 1
|
||||
|
||||
[MySQLi]
|
||||
mysqli.max_persistent = -1
|
||||
mysqli.allow_persistent = On
|
||||
mysqli.max_links = -1
|
||||
mysqli.default_port = 3306
|
||||
mysqli.default_socket =
|
||||
mysqli.default_host =
|
||||
mysqli.default_user =
|
||||
mysqli.default_pw =
|
||||
mysqli.reconnect = Off
|
||||
|
||||
[mysqlnd]
|
||||
mysqlnd.collect_statistics = On
|
||||
mysqlnd.collect_memory_statistics = On
|
||||
|
||||
[PostgreSQL]
|
||||
pgsql.allow_persistent = On
|
||||
pgsql.auto_reset_persistent = Off
|
||||
pgsql.max_persistent = -1
|
||||
pgsql.max_links = -1
|
||||
pgsql.ignore_notice = 0
|
||||
pgsql.log_notice = 0
|
||||
|
||||
[bcmath]
|
||||
bcmath.scale = 0
|
||||
|
||||
[Session]
|
||||
session.save_handler = files
|
||||
session.use_strict_mode = 0
|
||||
session.use_cookies = 1
|
||||
session.use_only_cookies = 1
|
||||
session.name = PHPSESSID
|
||||
session.auto_start = 0
|
||||
session.cookie_lifetime = 0
|
||||
session.cookie_path = /
|
||||
session.cookie_domain =
|
||||
session.cookie_httponly =
|
||||
session.cookie_samesite =
|
||||
session.serialize_handler = php
|
||||
session.gc_probability = 1
|
||||
session.gc_divisor = 1000
|
||||
session.gc_maxlifetime = 1440
|
||||
session.referer_check =
|
||||
session.cache_limiter = nocache
|
||||
session.cache_expire = 180
|
||||
session.use_trans_sid = 0
|
||||
session.sid_length = 26
|
||||
session.trans_sid_tags = "a=href,area=href,frame=src,form="
|
||||
session.sid_bits_per_character = 5
|
||||
|
||||
[Assertion]
|
||||
zend.assertions = 1
|
||||
|
||||
[Tidy]
|
||||
tidy.clean_output = Off
|
||||
|
||||
[soap]
|
||||
soap.wsdl_cache_enabled=1
|
||||
soap.wsdl_cache_dir="/tmp"
|
||||
soap.wsdl_cache_ttl=86400
|
||||
soap.wsdl_cache_limit = 5
|
||||
|
||||
[ldap]
|
||||
ldap.max_links = -1
|
||||
@ -1,135 +0,0 @@
|
||||
#!/bin/bash
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
set -vex
|
||||
|
||||
. ./taskcluster/docker/updatebot/updatebot-version.sh # Get UPDATEBOT_REVISION
|
||||
|
||||
HOME=$(python3 -c "import os;print(os.path.expanduser('~'))")
|
||||
export HOME
|
||||
GECKO_PATH="$PWD"
|
||||
UPDATEBOT_PATH="$MOZ_FETCHES_DIR/updatebot"
|
||||
|
||||
# MOZ_FETCHES_DIR is in Z:/ format. When we update the PATH we need to use
|
||||
# /z/ format. Fortunately, we can translate them like so:
|
||||
cd "$MOZ_FETCHES_DIR"
|
||||
MOZ_FETCHES_PATH="$PWD"
|
||||
|
||||
#########################################################
|
||||
# Install dependencies
|
||||
|
||||
# Move depot_tools
|
||||
cd "$MOZ_FETCHES_DIR"
|
||||
mv depot_tools.git depot_tools
|
||||
|
||||
|
||||
# Generating a new version of the preloaded depot_tools download can be done by:
|
||||
# 1) Running the task, uncommenting the variable assignment below, uncommenting the
|
||||
# _GENERATE_DEPOT_TOOLS_BINARIES_ section in taskcluster/ci/updatebot/kind.yml,
|
||||
# and ensuring that an angle update will actually take place (so it downloads the depot_tools)
|
||||
# 2) Downloading and sanity-checking the depot_tools-preloaded-binaries-GIT_HASH-DATE.zip artifact
|
||||
# 3) Adding it to tooltool
|
||||
# 4) Updating the updatebot manifest
|
||||
# Note that even for the same git revision the downloaded tools can change, so they are tagged
|
||||
# with both the git hash and the date it was generated
|
||||
|
||||
# export GENERATE_DEPOT_TOOLS_BINARIES=1
|
||||
|
||||
if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then
|
||||
cp -r depot_tools depot_tools-from-git
|
||||
fi
|
||||
|
||||
# Git is at /c/Program Files/Git/cmd/git.exe
|
||||
# It's in PATH for this script (confusingly) but not in PATH so we need to add it
|
||||
export PATH="/c/Program Files/Git/cmd:$PATH"
|
||||
|
||||
# php & arcanist
|
||||
if [ -n "$TOOLTOOL_MANIFEST" ]; then
|
||||
. "$GECKO_PATH/taskcluster/scripts/misc/tooltool-download.sh"
|
||||
fi
|
||||
|
||||
cp "$MOZ_FETCHES_DIR/vcruntime140.dll" "$MOZ_FETCHES_DIR/php-win"
|
||||
cp "$GECKO_PATH/taskcluster/docker/updatebot/windows-php.ini" "$MOZ_FETCHES_DIR/php-win/php.ini"
|
||||
|
||||
cd "$MOZ_FETCHES_DIR/arcanist.git"
|
||||
patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_windows_stream.patch"
|
||||
patch -p1 < "$GECKO_PATH/taskcluster/docker/updatebot/arcanist_patch_size.patch"
|
||||
cd "$MOZ_FETCHES_DIR"
|
||||
|
||||
export PATH="$MOZ_FETCHES_PATH/php-win:$PATH"
|
||||
export PATH="$MOZ_FETCHES_PATH/arcanist.git/bin:$PATH"
|
||||
|
||||
# get Updatebot
|
||||
cd "$MOZ_FETCHES_DIR"
|
||||
git clone https://github.com/mozilla-services/updatebot.git
|
||||
cd updatebot
|
||||
git checkout "$UPDATEBOT_REVISION"
|
||||
|
||||
# base python needs
|
||||
python3 -m pip install --no-warn-script-location --user -U pip
|
||||
python3 -m pip install --no-warn-script-location --user poetry wheel requests setuptools
|
||||
|
||||
# updatebot dependencies
|
||||
cd "$UPDATEBOT_PATH"
|
||||
python3 -m poetry install
|
||||
|
||||
# taskcluster secrets and writing out localconfig
|
||||
cd "$GECKO_PATH"
|
||||
python3 ./taskcluster/docker/updatebot/run.py "$GECKO_PATH" "$UPDATEBOT_PATH" "$MOZ_FETCHES_PATH"
|
||||
|
||||
# mercurial configuration
|
||||
cp "$GECKO_PATH/taskcluster/docker/updatebot/hgrc" "$HOME/.hgrc"
|
||||
# Windows is not happy with $HOME in the hgrc so we need to do a hack to replace it
|
||||
# with the actual value
|
||||
( echo "cat <<EOF" ; cat "$HOME/.hgrc" ) | sh > tmp
|
||||
mv tmp "$HOME/.hgrc"
|
||||
|
||||
# ssh known hosts
|
||||
cp "$GECKO_PATH/taskcluster/docker/push-to-try/known_hosts" "$HOME/ssh_known_hosts"
|
||||
|
||||
#########################################################
|
||||
# Run it
|
||||
export PYTHONIOENCODING=utf8
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
cd "$UPDATEBOT_PATH"
|
||||
python3 -m poetry run python3 ./automation.py
|
||||
|
||||
#########################################################
|
||||
if test -n "$GENERATE_DEPOT_TOOLS_BINARIES"; then
|
||||
# Artifacts
|
||||
|
||||
cd "$MOZ_FETCHES_PATH"
|
||||
mv depot_tools depot_tools-from-tc
|
||||
|
||||
# Clean out unneeded files
|
||||
# Need to use cmd because for some reason rm from bash throws 'Access Denied'
|
||||
cmd '/c for /d /r %i in (*__pycache__) do rmdir /s /q %i'
|
||||
rm -rf depot_tools-from-git/.git || true
|
||||
|
||||
# Delete the files that are already in git
|
||||
find depot_tools-from-git -mindepth 1 -maxdepth 1 | sed s/depot_tools-from-git/depot_tools-from-tc/ | while read -r d; do rm -rf "$d"; done
|
||||
|
||||
# Make the artifact
|
||||
rm -rf depot_tools-preloaded-binaries #remove it if it existed (i.e. we probably have one from tooltool already)
|
||||
mv depot_tools-from-tc depot_tools-preloaded-binaries
|
||||
|
||||
# zip can't add symbolic links, and exits with an error code. || true avoids a script crash
|
||||
zip -r depot_tools-preloaded-binaries.zip depot_tools-preloaded-binaries/ || true
|
||||
|
||||
# Convoluted way to get the git hash, because we don't have a .git directory
|
||||
# Adding extra print statements just in case we need to debug it
|
||||
GIT_HASH=$(grep depot_tools -A 1 "$GECKO_PATH/taskcluster/ci/fetch/updatebot.yml" | tee /dev/tty | grep revision | tee /dev/tty | awk -F': *' '{print $2}' | tee /dev/tty)
|
||||
DATE=$(date -I)
|
||||
mv depot_tools-preloaded-binaries.zip "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip"
|
||||
|
||||
# Put the artifact into the directory we will look for it
|
||||
mkdir -p "$GECKO_PATH/obj-build/depot_tools" || true
|
||||
mv "depot_tools-preloaded-binaries-$GIT_HASH-$DATE.zip" "$GECKO_PATH/obj-build/depot_tools"
|
||||
fi
|
||||
|
||||
#########################################################
|
||||
echo "Killing SQL Proxy"
|
||||
taskkill -f -im cloud_sql_proxy.exe || true
|
||||
@ -705,10 +705,6 @@ scriptworker-canary
|
||||
-------------------
|
||||
Push tasks to try to test new scriptworker deployments.
|
||||
|
||||
updatebot
|
||||
------------------
|
||||
Check for updates to (supported) third party libraries, and manage their lifecycle.
|
||||
|
||||
fuzzing
|
||||
-------
|
||||
|
||||
|
||||
@ -993,13 +993,6 @@ def target_tasks_build_linux64_clang_trunk_perf(
|
||||
return [l for l, t in full_task_graph.tasks.items() if filter(t.label)]
|
||||
|
||||
|
||||
# Run Updatebot's cron job 4 times daily.
|
||||
@_target_task("updatebot_cron")
|
||||
def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config):
|
||||
"""Select tasks required to run Updatebot's cron job"""
|
||||
return ["updatebot-cron"]
|
||||
|
||||
|
||||
@_target_task("customv8_update")
|
||||
def target_tasks_customv8_update(full_task_graph, parameters, graph_config):
|
||||
"""Select tasks required for building latest d8/v8 version."""
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user