Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 13:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1618188 - stop updating the mozilla-central copy of blocklist.xml, r=RyanVM

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D64932

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Gijs Kruitbosch 2020-03-09 11:58:32 +00:00
parent 558c5bec85
commit 03b081f658
4 changed files with 12 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
taskcluster/ci/repo-update/kind.yml
View File

@ -20,10 +20,6 @@ job-defaults:
by-project:
mozilla-(central|beta|esr.*): "1"
default: ""
DO_BLOCKLIST:
by-project:
mozilla-(central|beta|release|esr.*): "1"
default: ""
DO_REMOTE_SETTINGS:
by-project:
mozilla-(central|beta|release|esr.*): "1"
@ -41,7 +37,7 @@ job-defaults:
jobs:
hsts-hpkp-blocklist:
name: periodic_file_update
description: HSTS, HPKP, Blocklist and remote settings update
description: HSTS, HPKP and remote settings update
run-on-projects: [] # Only run via cron
treeherder:
kind: build
@ -67,9 +63,6 @@ jobs:
- name: 'public/build/StaticHPKPins.h.diff'
path: '/home/worker/artifacts/StaticHPKPins.h.diff'
type: file
- name: 'public/build/blocklist.diff'
path: '/home/worker/artifacts/blocklist.diff'
type: file
- name: 'public/build/remote-settings.diff'
path: '/home/worker/artifacts/remote-settings.diff'
type: file

11
taskcluster/docker/periodic-updates/README.md
View File

@ -1,7 +1,7 @@
==Periodic File Updates==
This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklist.xml, and
This docker image examines the in-tree files for HSTS preload data, HPKP pinning and blocklisting, and
will produce a diff for each necessary to update the in-tree files.
If given a conduit API token, it will also use the arcanist client to submit the commits for review.
@ -12,13 +12,13 @@ If given a conduit API token, it will also use the arcanist client to submit the
```sh
docker build -t hsts-local --no-cache --rm .
docker run -e DO_HSTS=1 -e DO_HPKP=1 -e DO_BLOCKLIST=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
docker run -e DO_HSTS=1 -e DO_HPKP=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
```
HSTS checks will only be run if the `DO_HSTS` environment variable is set.
Likewise for `DO_HPKP` and the HPKP checks, and `DO_BLOCKLIST` and the
blocklist checks. Environment variables are used rather than command line
arguments to make constructing taskcluster tasks easier.
Likewise for `DO_HPKP` and the HPKP checks. Environment variables are used
rather than command line arguments to make constructing taskcluster tasks
easier.
To prevent a full build when landing with Phabricator, set the `DONTBUILD`
environment variable.
@ -75,7 +75,6 @@ payload:
env:
DO_HSTS: 1
DO_HPKP: 1
DO_BLOCKLIST: 1
PRODUCT: firefox
BRANCH: mozilla-central
USE_MOZILLA_CENTRAL: 1

7
taskcluster/docker/periodic-updates/runme.sh
View File

@ -3,7 +3,7 @@
set -xe
# Things to be set by task definition.
# --pinset --hsts --hpkp --blocklist
# --pinset --hsts --hpkp
# -b branch
# --use-mozilla-central
# -p firefox
@ -32,11 +32,6 @@ then
PARAMS="${PARAMS} --hpkp"
fi
if [ -n "${DO_BLOCKLIST}" ]
then
PARAMS="${PARAMS} --blocklist"
fi
if [ -n "${DO_REMOTE_SETTINGS}" ]
then
PARAMS="${PARAMS} --remote-settings"

73
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
View File

@ -14,7 +14,7 @@ Usage: $(basename "$0") [-p product]
# Use archive.m.o instead of the taskcluster index to get xpcshell
[--use-ftp-builds]
# One (or more) of the following actions must be specified.
--hsts | --hpkp | --blocklist
--hsts | --hpkp | --remote-settings | --suffix-list
-b branch
EOF
@ -30,8 +30,6 @@ APPROVAL=false
COMMIT_AUTHOR='ffxbld <ffxbld@mozilla.com>'
REPODIR=''
APP_DIR=''
APP_ID=''
APP_NAME=''
HGHOST="hg.mozilla.org"
STAGEHOST="archive.mozilla.org"
WGET="wget -nv"
@ -65,13 +63,6 @@ HPKP_PRELOAD_INPUT="${DATADIR}/${HPKP_PRELOAD_INC}"
HPKP_PRELOAD_OUTPUT="${DATADIR}/${HPKP_PRELOAD_INC}.out"
HPKP_UPDATED=false
DO_BLOCKLIST=false
BLOCKLIST_URL_AMO=''
BLOCKLIST_URL_HG=''
BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
BLOCKLIST_UPDATED=false
DO_REMOTE_SETTINGS=false
REMOTE_SETTINGS_SERVER=''
REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
@ -90,7 +81,6 @@ ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
# Defaults
HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
SUFFIX_LIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${SUFFIX_LIST_DIFF_ARTIFACT:-"effective_tld_names.diff"}"
@ -298,7 +288,7 @@ function compare_suffix_lists {
rm -f "${HG_SUFFIX_LOCAL}"
${WGET} -O "${HG_SUFFIX_LOCAL}" "${HG_SUFFIX_URL}"
echo "INFO: diffing in-tree blocklist against the blocklist from AMO..."
echo "INFO: diffing in-tree suffix list against the suffix list from AMO..."
${DIFF} ${GITHUB_SUFFIX_LOCAL} ${HG_SUFFIX_LOCAL} | tee "${SUFFIX_LIST_DIFF_ARTIFACT}"
if [ -s "${SUFFIX_LIST_DIFF_ARTIFACT}" ]
then
@ -307,37 +297,6 @@ function compare_suffix_lists {
return 1
}
# Downloads the current in-tree blocklist file.
# Downloads the current blocklist file from AMO.
# Compares the AMO blocklist with the in-tree blocklist to determine whether we need to update.
function compare_blocklist_files {
BLOCKLIST_URL_AMO="https://blocklist.addons.mozilla.org/blocklist/3/${APP_ID}/${VERSION}/${APP_NAME}/20090105024647/blocklist-sync/en-US/nightly/blocklist-sync/default/default/"
BLOCKLIST_URL_HG="${HGREPO}/raw-file/default/${APP_DIR}/app/blocklist.xml"
cd "${BASEDIR}"
rm -f ${BLOCKLIST_LOCAL_AMO}
echo "INFO: ${WGET} -O ${BLOCKLIST_LOCAL_AMO} ${BLOCKLIST_URL_AMO}"
${WGET} -O "${BLOCKLIST_LOCAL_AMO}" "${BLOCKLIST_URL_AMO}"
rm -f ${BLOCKLIST_LOCAL_HG}
echo "INFO: ${WGET} -O ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_URL_HG}"
${WGET} -O "${BLOCKLIST_LOCAL_HG}" "${BLOCKLIST_URL_HG}"
# The downloaded files should be non-empty and have a valid xml header
# if they were retrieved properly, and some random HTML garbage if not.
# set -x catches these
is_valid_xml ${BLOCKLIST_LOCAL_AMO}
is_valid_xml ${BLOCKLIST_LOCAL_HG}
echo "INFO: diffing in-tree blocklist against the blocklist from AMO..."
${DIFF} ${BLOCKLIST_LOCAL_HG} ${BLOCKLIST_LOCAL_AMO} | tee "${BLOCKLIST_DIFF_ARTIFACT}"
if [ -s "${BLOCKLIST_DIFF_ARTIFACT}" ]
then
return 0
fi
return 1
}
function compare_remote_settings_files {
REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
@ -398,11 +357,6 @@ function stage_hpkp_files {
cp -f "${HPKP_PRELOAD_OUTPUT}" "${REPODIR}/security/manager/ssl/${HPKP_PRELOAD_INC}"
}
function stage_blocklist_files {
cd "${BASEDIR}"
cp -f ${BLOCKLIST_LOCAL_AMO} ${REPODIR}/${APP_DIR}/app/blocklist.xml
}
function stage_remote_settings_files {
cd "${BASEDIR}"
cp -a "${REMOTE_SETTINGS_OUTPUT}"/* "${REPODIR}${REMOTE_SETTINGS_DIR}"
@ -458,7 +412,6 @@ while [ $# -gt 0 ]; do
--pinset) DO_PRELOAD_PINSET=true ;;
--hsts) DO_HSTS=true ;;
--hpkp) DO_HPKP=true ;;
--blocklist) DO_BLOCKLIST=true ;;
--remote-settings) DO_REMOTE_SETTINGS=true ;;
--suffix-list) DO_SUFFIX_LIST=true ;;
-r) REPODIR="$2"; shift ;;
@ -479,9 +432,9 @@ if [ "${BRANCH}" == "" ]; then
fi
# Must choose at least one update action.
if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ]
if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ] && [ "$DO_SUFFIX_LIST" == "false" ]
then
echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
echo "Error: you must specify at least one action from: --hsts, --hpkp, --remote-settings, or --suffix-list" >&2
usage
exit 13
fi
@ -490,14 +443,10 @@ fi
case "${PRODUCT}" in
thunderbird)
APP_DIR="mail"
APP_ID="%7B3550f703-e582-4d05-9a08-453d09bdfdc6%7D"
APP_NAME="Thunderbird"
COMMIT_AUTHOR="tbirdbld <tbirdbld@thunderbird.net>"
;;
firefox)
APP_DIR="browser"
APP_ID="%7Bec8030f7-c20a-464f-9b0e-13a3a9e97384%7D"
APP_NAME="Firefox"
;;
*)
echo "Error: Invalid product specified"
@ -570,12 +519,6 @@ if [ "${DO_HPKP}" == "true" ]; then
HPKP_UPDATED=true
fi
fi
if [ "${DO_BLOCKLIST}" == "true" ]; then
if compare_blocklist_files
then
BLOCKLIST_UPDATED=true
fi
fi
if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
if compare_remote_settings_files
then
@ -590,7 +533,7 @@ if [ "${DO_SUFFIX_LIST}" == "true" ]; then
fi
if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then
if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ] && [ "${SUFFIX_LIST_UPDATED}" == "false" ]; then
echo "INFO: no updates required. Exiting."
exit 0
else
@ -615,12 +558,6 @@ then
COMMIT_MESSAGE="${COMMIT_MESSAGE} HPKP"
fi
if [ "${BLOCKLIST_UPDATED}" == "true" ]
then
stage_blocklist_files
COMMIT_MESSAGE="${COMMIT_MESSAGE} blocklist"
fi
if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
then
stage_remote_settings_files