Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-19 16:25:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1636158 - Update the addons-mlbf.bin attachment via periodic_file_updates.sh r=sfraser

Browse Source 
Tested by running the following code and verifying that the script runs
successfully and generates a reasonable diff with the latest metadata
and matching attachment content:

```
$ cd taskcluster/docker/periodic-updates/
$ tar -czf- -s "#$(git rev-parse --show-cdup)#topsrcdir/#" . $(awk -v d="$(git rev-parse --show-cdup)" '/^# %include/{print d$3}' Dockerfile) | docker build - -t hsts-local
$ docker run -e DO_REMOTE_SETTINGS=1 -e PRODUCT="firefox" -e BRANCH="mozilla-central" -e USE_MOZILLA_CENTRAL=1 hsts-local
```

After that, I opened a shell in the container, built Firefox and ran the
following test, to confirm that the updated records are used correctly:

```
mach test toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
```

Differential Revision: https://phabricator.services.mozilla.com/D76158
This commit is contained in:
Rob Wu 2020-05-22 01:37:37 +00:00
parent ac650df53f
commit 27981af497
2 changed files with 65 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
View File

@ -324,6 +324,22 @@ function compare_remote_settings_files {
local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
${WGET} -qO "$local_location_output" "$remote_records_url"
# 5. Download attachments if needed.
if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then
# Find the attachment with the most recent generation_time, like _updateMLBF in Blocklist.jsm.
# The server should return one "bloomfilter-base" record, but in case it returns multiple,
# return the most recent one. The server may send multiple entries if we ever decide to use
# the "filter_expression" feature of Remote Settings to send different records to specific
# channels. In that case this code should be updated to recognize the filter expression,
# but until we do, simply select the most recent record - can't go wrong with that.
# Note that "attachment_type" and "generation_time" are specific to addons-bloomfilters.
update_remote_settings_attachment "${bucket}" "${collection}" addons-mlbf.bin \
'map(select(.attachment_type == "bloomfilter-base")) | sort_by(.generation_time) | last'
fi
# Here is an example to download an attachment with record identifier "ID":
# update_remote_settings_attachment "${bucket}" "${collection}" ID '.[] | select(.id == "ID")'
# NOTE: The downloaded data is not validated. xpcshell should be used for that.
done
echo "INFO: diffing old/new remote settings dumps..."
@ -335,6 +351,54 @@ function compare_remote_settings_files {
return 1
}
# Helper for compare_remote_settings_files to download attachments from remote settings.
# The format and location is documented at:
# https://firefox-source-docs.mozilla.org/services/common/services/RemoteSettings.html#packaging-attachments
function update_remote_settings_attachment() {
local bucket=$1
local collection=$2
local attachment_id=$3
# $4 is a jq filter on the arrays that should return one record with the attachment
local jq_attachment_selector=".data | map(select(.attachment)) | $4"
# These paths match _readAttachmentDump in services/settings/Attachments.jsm.
local path_to_attachment="${bucket}/${collection}/${attachment_id}"
local path_to_meta="${bucket}/${collection}/${attachment_id}.meta.json"
local old_meta="$REMOTE_SETTINGS_INPUT/${path_to_meta}"
local new_meta="$REMOTE_SETTINGS_OUTPUT/${path_to_meta}"
# Those files should have been created by compare_remote_settings_files before the function call.
local local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
local local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
# Compute the metadata based on already-downloaded records.
mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}/${collection}"
${JQ} -cj <"$local_location_input" "${jq_attachment_selector}" > "${old_meta}"
mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}"
${JQ} -cj <"$local_location_output" "${jq_attachment_selector}" > "${new_meta}"
if cmp --silent "${old_meta}" "${new_meta}" ; then
# Metadata not changed, don't bother downloading the attachments themselves.
return
fi
# Metadata changed. Download attachments.
echo "INFO: Downloading updated remote settings dump: ${bucket}/${collection}/${attachment_id}"
# Overwrited old_meta with the actual file from the repo. The content should be equivalent,
# but can have minor differences (e.g. different line endings) if the checked in file was not
# generated by this script (e.g. manually checked in).
${WGET} -qO "${old_meta}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_meta}"
${WGET} -qO "${REMOTE_SETTINGS_INPUT}/${path_to_attachment}" "${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${path_to_attachment}"
if [ -z "${ATTACHMENT_BASE_URL}" ] ; then
ATTACHMENT_BASE_URL=$(${WGET} -qO- "${REMOTE_SETTINGS_SERVER}" | ${JQ} -r .capabilities.attachments.base_url)
fi
attachment_path_from_meta=$(${JQ} -r < "${new_meta}" .attachment.location)
${WGET} -qO "${REMOTE_SETTINGS_OUTPUT}/${path_to_attachment}" "${ATTACHMENT_BASE_URL}${attachment_path_from_meta}"
}
# Clones an hg repo
function clone_repo {
cd "${BASEDIR}"

1
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
View File

@ -9,6 +9,7 @@
*/
Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
Services.prefs.setBoolPref("extensions.blocklist.useMLBF.stashes", true);
const { ExtensionBlocklist: ExtensionBlocklistMLBF } = Blocklist;