From a43f7613d3c024b620bdf23507b11a2b3b4a86a8 Mon Sep 17 00:00:00 2001
From: Mathieu Leplatre <mathieu@mozilla.com>
Date: Wed, 16 May 2018 13:03:22 +0200
Subject: [PATCH] Bug 1451040 - Download remote settings dumps regularly
 r=sfraser

MozReview-Commit-ID: BCCEplOq1O8

--HG--
extra : rebase_source : 4a9840b7a36b1d08e3fa012fcad755516e292ec2
---
 taskcluster/ci/repo-update/kind.yml           |  6 +-
 taskcluster/docker/periodic-updates/runme.sh  |  5 ++
 .../scripts/periodic_file_updates.sh          | 89 ++++++++++++++++++-
 3 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/taskcluster/ci/repo-update/kind.yml b/taskcluster/ci/repo-update/kind.yml
index 32aa9aee707e..7be1f6da3b9d 100644
--- a/taskcluster/ci/repo-update/kind.yml
+++ b/taskcluster/ci/repo-update/kind.yml
@@ -11,7 +11,7 @@ transforms:
 jobs:
    hsts-hpkp-blocklist:
       name: periodic_file_update
-      description: HSTS, HPKP, and Blocklist update
+      description: HSTS, HPKP, Blocklist and remote settings update
       run-on-projects: []  # Only run via cron
       treeherder:
          kind: build
@@ -28,6 +28,7 @@ jobs:
             DO_HSTS: "1"
             DO_HPKP: "1"
             DO_BLOCKLIST: "1"
+            DO_REMOTE_SETTINGS: "1"
             USE_MOZILLA_CENTRAL: "1"
             BRANCH: mozilla-central
             PRODUCT: firefox
@@ -45,5 +46,8 @@ jobs:
             - name: 'public/build/blocklist.diff'
               path: '/home/worker/artifacts/blocklist.diff'
               type: file
+            - name: 'public/build/remote-settings.diff'
+              path: '/home/worker/artifacts/remote-settings.diff'
+              type: file
       scopes:
          - secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token
diff --git a/taskcluster/docker/periodic-updates/runme.sh b/taskcluster/docker/periodic-updates/runme.sh
index 45f4197a7c08..d559dac3dd0c 100755
--- a/taskcluster/docker/periodic-updates/runme.sh
+++ b/taskcluster/docker/periodic-updates/runme.sh
@@ -37,6 +37,11 @@ then
   PARAMS="${PARAMS} --blocklist"
 fi
 
+if [ ! -z "${DO_REMOTE_SETTINGS}" ]
+then
+  PARAMS="${PARAMS} --remote-settings"
+fi
+
 export ARTIFACTS_DIR="/home/worker/artifacts"
 mkdir -p "$ARTIFACTS_DIR"
 
diff --git a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
index e4903ba7f04c..63d0c420a833 100755
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@@ -74,11 +74,19 @@ BLOCKLIST_LOCAL_AMO="blocklist_amo.xml"
 BLOCKLIST_LOCAL_HG="blocklist_hg.xml"
 BLOCKLIST_UPDATED=false
 
-ARTIFACTS_DIR="${ARTIFACTS_DIR:-'.'}"
+DO_REMOTE_SETTINGS=false
+REMOTE_SETTINGS_SERVER=''
+REMOTE_SETTINGS_INPUT="${DATADIR}/remote-settings.in"
+REMOTE_SETTINGS_OUTPUT="${DATADIR}/remote-settings.out"
+REMOTE_SETTINGS_DIR="/services/settings/dumps"
+REMOTE_SETTINGS_UPDATED=false
+
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-.}"
 # Defaults
 HSTS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HSTS_DIFF_ARTIFACT:-"nsSTSPreloadList.diff"}"
 HPKP_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${HPKP_DIFF_ARTIFACT:-"StaticHPKPins.h.diff"}"
 BLOCKLIST_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${BLOCKLIST_DIFF_ARTIFACT:-"blocklist.diff"}"
+REMOTE_SETTINGS_DIFF_ARTIFACT="${ARTIFACTS_DIR}/${REMOTE_SETTINGS_DIFF_ARTIFACT:-"remote-settings.diff"}"
 
 
 # Get the current in-tree version for a code branch.
@@ -299,6 +307,44 @@ function compare_blocklist_files {
   return 1
 }
 
+function compare_remote_settings_files {
+  REMOTE_SETTINGS_SERVER="https://firefox.settings.services.mozilla.com/v1"
+
+  # 1. List remote settings collections from server.
+  echo "INFO: fetch remote settings list from server"
+  ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
+      ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
+      # 2. For each entry ${bucket, collection}
+      while IFS="/" read -r bucket collection; do
+
+        # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
+        hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
+        local_location_input="$REMOTE_SETTINGS_INPUT/${bucket}/${collection}.json"
+        mkdir -p "$REMOTE_SETTINGS_INPUT/${bucket}"
+        ${WGET} -qO "$local_location_input" "$hg_dump_url"
+        if [ $? -eq 8 ]; then
+          # We don't keep any dump for this collection, skip it.
+          # Try to clean up in case no collection in this bucket has dump.
+          rmdir "$REMOTE_SETTINGS_INPUT/${bucket}" --ignore-fail-on-non-empty
+          continue
+        fi
+
+        # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
+        remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+        local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
+        mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
+        ${WGET} -qO "$local_location_output" "$remote_records_url"
+      done
+
+  echo "INFO: diffing old/new remote settings dumps..."
+  ${DIFF} -r "${REMOTE_SETTINGS_INPUT}" "${REMOTE_SETTINGS_OUTPUT}" > "${REMOTE_SETTINGS_DIFF_ARTIFACT}"
+  if [ -s "${REMOTE_SETTINGS_DIFF_ARTIFACT}" ]
+  then
+    return 0
+  fi
+  return 1
+}
+
 function clone_build_tools {
     rm -fr "${TOOLSDIR}"
     CLONE_CMD="${HG} clone https://hg.mozilla.org/build/tools ${TOOLSDIR}"
@@ -393,6 +439,28 @@ function commit_blocklist_files {
   ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
 }
 
+# Copies new remote settings dump files in place, and commits them.
+function commit_remote_settings_files {
+  cd "${BASEDIR}"
+  cp -a "${REMOTE_SETTINGS_OUTPUT}/*" "${REPODIR}${REMOTE_SETTINGS_DIR}"
+
+  COMMIT_MESSAGE="No bug, Automated remote settings update"
+  if [ -n "${TASK_ID}" ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} from task ${TASK_ID}"
+  fi
+  if [ ${DONTBUILD} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - (DONTBUILD)"
+  fi
+  if [ ${CLOSED_TREE} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - CLOSED TREE"
+  fi
+  if [ ${APPROVAL} == true ]; then
+    COMMIT_MESSAGE="${COMMIT_MESSAGE} - a=remote-settings-update"
+  fi
+  echo "INFO: committing remote settings changes"
+  ${HG} -R ${REPODIR} commit -u "${HG_SSH_USER}" -m "${COMMIT_MESSAGE}"
+}
+
 # Push all pending commits to Phabricator
 function push_repo {
   cd "${REPODIR}"
@@ -440,6 +508,7 @@ while [ $# -gt 0 ]; do
     --hsts) DO_HSTS=true ;;
     --hpkp) DO_HPKP=true ;;
     --blocklist) DO_BLOCKLIST=true ;;
+    --remote-settings) DO_REMOTE_SETTINGS=true ;;
     -r) REPODIR="$2"; shift ;;
     --use-mozilla-central) USE_MC=true ;;
     --use-ftp-builds) USE_TC=false ;;
@@ -458,9 +527,9 @@ if [ "${BRANCH}" == "" ]; then
 fi
 
 # Must choose at least one update action.
-if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ]
+if [ "$DO_HSTS" == "false" ] && [ "$DO_HPKP" == "false" ] && [ "$DO_BLOCKLIST" == "false" ] && [ "$DO_REMOTE_SETTINGS" == "false" ]
 then
-  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist" >&2
+  echo "Error: you must specify at least one action from: --hsts, --hpkp, --blocklist, --remote-settings" >&2
   usage
   exit 13
 fi
@@ -543,8 +612,14 @@ if [ "${DO_BLOCKLIST}" == "true" ]; then
     BLOCKLIST_UPDATED=true
   fi
 fi
+if [ "${DO_REMOTE_SETTINGS}" == "true" ]; then
+  if compare_remote_settings_files
+  then
+    REMOTE_SETTINGS_UPDATED=true
+  fi
+fi
 
-if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ]; then
+if [ "${HSTS_UPDATED}" == "false" ] && [ "${HPKP_UPDATED}" == "false" ] && [ "${BLOCKLIST_UPDATED}" == "false" ] && [ "${REMOTE_SETTINGS_UPDATED}" == "false" ]; then
   echo "INFO: no updates required. Exiting."
   exit 0
 else
@@ -578,6 +653,12 @@ then
   MUST_PUSH=true
 fi
 
+if [ "${REMOTE_SETTINGS_UPDATED}" == "true" ]
+then
+  commit_remote_settings_files
+  MUST_PUSH=true
+fi
+
 if [ -n "${MUST_PUSH}" ]
 then
   push_repo