Bug 1563226 - Download the Public Suffix List using Remote Settings r=leplatrem

Differential Revision: https://phabricator.services.mozilla.com/D42469

--HG--
extra : moz-landing-system : lando

browser/components/BrowserGlue.jsm

@@ -534,6 +534,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   PluralForm: "resource://gre/modules/PluralForm.jsm",
   PrivateBrowsingUtils: "resource://gre/modules/PrivateBrowsingUtils.jsm",
   ProcessHangMonitor: "resource:///modules/ProcessHangMonitor.jsm",
+  PublicSuffixList: "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm",
   RemoteSettings: "resource://services-settings/remote-settings.js",
   RemoteSecuritySettings:
     "resource://gre/modules/psm/RemoteSecuritySettings.jsm",
@@ -2205,6 +2206,10 @@ BrowserGlue.prototype = {
       RemoteSettings.init();
     });
 
+    Services.tm.idleDispatchToMainThread(() => {
+      PublicSuffixList.init();
+    });
+
     Services.tm.idleDispatchToMainThread(() => {
       RemoteSecuritySettings.init();
     });

netwerk/dns/PublicSuffixList.jsm

@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { RemoteSettings } = ChromeUtils.import(
+  "resource://services-settings/remote-settings.js"
+);
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const FileUtils = ChromeUtils.import("resource://gre/modules/FileUtils.jsm")
+  .FileUtils;
+
+const EXPORTED_SYMBOLS = ["PublicSuffixList"];
+
+const RECORD_ID = "tld-dafsa";
+const SIGNAL = "public-suffix-list-updated";
+
+const PublicSuffixList = {
+  CLIENT: RemoteSettings("public-suffix-list"),
+
+  init() {
+    this.CLIENT.on("sync", this.onUpdate.bind(this));
+    /* We have a single record for this collection. Let's see if we already have it locally.
+     * Note that on startup, we don't need to synchronize immediately on new profiles.
+     */
+    this.CLIENT.get({ syncIfEmpty: false, filters: { id: RECORD_ID } })
+      .then(async records => {
+        if (records.length == 1) {
+          const fileURI = await this.CLIENT.attachments.download(records[0]);
+          // Send a signal so that the C++ code loads the updated list on startup.
+          this.notifyUpdate(fileURI);
+        }
+      })
+      .catch(err => console.error(err));
+  },
+
+  /**
+   * This method returns the path to the file based on the file uri received
+   * Example:
+   * On windows "file://C:/Users/AppData/main/public-suffix-list/dafsa.bin"
+   * will be converted to "C:\\Users\\main\\public-suffix-list\\dafsa.bin
+   *
+   * On macOS/linux "file:///home/main/public-suffix-list/dafsa.bin"
+   * will be converted to "/home/main/public-suffix-list/dafsa.bin"
+   */
+  getFilePath(fileURI) {
+    const uri = Services.io.newURI(fileURI);
+    const file = uri.QueryInterface(Ci.nsIFileURL).file;
+    return file.path;
+  },
+
+  notifyUpdate(fileURI) {
+    const filePath = this.getFilePath(fileURI);
+    const nsifile = new FileUtils.File(filePath);
+    /* Send a signal to be read by the C++, the method
+     * ::Observe(nsISupports* aSubject, const char* aTopic, const char16_t* aData)
+     * at netwerk/dns/nsEffectiveTLDService.cpp
+     */
+    Services.obs.notifyObservers(
+      nsifile, // aSubject
+      SIGNAL, // aTopic
+      filePath // aData
+    );
+  },
+
+  async onUpdate({ data: { created, updated, deleted } }) {
+    // In theory, this will never happen, we will never delete the record.
+    if (deleted.length == 1) {
+      await this.CLIENT.attachments.delete(deleted[0]);
+    }
+    // Handle creation and update the same way
+    const changed = created.concat(updated.map(u => u.new));
+    /* In theory, we should never have more than one record. And if we receive
+     * this event, it's because the single record was updated.
+     */
+    if (changed.length != 1) {
+      console.warn("Unsupported sync event for Public Suffix List");
+      return;
+    }
+    // Download the updated file.
+    const fileURI = await this.CLIENT.attachments.download(changed[0]);
+    // Notify the C++ part to reload it from disk.
+    this.notifyUpdate(fileURI);
+  },
+};

netwerk/dns/moz.build

@@ -9,6 +9,7 @@ with Files('**'):
 
 DIRS += [
     'mdns',
+    'tests'
 ]
 
 XPIDL_SOURCES += [
@@ -23,6 +24,12 @@ XPIDL_SOURCES += [
 
 XPIDL_MODULE = 'necko_dns'
 
+EXTRA_JS_MODULES['netwerk-dns'] += [
+    'PublicSuffixList.jsm',
+]
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
+
 EXPORTS += [
     'nsEffectiveTLDService.h',
 ]

netwerk/dns/tests/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'unit'
+]

netwerk/dns/tests/unit/data/fake_public_suffix_list.dat

@@ -0,0 +1,57 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// This is a fake suffix list created only for the purposes of testing,
+// The real PSL is rather large thus this file is kept small by cutting out most of the suffixes
+
+// The Original list can be found at https://publicsuffix.org/list/public_suffix_list.dat,
+// Learn more about the PSL at https://publicsuffix.org.
+
+// .xpcshelltest is the fake domain created specifically for the the tests while
+// the others are ripped from the real list, serving as examples.
+
+// This fake public suffix list was used to create the binary file fake_remote_dafsa.bin
+// The binary is built at compile time and can be found at
+// obj-dir/_tests/xpcshell/netwerk/dns/tests/unit/data/
+
+// The list created with help of netwerk/dns/prepare_tlds.py and xpcom/ds/tools/make_dafsa.py
+// The build directive for the binary is at moz.build at netwerk/dns/tests/unit/data/
+
+
+
+// ===BEGIN ICANN DOMAINS===
+
+// xpcshelltest : Used in tests
+xpcshelltest
+website.xpcshelltest
+com.xpcshelltest
+edu.xpcshelltest
+gov.xpcshelltest
+net.xpcshelltest
+mil.xpcshelltest
+org.xpcshelltest
+
+// ac : https://en.wikipedia.org/wiki/.ac
+ac
+coc.ac
+com.ac
+edu.ac
+gov.ac
+net.ac
+mil.ac
+org.ac
+
+// bj : https://en.wikipedia.org/wiki/.bj
+bj
+asso.bj
+barreau.bj
+gouv.bj
+
+// bm : http://www.bermudanic.bm/dnr-text.txt
+bm
+com.bm
+edu.bm
+gov.bm
+net.bm
+org.bm

netwerk/dns/tests/unit/data/moz.build

@@ -0,0 +1,14 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+GENERATED_FILES = ['fake_remote_dafsa.bin']
+
+fake_remote_dafsa = GENERATED_FILES['fake_remote_dafsa.bin']
+fake_remote_dafsa.script = '../../../prepare_tlds.py'
+fake_remote_dafsa.inputs = ['fake_public_suffix_list.dat']
+fake_remote_dafsa.flags = ['bin']
+
+TEST_HARNESS_FILES.xpcshell.netwerk.dns.tests.unit.data += ['!fake_remote_dafsa.bin']

netwerk/dns/tests/unit/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'data'
+]

netwerk/dns/tests/unit/test_PublicSuffixList.js

@@ -0,0 +1,167 @@
+"use strict";
+
+const { PublicSuffixList } = ChromeUtils.import(
+  "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm"
+);
+const { TestUtils } = ChromeUtils.import(
+  "resource://testing-common/TestUtils.jsm"
+);
+
+const CLIENT = PublicSuffixList.CLIENT;
+const SIGNAL = "public-suffix-list-updated";
+
+const PAYLOAD_UPDATED_RECORDS = {
+  current: [{ id: "tld-dafsa", "commit-hash": "current-commit-hash" }],
+  created: [],
+  updated: [
+    {
+      old: { id: "tld-dafsa", "commit-hash": "current-commit-hash" },
+      new: { id: "tld-dafsa", "commit-hash": "new-commit-hash" },
+    },
+  ],
+  deleted: [],
+};
+const PAYLOAD_CREATED_RECORDS = {
+  current: [],
+  created: [
+    {
+      id: "tld-dafsa",
+      "commit-hash": "new-commit-hash",
+      attachment: {},
+    },
+  ],
+  updated: [],
+  deleted: [],
+};
+const PAYLOAD_UPDATED_AND_CREATED_RECORDS = {
+  current: [{ id: "tld-dafsa", "commit-hash": "current-commit-hash" }],
+  created: [{ id: "tld-dafsa", "commit-hash": "another-commit-hash" }],
+  updated: [
+    {
+      old: { id: "tld-dafsa", "commit-hash": "current-commit-hash" },
+      new: { id: "tld-dafsa", "commit-hash": "new-commit-hash" },
+    },
+  ],
+  deleted: [],
+};
+
+const fakeDafsaBinFile = do_get_file("data/fake_remote_dafsa.bin");
+const mockedFilePath = fakeDafsaBinFile.path;
+
+/**
+ * downloadCalled is used by mockDownload() and resetMockDownload()
+ * to keep track weather CLIENT.attachments.download is called or not
+ * downloadBackup will help restore CLIENT.attachments.download to original definition
+ * notifyUpdateBackup will help restore PublicSuffixList.notifyUpdate to original definition
+ */
+let downloadCalled = false;
+const downloadBackup = CLIENT.attachments.download;
+
+// returns a fake fileURI and sends a signal with filePath and no nsifile
+const mockDownload = () => {
+  downloadCalled = false;
+  CLIENT.attachments.download = async rec => {
+    downloadCalled = true;
+    return `file://${mockedFilePath}`; // Create a fake file URI
+  };
+};
+
+// resetMockDownload() must be run at the end of the test that uses mockDownload()
+const resetMockDownload = () => {
+  CLIENT.attachments.download = downloadBackup;
+};
+
+add_task(async () => {
+  info("File path sent when record is in DB.");
+
+  const collection = await CLIENT.openCollection();
+  await collection.clear(); // Make sure there's no record initially
+  await collection.create(
+    {
+      id: "tld-dafsa",
+      "commit-hash": "fake-commit-hash",
+      attachment: {},
+    },
+    { synced: true }
+  );
+
+  mockDownload();
+
+  const promiseSignal = TestUtils.topicObserved(SIGNAL);
+  await PublicSuffixList.init();
+  const observed = await promiseSignal;
+
+  Assert.equal(
+    observed[1],
+    mockedFilePath,
+    "File path sent when record is in DB."
+  );
+  await collection.clear(); // Clean up the mockDownloaded record
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("File path sent when record updated.");
+
+  mockDownload();
+
+  const promiseSignal = TestUtils.topicObserved(SIGNAL);
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_RECORDS });
+  const observed = await promiseSignal;
+
+  Assert.equal(
+    observed[1],
+    mockedFilePath,
+    "File path sent when record updated."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("Attachment downloaded when record created.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_CREATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    true,
+    "Attachment downloaded when record created."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("Attachment downloaded when record updated.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    true,
+    "Attachment downloaded when record updated."
+  );
+  resetMockDownload();
+});
+
+add_task(async () => {
+  info("No download when more than one record is changed.");
+
+  mockDownload();
+
+  await PublicSuffixList.init();
+  await CLIENT.emit("sync", { data: PAYLOAD_UPDATED_AND_CREATED_RECORDS });
+
+  Assert.equal(
+    downloadCalled,
+    false,
+    "No download when more than one record is changed."
+  );
+  resetMockDownload();
+});

netwerk/dns/tests/unit/xpcshell.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+head = ../../../../services/common/tests/unit/head_global.js ../../../../services/common/tests/unit/head_helpers.js
+firefox-appdir = browser
+support-files = data/**
+
+[test_PublicSuffixList.js]