Bug 1543598 - Move OneCRL and Pinning blocklist clients out of services r=jcj,glasserc

Differential Revision: https://phabricator.services.mozilla.com/D32297

--HG--
rename : services/common/tests/unit/test_blocklist_onecrl.js => security/manager/ssl/tests/unit/test_blocklist_onecrl.js
rename : services/common/tests/unit/test_blocklist_pinning.js => security/manager/ssl/tests/unit/test_blocklist_pinning.js
extra : moz-landing-system : lando
This commit is contained in:
Mathieu Leplatre 2019-06-11 10:14:40 +00:00
parent 00134913e3
commit a102f01554
11 changed files with 522 additions and 545 deletions

View File

@ -3,10 +3,11 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const EXPORTED_SYMBOLS = ["RemoteSecuritySettings"];
const EXPORTED_SYMBOLS = [
"RemoteSecuritySettings",
];
const {RemoteSettings} = ChromeUtils.import("resource://services-settings/remote-settings.js");
ChromeUtils.defineModuleGetter(this, "BlocklistClients", "resource://services-common/blocklist-clients.js");
const {AppConstants} = ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");
@ -26,6 +27,17 @@ const INTERMEDIATES_PENDING_TELEMETRY = "security.intermediate_preloading_num
const INTERMEDIATES_PRELOADED_TELEMETRY = "security.intermediate_preloading_num_preloaded";
const INTERMEDIATES_UPDATE_MS_TELEMETRY = "INTERMEDIATE_PRELOADING_UPDATE_TIME_MS";
const ONECRL_BUCKET_PREF = "services.settings.security.onecrl.bucket";
const ONECRL_COLLECTION_PREF = "services.settings.security.onecrl.collection";
const ONECRL_SIGNER_PREF = "services.settings.security.onecrl.signer";
const ONECRL_CHECKED_PREF = "services.settings.security.onecrl.checked";
const PINNING_ENABLED_PREF = "services.blocklist.pinning.enabled";
const PINNING_BUCKET_PREF = "services.blocklist.pinning.bucket";
const PINNING_COLLECTION_PREF = "services.blocklist.pinning.collection";
const PINNING_CHECKED_SECONDS_PREF = "services.blocklist.pinning.checked";
const PINNING_SIGNER_PREF = "services.blocklist.pinning.signer";
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => new TextDecoder());
@ -103,304 +115,482 @@ class CertInfo {
}
CertInfo.prototype.QueryInterface = ChromeUtils.generateQI([Ci.nsICertInfo]);
this.RemoteSecuritySettings = class RemoteSecuritySettings {
/**
* Initialize the clients (cheap instantiation) and setup their sync event.
* This static method is called from BrowserGlue.jsm soon after startup.
*/
static init() {
// In Bug 1543598, the OneCRL and Pinning clients will be moved in this module.
BlocklistClients.initialize();
class RevocationState {
constructor(state) {
this.state = state;
}
}
if (AppConstants.MOZ_NEW_CERT_STORAGE) {
new RemoteSecuritySettings();
class IssuerAndSerialRevocationState extends RevocationState {
constructor(issuer, serial, state) {
super(state);
this.issuer = issuer;
this.serial = serial;
}
}
IssuerAndSerialRevocationState.prototype.QueryInterface =
ChromeUtils.generateQI([Ci.nsIIssuerAndSerialRevocationState]);
class SubjectAndPubKeyRevocationState extends RevocationState {
constructor(subject, pubKey, state) {
super(state);
this.subject = subject;
this.pubKey = pubKey;
}
}
SubjectAndPubKeyRevocationState.prototype.QueryInterface =
ChromeUtils.generateQI([Ci.nsISubjectAndPubKeyRevocationState]);
function setRevocations(certStorage, revocations) {
return new Promise((resolve) =>
certStorage.setRevocations(revocations, resolve)
);
}
/**
* Revoke the appropriate certificates based on the records from the blocklist.
*
* @param {Object} data Current records in the local db.
*/
const updateCertBlocklist = AppConstants.MOZ_NEW_CERT_STORAGE ?
async function ({ data: { current, created, updated, deleted } }) {
const certList = Cc["@mozilla.org/security/certstorage;1"]
.getService(Ci.nsICertStorage);
let items = [];
// See if we have prior revocation data (this can happen when we can't open
// the database and we have to re-create it (see bug 1546361)).
let hasPriorRevocationData = await new Promise((resolve) => {
certList.hasPriorData(Ci.nsICertStorage.DATA_TYPE_REVOCATION, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
// If calling hasPriorData failed, assume we need to reload
// everything (even though it's unlikely doing so will succeed).
resolve(false);
}
});
});
// If we don't have prior data, make it so we re-load everything.
if (!hasPriorRevocationData) {
deleted = [];
updated = [];
created = current;
}
for (let item of deleted) {
if (item.issuerName && item.serialNumber) {
items.push(new IssuerAndSerialRevocationState(item.issuerName,
item.serialNumber, Ci.nsICertStorage.STATE_UNSET));
} else if (item.subject && item.pubKeyHash) {
items.push(new SubjectAndPubKeyRevocationState(item.subject,
item.pubKeyHash, Ci.nsICertStorage.STATE_UNSET));
}
}
constructor() {
this.client = RemoteSettings(Services.prefs.getCharPref(INTERMEDIATES_COLLECTION_PREF), {
bucketNamePref: INTERMEDIATES_BUCKET_PREF,
lastCheckTimePref: INTERMEDIATES_CHECKED_SECONDS_PREF,
signerName: Services.prefs.getCharPref(INTERMEDIATES_SIGNER_PREF),
localFields: ["cert_import_complete"],
});
const toAdd = created.concat(updated.map(u => u.new));
this.client.on("sync", this.onSync.bind(this));
Services.obs.addObserver(this.onObservePollEnd.bind(this),
"remote-settings:changes-poll-end");
log.debug("Intermediate Preloading: constructor");
for (let item of toAdd) {
if (item.issuerName && item.serialNumber) {
items.push(new IssuerAndSerialRevocationState(item.issuerName,
item.serialNumber, Ci.nsICertStorage.STATE_ENFORCE));
} else if (item.subject && item.pubKeyHash) {
items.push(new SubjectAndPubKeyRevocationState(item.subject,
item.pubKeyHash, Ci.nsICertStorage.STATE_ENFORCE));
}
}
async updatePreloadedIntermediates() {
// Bug 1429800: once the CertStateService has the correct interface, also
// store the whitelist status and crlite enrollment status
if (!Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true)) {
log.debug("Intermediate Preloading is disabled");
Services.obs.notifyObservers(null, "remote-security-settings:intermediates-updated", "disabled");
return;
}
// Download attachments that are awaiting download, up to a max.
const maxDownloadsPerRun = Services.prefs.getIntPref(INTERMEDIATES_DL_PER_POLL_PREF, 100);
// Bug 1519256: Move this to a separate method that's on a separate timer
// with a higher frequency (so we can attempt to download outstanding
// certs more than once daily)
// See if we have prior cert data (this can happen when we can't open the database and we
// have to re-create it (see bug 1546361)).
const certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hasPriorCertData = await new Promise((resolve) => {
certStorage.hasPriorData(Ci.nsICertStorage.DATA_TYPE_CERTIFICATE, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
// If calling hasPriorData failed, assume we need to reload everything (even though
// it's unlikely doing so will succeed).
resolve(false);
}
});
});
const col = await this.client.openCollection();
// If we don't have prior data, make it so we re-load everything.
if (!hasPriorCertData) {
let { data: toUpdate } = await col.list();
let promises = [];
toUpdate.forEach((record) => {
record.cert_import_complete = false;
promises.push(col.update(record));
});
await Promise.all(promises);
}
const { data: current } = await col.list();
const waiting = current.filter(record => !record.cert_import_complete);
log.debug(`There are ${waiting.length} intermediates awaiting download.`);
TelemetryStopwatch.start(INTERMEDIATES_UPDATE_MS_TELEMETRY);
let toDownload = waiting.slice(0, maxDownloadsPerRun);
let recordsCertsAndSubjects = await Promise.all(
toDownload.map(record => this.maybeDownloadAttachment(record)));
let certInfos = [];
let recordsToUpdate = [];
for (let {record, cert, subject} of recordsCertsAndSubjects) {
if (cert && subject) {
certInfos.push(new CertInfo(cert, subject));
recordsToUpdate.push(record);
}
}
let result = await new Promise((resolve) => {
certStorage.addCerts(certInfos, resolve);
}).catch((err) => err);
if (result != Cr.NS_OK) {
Cu.reportError(`certStorage.addCerts failed: ${result}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToUpdateDB");
return;
}
await Promise.all(recordsToUpdate.map((record) => {
record.cert_import_complete = true;
return col.update(record);
}));
const { data: finalCurrent } = await col.list();
const finalWaiting = finalCurrent.filter(record => !record.cert_import_complete);
const countPreloaded = finalCurrent.length - finalWaiting.length;
TelemetryStopwatch.finish(INTERMEDIATES_UPDATE_MS_TELEMETRY);
Services.telemetry.scalarSet(INTERMEDIATES_PRELOADED_TELEMETRY,
countPreloaded);
Services.telemetry.scalarSet(INTERMEDIATES_PENDING_TELEMETRY,
finalWaiting.length);
Services.obs.notifyObservers(null, "remote-security-settings:intermediates-updated",
"success");
try {
await setRevocations(certList, items);
} catch (e) {
Cu.reportError(e);
}
async onObservePollEnd(subject, topic, data) {
log.debug(`onObservePollEnd ${subject} ${topic}`);
try {
await this.updatePreloadedIntermediates();
} catch (err) {
log.warn(`Unable to update intermediate preloads: ${err}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToObserve");
}
}
// This method returns a promise to RemoteSettingsClient.maybeSync method.
async onSync({ data: { current, created, updated, deleted } }) {
if (!Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true)) {
log.debug("Intermediate Preloading is disabled");
return;
}
log.debug(`Removing ${deleted.length} Intermediate certificates`);
await this.removeCerts(deleted);
let certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hasPriorCRLiteData = await new Promise((resolve) => {
certStorage.hasPriorData(Ci.nsICertStorage.DATA_TYPE_CRLITE, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
resolve(false);
}
});
});
if (!hasPriorCRLiteData) {
deleted = [];
updated = [];
created = current;
}
const toAdd = created.concat(updated.map(u => u.new));
let entries = [];
for (let entry of deleted) {
entries.push(new CRLiteState(entry.subjectDN, entry.pubKeyHash,
Ci.nsICertStorage.STATE_UNSET));
}
for (let entry of toAdd) {
entries.push(new CRLiteState(entry.subjectDN, entry.pubKeyHash,
entry.crlite_enrolled ? Ci.nsICertStorage.STATE_ENFORCE
: Ci.nsICertStorage.STATE_UNSET));
}
await new Promise((resolve) => certStorage.setCRLiteState(entries, resolve));
}
/**
* Downloads the attachment data of the given record. Does not retry,
* leaving that to the caller.
* @param {AttachmentRecord} record The data to obtain
* @return {Promise} resolves to a Uint8Array on success
*/
async _downloadAttachmentBytes(record) {
const {attachment: {location}} = record;
const remoteFilePath = (await baseAttachmentsURL) + location;
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
return fetch(remoteFilePath, {
headers,
credentials: "omit",
}).then(resp => {
log.debug(`Download fetch completed: ${resp.ok} ${resp.status}`);
if (!resp.ok) {
Cu.reportError(`Failed to fetch ${remoteFilePath}: ${resp.status}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToFetch");
return Promise.reject();
}
return resp.arrayBuffer();
})
.then(buffer => new Uint8Array(buffer));
}
/**
* Attempts to download the attachment, assuming it's not been processed
* already. Does not retry, and always resolves (e.g., does not reject upon
* failure.) Errors are reported via Cu.reportError.
* @param {AttachmentRecord} record defines which data to obtain
* @return {Promise} a Promise that will resolve to an object with the properties
* record, cert, and subject. record is the original record.
* cert is the base64-encoded bytes of the downloaded certificate (if
* downloading was successful), and null otherwise.
* subject is the base64-encoded bytes of the subject distinguished
* name of the same.
*/
async maybeDownloadAttachment(record) {
const {attachment: {hash, size}} = record;
let result = { record, cert: null, subject: null };
let attachmentData;
} : async function ({ data: { current: records } }) {
const certList = Cc["@mozilla.org/security/certblocklist;1"]
.getService(Ci.nsICertBlocklist);
for (let item of records) {
try {
attachmentData = await this._downloadAttachmentBytes(record);
} catch (err) {
Cu.reportError(`Failed to download attachment: ${err}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToDownloadMisc");
return result;
if (item.issuerName && item.serialNumber) {
certList.revokeCertByIssuerAndSerial(item.issuerName,
item.serialNumber);
} else if (item.subject && item.pubKeyHash) {
certList.revokeCertBySubjectAndPubKey(item.subject,
item.pubKeyHash);
}
} catch (e) {
// prevent errors relating to individual blocklist entries from
// causing sync to fail. We will accumulate telemetry on these failures in
// bug 1254099.
Cu.reportError(e);
}
}
certList.saveEntries();
};
if (!attachmentData || attachmentData.length == 0) {
// Bug 1519273 - Log telemetry for these rejections
log.debug(`Empty attachment. Hash=${hash}`);
/**
* Modify the appropriate security pins based on records from the remote
* collection.
*
* @param {Object} data Current records in the local db.
*/
async function updatePinningList({ data: { current: records } }) {
if (!Services.prefs.getBoolPref(PINNING_ENABLED_PREF)) {
return;
}
const siteSecurityService = Cc["@mozilla.org/ssservice;1"]
.getService(Ci.nsISiteSecurityService);
// clear the current preload list
siteSecurityService.clearPreloads();
// write each KeyPin entry to the preload list
for (let item of records) {
try {
const { pinType, pins = [], versions } = item;
if (versions.includes(Services.appinfo.version)) {
if (pinType == "KeyPin" && pins.length) {
siteSecurityService.setKeyPins(item.hostName,
item.includeSubdomains,
item.expires,
pins.length,
pins, true);
}
if (pinType == "STSPin") {
siteSecurityService.setHSTSPreload(item.hostName,
item.includeSubdomains,
item.expires);
}
}
} catch (e) {
// prevent errors relating to individual preload entries from causing
// sync to fail. We will accumulate telemetry for such failures in bug
// 1254099.
Cu.reportError(e);
}
}
}
var RemoteSecuritySettings = {
/**
* Initialize the clients (cheap instantiation) and setup their sync event.
* This static method is called from BrowserGlue.jsm soon after startup.
*
* @returns {Object} intantiated clients for security remote settings.
*/
init() {
const OneCRLBlocklistClient = RemoteSettings(Services.prefs.getCharPref(ONECRL_COLLECTION_PREF), {
bucketNamePref: ONECRL_BUCKET_PREF,
lastCheckTimePref: ONECRL_CHECKED_PREF,
signerName: Services.prefs.getCharPref(ONECRL_SIGNER_PREF),
});
OneCRLBlocklistClient.on("sync", updateCertBlocklist);
const PinningBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PINNING_COLLECTION_PREF), {
bucketNamePref: PINNING_BUCKET_PREF,
lastCheckTimePref: PINNING_CHECKED_SECONDS_PREF,
signerName: Services.prefs.getCharPref(PINNING_SIGNER_PREF),
});
PinningBlocklistClient.on("sync", updatePinningList);
let IntermediatePreloadsClient;
if (AppConstants.MOZ_NEW_CERT_STORAGE) {
IntermediatePreloadsClient = new IntermediatePreloads();
}
return {
OneCRLBlocklistClient,
PinningBlocklistClient,
IntermediatePreloadsClient,
};
},
};
class IntermediatePreloads {
constructor() {
this.client = RemoteSettings(Services.prefs.getCharPref(INTERMEDIATES_COLLECTION_PREF), {
bucketNamePref: INTERMEDIATES_BUCKET_PREF,
lastCheckTimePref: INTERMEDIATES_CHECKED_SECONDS_PREF,
signerName: Services.prefs.getCharPref(INTERMEDIATES_SIGNER_PREF),
localFields: ["cert_import_complete"],
});
this.client.on("sync", this.onSync.bind(this));
Services.obs.addObserver(this.onObservePollEnd.bind(this),
"remote-settings:changes-poll-end");
log.debug("Intermediate Preloading: constructor");
}
async updatePreloadedIntermediates() {
// Bug 1429800: once the CertStateService has the correct interface, also
// store the whitelist status and crlite enrollment status
if (!Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true)) {
log.debug("Intermediate Preloading is disabled");
Services.obs.notifyObservers(null, "remote-security-settings:intermediates-updated", "disabled");
return;
}
// Download attachments that are awaiting download, up to a max.
const maxDownloadsPerRun = Services.prefs.getIntPref(INTERMEDIATES_DL_PER_POLL_PREF, 100);
// Bug 1519256: Move this to a separate method that's on a separate timer
// with a higher frequency (so we can attempt to download outstanding
// certs more than once daily)
// See if we have prior cert data (this can happen when we can't open the database and we
// have to re-create it (see bug 1546361)).
const certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hasPriorCertData = await new Promise((resolve) => {
certStorage.hasPriorData(Ci.nsICertStorage.DATA_TYPE_CERTIFICATE, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
// If calling hasPriorData failed, assume we need to reload everything (even though
// it's unlikely doing so will succeed).
resolve(false);
}
});
});
const col = await this.client.openCollection();
// If we don't have prior data, make it so we re-load everything.
if (!hasPriorCertData) {
let { data: toUpdate } = await col.list();
let promises = [];
toUpdate.forEach((record) => {
record.cert_import_complete = false;
promises.push(col.update(record));
});
await Promise.all(promises);
}
const { data: current } = await col.list();
const waiting = current.filter(record => !record.cert_import_complete);
log.debug(`There are ${waiting.length} intermediates awaiting download.`);
TelemetryStopwatch.start(INTERMEDIATES_UPDATE_MS_TELEMETRY);
let toDownload = waiting.slice(0, maxDownloadsPerRun);
let recordsCertsAndSubjects = await Promise.all(
toDownload.map(record => this.maybeDownloadAttachment(record)));
let certInfos = [];
let recordsToUpdate = [];
for (let {record, cert, subject} of recordsCertsAndSubjects) {
if (cert && subject) {
certInfos.push(new CertInfo(cert, subject));
recordsToUpdate.push(record);
}
}
let result = await new Promise((resolve) => {
certStorage.addCerts(certInfos, resolve);
}).catch((err) => err);
if (result != Cr.NS_OK) {
Cu.reportError(`certStorage.addCerts failed: ${result}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToUpdateDB");
return;
}
await Promise.all(recordsToUpdate.map((record) => {
record.cert_import_complete = true;
return col.update(record);
}));
const { data: finalCurrent } = await col.list();
const finalWaiting = finalCurrent.filter(record => !record.cert_import_complete);
const countPreloaded = finalCurrent.length - finalWaiting.length;
TelemetryStopwatch.finish(INTERMEDIATES_UPDATE_MS_TELEMETRY);
Services.telemetry.scalarSet(INTERMEDIATES_PRELOADED_TELEMETRY,
countPreloaded);
Services.telemetry.scalarSet(INTERMEDIATES_PENDING_TELEMETRY,
finalWaiting.length);
Services.obs.notifyObservers(null, "remote-security-settings:intermediates-updated",
"success");
}
async onObservePollEnd(subject, topic, data) {
log.debug(`onObservePollEnd ${subject} ${topic}`);
try {
await this.updatePreloadedIntermediates();
} catch (err) {
log.warn(`Unable to update intermediate preloads: ${err}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToObserve");
}
}
// This method returns a promise to RemoteSettingsClient.maybeSync method.
async onSync({ data: { current, created, updated, deleted } }) {
if (!Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true)) {
log.debug("Intermediate Preloading is disabled");
return;
}
log.debug(`Removing ${deleted.length} Intermediate certificates`);
await this.removeCerts(deleted);
let certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hasPriorCRLiteData = await new Promise((resolve) => {
certStorage.hasPriorData(Ci.nsICertStorage.DATA_TYPE_CRLITE, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
resolve(false);
}
});
});
if (!hasPriorCRLiteData) {
deleted = [];
updated = [];
created = current;
}
const toAdd = created.concat(updated.map(u => u.new));
let entries = [];
for (let entry of deleted) {
entries.push(new CRLiteState(entry.subjectDN, entry.pubKeyHash,
Ci.nsICertStorage.STATE_UNSET));
}
for (let entry of toAdd) {
entries.push(new CRLiteState(entry.subjectDN, entry.pubKeyHash,
entry.crlite_enrolled ? Ci.nsICertStorage.STATE_ENFORCE
: Ci.nsICertStorage.STATE_UNSET));
}
await new Promise((resolve) => certStorage.setCRLiteState(entries, resolve));
}
/**
* Downloads the attachment data of the given record. Does not retry,
* leaving that to the caller.
* @param {AttachmentRecord} record The data to obtain
* @return {Promise} resolves to a Uint8Array on success
*/
async _downloadAttachmentBytes(record) {
const {attachment: {location}} = record;
const remoteFilePath = (await baseAttachmentsURL) + location;
const headers = new Headers();
headers.set("Accept-Encoding", "gzip");
return fetch(remoteFilePath, {
headers,
credentials: "omit",
}).then(resp => {
log.debug(`Download fetch completed: ${resp.ok} ${resp.status}`);
if (!resp.ok) {
Cu.reportError(`Failed to fetch ${remoteFilePath}: ${resp.status}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("emptyAttachment");
.add("failedToFetch");
return result;
return Promise.reject();
}
return resp.arrayBuffer();
})
.then(buffer => new Uint8Array(buffer));
}
// check the length
if (attachmentData.length !== size) {
log.debug(`Unexpected attachment length. Hash=${hash} Lengths ${attachmentData.length} != ${size}`);
/**
* Attempts to download the attachment, assuming it's not been processed
* already. Does not retry, and always resolves (e.g., does not reject upon
* failure.) Errors are reported via Cu.reportError.
* @param {AttachmentRecord} record defines which data to obtain
* @return {Promise} a Promise that will resolve to an object with the properties
* record, cert, and subject. record is the original record.
* cert is the base64-encoded bytes of the downloaded certificate (if
* downloading was successful), and null otherwise.
* subject is the base64-encoded bytes of the subject distinguished
* name of the same.
*/
async maybeDownloadAttachment(record) {
const {attachment: {hash, size}} = record;
let result = { record, cert: null, subject: null };
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("unexpectedLength");
return result;
}
// check the hash
let dataAsString = gTextDecoder.decode(attachmentData);
let calculatedHash = getHash(dataAsString);
if (calculatedHash !== hash) {
log.warn(`Invalid hash. CalculatedHash=${calculatedHash}, Hash=${hash}, data=${dataAsString}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("unexpectedHash");
return result;
}
log.debug(`downloaded cert with hash=${hash}, size=${size}`);
let certBase64;
let subjectBase64;
try {
// split off the header and footer
certBase64 = dataAsString.split("-----")[2].replace(/\s/g, "");
// get an array of bytes so we can use X509.jsm
let certBytes = stringToBytes(atob(certBase64));
let cert = new X509.Certificate();
cert.parse(certBytes);
// get the DER-encoded subject and get a base64-encoded string from it
// TODO(bug 1542028): add getters for _der and _bytes
subjectBase64 = btoa(bytesToString(cert.tbsCertificate.subject._der._bytes));
} catch (err) {
Cu.reportError(`Failed to decode cert: ${err}`);
// Re-purpose the "failedToUpdateNSS" telemetry tag as "failed to
// decode preloaded intermediate certificate"
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToUpdateNSS");
return result;
}
result.cert = certBase64;
result.subject = subjectBase64;
let attachmentData;
try {
attachmentData = await this._downloadAttachmentBytes(record);
} catch (err) {
Cu.reportError(`Failed to download attachment: ${err}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToDownloadMisc");
return result;
}
async maybeSync(expectedTimestamp, options) {
return this.client.maybeSync(expectedTimestamp, options);
if (!attachmentData || attachmentData.length == 0) {
// Bug 1519273 - Log telemetry for these rejections
log.debug(`Empty attachment. Hash=${hash}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("emptyAttachment");
return result;
}
async removeCerts(recordsToRemove) {
let certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hashes = recordsToRemove.map(record => record.derHash);
let result = await new Promise((resolve) => {
certStorage.removeCertsByHashes(hashes, resolve);
}).catch((err) => err);
if (result != Cr.NS_OK) {
Cu.reportError(`Failed to remove some intermediate certificates`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToRemove");
}
// check the length
if (attachmentData.length !== size) {
log.debug(`Unexpected attachment length. Hash=${hash} Lengths ${attachmentData.length} != ${size}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("unexpectedLength");
return result;
}
};
// check the hash
let dataAsString = gTextDecoder.decode(attachmentData);
let calculatedHash = getHash(dataAsString);
if (calculatedHash !== hash) {
log.warn(`Invalid hash. CalculatedHash=${calculatedHash}, Hash=${hash}, data=${dataAsString}`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("unexpectedHash");
return result;
}
log.debug(`downloaded cert with hash=${hash}, size=${size}`);
let certBase64;
let subjectBase64;
try {
// split off the header and footer
certBase64 = dataAsString.split("-----")[2].replace(/\s/g, "");
// get an array of bytes so we can use X509.jsm
let certBytes = stringToBytes(atob(certBase64));
let cert = new X509.Certificate();
cert.parse(certBytes);
// get the DER-encoded subject and get a base64-encoded string from it
// TODO(bug 1542028): add getters for _der and _bytes
subjectBase64 = btoa(bytesToString(cert.tbsCertificate.subject._der._bytes));
} catch (err) {
Cu.reportError(`Failed to decode cert: ${err}`);
// Re-purpose the "failedToUpdateNSS" telemetry tag as "failed to
// decode preloaded intermediate certificate"
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToUpdateNSS");
return result;
}
result.cert = certBase64;
result.subject = subjectBase64;
return result;
}
async maybeSync(expectedTimestamp, options) {
return this.client.maybeSync(expectedTimestamp, options);
}
async removeCerts(recordsToRemove) {
let certStorage = Cc["@mozilla.org/security/certstorage;1"].getService(Ci.nsICertStorage);
let hashes = recordsToRemove.map(record => record.derHash);
let result = await new Promise((resolve) => {
certStorage.removeCertsByHashes(hashes, resolve);
}).catch((err) => err);
if (result != Cr.NS_OK) {
Cu.reportError(`Failed to remove some intermediate certificates`);
Services.telemetry.getHistogramById(INTERMEDIATES_ERRORS_TELEMETRY)
.add("failedToRemove");
}
}
}

View File

@ -1,11 +1,19 @@
"use strict";
const { BlocklistClients } = ChromeUtils.import("resource://services-common/blocklist-clients.js");
const { AddonTestUtils } = ChromeUtils.import("resource://testing-common/AddonTestUtils.jsm");
const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm");
const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js");
const { OneCRLBlocklistClient } = BlocklistClients.initialize();
const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
const { RemoteSecuritySettings } = ChromeUtils.import("resource://gre/modules/psm/RemoteSecuritySettings.jsm");
const { OneCRLBlocklistClient } = RemoteSecuritySettings.init();
const global = this;
function run_test() {
// Initialize app, user profile etc.
AddonTestUtils.init(global);
AddonTestUtils.createAppInfo("XPCShell", "xpcshell@tests.mozilla.org", "1", "");
AddonTestUtils.promiseStartupManager().then(run_next_test);
}
add_task(async function test_uses_a_custom_signer() {
Assert.notEqual(OneCRLBlocklistClient.signerName, RemoteSettings("not-specified").signerName);
@ -16,16 +24,7 @@ add_task(async function test_has_initial_dump() {
});
add_task(async function test_default_jexl_filter_is_used() {
const countInDump = (await OneCRLBlocklistClient.get()).length;
// Create two fake records, one whose target expression is falsy (and will
// this be filtered by `.get()`) and another one with a truthy filter.
const collection = await OneCRLBlocklistClient.openCollection();
await collection.create({ filter_expression: "1 == 2" }); // filtered.
await collection.create({ filter_expression: "1 == 1" });
await collection.db.saveLastModified(42); // Fake sync state: prevent from loading JSON dump.
Assert.equal((await OneCRLBlocklistClient.get()).length, countInDump + 1);
Assert.deepEqual(OneCRLBlocklistClient.filterFunc, RemoteSettings("not-specified").filterFunc);
});
add_task({

View File

@ -1,15 +1,13 @@
"use strict";
const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");
const {BlocklistClients} = ChromeUtils.import("resource://services-common/blocklist-clients.js");
const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm");
const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js");
const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
const { RemoteSecuritySettings } = ChromeUtils.import("resource://gre/modules/psm/RemoteSecuritySettings.jsm");
const sss = Cc["@mozilla.org/ssservice;1"]
.getService(Ci.nsISiteSecurityService);
const { PinningBlocklistClient } = BlocklistClients.initialize();
const { PinningBlocklistClient } = RemoteSecuritySettings.init();
add_task(async function test_uses_a_custom_signer() {
@ -25,16 +23,7 @@ add_task(async function test_pinning_has_initial_dump() {
});
add_task(async function test_default_jexl_filter_is_used() {
const countInDump = (await PinningBlocklistClient.get()).length;
// Create two fake records, one whose target expression is falsy (and will
// this be filtered by `.get()`) and another one with a truthy filter.
const collection = await PinningBlocklistClient.openCollection();
await collection.create({ filter_expression: "1 == 2" }); // filtered.
await collection.create({ filter_expression: "1 == 1" });
await collection.db.saveLastModified(42); // Fake sync state: prevent from loading JSON dump.
Assert.equal((await PinningBlocklistClient.get()).length, countInDump + 1);
Assert.deepEqual(PinningBlocklistClient.filterFunc, RemoteSettings("not-specified").filterFunc);
});
add_task(async function test_no_pins_by_default() {
@ -164,6 +153,8 @@ add_task(async function test_bad_entries() {
"expires": new Date().getTime() + 1000000,
}, // missing versions.
];
// The event listener will catch any error, and won't throw.
// See https://bugzilla.mozilla.org/show_bug.cgi?id=1554939
await PinningBlocklistClient.emit("sync", { data: { current }});
ok(!sss.isSecureURI(sss.HEADER_HPKP,

View File

@ -12,8 +12,23 @@
// * it does a sanity check to ensure other cert verifier behavior is
// unmodified
const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js");
const { BlocklistClients } = ChromeUtils.import("resource://services-common/blocklist-clients.js");
const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm", {});
const { RemoteSecuritySettings } = ChromeUtils.import("resource://gre/modules/psm/RemoteSecuritySettings.jsm");
// First, we need to setup appInfo for the blocklist service to work
var id = "xpcshell@tests.mozilla.org";
var appName = "XPCShell";
var version = "1";
var platformVersion = "1.9.2";
ChromeUtils.import("resource://testing-common/AppInfo.jsm", this);
/* global updateAppInfo:false */ // Imported via AppInfo.jsm.
updateAppInfo({
name: appName,
ID: id,
version,
platformVersion: platformVersion ? platformVersion : "1.0",
crashReporter: true,
});
// we need to ensure we setup revocation data before certDB, or we'll start with
// no revocation.txt in the profile
@ -105,7 +120,7 @@ function load_cert(cert, trust) {
}
async function update_blocklist() {
const { OneCRLBlocklistClient } = BlocklistClients.initialize();
const { OneCRLBlocklistClient } = RemoteSecuritySettings.init();
const fakeEvent = {
current: certBlocklist, // with old .txt revocations.

View File

@ -8,15 +8,12 @@
do_get_profile(); // must be called before getting nsIX509CertDB
const {RemoteSettings} = ChromeUtils.import("resource://services-settings/remote-settings.js");
const {RemoteSecuritySettings} = ChromeUtils.import("resource://gre/modules/psm/RemoteSecuritySettings.jsm");
const {TestUtils} = ChromeUtils.import("resource://testing-common/TestUtils.jsm");
const {TelemetryTestUtils} = ChromeUtils.import("resource://testing-common/TelemetryTestUtils.jsm");
const {X509} = ChromeUtils.import("resource://gre/modules/psm/X509.jsm");
let remoteSecSetting;
if (AppConstants.MOZ_NEW_CERT_STORAGE) {
const {RemoteSecuritySettings} = ChromeUtils.import("resource://gre/modules/psm/RemoteSecuritySettings.jsm");
remoteSecSetting = new RemoteSecuritySettings();
}
const {IntermediatePreloadsClient} = RemoteSecuritySettings.init();
let server;
@ -86,7 +83,7 @@ async function syncAndDownload(filenames, options = {}) {
clear = true,
} = options;
const localDB = await remoteSecSetting.client.openCollection();
const localDB = await IntermediatePreloadsClient.client.openCollection();
if (clear) {
await localDB.clear();
}
@ -134,7 +131,7 @@ async function syncAndDownload(filenames, options = {}) {
* Return the list of records whose attachmnet was downloaded.
*/
async function locallyDownloaded() {
return remoteSecSetting.client.get({
return IntermediatePreloadsClient.client.get({
filters: { cert_import_complete: true },
syncIfEmpty: false,
});
@ -290,11 +287,11 @@ add_task({
await checkCertErrorGeneric(certDB, ee_cert, PRErrorCodeSuccess,
certificateUsageSSLServer);
let localDB = await remoteSecSetting.client.openCollection();
let localDB = await IntermediatePreloadsClient.client.openCollection();
let { data } = await localDB.list();
ok(data.length > 0, "should have some entries");
// simulate a sync (syncAndDownload doesn't actually... sync.)
await remoteSecSetting.client.emit("sync", {
await IntermediatePreloadsClient.client.emit("sync", {
"data": {
current: data,
created: data,
@ -366,7 +363,7 @@ add_task({
equal((await locallyDownloaded()).length, 2, "There should have been 2 downloads");
let localDB = await remoteSecSetting.client.openCollection();
let localDB = await IntermediatePreloadsClient.client.openCollection();
let { data } = await localDB.list();
ok(data.length > 0, "should have some entries");
let subject = data[0].subjectDN;
@ -374,7 +371,7 @@ add_task({
let resultsBefore = certStorage.findCertsBySubject(stringToArray(atob(subject)));
equal(resultsBefore.length, 1, "should find the intermediate in cert storage before");
// simulate a sync where we deleted the entry
await remoteSecSetting.client.emit("sync", {
await IntermediatePreloadsClient.client.emit("sync", {
"data": {
current: [],
created: [],

View File

@ -1,6 +1,7 @@
[DEFAULT]
head = head_psm.js
tags = psm
firefox-appdir = browser
support-files =
bad_certs/**
ocsp_certs/**
@ -43,6 +44,12 @@ support-files =
[test_add_preexisting_cert.js]
[test_baseline_requirements_subject_common_name.js]
[test_blocklist_onecrl.js]
# Skip signature tests for Thunderbird (Bug 1341983).
skip-if = appname == "thunderbird"
tags = remote-settings blocklist
[test_blocklist_pinning.js]
tags = remote-settings blocklist
[test_broken_fips.js]
# FIPS has never been a thing on Android, so the workaround doesn't
# exist on that platform.

View File

@ -1,214 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = [
"BlocklistClients",
];
const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
const {Services} = ChromeUtils.import("resource://gre/modules/Services.jsm");
const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm");
ChromeUtils.defineModuleGetter(this, "RemoteSettings", "resource://services-settings/remote-settings.js");
ChromeUtils.defineModuleGetter(this, "jexlFilterFunc", "resource://services-settings/remote-settings.js");
const PREF_SECURITY_SETTINGS_ONECRL_BUCKET = "services.settings.security.onecrl.bucket";
const PREF_SECURITY_SETTINGS_ONECRL_COLLECTION = "services.settings.security.onecrl.collection";
const PREF_SECURITY_SETTINGS_ONECRL_SIGNER = "services.settings.security.onecrl.signer";
const PREF_SECURITY_SETTINGS_ONECRL_CHECKED = "services.settings.security.onecrl.checked";
const PREF_BLOCKLIST_PINNING_ENABLED = "services.blocklist.pinning.enabled";
const PREF_BLOCKLIST_PINNING_BUCKET = "services.blocklist.pinning.bucket";
const PREF_BLOCKLIST_PINNING_COLLECTION = "services.blocklist.pinning.collection";
const PREF_BLOCKLIST_PINNING_CHECKED_SECONDS = "services.blocklist.pinning.checked";
const PREF_BLOCKLIST_PINNING_SIGNER = "services.blocklist.pinning.signer";
class RevocationState {
constructor(state) {
this.state = state;
}
}
class IssuerAndSerialRevocationState extends RevocationState {
constructor(issuer, serial, state) {
super(state);
this.issuer = issuer;
this.serial = serial;
}
}
IssuerAndSerialRevocationState.prototype.QueryInterface =
ChromeUtils.generateQI([Ci.nsIIssuerAndSerialRevocationState]);
class SubjectAndPubKeyRevocationState extends RevocationState {
constructor(subject, pubKey, state) {
super(state);
this.subject = subject;
this.pubKey = pubKey;
}
}
SubjectAndPubKeyRevocationState.prototype.QueryInterface =
ChromeUtils.generateQI([Ci.nsISubjectAndPubKeyRevocationState]);
function setRevocations(certStorage, revocations) {
return new Promise((resolve) =>
certStorage.setRevocations(revocations, resolve)
);
}
/**
* Revoke the appropriate certificates based on the records from the blocklist.
*
* @param {Object} data Current records in the local db.
*/
const updateCertBlocklist = AppConstants.MOZ_NEW_CERT_STORAGE ?
async function({ data: { current, created, updated, deleted } }) {
const certList = Cc["@mozilla.org/security/certstorage;1"]
.getService(Ci.nsICertStorage);
let items = [];
// See if we have prior revocation data (this can happen when we can't open
// the database and we have to re-create it (see bug 1546361)).
let hasPriorRevocationData = await new Promise((resolve) => {
certList.hasPriorData(Ci.nsICertStorage.DATA_TYPE_REVOCATION, (rv, hasPriorData) => {
if (rv == Cr.NS_OK) {
resolve(hasPriorData);
} else {
// If calling hasPriorData failed, assume we need to reload
// everything (even though it's unlikely doing so will succeed).
resolve(false);
}
});
});
// If we don't have prior data, make it so we re-load everything.
if (!hasPriorRevocationData) {
deleted = [];
updated = [];
created = current;
}
for (let item of deleted) {
if (item.issuerName && item.serialNumber) {
items.push(new IssuerAndSerialRevocationState(item.issuerName,
item.serialNumber, Ci.nsICertStorage.STATE_UNSET));
} else if (item.subject && item.pubKeyHash) {
items.push(new SubjectAndPubKeyRevocationState(item.subject,
item.pubKeyHash, Ci.nsICertStorage.STATE_UNSET));
}
}
const toAdd = created.concat(updated.map(u => u.new));
for (let item of toAdd) {
if (item.issuerName && item.serialNumber) {
items.push(new IssuerAndSerialRevocationState(item.issuerName,
item.serialNumber, Ci.nsICertStorage.STATE_ENFORCE));
} else if (item.subject && item.pubKeyHash) {
items.push(new SubjectAndPubKeyRevocationState(item.subject,
item.pubKeyHash, Ci.nsICertStorage.STATE_ENFORCE));
}
}
try {
await setRevocations(certList, items);
} catch (e) {
Cu.reportError(e);
}
} : async function({ data: { current: records } }) {
const certList = Cc["@mozilla.org/security/certblocklist;1"]
.getService(Ci.nsICertBlocklist);
for (let item of records) {
try {
if (item.issuerName && item.serialNumber) {
certList.revokeCertByIssuerAndSerial(item.issuerName,
item.serialNumber);
} else if (item.subject && item.pubKeyHash) {
certList.revokeCertBySubjectAndPubKey(item.subject,
item.pubKeyHash);
}
} catch (e) {
// prevent errors relating to individual blocklist entries from
// causing sync to fail. We will accumulate telemetry on these failures in
// bug 1254099.
Cu.reportError(e);
}
}
certList.saveEntries();
};
/**
* Modify the appropriate security pins based on records from the remote
* collection.
*
* @param {Object} data Current records in the local db.
*/
async function updatePinningList({ data: { current: records } }) {
if (!Services.prefs.getBoolPref(PREF_BLOCKLIST_PINNING_ENABLED)) {
return;
}
const siteSecurityService = Cc["@mozilla.org/ssservice;1"]
.getService(Ci.nsISiteSecurityService);
// clear the current preload list
siteSecurityService.clearPreloads();
// write each KeyPin entry to the preload list
for (let item of records) {
try {
const {pinType, pins = [], versions} = item;
if (versions.includes(Services.appinfo.version)) {
if (pinType == "KeyPin" && pins.length) {
siteSecurityService.setKeyPins(item.hostName,
item.includeSubdomains,
item.expires,
pins.length,
pins, true);
}
if (pinType == "STSPin") {
siteSecurityService.setHSTSPreload(item.hostName,
item.includeSubdomains,
item.expires);
}
}
} catch (e) {
// prevent errors relating to individual preload entries from causing
// sync to fail. We will accumulate telemetry for such failures in bug
// 1254099.
}
}
}
var OneCRLBlocklistClient;
var PinningBlocklistClient;
function initialize(options = {}) {
const { verifySignature = true } = options;
OneCRLBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PREF_SECURITY_SETTINGS_ONECRL_COLLECTION), {
bucketNamePref: PREF_SECURITY_SETTINGS_ONECRL_BUCKET,
lastCheckTimePref: PREF_SECURITY_SETTINGS_ONECRL_CHECKED,
signerName: Services.prefs.getCharPref(PREF_SECURITY_SETTINGS_ONECRL_SIGNER),
});
OneCRLBlocklistClient.verifySignature = verifySignature;
OneCRLBlocklistClient.on("sync", updateCertBlocklist);
PinningBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PREF_BLOCKLIST_PINNING_COLLECTION), {
bucketNamePref: PREF_BLOCKLIST_PINNING_BUCKET,
lastCheckTimePref: PREF_BLOCKLIST_PINNING_CHECKED_SECONDS,
signerName: Services.prefs.getCharPref(PREF_BLOCKLIST_PINNING_SIGNER),
});
PinningBlocklistClient.verifySignature = verifySignature;
PinningBlocklistClient.on("sync", updatePinningList);
return {
OneCRLBlocklistClient,
PinningBlocklistClient,
};
}
let BlocklistClients = {initialize};

View File

@ -15,7 +15,6 @@ EXTRA_COMPONENTS += [
EXTRA_JS_MODULES['services-common'] += [
'async.js',
'blocklist-clients.js',
'kinto-http-client.js',
'kinto-offline-client.js',
'kinto-storage-adapter.js',

View File

@ -7,13 +7,6 @@ support-files =
# Test load modules first so syntax failures are caught early.
[test_load_modules.js]
[test_blocklist_onecrl.js]
# Skip signature tests for Thunderbird (Bug 1341983).
skip-if = appname == "thunderbird"
tags = blocklist
[test_blocklist_pinning.js]
tags = blocklist
[test_kinto.js]
tags = blocklist
[test_storage_adapter.js]

View File

@ -13,6 +13,8 @@ user_pref("media.gmp-manager.url.override", "http://%(server)s/dummy-gmp-manager
user_pref("toolkit.telemetry.server", "https://%(server)s/telemetry-dummy");
// Prevent Remote Settings to issue non local connections.
user_pref("services.settings.server", "http://localhost/remote-settings-dummy/v1");
// Prevent intermediate preloads to be downloaded on Remote Settings polling.
user_pref("security.remote_settings.intermediates.enabled", false);
// The process priority manager only shifts priorities when it has at least
// one active tab. xpcshell tabs don't have any active tabs, which would mean
// all processes would run at low priority, which is not desirable, so we

View File

@ -18,8 +18,6 @@
"async.js": ["Async"],
"AsyncSpellCheckTestHelper.jsm": ["onSpellCheck"],
"base-loader.js": ["Loader", "resolveURI", "Module", "Require", "unload"],
"blocklist-clients.js": ["BlocklistClients"],
"blocklist-updater.js": ["checkVersions", "addTestBlocklistClient"],
"bogus_element_type.jsm": [],
"bookmarks.js": ["BookmarksEngine", "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator", "BufferedBookmarksEngine"],
"bookmarks.jsm": ["PlacesItem", "Bookmark", "Separator", "Livemark", "BookmarkFolder", "DumpBookmarks"],