From 10570b80e68a1f00cbf06ae8e027816ece3dffe5 Mon Sep 17 00:00:00 2001 From: Marian Raiciof Date: Wed, 31 Jul 2019 16:31:42 +0000 Subject: [PATCH 001/101] Bug 1568232 - Run raptor cold page-load test job on fenix and refbrow. r=perftest-reviewers,davehunt Run fenix and refbrow raptor cold page-load tests once a day, through cron, at 3 AM. Differential Revision: https://phabricator.services.mozilla.com/D39485 --HG-- extra : moz-landing-system : lando --- .cron.yml | 17 +++++++++++++++++ taskcluster/taskgraph/target_tasks.py | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.cron.yml b/.cron.yml index b7ba1cb03cef..560f43b76381 100644 --- a/.cron.yml +++ b/.cron.yml @@ -193,3 +193,20 @@ jobs: # No default branch mozilla-central: - {weekday: 'Monday', hour: 10, minute: 0} + + - name: raptor-tp6m + job: + type: decision-task + treeherder-symbol: tp6m + target-tasks-method: raptor_tp6m + include-push-tasks: true + run-on-projects: + - mozilla-central + when: + - {weekday: 'Monday', hour: 3, minute: 0} + - {weekday: 'Tuesday', hour: 3, minute: 0} + - {weekday: 'Wednesday', hour: 3, minute: 0} + - {weekday: 'Thursday', hour: 3, minute: 0} + - {weekday: 'Friday', hour: 3, minute: 0} + - {weekday: 'Saturday', hour: 3, minute: 0} + - {weekday: 'Sunday', hour: 3, minute: 0} diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 983f25dd28c1..d5a942f3a9c5 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -759,3 +759,26 @@ def target_tasks_codereview(full_task_graph, parameters, graph_config): def target_tasks_nothing(full_task_graph, parameters, graph_config): """Select nothing, for DONTBUILD pushes""" return [] + + +@_target_task('raptor_tp6m') +def target_tasks_raptor_tp6m(full_task_graph, parameters, graph_config): + """ + Select tasks required for running raptor cold page-load tests on fenix and refbrow + """ + def filter(task): + platform = task.attributes.get('build_platform') + attributes = task.attributes + + if platform and 'android' not in platform: + return False + if attributes.get('unittest_suite') != 'raptor': + return False + try_name = attributes.get('raptor_try_name') + if '-cold' in try_name and 'pgo' in platform: + if '-1-refbrow-' in try_name: + return True + if '-1-fenix-' in try_name: + return True + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] From cb092c29dc1db249cab2eda0fbd8b72aaf0ac8de Mon Sep 17 00:00:00 2001 From: Alexandru Irimovici Date: Wed, 31 Jul 2019 16:36:21 +0000 Subject: [PATCH 002/101] Bug 1558456 - stop tracking >1080p youtube-playback-test on motoG5 r=sparky,perftest-reviewers,davehunt Add functionality for being able to send extra parameters for the test_url query of a test, directly from a taskcluster config. Also, the PR adds logic in the `setup-raptor` taskgraph transform for dynamically changing the subset of youtube-playback tests based on the platform and project Differential Revision: https://phabricator.services.mozilla.com/D39006 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/test/raptor.yml | 6 ++ taskcluster/taskgraph/transforms/tests.py | 37 +++++++++ .../mozharness/mozilla/testing/raptor.py | 9 +++ testing/raptor/raptor/cmdline.py | 2 + testing/raptor/raptor/manifest.py | 37 +++++++++ testing/raptor/test/conftest.py | 1 + testing/raptor/test/test_manifest.py | 76 ++++++++++++++++++- 7 files changed, 167 insertions(+), 1 deletion(-) diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml index f2aae234c1c2..b969dbb94ece 100644 --- a/taskcluster/ci/test/raptor.yml +++ b/taskcluster/ci/test/raptor.yml @@ -2259,6 +2259,9 @@ raptor-youtube-playback-geckoview: - --app=geckoview - --binary=org.mozilla.geckoview_example - --activity=org.mozilla.geckoview_example.GeckoViewActivity + # Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases + # - --test-url-params is dynamically added from setup_raptor transform depending + # on platform and project raptor-youtube-playback-fenix: description: "Raptor YouTube Playback on Fenix" @@ -2282,6 +2285,9 @@ raptor-youtube-playback-fenix: - --app=fenix - --binary-path=org.mozilla.fenix.performancetest - --activity=org.mozilla.fenix.IntentReceiverActivity + # Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases + # - --test-url-params is dynamically added from setup_raptor transform depending + # on platform and project raptor-tp6-1-firefox-cold: description: "Raptor tp6-1 cold page-load on Firefox" diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index fc0a746d4970..66eb02b9dbbf 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -664,6 +664,43 @@ def setup_raptor(config, tests): if test['require-signed-extensions']: extra_options.append('--is-release-build') + # add urlparams based on platform, test names and projects + testurlparams_by_platform_and_project = { + "android-hw-g5": [ + { + "branches": [], # For all branches + "testnames": ["youtube-playback"], + "urlparams": [ + # param used for excluding youtube-playback tests from executing + # it excludes the tests with videos >1080p + "exclude=1,2,9,10,17,18,21,22,26,28,30,32,39,40,47," + "48,55,56,63,64,71,72,79,80,83,84,89,90,95,96", + ] + }, + ] + } + + for platform, testurlparams_by_project_definitions \ + in testurlparams_by_platform_and_project.items(): + + if test['test-platform'].startswith(platform): + # For every platform it may have several definitions + for testurlparams_by_project in testurlparams_by_project_definitions: + # The test should contain at least one defined testname + if any( + testname in test['test-name'] + for testname in testurlparams_by_project['testnames'] + ): + branches = testurlparams_by_project['branches'] + if ( + branches == [] or + config.params.get('project') in branches or + config.params.is_try() and 'try' in branches + ): + params_query = '&'.join(testurlparams_by_project['urlparams']) + add_extra_params_option = "--test-url-params={}".format(params_query) + extra_options.append(add_extra_params_option) + yield test diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index 8f42bfb9b009..fdd4b59d7ac2 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -162,6 +162,11 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): "help": "The number of times a cold load test is repeated (for cold load tests only, " "where the browser is shutdown and restarted between test iterations)" }], + [["--test-url-params"], { + "action": "store", + "dest": "test_url_params", + "help": "Parameters to add to the test_url query string" + }], [["--host"], { "dest": "host", "help": "Hostname from which to serve urls (default: 127.0.0.1). " @@ -285,6 +290,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): self.gecko_profile_interval = self.config.get('gecko_profile_interval') self.gecko_profile_entries = self.config.get('gecko_profile_entries') self.test_packages_url = self.config.get('test_packages_url') + self.test_url_params = self.config.get('test_url_params') self.host = self.config.get('host') if self.host == 'HOST_IP': self.host = os.environ['HOST_IP'] @@ -412,6 +418,9 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): kw_options['symbolsPath'] = self.symbols_path if self.config.get('obj_path', None) is not None: kw_options['obj-path'] = self.config['obj_path'] + if self.test_url_params: + kw_options['test-url-params'] = self.test_url_params + kw_options.update(kw) if self.host: kw_options['host'] = self.host diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index f77af01f7c1e..1f2f652b3953 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -128,6 +128,8 @@ def create_parser(mach_interface=False): add_arg('--browser-cycles', dest="browser_cycles", type=int, help="The number of times a cold load test is repeated (for cold load tests only, " "where the browser is shutdown and restarted between test iterations)") + add_arg('--test-url-params', dest='test_url_params', + help="Parameters to add to the test_url query string") add_arg('--print-tests', action=_PrintTests, help="Print all available Raptor tests") add_arg('--debug-mode', dest="debug_mode", action="store_true", diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py index 157c6732f901..68b733a507c9 100644 --- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -5,6 +5,8 @@ from __future__ import absolute_import import json import os +from urlparse import parse_qs, urlsplit, urlunsplit +from urllib import urlencode, unquote from logger.logger import RaptorLogger from manifestparser import TestManifest @@ -125,6 +127,32 @@ def validate_test_ini(test_details): return valid_settings +def add_test_url_params(url, extra_params): + # add extra parameters to the test_url query string + # the values that already exist are re-written + + # urlsplit returns a result as a tuple like (scheme, netloc, path, query, fragment) + parsed_url = urlsplit(url) + + parsed_query_params = parse_qs(parsed_url.query) + parsed_extra_params = parse_qs(extra_params) + + for name, value in parsed_extra_params.items(): + # overwrite the old value + parsed_query_params[name] = value + + final_query_string = unquote(urlencode(parsed_query_params, doseq=True)) + + # reconstruct test_url with the changed query string + return urlunsplit(( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + final_query_string, + parsed_url.fragment + )) + + def write_test_settings_json(args, test_details, oskey): # write test settings json file with test details that the control # server will provide for the web ext @@ -347,6 +375,15 @@ def get_raptor_test_list(args, oskey): # either warm or cold-mode, initialize the starting current 'browser-cycle' next_test['browser_cycle'] = 1 + # if --test-url-params was provided on the command line, add the params to the test_url + # provided in the INI + if args.test_url_params is not None: + initial_test_url = next_test['test_url'] + next_test['test_url'] = add_test_url_params(initial_test_url, args.test_url_params) + LOG.info("adding extra test_url params (%s) as specified on cmd line " + "to the current test_url (%s), resulting: %s" % + (args.test_url_params, initial_test_url, next_test['test_url'])) + if next_test.get('use_live_sites', "false") == "true": # when using live sites we want to turn off playback LOG.info("using live sites so turning playback off!") diff --git a/testing/raptor/test/conftest.py b/testing/raptor/test/conftest.py index eeb0b89fc866..09531b3c2991 100644 --- a/testing/raptor/test/conftest.py +++ b/testing/raptor/test/conftest.py @@ -69,6 +69,7 @@ def create_args(): debug_mode=False, page_cycles=None, page_timeout=None, + test_url_params=None, host=None, run_local=True) diff --git a/testing/raptor/test/test_manifest.py b/testing/raptor/test/test_manifest.py index b687cafc1448..0e70c29b112d 100644 --- a/testing/raptor/test/test_manifest.py +++ b/testing/raptor/test/test_manifest.py @@ -6,13 +6,20 @@ import os import pytest import sys +from urlparse import parse_qs, urlsplit + # need this so raptor imports work both from /raptor and via mach here = os.path.abspath(os.path.dirname(__file__)) raptor_dir = os.path.join(os.path.dirname(here), 'raptor') sys.path.insert(0, raptor_dir) -from manifest import get_browser_test_list, validate_test_ini, get_raptor_test_list +from manifest import ( + add_test_url_params, + get_browser_test_list, + get_raptor_test_list, + validate_test_ini, +) # some test details (test INIs) @@ -285,6 +292,18 @@ def test_get_raptor_test_list_override_page_timeout(create_args): assert test_list[0]['page_timeout'] == 9999 +def test_get_raptor_test_list_add_test_url_params(create_args): + args = create_args(test="raptor-tp6-google-firefox", + test_url_params='c=4', + browser_cycles=1) + + test_list = get_raptor_test_list(args, mozinfo.os) + assert len(test_list) == 1 + assert test_list[0]['name'] == 'raptor-tp6-google-firefox' + query_params = parse_qs(urlsplit(test_list[0]['test_url']).query) + assert query_params.get('c') == ['4'] + + def test_get_raptor_test_list_refbrow(create_args): args = create_args(app="refbrow", test="raptor-speedometer", @@ -305,5 +324,60 @@ def test_get_raptor_test_list_fenix(create_args): assert len(test_list) == 0 +def test_add_test_url_params_with_single_extra_param(): + initial_test_url = 'http://test.com?a=1&b=2' + extra_params = 'c=3' + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {'a': ['1'], 'b': ['2'], 'c': ['3']} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_with_multiple_extra_param(): + initial_test_url = 'http://test.com?a=1&b=2' + extra_params = 'c=3&d=4' + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {'a': ['1'], 'b': ['2'], 'c': ['3'], 'd': ['4']} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_without_params_in_url(): + initial_test_url = 'http://test.com' + extra_params = 'c=3' + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {'c': ['3']} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_overwrites_single_param(): + initial_test_url = 'http://test.com?a=1&b=2' + extra_params = 'b=3' + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {'a': ['1'], 'b': ['3']} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + +def test_add_test_url_params_overwrites_multiple_param(): + initial_test_url = 'http://test.com?a=1&b=2&c=3' + extra_params = 'c=4&b=5' + + result = add_test_url_params(initial_test_url, extra_params) + + expected_params = {'a': ['1'], 'b': ['5'], 'c': ['4']} + actual_params = parse_qs(urlsplit(result).query) + assert actual_params == expected_params + + if __name__ == '__main__': mozunit.main() From fb65089aa272288b3c1887a0faef14206bd4fc3d Mon Sep 17 00:00:00 2001 From: Marian Raiciof Date: Wed, 31 Jul 2019 16:40:31 +0000 Subject: [PATCH 003/101] Bug 1570317 - Update raptor-scn-power-idle-bg-fenix to use the latest fenix index and binary. r=perftest-reviewers,alexandru.irimovici,davehunt Differential Revision: https://phabricator.services.mozilla.com/D40046 --HG-- extra : moz-landing-system : lando --- taskcluster/ci/test/raptor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml index b969dbb94ece..320db71cec9d 100644 --- a/taskcluster/ci/test/raptor.yml +++ b/taskcluster/ci/test/raptor.yml @@ -1708,10 +1708,10 @@ raptor-scn-power-idle-bg-fenix: target: by-test-platform: android-hw.*-aarch64.*/.*: - index: project.mobile.fenix.v2.branch.master.latest.raptor.aarch64 + index: project.mobile.fenix.v2.performance-test.latest.aarch64 name: target.apk default: - index: project.mobile.fenix.v2.branch.master.latest.raptor.arm + index: project.mobile.fenix.v2.performance-test.latest.arm name: target.apk tier: 3 max-run-time: 1800 @@ -1719,7 +1719,7 @@ raptor-scn-power-idle-bg-fenix: extra-options: - --test=raptor-scn-power-idle-bg - --app=fenix - - --binary-path=org.mozilla.fenix.raptor + - --binary-path=org.mozilla.fenix.performancetest - --power-test - --host HOST_IP - --activity=org.mozilla.fenix.IntentReceiverActivity From 356d25bd08818e62b9d35e1f4020c97ccde52398 Mon Sep 17 00:00:00 2001 From: Moritz Birghan Date: Wed, 31 Jul 2019 16:52:02 +0000 Subject: [PATCH 004/101] Bug 1360307 - Improves the arguments to mozilla::psm::InitializeNSS r=keeler Differential Revision: https://phabricator.services.mozilla.com/D39011 --HG-- extra : moz-landing-system : lando --- security/certverifier/NSSCertDBTrustDomain.cpp | 14 +++++++------- security/certverifier/NSSCertDBTrustDomain.h | 14 ++++++++++++-- security/manager/ssl/nsNSSComponent.cpp | 17 ++++++++++++----- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/security/certverifier/NSSCertDBTrustDomain.cpp b/security/certverifier/NSSCertDBTrustDomain.cpp index 2302fb3707c5..b48eb771e95d 100644 --- a/security/certverifier/NSSCertDBTrustDomain.cpp +++ b/security/certverifier/NSSCertDBTrustDomain.cpp @@ -1281,8 +1281,8 @@ void NSSCertDBTrustDomain::NoteAuxiliaryExtension(AuxiliaryExtension extension, } } -SECStatus InitializeNSS(const nsACString& dir, bool readOnly, - bool loadPKCS11Modules) { +SECStatus InitializeNSS(const nsACString& dir, NSSDBConfig nssDbConfig, + PKCS11DBConfig pkcs11DbConfig) { MOZ_ASSERT(NS_IsMainThread()); // The NSS_INIT_NOROOTINIT flag turns off the loading of the root certs @@ -1291,24 +1291,24 @@ SECStatus InitializeNSS(const nsACString& dir, bool readOnly, // Ubuntu 8.04, which loads any nonexistent "/libnssckbi.so" as // "/usr/lib/nss/libnssckbi.so". uint32_t flags = NSS_INIT_NOROOTINIT | NSS_INIT_OPTIMIZESPACE; - if (readOnly) { + if (nssDbConfig == NSSDBConfig::ReadOnly) { flags |= NSS_INIT_READONLY; } - if (!loadPKCS11Modules) { + if (pkcs11DbConfig == PKCS11DBConfig::DoNotLoadModules) { flags |= NSS_INIT_NOMODDB; } nsAutoCString dbTypeAndDirectory("sql:"); dbTypeAndDirectory.Append(dir); MOZ_LOG(gCertVerifierLog, LogLevel::Debug, - ("InitializeNSS(%s, %d, %d)", dbTypeAndDirectory.get(), readOnly, - loadPKCS11Modules)); + ("InitializeNSS(%s, %d, %d)", dbTypeAndDirectory.get(), nssDbConfig, + pkcs11DbConfig)); SECStatus srv = NSS_Initialize(dbTypeAndDirectory.get(), "", "", SECMOD_DB, flags); if (srv != SECSuccess) { return srv; } - if (!readOnly) { + if (nssDbConfig == NSSDBConfig::ReadWrite) { UniquePK11SlotInfo slot(PK11_GetInternalKeySlot()); if (!slot) { return SECFailure; diff --git a/security/certverifier/NSSCertDBTrustDomain.h b/security/certverifier/NSSCertDBTrustDomain.h index 3fba79a4168b..1486d3e066cd 100644 --- a/security/certverifier/NSSCertDBTrustDomain.h +++ b/security/certverifier/NSSCertDBTrustDomain.h @@ -28,6 +28,16 @@ enum class ValidityCheckingMode { CheckForEV = 1, }; +enum class NSSDBConfig { + ReadWrite = 0, + ReadOnly = 1, +}; + +enum class PKCS11DBConfig { + DoNotLoadModules = 0, + LoadModules = 1, +}; + // Policy options for matching id-Netscape-stepUp with id-kp-serverAuth (for CA // certificates only): // * Always match: the step-up OID is considered equivalent to serverAuth @@ -42,8 +52,8 @@ enum class NetscapeStepUpPolicy : uint32_t { NeverMatch = 3, }; -SECStatus InitializeNSS(const nsACString& dir, bool readOnly, - bool loadPKCS11Modules); +SECStatus InitializeNSS(const nsACString& dir, NSSDBConfig nssDbConfig, + PKCS11DBConfig pkcs11DbConfig); void DisableMD5(); diff --git a/security/manager/ssl/nsNSSComponent.cpp b/security/manager/ssl/nsNSSComponent.cpp index 593acfdbc2f1..86f08a02aa45 100644 --- a/security/manager/ssl/nsNSSComponent.cpp +++ b/security/manager/ssl/nsNSSComponent.cpp @@ -1571,7 +1571,10 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, #ifndef ANDROID PRErrorCode savedPRErrorCode1; #endif // ifndef ANDROID - SECStatus srv = ::mozilla::psm::InitializeNSS(profilePath, false, !safeMode); + PKCS11DBConfig safeModeDBConfig = + safeMode ? PKCS11DBConfig::DoNotLoadModules : PKCS11DBConfig::LoadModules; + SECStatus srv = ::mozilla::psm::InitializeNSS( + profilePath, NSSDBConfig::ReadWrite, safeModeDBConfig); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r/w mode")); #ifndef ANDROID @@ -1584,7 +1587,8 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, PRErrorCode savedPRErrorCode2; #endif // ifndef ANDROID // That failed. Try read-only mode. - srv = ::mozilla::psm::InitializeNSS(profilePath, true, !safeMode); + srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadOnly, + safeModeDBConfig); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r-o mode")); return NS_OK; @@ -1611,7 +1615,8 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, // problem, but for some reason the combination of read-only and no-moddb // flags causes NSS initialization to fail, so unfortunately we have to use // read-write mode. - srv = ::mozilla::psm::InitializeNSS(profilePath, false, false); + srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadWrite, + PKCS11DBConfig::DoNotLoadModules); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("FIPS may be the problem")); // Unload NSS so we can attempt to fix this situation for the user. @@ -1637,12 +1642,14 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, # endif return rv; } - srv = ::mozilla::psm::InitializeNSS(profilePath, false, true); + srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadWrite, + PKCS11DBConfig::LoadModules); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r/w mode")); return NS_OK; } - srv = ::mozilla::psm::InitializeNSS(profilePath, true, true); + srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadOnly, + PKCS11DBConfig::LoadModules); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r-o mode")); return NS_OK; From 7d609ae764b91b756c11e31b40cc64adaa2b76db Mon Sep 17 00:00:00 2001 From: Drew Willcoxon Date: Wed, 31 Jul 2019 17:00:07 +0000 Subject: [PATCH 005/101] Bug 1569812 - Quantumbar: Allow extensions to specify search engines by name, URL, or alias. r=mak `UrlbarResult` objects for search results have a `payload.engine` property, the name of the engine, but extensions won't always know the name Firefox uses internally or even any name at all. For example, the top sites API returns search results with aliases but not names. We should let extensions specify engine URLs and aliases, and we can use them to look up the engine name. The UrlbarResult.jsm change is unrelated but fixes a minor annoyance I found while working on the top-sites extension. My background script returns results with `isKeywordOffer: true`, but that causes an error in `UrlbarResult.payloadAndSimpleHighlights` because it converts string values to arrays and then expects non-string values to be arrays. Instead, it should be converting non-array values to arrays. Differential Revision: https://phabricator.services.mozilla.com/D39800 --HG-- extra : moz-landing-system : lando --- .../test/xpcshell/test_ext_urlbar.js | 181 ++++++++++++++++++ .../urlbar/UrlbarProviderExtension.jsm | 50 ++++- browser/components/urlbar/UrlbarResult.jsm | 4 +- 3 files changed, 226 insertions(+), 9 deletions(-) diff --git a/browser/components/extensions/test/xpcshell/test_ext_urlbar.js b/browser/components/extensions/test/xpcshell/test_ext_urlbar.js index d83b6abf3b04..2fd14c3cd40a 100644 --- a/browser/components/extensions/test/xpcshell/test_ext_urlbar.js +++ b/browser/components/extensions/test/xpcshell/test_ext_urlbar.js @@ -36,6 +36,7 @@ add_task(async function startup() { // engine from over the network. let engine = await Services.search.addEngineWithDetails("Test engine", { template: "http://example.com/?s=%S", + alias: "@testengine", }); Services.search.defaultEngine = engine; @@ -298,6 +299,186 @@ add_task(async function test_onProviderResultsRequested() { await ext.unload(); }); +// Extensions can specify search engines using engine names, aliases, and URLs. +add_task(async function test_onProviderResultsRequested_searchEngines() { + let ext = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["urlbar"], + }, + isPrivileged: true, + incognitoOverride: "spanning", + background() { + browser.urlbar.onBehaviorRequested.addListener(query => { + return "restricting"; + }, "test"); + browser.urlbar.onResultsRequested.addListener(query => { + return [ + { + type: "search", + source: "search", + payload: { + engine: "Test engine", + suggestion: "engine specified", + }, + }, + { + type: "search", + source: "search", + payload: { + keyword: "@testengine", + suggestion: "keyword specified", + }, + }, + { + type: "search", + source: "search", + payload: { + url: "http://example.com/?s", + suggestion: "url specified", + }, + }, + { + type: "search", + source: "search", + payload: { + engine: "Test engine", + keyword: "@testengine", + url: "http://example.com/?s", + suggestion: "engine, keyword, and url specified", + }, + }, + { + type: "search", + source: "search", + payload: { + keyword: "@testengine", + url: "http://example.com/?s", + suggestion: "keyword and url specified", + }, + }, + { + type: "search", + source: "search", + payload: { + suggestion: "no engine", + }, + }, + { + type: "search", + source: "search", + payload: { + engine: "bogus", + suggestion: "no matching engine", + }, + }, + { + type: "search", + source: "search", + payload: { + keyword: "@bogus", + suggestion: "no matching keyword", + }, + }, + { + type: "search", + source: "search", + payload: { + url: "http://bogus-no-search-engine.com/", + suggestion: "no matching url", + }, + }, + { + type: "search", + source: "search", + payload: { + url: "bogus", + suggestion: "invalid url", + }, + }, + { + type: "search", + source: "search", + payload: { + url: "foo:bar", + suggestion: "url with no hostname", + }, + }, + ]; + }, "test"); + }, + }); + await ext.startup(); + + // Run a query. + let context = new UrlbarQueryContext({ + allowAutofill: false, + isPrivate: false, + maxResults: 10, + searchString: "test", + }); + let controller = new UrlbarController({ + browserWindow: { + location: { + href: AppConstants.BROWSER_CHROME_URL, + }, + }, + }); + await controller.startQuery(context); + + // Check the results. The first several are valid and should include "Test + // engine" as the engine. The others don't specify an engine and are + // therefore invalid, so they should be ignored. + let expectedResults = [ + { + type: UrlbarUtils.RESULT_TYPE.SEARCH, + source: UrlbarUtils.RESULT_SOURCE.SEARCH, + engine: "Test engine", + title: "engine specified", + heuristic: false, + }, + { + type: UrlbarUtils.RESULT_TYPE.SEARCH, + source: UrlbarUtils.RESULT_SOURCE.SEARCH, + engine: "Test engine", + title: "keyword specified", + heuristic: false, + }, + { + type: UrlbarUtils.RESULT_TYPE.SEARCH, + source: UrlbarUtils.RESULT_SOURCE.SEARCH, + engine: "Test engine", + title: "url specified", + heuristic: false, + }, + { + type: UrlbarUtils.RESULT_TYPE.SEARCH, + source: UrlbarUtils.RESULT_SOURCE.SEARCH, + engine: "Test engine", + title: "engine, keyword, and url specified", + heuristic: false, + }, + { + type: UrlbarUtils.RESULT_TYPE.SEARCH, + source: UrlbarUtils.RESULT_SOURCE.SEARCH, + engine: "Test engine", + title: "keyword and url specified", + heuristic: false, + }, + ]; + + let actualResults = context.results.map(r => ({ + type: r.type, + source: r.source, + engine: r.payload.engine || null, + title: r.title, + heuristic: r.heuristic, + })); + + Assert.deepEqual(actualResults, expectedResults); + + await ext.unload(); +}); + // Adds two providers, one active and one inactive. Only the active provider // should be asked to return results. add_task(async function test_activeAndInactiveProviders() { diff --git a/browser/components/urlbar/UrlbarProviderExtension.jsm b/browser/components/urlbar/UrlbarProviderExtension.jsm index b08e16c97a09..1e30c6406565 100644 --- a/browser/components/urlbar/UrlbarProviderExtension.jsm +++ b/browser/components/urlbar/UrlbarProviderExtension.jsm @@ -15,6 +15,9 @@ const { XPCOMUtils } = ChromeUtils.import( "resource://gre/modules/XPCOMUtils.jsm" ); XPCOMUtils.defineLazyModuleGetters(this, { + PlacesSearchAutocompleteProvider: + "resource://gre/modules/PlacesSearchAutocompleteProvider.jsm", + Services: "resource://gre/modules/Services.jsm", SkippableTimer: "resource:///modules/UrlbarUtils.jsm", UrlbarProvider: "resource:///modules/UrlbarUtils.jsm", UrlbarProvidersManager: "resource:///modules/UrlbarProvidersManager.jsm", @@ -189,13 +192,12 @@ class UrlbarProviderExtension extends UrlbarProvider { let extResults = await this._notifyListener("resultsRequested", context); if (extResults) { for (let extResult of extResults) { - let result; - try { - result = this._makeUrlbarResult(context, extResult); - } catch (err) { - continue; + let result = await this._makeUrlbarResult(context, extResult).catch( + Cu.reportError + ); + if (result) { + addCallback(this, result); } - addCallback(this, result); } } } @@ -261,7 +263,41 @@ class UrlbarProviderExtension extends UrlbarProvider { * @returns {UrlbarResult} * The UrlbarResult object. */ - _makeUrlbarResult(context, extResult) { + async _makeUrlbarResult(context, extResult) { + // If the result is a search result, make sure its payload has a valid + // `engine` property, which is the name of an engine, and which we use later + // on to look up the nsISearchEngine. We allow the extension to specify the + // engine by its name, alias, or domain. Prefer aliases over domains since + // one domain can have many engines. + if (extResult.type == "search") { + let engine; + if (extResult.payload.engine) { + // Validate the engine name by looking it up. + engine = Services.search.getEngineByName(extResult.payload.engine); + } else if (extResult.payload.keyword) { + // Look up the engine by its alias. + engine = await PlacesSearchAutocompleteProvider.engineForAlias( + extResult.payload.keyword + ); + } else if (extResult.payload.url) { + // Look up the engine by its domain. + let host; + try { + host = new URL(extResult.payload.url).hostname; + } catch (err) {} + if (host) { + engine = await PlacesSearchAutocompleteProvider.engineForDomainPrefix( + host + ); + } + } + if (!engine) { + // No engine found. + throw new Error("Invalid or missing engine specified by extension"); + } + extResult.payload.engine = engine.name; + } + return new UrlbarResult( UrlbarProviderExtension.RESULT_TYPES[extResult.type], UrlbarProviderExtension.SOURCE_TYPES[extResult.source], diff --git a/browser/components/urlbar/UrlbarResult.jsm b/browser/components/urlbar/UrlbarResult.jsm index b59a67f67a1a..afdc04d5c83c 100644 --- a/browser/components/urlbar/UrlbarResult.jsm +++ b/browser/components/urlbar/UrlbarResult.jsm @@ -168,9 +168,9 @@ class UrlbarResult { * @returns {array} An array [payload, payloadHighlights]. */ static payloadAndSimpleHighlights(tokens, payloadInfo) { - // Convert string values in payloadInfo to [value, false] arrays. + // Convert scalar values in payloadInfo to [value] arrays. for (let [name, info] of Object.entries(payloadInfo)) { - if (typeof info == "string") { + if (!Array.isArray(info)) { payloadInfo[name] = [info]; } } From 3fe9b925aabed06c15a097294a89da2870b15bb7 Mon Sep 17 00:00:00 2001 From: Kristen Wright Date: Wed, 31 Jul 2019 13:53:52 +0000 Subject: [PATCH 006/101] Bug 1570173 - Add AutoSetHelperThreadContext to js::wasm::CompileTask::runTask(). r=jandem CompileTask can call js::AutoUnsafeCallWtihABI::AutoUnsafeCallWithABI which uses a thread-local context, so we've got to set one to the thread in CompileTask::runTask(). Differential Revision: https://phabricator.services.mozilla.com/D39982 --HG-- extra : moz-landing-system : lando --- js/src/wasm/WasmGenerator.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp index c825b58c5e36..4964e6eee209 100644 --- a/js/src/wasm/WasmGenerator.cpp +++ b/js/src/wasm/WasmGenerator.cpp @@ -1259,7 +1259,10 @@ bool ModuleGenerator::finishTier2(const Module& module) { return module.finishTier2(*linkData_, std::move(codeTier)); } -void CompileTask::runTask() { ExecuteCompileTaskFromHelperThread(this); } +void CompileTask::runTask() { + AutoSetHelperThreadContext needsContext; + ExecuteCompileTaskFromHelperThread(this); +} size_t CompiledCode::sizeOfExcludingThis( mozilla::MallocSizeOf mallocSizeOf) const { From 4b81ad069641d0b4c57e875b28830af0d95985cd Mon Sep 17 00:00:00 2001 From: lesleynorton Date: Wed, 31 Jul 2019 17:05:48 +0000 Subject: [PATCH 007/101] Bug 1549808: Add favicons to login list items. r=jaws,fluent-reviewers,flod Differential Revision: https://phabricator.services.mozilla.com/D38401 --HG-- extra : moz-landing-system : lando --- browser/components/BrowserGlue.jsm | 1 + .../aboutlogins/AboutLoginsChild.jsm | 3 + .../aboutlogins/AboutLoginsParent.jsm | 55 ++++++++++++++++--- .../aboutlogins/content/aboutLogins.html | 8 ++- .../aboutlogins/content/aboutLogins.js | 4 ++ .../content/components/login-item.css | 6 ++ .../content/components/login-item.js | 17 ++++++ .../content/components/login-list-item.js | 9 ++- .../content/components/login-list.css | 8 ++- .../content/components/login-list.js | 27 +++++++++ browser/locales/en-US/browser/aboutLogins.ftl | 6 ++ 11 files changed, 133 insertions(+), 11 deletions(-) diff --git a/browser/components/BrowserGlue.jsm b/browser/components/BrowserGlue.jsm index c6657d0f4930..bc32212274ed 100644 --- a/browser/components/BrowserGlue.jsm +++ b/browser/components/BrowserGlue.jsm @@ -154,6 +154,7 @@ let LEGACY_ACTORS = { "AboutLogins:LoginModified", "AboutLogins:LoginRemoved", "AboutLogins:MasterPasswordResponse", + "AboutLogins:SendFavicons", "AboutLogins:SyncState", "AboutLogins:UpdateBreaches", ], diff --git a/browser/components/aboutlogins/AboutLoginsChild.jsm b/browser/components/aboutlogins/AboutLoginsChild.jsm index 3dc54c6d42bd..8286d2b0b8dc 100644 --- a/browser/components/aboutlogins/AboutLoginsChild.jsm +++ b/browser/components/aboutlogins/AboutLoginsChild.jsm @@ -167,6 +167,9 @@ class AboutLoginsChild extends ActorChild { if (masterPasswordPromise) { masterPasswordPromise.resolve(message.data); } + case "AboutLogins:SendFavicons": + this.sendToContent("SendFavicons", message.data); + break; case "AboutLogins:SyncState": this.sendToContent("SyncState", message.data); break; diff --git a/browser/components/aboutlogins/AboutLoginsParent.jsm b/browser/components/aboutlogins/AboutLoginsParent.jsm index 4e5cd087a655..0a6025c834c5 100644 --- a/browser/components/aboutlogins/AboutLoginsParent.jsm +++ b/browser/components/aboutlogins/AboutLoginsParent.jsm @@ -35,6 +35,12 @@ ChromeUtils.defineModuleGetter( "resource://services-sync/UIState.jsm" ); +ChromeUtils.defineModuleGetter( + this, + "PlacesUtils", + "resource://gre/modules/PlacesUtils.jsm" +); + XPCOMUtils.defineLazyGetter(this, "log", () => { return LoginHelper.createLogger("AboutLoginsParent"); }); @@ -261,16 +267,19 @@ var AboutLoginsParent = { messageManager.sendAsyncMessage("AboutLogins:SyncState", syncState); this.updatePasswordSyncNotificationState(); - if (!BREACH_ALERTS_ENABLED) { - return; + if (BREACH_ALERTS_ENABLED) { + const breachesByLoginGUID = await LoginHelper.getBreachesForLogins( + logins + ); + messageManager.sendAsyncMessage( + "AboutLogins:UpdateBreaches", + breachesByLoginGUID + ); } - const breachesByLoginGUID = await LoginHelper.getBreachesForLogins( - logins - ); messageManager.sendAsyncMessage( - "AboutLogins:UpdateBreaches", - breachesByLoginGUID + "AboutLogins:SendFavicons", + await this.getAllFavicons(logins) ); } catch (ex) { if (ex.result != Cr.NS_ERROR_NOT_INITIALIZED) { @@ -373,6 +382,38 @@ var AboutLoginsParent = { } }, + async getFavicon(login) { + try { + const faviconData = await PlacesUtils.promiseFaviconData(login.hostname); + return { + faviconData, + title: login.title, + }; + } catch (ex) { + return null; + } + }, + + async getAllFavicons(logins) { + let favicons = await Promise.all( + logins.map(login => this.getFavicon(login)) + ); + let vanillaFavicons = {}; + for (let favicon of favicons) { + if (!favicon) { + continue; + } + try { + vanillaFavicons[favicon.title] = { + data: favicon.faviconData.data, + dataLen: favicon.faviconData.dataLen, + mimeType: favicon.faviconData.mimeType, + }; + } catch (ex) {} + } + return vanillaFavicons; + }, + showMasterPasswordLoginNotifications() { this.showNotifications({ id: MASTER_PASSWORD_NOTIFICATION_ID, diff --git a/browser/components/aboutlogins/content/aboutLogins.html b/browser/components/aboutlogins/content/aboutLogins.html index e45dbe24777e..aaf1e1b55b59 100644 --- a/browser/components/aboutlogins/content/aboutLogins.html +++ b/browser/components/aboutlogins/content/aboutLogins.html @@ -90,8 +90,11 @@ @@ -104,6 +107,7 @@
+

diff --git a/browser/components/aboutlogins/content/aboutLogins.js b/browser/components/aboutlogins/content/aboutLogins.js index f171f0d8ab49..946d98973d46 100644 --- a/browser/components/aboutlogins/content/aboutLogins.js +++ b/browser/components/aboutlogins/content/aboutLogins.js @@ -47,6 +47,10 @@ window.addEventListener("AboutLoginsChromeToContent", event => { gElements.loginItem.loginRemoved(event.detail.value); break; } + case "SendFavicons": { + gElements.loginList.addFavicons(event.detail.value); + break; + } case "SyncState": { gElements.fxAccountsButton.updateState(event.detail.value); break; diff --git a/browser/components/aboutlogins/content/components/login-item.css b/browser/components/aboutlogins/content/components/login-item.css index 730e05ee569a..82949677f5bc 100644 --- a/browser/components/aboutlogins/content/components/login-item.css +++ b/browser/components/aboutlogins/content/components/login-item.css @@ -41,6 +41,7 @@ .header { display: flex; + align-items: center; border-bottom: 1px solid var(--in-content-box-border-color); margin-bottom: 40px; } @@ -190,6 +191,11 @@ box-shadow: 0 0 0 4px var(--in-content-border-active-shadow); } +.login-item-favicon { + width: 38px; + margin-right: 12px; +} + .breach-alert { border-radius: 8px; border: 1px solid var(--in-content-border-color); diff --git a/browser/components/aboutlogins/content/components/login-item.js b/browser/components/aboutlogins/content/components/login-item.js index 93d13f7aad08..fd5ded9cacb4 100644 --- a/browser/components/aboutlogins/content/components/login-item.js +++ b/browser/components/aboutlogins/content/components/login-item.js @@ -54,6 +54,7 @@ export default class LoginItem extends HTMLElement { this._saveChangesButton = this.shadowRoot.querySelector( ".save-changes-button" ); + this._favicon = this.shadowRoot.querySelector(".login-item-favicon"); this._title = this.shadowRoot.querySelector(".login-item-title"); this._timeCreated = this.shadowRoot.querySelector(".time-created"); this._timeChanged = this.shadowRoot.querySelector(".time-changed"); @@ -74,6 +75,7 @@ export default class LoginItem extends HTMLElement { this._revealCheckbox.addEventListener("click", this); window.addEventListener("AboutLoginsCreateLogin", this); window.addEventListener("AboutLoginsInitialLoginSelected", this); + window.addEventListener("AboutLoginsLoadInitialFavicon", this); window.addEventListener("AboutLoginsLoginSelected", this); } @@ -97,6 +99,17 @@ export default class LoginItem extends HTMLElement { timeUsed: this._login.timeLastUsed || "", }); + if (this._login.faviconDataURI) { + this._favicon.src = this._login.faviconDataURI; + document.l10n.setAttributes(this._favicon, "login-favicon", { + title: this._login.title, + }); + } else { + // reset the src and alt attributes if the currently selected favicon doesn't have a favicon + this._favicon.src = "chrome://mozapps/skin/places/defaultFavicon.svg"; + this._favicon.alt = ""; + } + this._title.textContent = this._login.title; this._originInput.defaultValue = this._login.origin || ""; this._usernameInput.defaultValue = this._login.username || ""; @@ -125,6 +138,10 @@ export default class LoginItem extends HTMLElement { this.setLogin(event.detail, { skipFocusChange: true }); break; } + case "AboutLoginsLoadInitialFavicon": { + this.setLogin(event.detail, { skipFocusChange: true }); + break; + } case "AboutLoginsLoginSelected": { this.setLogin(event.detail); break; diff --git a/browser/components/aboutlogins/content/components/login-list-item.js b/browser/components/aboutlogins/content/components/login-list-item.js index a8532861eac0..2f3f627f4876 100644 --- a/browser/components/aboutlogins/content/components/login-list-item.js +++ b/browser/components/aboutlogins/content/components/login-list-item.js @@ -17,7 +17,7 @@ export default class LoginListItemFactory { let listItem = loginListItem.querySelector("li"); let title = loginListItem.querySelector(".title"); let username = loginListItem.querySelector(".username"); - + let favicon = loginListItem.querySelector(".favicon"); listItem.setAttribute("role", "option"); if (!login.guid) { @@ -45,6 +45,13 @@ export default class LoginListItemFactory { ); } + if (login.faviconDataURI) { + favicon.src = login.faviconDataURI; + document.l10n.setAttributes(favicon, "login-favicon", { + title: login.title, + }); + } + return listItem; } } diff --git a/browser/components/aboutlogins/content/components/login-list.css b/browser/components/aboutlogins/content/components/login-list.css index 642c7a9a3da4..b0a80ac9f7b4 100644 --- a/browser/components/aboutlogins/content/components/login-list.css +++ b/browser/components/aboutlogins/content/components/login-list.css @@ -53,7 +53,8 @@ ol { } .login-list-item { - display: block; + display: flex; + align-items: center; padding: 10px; padding-inline-end: 18px; padding-inline-start: 14px; @@ -91,6 +92,11 @@ ol { overflow: hidden; } +.favicon { + width: 16px; + margin-right: 14px; +} + .username { font-size: 0.85em; color: var(--in-content-deemphasized-text); diff --git a/browser/components/aboutlogins/content/components/login-list.js b/browser/components/aboutlogins/content/components/login-list.js index c76c21a2fd27..1b55e6bfca8c 100644 --- a/browser/components/aboutlogins/content/components/login-list.js +++ b/browser/components/aboutlogins/content/components/login-list.js @@ -217,6 +217,33 @@ export default class LoginList extends HTMLElement { } } + addFavicons(favicons) { + for (let login of this._logins) { + const favicon = favicons[login.title]; + if (favicon && login.title) { + favicon.uri = btoa( + new Uint8Array(favicon.data).reduce( + (data, byte) => data + String.fromCharCode(byte), + "" + ) + ); + let faviconDataURI = `data:${favicon.mimeType};base64,${favicon.uri}`; + login.faviconDataURI = faviconDataURI; + } + } + let selectedListItem = this._list.querySelector( + ".login-list-item.selected" + ); + if (selectedListItem) { + window.dispatchEvent( + new CustomEvent("AboutLoginsLoadInitialFavicon", { + detail: selectedListItem._login, + }) + ); + } + this.render(); + } + /** * @param {Map} breachesByLoginGUID A Map of breaches by login GUIDs used for displaying breached login indicators. */ diff --git a/browser/locales/en-US/browser/aboutLogins.ftl b/browser/locales/en-US/browser/aboutLogins.ftl index 2d8bfbb5e4b1..601537fee09d 100644 --- a/browser/locales/en-US/browser/aboutLogins.ftl +++ b/browser/locales/en-US/browser/aboutLogins.ftl @@ -9,6 +9,12 @@ login-filter = create-login-button = Create New Login +# This string is used as alternative text for favicon images. +# Variables: +# $title (String) - The title of the website associated with the favicon. +login-favicon = + .alt = Favicon for { $title } + fxaccounts-sign-in-text = Get your passwords on your other devices fxaccounts-sign-in-button = Sign in to { -sync-brand-short-name } From 08ddcd8a5ac25349a6d63cff2272b4d7f23368dc Mon Sep 17 00:00:00 2001 From: Gurzau Raul Date: Wed, 31 Jul 2019 20:22:43 +0300 Subject: [PATCH 008/101] Backed out changeset 4a66bfcbaca5 (bug 1360307) for build bustage at Logging.h on a CLOSED TREE. --- security/certverifier/NSSCertDBTrustDomain.cpp | 14 +++++++------- security/certverifier/NSSCertDBTrustDomain.h | 14 ++------------ security/manager/ssl/nsNSSComponent.cpp | 17 +++++------------ 3 files changed, 14 insertions(+), 31 deletions(-) diff --git a/security/certverifier/NSSCertDBTrustDomain.cpp b/security/certverifier/NSSCertDBTrustDomain.cpp index b48eb771e95d..2302fb3707c5 100644 --- a/security/certverifier/NSSCertDBTrustDomain.cpp +++ b/security/certverifier/NSSCertDBTrustDomain.cpp @@ -1281,8 +1281,8 @@ void NSSCertDBTrustDomain::NoteAuxiliaryExtension(AuxiliaryExtension extension, } } -SECStatus InitializeNSS(const nsACString& dir, NSSDBConfig nssDbConfig, - PKCS11DBConfig pkcs11DbConfig) { +SECStatus InitializeNSS(const nsACString& dir, bool readOnly, + bool loadPKCS11Modules) { MOZ_ASSERT(NS_IsMainThread()); // The NSS_INIT_NOROOTINIT flag turns off the loading of the root certs @@ -1291,24 +1291,24 @@ SECStatus InitializeNSS(const nsACString& dir, NSSDBConfig nssDbConfig, // Ubuntu 8.04, which loads any nonexistent "/libnssckbi.so" as // "/usr/lib/nss/libnssckbi.so". uint32_t flags = NSS_INIT_NOROOTINIT | NSS_INIT_OPTIMIZESPACE; - if (nssDbConfig == NSSDBConfig::ReadOnly) { + if (readOnly) { flags |= NSS_INIT_READONLY; } - if (pkcs11DbConfig == PKCS11DBConfig::DoNotLoadModules) { + if (!loadPKCS11Modules) { flags |= NSS_INIT_NOMODDB; } nsAutoCString dbTypeAndDirectory("sql:"); dbTypeAndDirectory.Append(dir); MOZ_LOG(gCertVerifierLog, LogLevel::Debug, - ("InitializeNSS(%s, %d, %d)", dbTypeAndDirectory.get(), nssDbConfig, - pkcs11DbConfig)); + ("InitializeNSS(%s, %d, %d)", dbTypeAndDirectory.get(), readOnly, + loadPKCS11Modules)); SECStatus srv = NSS_Initialize(dbTypeAndDirectory.get(), "", "", SECMOD_DB, flags); if (srv != SECSuccess) { return srv; } - if (nssDbConfig == NSSDBConfig::ReadWrite) { + if (!readOnly) { UniquePK11SlotInfo slot(PK11_GetInternalKeySlot()); if (!slot) { return SECFailure; diff --git a/security/certverifier/NSSCertDBTrustDomain.h b/security/certverifier/NSSCertDBTrustDomain.h index 1486d3e066cd..3fba79a4168b 100644 --- a/security/certverifier/NSSCertDBTrustDomain.h +++ b/security/certverifier/NSSCertDBTrustDomain.h @@ -28,16 +28,6 @@ enum class ValidityCheckingMode { CheckForEV = 1, }; -enum class NSSDBConfig { - ReadWrite = 0, - ReadOnly = 1, -}; - -enum class PKCS11DBConfig { - DoNotLoadModules = 0, - LoadModules = 1, -}; - // Policy options for matching id-Netscape-stepUp with id-kp-serverAuth (for CA // certificates only): // * Always match: the step-up OID is considered equivalent to serverAuth @@ -52,8 +42,8 @@ enum class NetscapeStepUpPolicy : uint32_t { NeverMatch = 3, }; -SECStatus InitializeNSS(const nsACString& dir, NSSDBConfig nssDbConfig, - PKCS11DBConfig pkcs11DbConfig); +SECStatus InitializeNSS(const nsACString& dir, bool readOnly, + bool loadPKCS11Modules); void DisableMD5(); diff --git a/security/manager/ssl/nsNSSComponent.cpp b/security/manager/ssl/nsNSSComponent.cpp index 86f08a02aa45..593acfdbc2f1 100644 --- a/security/manager/ssl/nsNSSComponent.cpp +++ b/security/manager/ssl/nsNSSComponent.cpp @@ -1571,10 +1571,7 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, #ifndef ANDROID PRErrorCode savedPRErrorCode1; #endif // ifndef ANDROID - PKCS11DBConfig safeModeDBConfig = - safeMode ? PKCS11DBConfig::DoNotLoadModules : PKCS11DBConfig::LoadModules; - SECStatus srv = ::mozilla::psm::InitializeNSS( - profilePath, NSSDBConfig::ReadWrite, safeModeDBConfig); + SECStatus srv = ::mozilla::psm::InitializeNSS(profilePath, false, !safeMode); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r/w mode")); #ifndef ANDROID @@ -1587,8 +1584,7 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, PRErrorCode savedPRErrorCode2; #endif // ifndef ANDROID // That failed. Try read-only mode. - srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadOnly, - safeModeDBConfig); + srv = ::mozilla::psm::InitializeNSS(profilePath, true, !safeMode); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized NSS in r-o mode")); return NS_OK; @@ -1615,8 +1611,7 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, // problem, but for some reason the combination of read-only and no-moddb // flags causes NSS initialization to fail, so unfortunately we have to use // read-write mode. - srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadWrite, - PKCS11DBConfig::DoNotLoadModules); + srv = ::mozilla::psm::InitializeNSS(profilePath, false, false); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("FIPS may be the problem")); // Unload NSS so we can attempt to fix this situation for the user. @@ -1642,14 +1637,12 @@ static nsresult InitializeNSSWithFallbacks(const nsACString& profilePath, # endif return rv; } - srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadWrite, - PKCS11DBConfig::LoadModules); + srv = ::mozilla::psm::InitializeNSS(profilePath, false, true); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r/w mode")); return NS_OK; } - srv = ::mozilla::psm::InitializeNSS(profilePath, NSSDBConfig::ReadOnly, - PKCS11DBConfig::LoadModules); + srv = ::mozilla::psm::InitializeNSS(profilePath, true, true); if (srv == SECSuccess) { MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("initialized in r-o mode")); return NS_OK; From 529478882fd9dd54d0bbcf00f44db791ca787c49 Mon Sep 17 00:00:00 2001 From: Daniel Holbert Date: Wed, 31 Jul 2019 16:59:51 +0000 Subject: [PATCH 009/101] Bug 1569475: Adjust MobileViewportManager::ClampZoom to check for NaN zoom values and use 1.0 instead. r=botond Also spam a NS_ERROR whenever this happens (in debug builds), so that we know to fix the caller to avoid division by zero. Differential Revision: https://phabricator.services.mozilla.com/D39801 --HG-- extra : moz-landing-system : lando --- layout/base/MobileViewportManager.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/layout/base/MobileViewportManager.cpp b/layout/base/MobileViewportManager.cpp index 06a078beeaad..5007da59a385 100644 --- a/layout/base/MobileViewportManager.cpp +++ b/layout/base/MobileViewportManager.cpp @@ -172,6 +172,11 @@ void MobileViewportManager::SetInitialViewport() { CSSToScreenScale MobileViewportManager::ClampZoom( const CSSToScreenScale& aZoom, const nsViewportInfo& aViewportInfo) const { CSSToScreenScale zoom = aZoom; + if (IsNaN(zoom.scale)) { + NS_ERROR("Don't pass NaN to ClampZoom; check caller for 0/0 division"); + zoom = CSSToScreenScale(1.0); + } + if (zoom < aViewportInfo.GetMinZoom()) { zoom = aViewportInfo.GetMinZoom(); MVM_LOG("%p: Clamped to %f\n", this, zoom.scale); From 7134e2b09b9320c1cf455d0f799520bc7168b4f4 Mon Sep 17 00:00:00 2001 From: David Walsh Date: Wed, 31 Jul 2019 16:48:32 +0000 Subject: [PATCH 010/101] Bug 1570129 - Miscellaneous code cleanup for debugger components r=jlast Super minor cleanup Differential Revision: https://phabricator.services.mozilla.com/D39962 --HG-- extra : moz-landing-system : lando --- .../components/Editor/ColumnBreakpoints.js | 12 +++---- .../src/components/Editor/ConditionalPanel.js | 3 +- .../src/components/Editor/menus/editor.js | 34 ++++++++----------- .../SecondaryPanes/EventListeners.js | 12 +++---- .../components/SecondaryPanes/Expressions.js | 14 ++++---- .../src/components/SecondaryPanes/Workers.js | 2 +- .../SecondaryPanes/XHRBreakpoints.js | 12 +++---- .../src/components/shared/SourceIcon.js | 10 +++--- .../src/workers/parser/getScopes/index.js | 18 +++++----- .../src/workers/parser/getScopes/visitor.js | 31 +++++++---------- .../src/workers/parser/utils/helpers.js | 14 ++++---- 11 files changed, 69 insertions(+), 93 deletions(-) diff --git a/devtools/client/debugger/src/components/Editor/ColumnBreakpoints.js b/devtools/client/debugger/src/components/Editor/ColumnBreakpoints.js index 0138db7f6d35..4c9a73bcf03b 100644 --- a/devtools/client/debugger/src/components/Editor/ColumnBreakpoints.js +++ b/devtools/client/debugger/src/components/Editor/ColumnBreakpoints.js @@ -63,13 +63,11 @@ class ColumnBreakpoints extends Component { } } -const mapStateToProps = state => { - return { - cx: getContext(state), - selectedSource: getSelectedSource(state), - columnBreakpoints: visibleColumnBreakpoints(state), - }; -}; +const mapStateToProps = state => ({ + cx: getContext(state), + selectedSource: getSelectedSource(state), + columnBreakpoints: visibleColumnBreakpoints(state), +}); export default connect( mapStateToProps, diff --git a/devtools/client/debugger/src/components/Editor/ConditionalPanel.js b/devtools/client/debugger/src/components/Editor/ConditionalPanel.js index 0f07d8a0dc8a..aa34cbbcfb2f 100644 --- a/devtools/client/debugger/src/components/Editor/ConditionalPanel.js +++ b/devtools/client/debugger/src/components/Editor/ConditionalPanel.js @@ -229,12 +229,11 @@ export class ConditionalPanel extends PureComponent { const mapStateToProps = state => { const location = getConditionalPanelLocation(state); - const log = getLogPointStatus(state); return { cx: getContext(state), breakpoint: getBreakpointForLocation(state, location), location, - log, + log: getLogPointStatus(state), }; }; diff --git a/devtools/client/debugger/src/components/Editor/menus/editor.js b/devtools/client/debugger/src/components/Editor/menus/editor.js index 4968adf87a9b..b6f7aa20ea20 100644 --- a/devtools/client/debugger/src/components/Editor/menus/editor.js +++ b/devtools/client/debugger/src/components/Editor/menus/editor.js @@ -51,17 +51,15 @@ export const continueToHereItem = ( const copyToClipboardItem = ( selectedContent: SourceContent, editorActions: EditorItemActions -) => { - return { - id: "node-menu-copy-to-clipboard", - label: L10N.getStr("copyToClipboard.label"), - accesskey: L10N.getStr("copyToClipboard.accesskey"), - disabled: false, - click: () => - selectedContent.type === "text" && - copyToTheClipboard(selectedContent.value), - }; -}; +) => ({ + id: "node-menu-copy-to-clipboard", + label: L10N.getStr("copyToClipboard.label"), + accesskey: L10N.getStr("copyToClipboard.accesskey"), + disabled: false, + click: () => + selectedContent.type === "text" && + copyToTheClipboard(selectedContent.value), +}); const copySourceItem = ( selectedSource: Source, @@ -158,14 +156,12 @@ const downloadFileItem = ( selectedSource: Source, selectedContent: SourceContent, editorActions: EditorItemActions -) => { - return { - id: "node-menu-download-file", - label: L10N.getStr("downloadFile.label"), - accesskey: L10N.getStr("downloadFile.accesskey"), - click: () => downloadFile(selectedContent, getFilename(selectedSource)), - }; -}; +) => ({ + id: "node-menu-download-file", + label: L10N.getStr("downloadFile.label"), + accesskey: L10N.getStr("downloadFile.accesskey"), + click: () => downloadFile(selectedContent, getFilename(selectedSource)), +}); export function editorMenuItems({ cx, diff --git a/devtools/client/debugger/src/components/SecondaryPanes/EventListeners.js b/devtools/client/debugger/src/components/SecondaryPanes/EventListeners.js index 38daf57b7838..633845d75cc4 100644 --- a/devtools/client/debugger/src/components/SecondaryPanes/EventListeners.js +++ b/devtools/client/debugger/src/components/SecondaryPanes/EventListeners.js @@ -159,13 +159,11 @@ class EventListeners extends Component { } } -const mapStateToProps = state => { - return { - activeEventListeners: getActiveEventListeners(state), - categories: getEventListenerBreakpointTypes(state), - expandedCategories: getEventListenerExpanded(state), - }; -}; +const mapStateToProps = state => ({ + activeEventListeners: getActiveEventListeners(state), + categories: getEventListenerBreakpointTypes(state), + expandedCategories: getEventListenerExpanded(state), +}); export default connect( mapStateToProps, diff --git a/devtools/client/debugger/src/components/SecondaryPanes/Expressions.js b/devtools/client/debugger/src/components/SecondaryPanes/Expressions.js index f509432c424f..ccced977c2ae 100644 --- a/devtools/client/debugger/src/components/SecondaryPanes/Expressions.js +++ b/devtools/client/debugger/src/components/SecondaryPanes/Expressions.js @@ -383,14 +383,12 @@ class Expressions extends Component { } } -const mapStateToProps = state => { - return { - cx: getThreadContext(state), - autocompleteMatches: getAutocompleteMatchset(state), - expressions: getExpressions(state), - expressionError: getExpressionError(state), - }; -}; +const mapStateToProps = state => ({ + cx: getThreadContext(state), + autocompleteMatches: getAutocompleteMatchset(state), + expressions: getExpressions(state), + expressionError: getExpressionError(state), +}); export default connect( mapStateToProps, diff --git a/devtools/client/debugger/src/components/SecondaryPanes/Workers.js b/devtools/client/debugger/src/components/SecondaryPanes/Workers.js index 0df157f16d4d..c0a2876038c6 100644 --- a/devtools/client/debugger/src/components/SecondaryPanes/Workers.js +++ b/devtools/client/debugger/src/components/SecondaryPanes/Workers.js @@ -34,7 +34,7 @@ export class Workers extends Component { onClick={() => openWorkerToolbox(thread)} >
- +
{getDisplayName(thread)}

diff --git a/devtools/client/debugger/src/components/SecondaryPanes/XHRBreakpoints.js b/devtools/client/debugger/src/components/SecondaryPanes/XHRBreakpoints.js index 478968d52062..53b2ad83ff9b 100644 --- a/devtools/client/debugger/src/components/SecondaryPanes/XHRBreakpoints.js +++ b/devtools/client/debugger/src/components/SecondaryPanes/XHRBreakpoints.js @@ -338,7 +338,7 @@ class XHRBreakpoints extends Component { return (