Bug 1526891 - Part 2: Make it possible to use MozURL by QuotaManager for all URIs (instead of using nsIPrincipal); r=nika,asuth

Differential Revision: https://phabricator.services.mozilla.com/D20906

Cargo.lock

@@ -1722,6 +1722,7 @@ dependencies = [
  "nserror 0.1.0",
  "nsstring 0.1.0",
  "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "xpcom 0.1.0",
 ]
 

caps/ContentPrincipal.cpp

@@ -152,7 +152,8 @@ nsresult ContentPrincipal::GenerateOriginNoSuffixFromURI(
        // about:blank is special since it can be generated from different
        // sources. We check for moz-safe-about:blank since origin is an
        // innermost URI.
-       !origin->GetSpecOrDefault().EqualsLiteral("moz-safe-about:blank"))) {
+       !StringBeginsWith(origin->GetSpecOrDefault(),
+                         NS_LITERAL_CSTRING("moz-safe-about:blank")))) {
     rv = origin->GetAsciiSpec(aOriginNoSuffix);
     NS_ENSURE_SUCCESS(rv, rv);
 

modules/libpref/init/StaticPrefList.h

@@ -2301,6 +2301,13 @@ VARCACHE_PREF(
   RelaxedAtomicBool, true
 )
 
+// Whether strict file origin policy is in effect.
+VARCACHE_PREF(
+  "security.fileuri.strict_origin_policy",
+   security_fileuri_strict_origin_policy,
+  RelaxedAtomicBool, true
+)
+
 //---------------------------------------------------------------------------
 // End of prefs
 //---------------------------------------------------------------------------

netwerk/base/mozurl/Cargo.toml

@@ -8,3 +8,4 @@ url = "1.7.2"
 nserror = { path = "../../../xpcom/rust/nserror" }
 nsstring = { path = "../../../xpcom/rust/nsstring" }
 xpcom = { path = "../../../xpcom/rust/xpcom" }
+uuid = { version = "0.6", features = ["v4"] }

netwerk/base/mozurl/MozURL.cpp

@@ -0,0 +1,13 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/StaticPrefs.h"
+
+extern "C" {
+
+bool Gecko_StrictFileOriginPolicy() {
+  return mozilla::StaticPrefs::security_fileuri_strict_origin_policy();
+}
+
+}

netwerk/base/mozurl/MozURL.h

@@ -61,10 +61,17 @@ class MozURL final {
   bool HasFragment() const { return mozurl_has_fragment(this); }
   nsDependentCSubstring Directory() const { return mozurl_directory(this); }
 
-  // WARNING: This does not match the definition of origins in nsIPrincipal for
-  // all URIs.
-  // XXX: Consider bringing these implementations in sync with one-another?
+  // This matches the definition of origins and base domains in nsIPrincipal for
+  // almost all URIs (some rare file:// URIs don't match and it would be hard to
+  // fix them). It definitely matches nsIPrincipal for URIs used in quota
+  // manager and there are checks in quota manager and its clients that prevent
+  // different definitions (see QuotaManager::IsPrincipalInfoValid).
+  // See also TestMozURL.cpp which enumerates a huge pile of URIs and checks
+  // that origin and base domain definitions are in sync.
   void Origin(nsACString& aOrigin) const { mozurl_origin(this, &aOrigin); }
+  nsresult BaseDomain(nsACString& aBaseDomain) const {
+    return mozurl_base_domain(this, &aBaseDomain);
+  }
 
   nsresult GetCommonBase(const MozURL* aOther, MozURL** aCommon) const {
     return mozurl_common_base(this, aOther, aCommon);

netwerk/base/mozurl/MozURL_ffi.h

@@ -59,6 +59,7 @@ MozURLSpecSlice mozurl_fragment(const mozilla::net::MozURL*);
 bool mozurl_has_fragment(const mozilla::net::MozURL*);
 MozURLSpecSlice mozurl_directory(const mozilla::net::MozURL*);
 void mozurl_origin(const mozilla::net::MozURL*, nsACString* aResult);
+nsresult mozurl_base_domain(const mozilla::net::MozURL*, nsACString* aResult);
 
 nsresult mozurl_common_base(const mozilla::net::MozURL* aUrl1,
                             const mozilla::net::MozURL* aUrl2,

netwerk/base/mozurl/moz.build

@@ -9,4 +9,8 @@ EXPORTS.mozilla.net += [
     'MozURL_ffi.h',
 ]
 
+SOURCES += [
+    'MozURL.cpp',
+]
+
 FINAL_LIBRARY = 'xul'

netwerk/base/mozurl/src/lib.rs

@@ -17,12 +17,19 @@ extern crate xpcom;
 use xpcom::{AtomicRefcnt, RefPtr, RefCounted};
 use xpcom::interfaces::nsrefcnt;
 
+extern crate uuid;
+use uuid::Uuid;
+
 use std::str;
 use std::ptr;
 use std::ops;
 use std::marker::PhantomData;
 use std::fmt::Write;
 
+extern "C" {
+  fn Gecko_StrictFileOriginPolicy() -> bool;
+}
+
 /// Helper macro. If the expression $e is Ok(t) evaluates to t, otherwise,
 /// returns NS_ERROR_MALFORMED_URI.
 macro_rules! try_or_malformed {
@@ -244,14 +251,115 @@ pub extern "C" fn mozurl_directory(url: &MozURL) -> SpecSlice {
   }
 }
 
+fn get_origin(url: &MozURL) -> Option<String> {
+  match url.scheme() {
+    "blob" | "ftp" | "http" | "https" | "ws" | "wss" =>
+      Some(url.origin().ascii_serialization()),
+    "indexeddb" | "moz-extension" | "resource" => {
+      let host = url.host_str().unwrap_or("");
+
+      let port = url.port().or_else(|| default_port(url.scheme()));
+
+      if port == default_port(url.scheme()) {
+        Some(format!("{}://{}", url.scheme(), host))
+      } else {
+        Some(format!("{}://{}:{}", url.scheme(), host, port.unwrap()))
+      }
+    },
+    "file" =>
+      if unsafe { Gecko_StrictFileOriginPolicy() } {
+        Some(url[..Position::AfterPath].to_owned())
+      } else {
+        Some("file://UNIVERSAL_FILE_URI_ORIGIN".to_owned())
+      },
+    "about" | "moz-safe-about" => Some(url[..Position::AfterPath].to_owned()),
+    _ => None
+  }
+}
+
 #[no_mangle]
 pub extern "C" fn mozurl_origin(url: &MozURL, origin: &mut nsACString) {
+  let origin_str = if !url.as_ref().starts_with("about:blank") {
+    get_origin(url)
+  } else {
+    None
+  };
+
+  let origin_str = origin_str.unwrap_or_else(|| {
+    // nsIPrincipal stores the uuid, so the same uuid is returned everytime.
+    // We can't do that for MozURL because it can be used across threads.
+    // Storing uuid would mutate the object which would cause races between
+    // threads.
+    format!("moz-nullprincipal:{{{}}}", Uuid::new_v4())
+  });
+
   // NOTE: Try to re-use the allocation we got from rust-url, and transfer
   // ownership of the buffer to C++.
-  let mut o = nsCString::from(url.origin().ascii_serialization());
+  let mut o = nsCString::from(origin_str);
   origin.take_from(&mut o);
 }
 
+fn get_base_domain(url: &MozURL) -> Result<Option<String>, nsresult> {
+  match url.scheme() {
+    "ftp" | "http" | "https" | "moz-extension" | "resource" => {
+      let third_party_util = xpcom::services::get_ThirdPartyUtil().unwrap();
+
+      let scheme = nsCString::from(url.scheme());
+
+      let mut host_str = url.host_str().unwrap_or("");
+
+      if host_str.starts_with('[') && host_str.ends_with(']') {
+        host_str = &host_str[1..host_str.len() - 1];
+      }
+
+      let host = nsCString::from(host_str);
+
+      unsafe {
+        let mut string = nsCString::new();
+        third_party_util.GetBaseDomainFromSchemeHost(&*scheme, &*host,
+            &mut *string).to_result()?;
+
+        // We know that GetBaseDomainFromSchemeHost returns AUTF8String, so just
+        // use unwrap().
+        Ok(Some(String::from_utf8(string.to_vec()).unwrap()))
+      }
+    },
+    "ws" | "wss" => Ok(Some(url.as_ref().to_owned())),
+    "file" =>
+      if unsafe { Gecko_StrictFileOriginPolicy() } {
+        Ok(Some(url.path().to_owned()))
+      } else {
+        Ok(Some("UNIVERSAL_FILE_URI_ORIGIN".to_owned()))
+      },
+    "about" | "moz-safe-about" | "indexeddb" => Ok(Some(url.as_ref().to_owned())),
+    _ => Ok(None)
+  }
+}
+
+#[no_mangle]
+pub extern "C" fn mozurl_base_domain(url: &MozURL, base_domain: &mut nsACString) -> nsresult {
+  let base_domain_str = if !url.as_ref().starts_with("about:blank") {
+    match get_base_domain(url) {
+      Ok(domain) => domain,
+      Err(rv) => return rv,
+    }
+  } else {
+    None
+  };
+
+  let base_domain_str = base_domain_str.unwrap_or_else(|| {
+    // See the comment in mozurl_origin about why we return a new uuid for
+    // "moz-nullprincipals".
+    format!("{{{}}}", Uuid::new_v4())
+  });
+
+
+  let mut bd = nsCString::from(base_domain_str);
+  base_domain.take_from(&mut bd);
+
+  NS_OK
+}
+
 // Helper macro for debug asserting that we're the only reference to MozURL.
 macro_rules! debug_assert_mut {
   ($e:expr) => {

netwerk/test/gtest/TestMozURL.cpp

@@ -1,9 +1,14 @@
 #include "gtest/gtest.h"
 #include "gtest/MozGTestBench.h"  // For MOZ_GTEST_BENCH
 
-#include "nsCOMPtr.h"
+#include <regex>
+#include "json/json.h"
 #include "mozilla/net/MozURL.h"
+#include "nsCOMPtr.h"
+#include "nsDirectoryServiceDefs.h"
+#include "nsIFile.h"
 
+using namespace mozilla;
 using namespace mozilla::net;
 
 TEST(TestMozURL, Getters) {
@@ -153,5 +158,230 @@ TEST(TestMozURL, Origin) {
       MozURL::Init(getter_AddRefs(url2), NS_LITERAL_CSTRING("file:///tmp/foo")),
       NS_OK);
   url2->Origin(out);
-  ASSERT_TRUE(out.EqualsLiteral("null"));
+  ASSERT_TRUE(out.EqualsLiteral("file:///tmp/foo"));
+
+  RefPtr<MozURL> url3;
+  ASSERT_EQ(
+      MozURL::Init(getter_AddRefs(url3),
+                   NS_LITERAL_CSTRING(
+                       "moz-extension://53711a8f-65ed-e742-9671-1f02e267c0bc/"
+                       "foo/bar.html")),
+      NS_OK);
+  url3->Origin(out);
+  ASSERT_TRUE(out.EqualsLiteral(
+      "moz-extension://53711a8f-65ed-e742-9671-1f02e267c0bc"));
+
+  RefPtr<MozURL> url4;
+  ASSERT_EQ(MozURL::Init(getter_AddRefs(url4),
+                         NS_LITERAL_CSTRING("resource://foo/bar.html")),
+            NS_OK);
+  url4->Origin(out);
+  ASSERT_TRUE(out.EqualsLiteral("resource://foo"));
+
+  RefPtr<MozURL> url5;
+  ASSERT_EQ(
+      MozURL::Init(getter_AddRefs(url5), NS_LITERAL_CSTRING("about:home")),
+      NS_OK);
+  url5->Origin(out);
+  ASSERT_TRUE(out.EqualsLiteral("about:home"));
+}
+
+TEST(TestMozURL, BaseDomain) {
+  nsAutoCString href("https://user:pass@example.net:1234/path?query#ref");
+  RefPtr<MozURL> url;
+  ASSERT_EQ(MozURL::Init(getter_AddRefs(url), href), NS_OK);
+
+  nsAutoCString out;
+  ASSERT_EQ(url->BaseDomain(out), NS_OK);
+  ASSERT_TRUE(out.EqualsLiteral("example.net"));
+
+  RefPtr<MozURL> url2;
+  ASSERT_EQ(
+      MozURL::Init(getter_AddRefs(url2), NS_LITERAL_CSTRING("file:///tmp/foo")),
+      NS_OK);
+  ASSERT_EQ(url2->BaseDomain(out), NS_OK);
+  ASSERT_TRUE(out.EqualsLiteral("/tmp/foo"));
+
+  RefPtr<MozURL> url3;
+  ASSERT_EQ(
+      MozURL::Init(getter_AddRefs(url3),
+                   NS_LITERAL_CSTRING(
+                       "moz-extension://53711a8f-65ed-e742-9671-1f02e267c0bc/"
+                       "foo/bar.html")),
+      NS_OK);
+  ASSERT_EQ(url3->BaseDomain(out), NS_OK);
+  ASSERT_TRUE(out.EqualsLiteral("53711a8f-65ed-e742-9671-1f02e267c0bc"));
+
+  RefPtr<MozURL> url4;
+  ASSERT_EQ(MozURL::Init(getter_AddRefs(url4),
+                         NS_LITERAL_CSTRING("resource://foo/bar.html")),
+            NS_OK);
+  ASSERT_EQ(url4->BaseDomain(out), NS_OK);
+  ASSERT_TRUE(out.EqualsLiteral("foo"));
+
+  RefPtr<MozURL> url5;
+  ASSERT_EQ(
+      MozURL::Init(getter_AddRefs(url5), NS_LITERAL_CSTRING("about:home")),
+      NS_OK);
+  ASSERT_EQ(url5->BaseDomain(out), NS_OK);
+  ASSERT_TRUE(out.EqualsLiteral("about:home"));
+}
+
+namespace {
+
+bool OriginMatchesExpectedOrigin(const nsACString& aOrigin,
+                                 const nsACString& aExpectedOrigin) {
+  if (aExpectedOrigin.Equals("null") &&
+      StringBeginsWith(aOrigin, NS_LITERAL_CSTRING("moz-nullprincipal"))) {
+    return true;
+  }
+  return aOrigin == aExpectedOrigin;
+}
+
+bool IsUUID(const nsACString& aString) {
+  if (!IsASCII(aString)) {
+    return false;
+  }
+
+  std::regex pattern("^\\{[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab"
+                     "][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}\\}$");
+  return regex_match(nsCString(aString).get(), pattern);
+}
+
+bool BaseDomainsEqual(const nsACString& aBaseDomain1,
+                      const nsACString& aBaseDomain2) {
+  if (IsUUID(aBaseDomain1) && IsUUID(aBaseDomain2)) {
+    return true;
+  }
+  return aBaseDomain1 == aBaseDomain2;
+}
+
+void CheckOrigin(const nsACString& aSpec, const nsACString& aBase,
+                 const nsACString& aOrigin) {
+  nsCOMPtr<nsIURI> baseUri;
+  nsresult rv = NS_NewURI(getter_AddRefs(baseUri), aBase);
+  ASSERT_EQ(rv, NS_OK);
+
+  nsCOMPtr<nsIURI> uri;
+  rv = NS_NewURI(getter_AddRefs(uri), aSpec, nullptr, baseUri);
+  ASSERT_EQ(rv, NS_OK);
+
+  OriginAttributes attrs;
+
+  nsCOMPtr<nsIPrincipal> principal =
+      BasePrincipal::CreateCodebasePrincipal(uri, attrs);
+  ASSERT_TRUE(principal);
+
+  nsCString origin;
+  rv = principal->GetOriginNoSuffix(origin);
+  ASSERT_EQ(rv, NS_OK);
+
+  EXPECT_TRUE(OriginMatchesExpectedOrigin(origin, aOrigin));
+
+  nsCString baseDomain;
+  rv = principal->GetBaseDomain(baseDomain);
+
+  bool baseDomainSucceeded = NS_SUCCEEDED(rv);
+
+  RefPtr<MozURL> baseUrl;
+  ASSERT_EQ(MozURL::Init(getter_AddRefs(baseUrl), aBase), NS_OK);
+
+  RefPtr<MozURL> url;
+  ASSERT_EQ(MozURL::Init(getter_AddRefs(url), aSpec, baseUrl), NS_OK);
+
+  url->Origin(origin);
+
+  EXPECT_TRUE(OriginMatchesExpectedOrigin(origin, aOrigin));
+
+  nsCString baseDomain2;
+  rv = url->BaseDomain(baseDomain2);
+
+  bool baseDomain2Succeeded = NS_SUCCEEDED(rv);
+
+  EXPECT_TRUE(baseDomainSucceeded == baseDomain2Succeeded);
+
+  if (baseDomainSucceeded) {
+    EXPECT_TRUE(BaseDomainsEqual(baseDomain, baseDomain2));
+  }
+}
+
+}  // namespace
+
+TEST(TestMozURL, UrlTestData) {
+  nsCOMPtr<nsIFile> file;
+  nsresult rv =
+      NS_GetSpecialDirectory(NS_OS_CURRENT_WORKING_DIR, getter_AddRefs(file));
+  ASSERT_EQ(rv, NS_OK);
+
+  rv = file->Append(NS_LITERAL_STRING("urltestdata.json"));
+  ASSERT_EQ(rv, NS_OK);
+
+  bool exists;
+  rv = file->Exists(&exists);
+  ASSERT_EQ(rv, NS_OK);
+
+  ASSERT_TRUE(exists);
+
+  nsCOMPtr<nsIInputStream> stream;
+  rv = NS_NewLocalFileInputStream(getter_AddRefs(stream), file);
+  ASSERT_EQ(rv, NS_OK);
+
+  nsCOMPtr<nsIInputStream> bufferedStream;
+  rv = NS_NewBufferedInputStream(getter_AddRefs(bufferedStream),
+                                 stream.forget(), 4096);
+  ASSERT_EQ(rv, NS_OK);
+
+  nsCString data;
+  rv = NS_ConsumeStream(bufferedStream, UINT32_MAX, data);
+  ASSERT_EQ(rv, NS_OK);
+
+  Json::Reader reader;
+  Json::Value root;
+  ASSERT_TRUE(reader.parse(data.BeginReading(), data.EndReading(), root));
+  ASSERT_TRUE(root.isArray());
+
+  for (uint32_t index = 0; index < root.size(); index++) {
+    const Json::Value& item = root[index];
+
+    if (!item.isObject()) {
+      continue;
+    }
+
+    const Json::Value& skip = item["skip"];
+    ASSERT_TRUE(skip.isNull() || skip.isBool());
+    if (skip.isBool() && skip.asBool()) {
+      continue;
+    }
+
+    const Json::Value& failure = item["failure"];
+    ASSERT_TRUE(failure.isNull() || failure.isBool());
+    if (failure.isBool() && failure.asBool()) {
+      continue;
+    }
+
+    const Json::Value& origin = item["origin"];
+    ASSERT_TRUE(origin.isNull() || origin.isString());
+    if (origin.isNull()) {
+      continue;
+    }
+    const char* originBegin;
+    const char* originEnd;
+    origin.getString(&originBegin, &originEnd);
+
+    const Json::Value& base = item["base"];
+    ASSERT_TRUE(base.isString());
+    const char* baseBegin;
+    const char* baseEnd;
+    base.getString(&baseBegin, &baseEnd);
+
+    const Json::Value& input = item["input"];
+    ASSERT_TRUE(input.isString());
+    const char* inputBegin;
+    const char* inputEnd;
+    input.getString(&inputBegin, &inputEnd);
+
+    CheckOrigin(nsDependentCString(inputBegin, inputEnd),
+                nsDependentCString(baseBegin, baseEnd),
+                nsDependentCString(originBegin, originEnd));
+  }
 }

netwerk/test/gtest/moz.build

@@ -20,12 +20,17 @@ UNIFIED_SOURCES += [
     'TestURIMutator.cpp',
 ]
 
+TEST_HARNESS_FILES.gtest += [
+    'urltestdata.json',
+]
+
 TEST_DIRS += [
     'parse-ftp',
 ]
 
 LOCAL_INCLUDES += [
     '/netwerk/base',
+    '/toolkit/components/jsoncpp/include',
     '/xpcom/tests/gtest',
 ]