Merge mozilla-inbound to mozilla-central. a=merge

CLOBBER

@@ -22,4 +22,4 @@
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Merge day clobber
+Bug 1466471 - Update to ICU 62 requires clobber

browser/base/content/tabbrowser.js

@@ -2231,8 +2231,6 @@ window._gBrowser = {
       ContextualIdentityService.setTabStyle(t);
     }
 
-    t.setAttribute("onerror", "this.removeAttribute('image');");
-
     if (aSkipBackgroundNotify) {
       t.setAttribute("skipbackgroundnotify", true);
     }

browser/base/content/tabbrowser.xml

@@ -111,7 +111,6 @@
 
           var tab = this.firstChild;
           tab.label = this.emptyTabTitle;
-          tab.setAttribute("onerror", "this.removeAttribute('image');");
 
           window.addEventListener("resize", this);
 

config/external/icu/common/sources.mozbuild

@@ -70,6 +70,7 @@ SOURCES += [
    '/intl/icu/source/common/servslkf.cpp',
    '/intl/icu/source/common/sharedobject.cpp',
    '/intl/icu/source/common/simpleformatter.cpp',
+   '/intl/icu/source/common/static_unicode_sets.cpp',
    '/intl/icu/source/common/stringpiece.cpp',
    '/intl/icu/source/common/stringtriebuilder.cpp',
    '/intl/icu/source/common/uarrsort.cpp',

config/external/icu/i18n/sources.mozbuild

@@ -1,6 +1,5 @@
 # THIS FILE IS GENERATED BY /intl/icu_sources_data.py DO NOT EDIT
 SOURCES += [
-   '/intl/icu/source/i18n/affixpatternparser.cpp',
    '/intl/icu/source/i18n/alphaindex.cpp',
    '/intl/icu/source/i18n/anytrans.cpp',
    '/intl/icu/source/i18n/astro.cpp',
@@ -54,22 +53,14 @@ SOURCES += [
    '/intl/icu/source/i18n/dayperiodrules.cpp',
    '/intl/icu/source/i18n/dcfmtsym.cpp',
    '/intl/icu/source/i18n/decContext.cpp',
-   '/intl/icu/source/i18n/decfmtst.cpp',
-   '/intl/icu/source/i18n/decimalformatpattern.cpp',
    '/intl/icu/source/i18n/decimfmt.cpp',
-   '/intl/icu/source/i18n/decimfmtimpl.cpp',
    '/intl/icu/source/i18n/decNumber.cpp',
-   '/intl/icu/source/i18n/digitaffix.cpp',
-   '/intl/icu/source/i18n/digitaffixesandpadding.cpp',
-   '/intl/icu/source/i18n/digitformatter.cpp',
-   '/intl/icu/source/i18n/digitgrouping.cpp',
-   '/intl/icu/source/i18n/digitinterval.cpp',
-   '/intl/icu/source/i18n/digitlst.cpp',
    '/intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp',
    '/intl/icu/source/i18n/double-conversion-bignum.cpp',
    '/intl/icu/source/i18n/double-conversion-cached-powers.cpp',
    '/intl/icu/source/i18n/double-conversion-diy-fp.cpp',
    '/intl/icu/source/i18n/double-conversion-fast-dtoa.cpp',
+   '/intl/icu/source/i18n/double-conversion-strtod.cpp',
    '/intl/icu/source/i18n/double-conversion.cpp',
    '/intl/icu/source/i18n/dtfmtsym.cpp',
    '/intl/icu/source/i18n/dtitvfmt.cpp',
@@ -104,7 +95,10 @@ SOURCES += [
    '/intl/icu/source/i18n/nounit.cpp',
    '/intl/icu/source/i18n/nultrans.cpp',
    '/intl/icu/source/i18n/number_affixutils.cpp',
+   '/intl/icu/source/i18n/number_asformat.cpp',
+   '/intl/icu/source/i18n/number_capi.cpp',
    '/intl/icu/source/i18n/number_compact.cpp',
+   '/intl/icu/source/i18n/number_currencysymbols.cpp',
    '/intl/icu/source/i18n/number_decimalquantity.cpp',
    '/intl/icu/source/i18n/number_decimfmtprops.cpp',
    '/intl/icu/source/i18n/number_fluent.cpp',
@@ -112,22 +106,34 @@ SOURCES += [
    '/intl/icu/source/i18n/number_grouping.cpp',
    '/intl/icu/source/i18n/number_integerwidth.cpp',
    '/intl/icu/source/i18n/number_longnames.cpp',
+   '/intl/icu/source/i18n/number_mapper.cpp',
    '/intl/icu/source/i18n/number_modifiers.cpp',
+   '/intl/icu/source/i18n/number_multiplier.cpp',
    '/intl/icu/source/i18n/number_notation.cpp',
    '/intl/icu/source/i18n/number_padding.cpp',
    '/intl/icu/source/i18n/number_patternmodifier.cpp',
    '/intl/icu/source/i18n/number_patternstring.cpp',
    '/intl/icu/source/i18n/number_rounding.cpp',
    '/intl/icu/source/i18n/number_scientific.cpp',
+   '/intl/icu/source/i18n/number_skeletons.cpp',
    '/intl/icu/source/i18n/number_stringbuilder.cpp',
+   '/intl/icu/source/i18n/number_utils.cpp',
    '/intl/icu/source/i18n/numfmt.cpp',
+   '/intl/icu/source/i18n/numparse_affixes.cpp',
+   '/intl/icu/source/i18n/numparse_compositions.cpp',
+   '/intl/icu/source/i18n/numparse_currency.cpp',
+   '/intl/icu/source/i18n/numparse_decimal.cpp',
+   '/intl/icu/source/i18n/numparse_impl.cpp',
+   '/intl/icu/source/i18n/numparse_parsednumber.cpp',
+   '/intl/icu/source/i18n/numparse_scientific.cpp',
+   '/intl/icu/source/i18n/numparse_stringsegment.cpp',
+   '/intl/icu/source/i18n/numparse_symbols.cpp',
+   '/intl/icu/source/i18n/numparse_validators.cpp',
    '/intl/icu/source/i18n/numsys.cpp',
    '/intl/icu/source/i18n/olsontz.cpp',
    '/intl/icu/source/i18n/persncal.cpp',
-   '/intl/icu/source/i18n/pluralaffix.cpp',
    '/intl/icu/source/i18n/plurfmt.cpp',
    '/intl/icu/source/i18n/plurrule.cpp',
-   '/intl/icu/source/i18n/precision.cpp',
    '/intl/icu/source/i18n/quant.cpp',
    '/intl/icu/source/i18n/quantityformatter.cpp',
    '/intl/icu/source/i18n/rbnf.cpp',
@@ -154,7 +160,6 @@ SOURCES += [
    '/intl/icu/source/i18n/selfmt.cpp',
    '/intl/icu/source/i18n/sharedbreakiterator.cpp',
    '/intl/icu/source/i18n/simpletz.cpp',
-   '/intl/icu/source/i18n/smallintformatter.cpp',
    '/intl/icu/source/i18n/smpdtfmt.cpp',
    '/intl/icu/source/i18n/smpdtfst.cpp',
    '/intl/icu/source/i18n/sortkey.cpp',
@@ -210,8 +215,6 @@ SOURCES += [
    '/intl/icu/source/i18n/utf8collationiterator.cpp',
    '/intl/icu/source/i18n/utmscale.cpp',
    '/intl/icu/source/i18n/utrans.cpp',
-   '/intl/icu/source/i18n/valueformatter.cpp',
-   '/intl/icu/source/i18n/visibledigits.cpp',
    '/intl/icu/source/i18n/vtzone.cpp',
    '/intl/icu/source/i18n/vzone.cpp',
    '/intl/icu/source/i18n/windtfmt.cpp',
@@ -293,6 +296,7 @@ EXPORTS.unicode += [
    '/intl/icu/source/i18n/unicode/umsg.h',
    '/intl/icu/source/i18n/unicode/unirepl.h',
    '/intl/icu/source/i18n/unicode/unum.h',
+   '/intl/icu/source/i18n/unicode/unumberformatter.h',
    '/intl/icu/source/i18n/unicode/unumsys.h',
    '/intl/icu/source/i18n/unicode/upluralrules.h',
    '/intl/icu/source/i18n/unicode/uregex.h',

dom/base/nsContentUtils.cpp

@@ -515,41 +515,6 @@ EventListenerManagerHashClearEntry(PLDHashTable *table, PLDHashEntryHdr *entry)
   lm->~EventListenerManagerMapEntry();
 }
 
-static bool
-IsThirdPartyWindowOrChannel(nsPIDOMWindowInner* aWindow,
-                            nsIChannel* aChannel,
-                            nsIURI* aURI)
-{
-  MOZ_ASSERT(!aWindow || !aChannel,
-             "A window and channel should not both be provided.");
-
-  nsCOMPtr<mozIThirdPartyUtil> thirdPartyUtil = services::GetThirdPartyUtil();
-  if (!thirdPartyUtil) {
-    return false;
-  }
-
-  // In the absence of a window or channel, we assume that we are first-party.
-  bool thirdParty = false;
-
-  if (aWindow) {
-    Unused << thirdPartyUtil->IsThirdPartyWindow(aWindow->GetOuterWindow(),
-                                                 aURI,
-                                                 &thirdParty);
-  }
-
-  if (aChannel) {
-    // Note, we must call IsThirdPartyChannel() here and not just try to
-    // use nsILoadInfo.isThirdPartyContext.  That nsILoadInfo property only
-    // indicates if the parent loading window is third party or not.  We
-    // want to check the channel URI against the loading principal as well.
-    Unused << thirdPartyUtil->IsThirdPartyChannel(aChannel,
-                                                  nullptr,
-                                                  &thirdParty);
-  }
-
-  return thirdParty;
-}
-
 class SameOriginCheckerImpl final : public nsIChannelEventSink,
                                     public nsIInterfaceRequestor
 {
@@ -8837,6 +8802,42 @@ nsContentUtils::GetCookieBehaviorForPrincipal(nsIPrincipal* aPrincipal,
   }
 }
 
+// static public
+bool
+nsContentUtils::IsThirdPartyWindowOrChannel(nsPIDOMWindowInner* aWindow,
+                                            nsIChannel* aChannel,
+                                            nsIURI* aURI)
+{
+  MOZ_ASSERT(!aWindow || !aChannel,
+             "A window and channel should not both be provided.");
+
+  nsCOMPtr<mozIThirdPartyUtil> thirdPartyUtil = services::GetThirdPartyUtil();
+  if (!thirdPartyUtil) {
+    return false;
+  }
+
+  // In the absence of a window or channel, we assume that we are first-party.
+  bool thirdParty = false;
+
+  if (aWindow) {
+    Unused << thirdPartyUtil->IsThirdPartyWindow(aWindow->GetOuterWindow(),
+                                                 aURI,
+                                                 &thirdParty);
+  }
+
+  if (aChannel) {
+    // Note, we must call IsThirdPartyChannel() here and not just try to
+    // use nsILoadInfo.isThirdPartyContext.  That nsILoadInfo property only
+    // indicates if the parent loading window is third party or not.  We
+    // want to check the channel URI against the loading principal as well.
+    Unused << thirdPartyUtil->IsThirdPartyChannel(aChannel,
+                                                  nullptr,
+                                                  &thirdParty);
+  }
+
+  return thirdParty;
+}
+
 // static public
 bool
 nsContentUtils::StorageDisabledByAntiTracking(nsPIDOMWindowInner* aWindow,
@@ -8852,19 +8853,54 @@ nsContentUtils::StorageDisabledByAntiTracking(nsPIDOMWindowInner* aWindow,
     return false;
   }
 
-  nsCOMPtr<nsIChannel> channel;
-
-  // aChannel and aWindow are mutually exclusive.
-  channel = aChannel;
   if (aWindow) {
+    nsCOMPtr<nsIHttpChannel> httpChannel;
     nsIDocument* document = aWindow->GetExtantDoc();
     if (document) {
-      channel = document->GetChannel();
+      httpChannel = do_QueryInterface(document->GetChannel());
     }
+
+    // If this is not a tracking resource, nothing is disabled.
+    if (!httpChannel || !httpChannel->GetIsTrackingResource()) {
+      return false;
+    }
+
+    // Maybe we want to grant this origin.
+    nsIURI* documentURI = aURI ? aURI : aWindow->GetDocumentURI();
+    if (documentURI &&
+        nsGlobalWindowInner::Cast(aWindow)->IsFirstPartyStorageAccessGrantedFor(documentURI)) {
+      return false;
+    }
+
+    return true;
   }
 
-  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(channel);
-  return httpChannel && httpChannel->GetIsTrackingResource();
+  // aChannel and aWindow are mutually exclusive.
+  MOZ_ASSERT(aChannel);
+
+  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aChannel);
+  if (!httpChannel) {
+    return false;
+  }
+
+  // If this is not a tracking resource, nothing is disabled.
+  if (!httpChannel->GetIsTrackingResource()) {
+    return false;
+  }
+
+  nsCOMPtr<nsIURI> uri;
+  nsresult rv = httpChannel->GetURI(getter_AddRefs(uri));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return false;
+  }
+
+  nsCOMPtr<nsILoadInfo> loadInfo;
+  rv = aChannel->GetLoadInfo(getter_AddRefs(loadInfo));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return false;
+  }
+
+  return !loadInfo->IsFirstPartyStorageAccessGrantedFor(uri);
 }
 
 // static, private

dom/base/nsContentUtils.h

@@ -2957,6 +2957,13 @@ public:
                                             nsIChannel* aChannel,
                                             nsIURI* aURI);
 
+  /*
+   * Returns true if this window/channel is a 3rd party context.
+   */
+  static bool IsThirdPartyWindowOrChannel(nsPIDOMWindowInner* aWindow,
+                                          nsIChannel* aChannel,
+                                          nsIURI* aURI);
+
   /*
    * Serializes a HTML nsINode into its markup representation.
    */

dom/base/nsDocument.cpp

@@ -23,6 +23,7 @@
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/Likely.h"
 #include "mozilla/PresShell.h"
+#include "mozilla/StaticPrefs.h"
 #include "mozilla/URLExtraData.h"
 #include <algorithm>
 
@@ -12415,10 +12416,62 @@ nsIDocument::NotifyUserGestureActivation()
             LogLevel::Debug,
             ("Document %p has been activated by user.", this));
     doc->mUserGestureActivated = true;
+    doc->MaybeAllowStorageForOpener();
     doc = doc->GetSameTypeParentDocument();
   }
 }
 
+void
+nsIDocument::MaybeAllowStorageForOpener()
+{
+  if (!StaticPrefs::privacy_restrict3rdpartystorage_enabled()) {
+    return;
+  }
+
+  // This will probably change for project fission, but currently this document
+  // and the opener are on the same process. In the future, we should make this
+  // part async.
+
+  nsPIDOMWindowInner* inner = GetInnerWindow();
+  if (NS_WARN_IF(!inner)) {
+    return;
+  }
+
+  nsCOMPtr<nsPIDOMWindowOuter> outer = inner->GetOuterWindow();
+  if (NS_WARN_IF(!outer)) {
+    return;
+  }
+
+  nsCOMPtr<nsPIDOMWindowOuter> outerOpener = outer->GetOpener();
+  if (NS_WARN_IF(!outerOpener)) {
+    return;
+  }
+
+  nsPIDOMWindowInner* openerInner = outerOpener->GetCurrentInnerWindow();
+  if (NS_WARN_IF(!openerInner)) {
+    return;
+  }
+
+  // No 3rd party.
+  if (!nsContentUtils::IsThirdPartyWindowOrChannel(openerInner, nullptr,
+                                                   nullptr)) {
+    return;
+  }
+
+  nsCOMPtr<nsIURI> uri = GetDocumentURI();
+  if (NS_WARN_IF(!uri)) {
+    return;
+  }
+
+  nsAutoString origin;
+  nsresult rv = nsContentUtils::GetUTFOrigin(uri, origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  nsGlobalWindowInner::Cast(openerInner)->AddFirstPartyStorageAccessGrantedFor(origin);
+}
+
 bool
 nsIDocument::HasBeenUserGestureActivated()
 {

dom/base/nsGlobalWindowInner.cpp

@@ -36,6 +36,7 @@
 #if defined(MOZ_WIDGET_ANDROID)
 #include "mozilla/dom/WindowOrientationObserver.h"
 #endif
+#include "mozilla/StaticPrefs.h"
 #include "nsDOMOfflineResourceList.h"
 #include "nsError.h"
 #include "nsIIdleService.h"
@@ -48,6 +49,7 @@
 #include "nsIDocShellTreeOwner.h"
 #include "nsIDocumentLoader.h"
 #include "nsIInterfaceRequestorUtils.h"
+#include "nsIPermission.h"
 #include "nsIPermissionManager.h"
 #include "nsIScriptContext.h"
 #include "nsIScriptTimeoutHandler.h"
@@ -332,6 +334,10 @@ using mozilla::dom::cache::CacheStorage;
 // Min idle notification time in seconds.
 #define MIN_IDLE_NOTIFICATION_TIME_S 1
 
+// Anti-tracking permission expiration
+#define ANTITRACKING_EXPIRATION 2592000000 // 30 days.
+#define ANTITRACKING_PERM_KEY "3rdPartyStorage"
+
 static LazyLogModule gDOMLeakPRLogInner("DOMLeakInner");
 
 static bool                 gIdleObserversAPIFuzzTimeDisabled = false;
@@ -916,7 +922,8 @@ nsGlobalWindowInner::nsGlobalWindowInner(nsGlobalWindowOuter *aOuterWindow)
     mObservingDidRefresh(false),
     mIteratingDocumentFlushedResolvers(false),
     mCanSkipCCGeneration(0),
-    mBeforeUnloadListenerCount(0)
+    mBeforeUnloadListenerCount(0),
+    mStorageGrantedOriginPopulated(false)
 {
   mIsInnerWindow = true;
 
@@ -1234,6 +1241,7 @@ nsGlobalWindowInner::FreeInnerObjects()
   }
 
   UnlinkHostObjectURIs();
+  ReleaseFirstPartyStorageAccessGrantedOrigins();
 
   NotifyWindowIDDestroyed("inner-window-destroyed");
 
@@ -1557,6 +1565,7 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(nsGlobalWindowInner)
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mIntlUtils)
 
   tmp->UnlinkHostObjectURIs();
+  tmp->ReleaseFirstPartyStorageAccessGrantedOrigins();
 
   NS_IMPL_CYCLE_COLLECTION_UNLINK(mIdleRequestExecutor)
 
@@ -5194,17 +5203,8 @@ nsGlobalWindowInner::GetCaches(ErrorResult& aRv)
   if (!mCacheStorage) {
     bool forceTrustedOrigin =
       GetOuterWindow()->GetServiceWorkersTestingEnabled();
-
-    nsContentUtils::StorageAccess access =
-      nsContentUtils::StorageAllowedForWindow(this);
-
-    // We don't block the cache API when being told to only allow storage for the
-    // current session.
-    bool storageBlocked = access <= nsContentUtils::StorageAccess::ePrivateBrowsing;
-
     mCacheStorage = CacheStorage::CreateOnMainThread(cache::DEFAULT_NAMESPACE,
                                                      this, GetPrincipal(),
-                                                     storageBlocked,
                                                      forceTrustedOrigin, aRv);
   }
 
@@ -6515,6 +6515,34 @@ nsGlobalWindowInner::GetParentInternal()
   return outer->GetParentInternal();
 }
 
+nsIPrincipal*
+nsGlobalWindowInner::GetTopLevelStorageAreaPrincipal()
+{
+  nsPIDOMWindowOuter* outerWindow = GetParentInternal();
+  if (!outerWindow) {
+    // No outer window available!
+    return nullptr;
+  }
+
+  if (!outerWindow->IsTopLevelWindow()) {
+    return nullptr;
+  }
+
+  nsPIDOMWindowInner* innerWindow = outerWindow->GetCurrentInnerWindow();
+  if (NS_WARN_IF(!innerWindow)) {
+    return nullptr;
+  }
+
+  nsIPrincipal* parentPrincipal =
+    nsGlobalWindowInner::Cast(innerWindow)->GetPrincipal();
+  if (NS_WARN_IF(!parentPrincipal)) {
+    return nullptr;
+  }
+
+  return parentPrincipal;
+}
+
+
 //*****************************************************************************
 // nsGlobalWindowInner: Timeout Functions
 //*****************************************************************************
@@ -8034,6 +8062,285 @@ nsGlobalWindowInner::GetRegionalPrefsLocales(nsTArray<nsString>& aLocales)
   }
 }
 
+void
+nsGlobalWindowInner::AddFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin,
+                                                          bool aOverwritten)
+{
+  MOZ_ASSERT(StaticPrefs::privacy_restrict3rdpartystorage_enabled());
+
+  if (aOverwritten) {
+    SaveFirstPartyStorageAccessGrantedFor(aOrigin);
+  }
+
+  for (StorageGrantedOrigin& data : mStorageGrantedOrigins) {
+    if (data.mOrigin == aOrigin) {
+      data.mOverwritten = aOverwritten;
+      return;
+    }
+  }
+
+  bool wasAllowed =
+    nsContentUtils::StorageDisabledByAntiTracking(this, nullptr, nullptr);
+
+  StorageGrantedOrigin* data = mStorageGrantedOrigins.AppendElement();
+  data->mOrigin = aOrigin;
+  data->mOverwritten = aOverwritten;
+
+  if (!wasAllowed &&
+      nsContentUtils::StorageDisabledByAntiTracking(this, nullptr, nullptr)) {
+    PropagateFirstPartyStorageAccessGrantedToWorkers(this);
+  }
+
+  // Let's store the origin in the loadInfo as well.
+  if (mDoc) {
+    nsCOMPtr<nsIChannel> channel = mDoc->GetChannel();
+    if (channel) {
+      nsCOMPtr<nsILoadInfo> loadInfo = channel->GetLoadInfo();
+      if (loadInfo) {
+        loadInfo->AddFirstPartyStorageAccessGrantedFor(aOrigin);
+      }
+    }
+  }
+}
+
+void
+nsGlobalWindowInner::GetFirstPartyStorageAccessGrantedOrigins(nsTArray<nsString>& aOrigin)
+{
+  aOrigin.Clear();
+
+  if (!StaticPrefs::privacy_restrict3rdpartystorage_enabled()) {
+    return;
+  }
+
+  MaybeRestoreFirstPartyStorageAccessGrantedOrigins();
+  for (const StorageGrantedOrigin& data : mStorageGrantedOrigins) {
+    aOrigin.AppendElement(data.mOrigin);
+  }
+}
+
+bool
+nsGlobalWindowInner::IsFirstPartyStorageAccessGrantedFor(nsIURI* aURI)
+{
+  MOZ_ASSERT(aURI);
+
+  if (!StaticPrefs::privacy_restrict3rdpartystorage_enabled()) {
+    return true;
+  }
+
+  MaybeRestoreFirstPartyStorageAccessGrantedOrigins();
+
+  if (mStorageGrantedOrigins.IsEmpty()) {
+    return false;
+  }
+
+  nsAutoString origin;
+  nsresult rv = nsContentUtils::GetUTFOrigin(aURI, origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return false;
+  }
+
+  for (const StorageGrantedOrigin& data : mStorageGrantedOrigins) {
+    if (data.mOrigin.Equals(origin)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void
+nsGlobalWindowInner::ReleaseFirstPartyStorageAccessGrantedOrigins()
+{
+  mStorageGrantedOriginPopulated = false;
+  mStorageGrantedOrigins.Clear();
+
+}
+
+namespace mozilla {
+namespace dom {
+
+extern void
+SendFirstPartyStorageAccessGrantedForOriginToParentProcess(nsIPrincipal* aPrincipal,
+                                                           const nsACString& aParentOrigin,
+                                                           const nsACString& aGrantedOrigin);
+
+} // namespace dom
+} // namespace mozilla
+
+void
+nsGlobalWindowInner::SaveFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin)
+{
+  MOZ_ASSERT(StaticPrefs::privacy_restrict3rdpartystorage_enabled());
+
+  // Now we need the principal and the origin of the parent window.
+  nsIPrincipal* parentPrincipal = GetTopLevelStorageAreaPrincipal();
+  if (NS_WARN_IF(!parentPrincipal)) {
+    return;
+  }
+
+  nsAutoCString parentOrigin;
+  nsresult rv = parentPrincipal->GetOriginNoSuffix(parentOrigin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  // Let's take the principal and the origin of the current window.
+  nsIPrincipal* principal = GetPrincipal();
+  if (NS_WARN_IF(!principal)) {
+    return;
+  }
+
+  NS_ConvertUTF16toUTF8 grantedOrigin(aOrigin);
+
+  if (XRE_IsParentProcess()) {
+    SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(principal,
+                                                               parentOrigin,
+                                                               grantedOrigin);
+    return;
+  }
+
+  // We have this external function because ContentChild includes windows.h and
+  // for this reason it cannot be included here.
+  SendFirstPartyStorageAccessGrantedForOriginToParentProcess(principal,
+                                                             parentOrigin,
+                                                             grantedOrigin);
+}
+
+/* static */ void
+nsGlobalWindowInner::SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(nsIPrincipal* aPrincipal,
+                                                                                const nsCString& aParentOrigin,
+                                                                                const nsCString& aGrantedOrigin)
+{
+  MOZ_ASSERT(XRE_IsParentProcess());
+  MOZ_ASSERT(aPrincipal);
+
+  nsAutoCString origin;
+  nsresult rv = aPrincipal->GetOriginNoSuffix(origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  nsCOMPtr<nsIPermissionManager> pm = services::GetPermissionManager();
+  if (NS_WARN_IF(!pm)) {
+    return;
+  }
+
+  int64_t when = (PR_Now() / PR_USEC_PER_MSEC) + ANTITRACKING_EXPIRATION;
+
+  // We store a permission for the 3rd party principal, to know that we grant
+  // the storage permission when loaded by the current parent origin.
+  nsAutoCString type;
+  if (origin == aGrantedOrigin) {
+    type = nsPrintfCString(ANTITRACKING_PERM_KEY "^%s", aParentOrigin.get());
+  } else {
+    type = nsPrintfCString(ANTITRACKING_PERM_KEY "^%s^%s", aParentOrigin.get(),
+                           aGrantedOrigin.get());
+  }
+
+  rv = pm->AddFromPrincipal(aPrincipal, type.get(),
+                            nsIPermissionManager::ALLOW_ACTION,
+                            nsIPermissionManager::EXPIRE_TIME, when);
+  Unused << NS_WARN_IF(NS_FAILED(rv));
+}
+
+void
+nsGlobalWindowInner::MaybeRestoreFirstPartyStorageAccessGrantedOrigins()
+{
+  if (!StaticPrefs::privacy_restrict3rdpartystorage_enabled()) {
+    return;
+  }
+
+  if (mStorageGrantedOriginPopulated) {
+    return;
+  }
+
+  mStorageGrantedOriginPopulated = true;
+
+  // Now we need the principal and the origin of the parent window.
+  nsIPrincipal* parentPrincipal = GetTopLevelStorageAreaPrincipal();
+  if (!parentPrincipal) {
+    // No parent window.
+    return;
+  }
+
+  nsAutoCString parentOrigin;
+  nsresult rv = parentPrincipal->GetOriginNoSuffix(parentOrigin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  // Let's take the principal and the origin of the current window.
+  nsIPrincipal* principal = GetPrincipal();
+  if (NS_WARN_IF(!principal)) {
+    return;
+  }
+
+  nsAutoCString origin;
+  rv = principal->GetOriginNoSuffix(origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  nsCOMPtr<nsIPermissionManager> pm = services::GetPermissionManager();
+  if (NS_WARN_IF(!pm)) {
+    return;
+  }
+
+  nsCOMPtr<nsISimpleEnumerator> enumerator;
+  rv = pm->GetAllForPrincipal(principal, getter_AddRefs(enumerator));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  bool more = false;
+  nsCOMPtr<nsISupports> iter;
+  nsCOMPtr<nsIPermission> perm;
+  while (NS_SUCCEEDED(enumerator->HasMoreElements(&more)) && more) {
+    rv = enumerator->GetNext(getter_AddRefs(iter));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return;
+    }
+
+    perm = do_QueryInterface(iter);
+    if (NS_WARN_IF(!perm)) {
+      return;
+    }
+
+    nsAutoCString type;
+    rv = perm->GetType(type);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return;
+    }
+
+    if (!StringBeginsWith(type, NS_LITERAL_CSTRING(ANTITRACKING_PERM_KEY "^"))) {
+      continue;
+    }
+
+    nsCCharSeparatedTokenizer token(type, '^');
+    MOZ_ASSERT(token.hasMoreTokens());
+    auto value = token.nextToken();
+    MOZ_ASSERT(value.EqualsLiteral(ANTITRACKING_PERM_KEY));
+
+    nsAutoCString originA;
+    if (token.hasMoreTokens()) {
+      originA = token.nextToken();
+    }
+
+    // This permission was granted for another top-level window.
+    if (originA != parentOrigin) {
+      continue;
+    }
+
+    nsAutoCString originB;
+    if (token.hasMoreTokens()) {
+      originB = token.nextToken();
+    }
+
+    AddFirstPartyStorageAccessGrantedFor(NS_ConvertUTF8toUTF16(originB.IsEmpty() ? origin : originB),
+                                         false /* no overwrite */);
+  }
+}
+
 IntlUtils*
 nsGlobalWindowInner::GetIntlUtils(ErrorResult& aError)
 {

dom/base/nsGlobalWindowInner.h

@@ -717,6 +717,20 @@ public:
   mozilla::dom::IntlUtils*
   GetIntlUtils(mozilla::ErrorResult& aRv);
 
+  void
+  AddFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin, bool aOverwritten = true);
+
+  void
+  GetFirstPartyStorageAccessGrantedOrigins(nsTArray<nsString>& aOrigins);
+
+  bool
+  IsFirstPartyStorageAccessGrantedFor(nsIURI* aURI);
+
+  static void
+  SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(nsIPrincipal* aPrincipal,
+                                                             const nsCString& aParentOrigin,
+                                                             const nsCString& aGrantedOrigin);
+
 public:
   void Alert(nsIPrincipal& aSubjectPrincipal,
              mozilla::ErrorResult& aError);
@@ -1059,6 +1073,15 @@ protected:
                       mozilla::dom::CallerType aCallerType,
                       mozilla::ErrorResult& aError);
 
+  void
+  ReleaseFirstPartyStorageAccessGrantedOrigins();
+
+  void
+  SaveFirstPartyStorageAccessGrantedFor(const nsAString& aOrigin);
+
+  void
+  MaybeRestoreFirstPartyStorageAccessGrantedOrigins();
+
   // Array of idle observers that are notified of idle events.
   nsTObserverArray<IdleObserverHolder> mIdleObservers;
 
@@ -1106,6 +1129,9 @@ protected:
   // Get the parent, returns null if this is a toplevel window
   nsPIDOMWindowOuter* GetParentInternal();
 
+  // Get the parent principal, returns null if this is a toplevel window.
+  nsIPrincipal* GetTopLevelStorageAreaPrincipal();
+
 public:
   // popup tracking
   bool IsPopupSpamWindow();
@@ -1476,6 +1502,13 @@ protected:
 
   nsTArray<mozilla::UniquePtr<PromiseDocumentFlushedResolver>> mDocumentFlushedResolvers;
 
+  struct StorageGrantedOrigin {
+    nsString mOrigin;
+    bool mOverwritten;
+  };
+  nsTArray<StorageGrantedOrigin> mStorageGrantedOrigins;
+  bool mStorageGrantedOriginPopulated;
+
   static InnerWindowByIdTable* sInnerWindowsById;
 
   // Members in the mChromeFields member should only be used in chrome windows.

dom/base/nsIDocument.h

@@ -3682,6 +3682,8 @@ protected:
   // Return the same type parent docuement if exists, or return null.
   nsIDocument* GetSameTypeParentDocument();
 
+  void MaybeAllowStorageForOpener();
+
   // Helpers for GetElementsByName.
   static bool MatchNameAttribute(mozilla::dom::Element* aElement,
                                  int32_t aNamespaceID,

dom/cache/CacheStorage.cpp

@@ -139,19 +139,13 @@ IsTrusted(const PrincipalInfo& aPrincipalInfo, bool aTestingPrefEnabled)
 // static
 already_AddRefed<CacheStorage>
 CacheStorage::CreateOnMainThread(Namespace aNamespace, nsIGlobalObject* aGlobal,
-                                 nsIPrincipal* aPrincipal, bool aStorageDisabled,
+                                 nsIPrincipal* aPrincipal,
                                  bool aForceTrustedOrigin, ErrorResult& aRv)
 {
   MOZ_DIAGNOSTIC_ASSERT(aGlobal);
   MOZ_DIAGNOSTIC_ASSERT(aPrincipal);
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (aStorageDisabled) {
-    NS_WARNING("CacheStorage has been disabled.");
-    RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR);
-    return ref.forget();
-  }
-
   PrincipalInfo principalInfo;
   nsresult rv = PrincipalToPrincipalInfo(aPrincipal, &principalInfo);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -170,7 +164,7 @@ CacheStorage::CreateOnMainThread(Namespace aNamespace, nsIGlobalObject* aGlobal,
   }
 
   RefPtr<CacheStorage> ref = new CacheStorage(aNamespace, aGlobal,
-                                                principalInfo, nullptr);
+                                              principalInfo, nullptr);
   return ref.forget();
 }
 
@@ -183,12 +177,6 @@ CacheStorage::CreateOnWorker(Namespace aNamespace, nsIGlobalObject* aGlobal,
   MOZ_DIAGNOSTIC_ASSERT(aWorkerPrivate);
   aWorkerPrivate->AssertIsOnWorkerThread();
 
-  if (!aWorkerPrivate->IsStorageAllowed()) {
-    NS_WARNING("CacheStorage is not allowed.");
-    RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR);
-    return ref.forget();
-  }
-
   if (aWorkerPrivate->GetOriginAttributes().mPrivateBrowsingId > 0) {
     NS_WARNING("CacheStorage not supported during private browsing.");
     RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR);
@@ -255,7 +243,6 @@ CacheStorage::DefineCaches(JSContext* aCx, JS::Handle<JSObject*> aGlobal)
   ErrorResult rv;
   RefPtr<CacheStorage> storage =
     CreateOnMainThread(DEFAULT_NAMESPACE, xpc::NativeGlobal(aGlobal), principal,
-                       false, /* private browsing */
                        true,  /* force trusted */
                        rv);
   if (NS_WARN_IF(rv.MaybeSetPendingException(aCx))) {
@@ -319,6 +306,11 @@ CacheStorage::Match(JSContext* aCx, const RequestOrUSVString& aRequest,
 {
   NS_ASSERT_OWNINGTHREAD(CacheStorage);
 
+  if (!HasStorageAccess()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
   if (NS_WARN_IF(NS_FAILED(mStatus))) {
     aRv.Throw(mStatus);
     return nullptr;
@@ -353,6 +345,11 @@ CacheStorage::Has(const nsAString& aKey, ErrorResult& aRv)
 {
   NS_ASSERT_OWNINGTHREAD(CacheStorage);
 
+  if (!HasStorageAccess()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
   if (NS_WARN_IF(NS_FAILED(mStatus))) {
     aRv.Throw(mStatus);
     return nullptr;
@@ -377,6 +374,11 @@ CacheStorage::Open(const nsAString& aKey, ErrorResult& aRv)
 {
   NS_ASSERT_OWNINGTHREAD(CacheStorage);
 
+  if (!HasStorageAccess()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
   if (NS_WARN_IF(NS_FAILED(mStatus))) {
     aRv.Throw(mStatus);
     return nullptr;
@@ -401,6 +403,11 @@ CacheStorage::Delete(const nsAString& aKey, ErrorResult& aRv)
 {
   NS_ASSERT_OWNINGTHREAD(CacheStorage);
 
+  if (!HasStorageAccess()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
   if (NS_WARN_IF(NS_FAILED(mStatus))) {
     aRv.Throw(mStatus);
     return nullptr;
@@ -425,6 +432,11 @@ CacheStorage::Keys(ErrorResult& aRv)
 {
   NS_ASSERT_OWNINGTHREAD(CacheStorage);
 
+  if (!HasStorageAccess()) {
+    aRv.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return nullptr;
+  }
+
   if (NS_WARN_IF(NS_FAILED(mStatus))) {
     aRv.Throw(mStatus);
     return nullptr;
@@ -475,9 +487,14 @@ CacheStorage::Constructor(const GlobalObject& aGlobal,
     }
   }
 
+  if (privateBrowsing) {
+    RefPtr<CacheStorage> ref = new CacheStorage(NS_ERROR_DOM_SECURITY_ERR);
+    return ref.forget();
+  }
+
   // Create a CacheStorage object bypassing the trusted origin checks
   // since this is a chrome-only constructor.
-  return CreateOnMainThread(ns, global, aPrincipal, privateBrowsing,
+  return CreateOnMainThread(ns, global, aPrincipal,
                             true /* force trusted origin */, aRv);
 }
 
@@ -569,6 +586,28 @@ CacheStorage::GetOpenMode() const
   return mNamespace == CHROME_ONLY_NAMESPACE ? OpenMode::Eager : OpenMode::Lazy;
 }
 
+bool
+CacheStorage::HasStorageAccess() const
+{
+  NS_ASSERT_OWNINGTHREAD(CacheStorage);
+
+  if (NS_IsMainThread()) {
+    nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(mGlobal);
+    if (NS_WARN_IF(!window)) {
+      return true;
+    }
+
+    nsContentUtils::StorageAccess access =
+      nsContentUtils::StorageAllowedForWindow(window);
+    return access > nsContentUtils::StorageAccess::ePrivateBrowsing;
+  }
+
+  WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate();
+  MOZ_ASSERT(workerPrivate);
+
+  return workerPrivate->IsStorageAllowed();
+}
+
 } // namespace cache
 } // namespace dom
 } // namespace mozilla

dom/cache/CacheStorage.h

@@ -45,8 +45,8 @@ class CacheStorage final : public nsISupports
 public:
   static already_AddRefed<CacheStorage>
   CreateOnMainThread(Namespace aNamespace, nsIGlobalObject* aGlobal,
-                     nsIPrincipal* aPrincipal, bool aStorageDisabled,
-                     bool aForceTrustedOrigin, ErrorResult& aRv);
+                     nsIPrincipal* aPrincipal, bool aForceTrustedOrigin,
+                     ErrorResult& aRv);
 
   static already_AddRefed<CacheStorage>
   CreateOnWorker(Namespace aNamespace, nsIGlobalObject* aGlobal,
@@ -98,6 +98,9 @@ private:
   OpenMode
   GetOpenMode() const;
 
+  bool
+  HasStorageAccess() const;
+
   const Namespace mNamespace;
   nsCOMPtr<nsIGlobalObject> mGlobal;
   UniquePtr<mozilla::ipc::PrincipalInfo> mPrincipalInfo;

dom/html/nsHTMLDocument.cpp

@@ -3434,3 +3434,9 @@ nsHTMLDocument::GetFormsAndFormControls(nsContentList** aFormList,
   NS_ADDREF(*aFormList = holder->mFormList);
   NS_ADDREF(*aFormControlList = holder->mFormControlList);
 }
+
+void
+nsHTMLDocument::UserInteractionForTesting()
+{
+  NotifyUserGestureActivation();
+}

dom/html/nsHTMLDocument.h

@@ -225,6 +225,9 @@ public:
 
   void GetFormsAndFormControls(nsContentList** aFormList,
                                nsContentList** aFormControlList);
+
+  void UserInteractionForTesting();
+
 protected:
   ~nsHTMLDocument();
 

dom/ipc/ContentChild.cpp

@@ -2663,6 +2663,20 @@ ContentChild::RecvAddPermission(const IPC::Permission& permission)
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+ContentChild::RecvRemoveAllPermissions()
+{
+  nsCOMPtr<nsIPermissionManager> permissionManagerIface =
+    services::GetPermissionManager();
+  nsPermissionManager* permissionManager =
+    static_cast<nsPermissionManager*>(permissionManagerIface.get());
+  MOZ_ASSERT(permissionManager,
+         "We have no permissionManager in the Content process !");
+
+  permissionManager->RemoveAllFromIPC();
+  return IPC_OK();
+}
+
 mozilla::ipc::IPCResult
 ContentChild::RecvFlushMemory(const nsString& reason)
 {
@@ -3265,6 +3279,25 @@ NextWindowID()
   return (processBits << kWindowIDWindowBits) | windowBits;
 }
 
+// This code goes here rather than nsGlobalWindow.cpp because nsGlobalWindow.cpp
+// can't include ContentChild.h since it includes windows.h.
+void
+SendFirstPartyStorageAccessGrantedForOriginToParentProcess(nsIPrincipal* aPrincipal,
+                                                           const nsACString& aParentOrigin,
+                                                           const nsACString& aGrantedOrigin)
+{
+  MOZ_ASSERT(!XRE_IsParentProcess());
+
+  ContentChild* cc = ContentChild::GetSingleton();
+  MOZ_ASSERT(cc);
+
+  // This is not really secure, because here we have the content process sending
+  // the request of storing a permission.
+  Unused << cc->SendFirstPartyStorageAccessGrantedForOrigin(IPC::Principal(aPrincipal),
+                                                            nsCString(aParentOrigin),
+                                                            nsCString(aGrantedOrigin));
+}
+
 mozilla::ipc::IPCResult
 ContentChild::RecvInvokeDragSession(nsTArray<IPCDataTransfer>&& aTransfers,
                                     const uint32_t& aAction)

dom/ipc/ContentChild.h

@@ -411,6 +411,8 @@ public:
 
   virtual mozilla::ipc::IPCResult RecvAddPermission(const IPC::Permission& permission) override;
 
+  virtual mozilla::ipc::IPCResult RecvRemoveAllPermissions() override;
+
   virtual mozilla::ipc::IPCResult RecvFlushMemory(const nsString& reason) override;
 
   virtual mozilla::ipc::IPCResult RecvActivateA11y(const uint32_t& aMainChromeTid,
@@ -846,6 +848,11 @@ private:
 uint64_t
 NextWindowID();
 
+void
+SendFirstPartyStorageAccessGrantedForOriginToParentProcess(nsIPrincipal* aPrincipal,
+                                                           const nsACString& aParentOrigin,
+                                                           const nsACString& aGrantedOrigin);
+
 } // namespace dom
 } // namespace mozilla
 

dom/ipc/ContentParent.cpp

@@ -115,6 +115,7 @@
 #include "nsDebugImpl.h"
 #include "nsFrameLoader.h"
 #include "nsFrameMessageManager.h"
+#include "nsGlobalWindowInner.h"
 #include "nsHashPropertyBag.h"
 #include "nsIAlertsService.h"
 #include "nsIClipboard.h"
@@ -5750,3 +5751,14 @@ ContentParent::RecvBHRThreadHang(const HangDetails& aDetails)
   }
   return IPC_OK();
 }
+
+mozilla::ipc::IPCResult
+ContentParent::RecvFirstPartyStorageAccessGrantedForOrigin(const Principal& aPrincipal,
+                                                           const nsCString& aParentOrigin,
+                                                           const nsCString& aGrantedOrigin)
+{
+  nsGlobalWindowInner::SaveFirstPartyStorageAccessGrantedForOriginOnParentProcess(aPrincipal,
+                                                                                  aParentOrigin,
+                                                                                  aGrantedOrigin);
+  return IPC_OK();
+}

dom/ipc/ContentParent.h

@@ -1224,6 +1224,11 @@ public:
   virtual mozilla::ipc::IPCResult RecvBHRThreadHang(
     const HangDetails& aHangDetails) override;
 
+  virtual mozilla::ipc::IPCResult
+  RecvFirstPartyStorageAccessGrantedForOrigin(const Principal& aPrincipal,
+                                              const nsCString& aParentOrigin,
+                                              const nsCString& aGrantedOrigin) override;
+
   // Notify the ContentChild to enable the input event prioritization when
   // initializing.
   void MaybeEnableRemoteInputEventQueue();

dom/ipc/PContent.ipdl

@@ -468,6 +468,7 @@ child:
 
     // nsIPermissionManager messages
     async AddPermission(Permission permission);
+    async RemoveAllPermissions();
 
     async FlushMemory(nsString reason);
 
@@ -1151,6 +1152,15 @@ parent:
     async BHRThreadHang(HangDetails aHangDetails);
 
     async AddPerformanceMetrics(nsID aID, PerformanceInfo[] aMetrics);
+
+    /*
+     * A 3rd party context (aPrincipal) has received the permission granted to
+     * have access to aGrantedOrigin when loaded by aParentOrigin.
+     */
+    async FirstPartyStorageAccessGrantedForOrigin(Principal aPrincipal,
+                                                  nsCString aParentOrigin,
+                                                  nsCString aGrantedOrigin);
+
 both:
      async AsyncMessage(nsString aMessage, CpowEntry[] aCpows,
                         Principal aPrincipal, ClonedMessageData aData);

dom/security/test/cors/test_CrossSiteXHR.html

@@ -1122,7 +1122,25 @@ function* runTest() {
                     },
                     ],
            },
-           { pass: 0,
+           { pass: 1,
+             method: "POST",
+             body: "hi there",
+             headers: { "Content-Type": "text/plain",
+                        "my-header": "myValue",
+                      },
+             hops: [{ server: "http://example.org",
+                    },
+                    { server: "http://example.com",
+                      allowOrigin: origin,
+                      allowHeaders: "my-header",
+                    },
+                    { server: "http://sub1.test1.example.org",
+                      allowOrigin: "*",
+                      allowHeaders: "my-header",
+                    },
+                    ],
+           },
+           { pass: 1,
              method: "POST",
              body: "hi there",
              headers: { "Content-Type": "text/plain",
@@ -1196,7 +1214,21 @@ function* runTest() {
                     },
                     ],
            },
-           { pass: 0,
+           { pass: 1,
+             method: "DELETE",
+             hops: [{ server: "http://example.org",
+                    },
+                    { server: "http://example.com",
+                      allowOrigin: origin,
+                      allowMethods: "DELETE",
+                    },
+                    { server: "http://sub1.test1.example.org",
+                      allowOrigin: "*",
+                      allowMethods: "DELETE",
+                    },
+                    ],
+           },
+           { pass: 1,
              method: "DELETE",
              hops: [{ server: "http://example.org",
                     },

dom/serviceworkers/ServiceWorkerScriptCache.cpp

@@ -77,7 +77,6 @@ CreateCacheStorage(JSContext* aCx, nsIPrincipal* aPrincipal, ErrorResult& aRv,
   // to revalidate is not available now.
   return CacheStorage::CreateOnMainThread(cache::CHROME_ONLY_NAMESPACE,
                                           sandboxGlobalObject, aPrincipal,
-                                          false /* private browsing */,
                                           true /* force trusted origin */,
                                           aRv);
 }

dom/webidl/HTMLDocument.webidl

@@ -73,4 +73,7 @@ partial interface HTMLDocument {
    */
   [ChromeOnly, Pure]
   readonly attribute NodeList blockedTrackingNodes;
+
+  [ChromeOnly]
+  void userInteractionForTesting();
 };

dom/workers/RuntimeService.cpp

@@ -53,6 +53,7 @@
 #include "mozilla/Preferences.h"
 #include "mozilla/dom/Navigator.h"
 #include "mozilla/Monitor.h"
+#include "mozilla/StaticPrefs.h"
 #include "nsContentUtils.h"
 #include "nsCycleCollector.h"
 #include "nsDOMJSUtils.h"
@@ -2237,6 +2238,21 @@ RuntimeService::ResumeWorkersForWindow(nsPIDOMWindowInner* aWindow)
   }
 }
 
+void
+RuntimeService::PropagateFirstPartyStorageAccessGranted(nsPIDOMWindowInner* aWindow)
+{
+  AssertIsOnMainThread();
+  MOZ_ASSERT(aWindow);
+  MOZ_ASSERT(StaticPrefs::privacy_restrict3rdpartystorage_enabled());
+
+  nsTArray<WorkerPrivate*> workers;
+  GetWorkersForWindow(aWindow, workers);
+
+  for (uint32_t index = 0; index < workers.Length(); index++) {
+    workers[index]->PropagateFirstPartyStorageAccessGranted();
+  }
+}
+
 nsresult
 RuntimeService::CreateSharedWorker(const GlobalObject& aGlobal,
                                    const nsAString& aScriptURL,
@@ -2833,6 +2849,18 @@ ResumeWorkersForWindow(nsPIDOMWindowInner* aWindow)
   }
 }
 
+void
+PropagateFirstPartyStorageAccessGrantedToWorkers(nsPIDOMWindowInner* aWindow)
+{
+  AssertIsOnMainThread();
+  MOZ_ASSERT(StaticPrefs::privacy_restrict3rdpartystorage_enabled());
+
+  RuntimeService* runtime = RuntimeService::GetService();
+  if (runtime) {
+    runtime->PropagateFirstPartyStorageAccessGranted(aWindow);
+  }
+}
+
 WorkerPrivate*
 GetWorkerPrivateFromContext(JSContext* aCx)
 {

dom/workers/RuntimeService.h

@@ -152,6 +152,9 @@ public:
   void
   ResumeWorkersForWindow(nsPIDOMWindowInner* aWindow);
 
+  void
+  PropagateFirstPartyStorageAccessGranted(nsPIDOMWindowInner* aWindow);
+
   nsresult
   CreateSharedWorker(const GlobalObject& aGlobal,
                      const nsAString& aScriptURL,

dom/workers/ScriptLoader.cpp

@@ -1540,7 +1540,6 @@ CacheCreator::CreateCacheStorage(nsIPrincipal* aPrincipal)
     CacheStorage::CreateOnMainThread(mozilla::dom::cache::CHROME_ONLY_NAMESPACE,
                                      mSandboxGlobalObject,
                                      aPrincipal,
-                                     false, /* privateBrowsing can't be true here */
                                      true /* force trusted origin */,
                                      error);
   if (NS_WARN_IF(error.Failed())) {

dom/workers/WorkerCommon.h

@@ -53,6 +53,9 @@ SuspendWorkersForWindow(nsPIDOMWindowInner* aWindow);
 void
 ResumeWorkersForWindow(nsPIDOMWindowInner* aWindow);
 
+void
+PropagateFirstPartyStorageAccessGrantedToWorkers(nsPIDOMWindowInner* aWindow);
+
 // All of these are implemented in WorkerScope.cpp
 
 bool

dom/workers/WorkerLoadInfo.cpp

@@ -94,6 +94,7 @@ WorkerLoadInfo::WorkerLoadInfo()
   , mXHRParamsAllowed(false)
   , mPrincipalIsSystem(false)
   , mStorageAllowed(false)
+  , mFirstPartyStorageAccessGranted(false)
   , mServiceWorkersTestingInWindow(false)
 {
   MOZ_COUNT_CTOR(WorkerLoadInfo);
@@ -154,6 +155,7 @@ WorkerLoadInfo::StealFrom(WorkerLoadInfo& aOther)
   mXHRParamsAllowed = aOther.mXHRParamsAllowed;
   mPrincipalIsSystem = aOther.mPrincipalIsSystem;
   mStorageAllowed = aOther.mStorageAllowed;
+  mFirstPartyStorageAccessGranted = aOther.mFirstPartyStorageAccessGranted;
   mServiceWorkersTestingInWindow = aOther.mServiceWorkersTestingInWindow;
   mOriginAttributes = aOther.mOriginAttributes;
   mParentController = aOther.mParentController;

dom/workers/WorkerLoadInfo.h

@@ -103,6 +103,7 @@ struct WorkerLoadInfo
   bool mXHRParamsAllowed;
   bool mPrincipalIsSystem;
   bool mStorageAllowed;
+  bool mFirstPartyStorageAccessGranted;
   bool mServiceWorkersTestingInWindow;
   OriginAttributes mOriginAttributes;
 

dom/workers/WorkerPrivate.cpp

@@ -569,6 +569,22 @@ private:
   }
 };
 
+class PropagateFirstPartyStorageAccessGrantedRunnable final : public WorkerControlRunnable
+{
+public:
+  explicit PropagateFirstPartyStorageAccessGrantedRunnable(WorkerPrivate* aWorkerPrivate)
+    : WorkerControlRunnable(aWorkerPrivate, WorkerThreadUnchangedBusyCount)
+  {}
+
+private:
+  bool
+  WorkerRun(JSContext* aCx, WorkerPrivate* aWorkerPrivate) override
+  {
+    aWorkerPrivate->PropagateFirstPartyStorageAccessGrantedInternal();
+    return true;
+  }
+};
+
 class ReportErrorToConsoleRunnable final : public WorkerRunnable
 {
   const char* mMessage;
@@ -1992,6 +2008,24 @@ WorkerPrivate::ParentWindowResumed()
   }
 }
 
+void
+WorkerPrivate::PropagateFirstPartyStorageAccessGranted()
+{
+  AssertIsOnParentThread();
+
+  {
+    MutexAutoLock lock(mMutex);
+
+    if (mParentStatus >= Terminating) {
+      return;
+    }
+  }
+
+  RefPtr<PropagateFirstPartyStorageAccessGrantedRunnable> runnable =
+    new PropagateFirstPartyStorageAccessGrantedRunnable(this);
+  Unused << NS_WARN_IF(!runnable->Dispatch());
+}
+
 bool
 WorkerPrivate::Close()
 {
@@ -3943,6 +3977,18 @@ WorkerPrivate::ThawInternal()
   return true;
 }
 
+void
+WorkerPrivate::PropagateFirstPartyStorageAccessGrantedInternal()
+{
+  AssertIsOnWorkerThread();
+
+  mLoadInfo.mFirstPartyStorageAccessGranted = true;
+
+  for (uint32_t index = 0; index < mChildWorkers.Length(); index++) {
+    mChildWorkers[index]->PropagateFirstPartyStorageAccessGranted();
+  }
+}
+
 void
 WorkerPrivate::TraverseTimeouts(nsCycleCollectionTraversalCallback& cb)
 {

dom/workers/WorkerPrivate.h

@@ -268,6 +268,9 @@ public:
   bool
   ThawInternal();
 
+  void
+  PropagateFirstPartyStorageAccessGrantedInternal();
+
   void
   TraverseTimeouts(nsCycleCollectionTraversalCallback& aCallback);
 
@@ -1030,7 +1033,8 @@ public:
   bool
   IsStorageAllowed() const
   {
-    return mLoadInfo.mStorageAllowed;
+    AssertIsOnWorkerThread();
+    return mLoadInfo.mStorageAllowed || mLoadInfo.mFirstPartyStorageAccessGranted;
   }
 
   const OriginAttributes&
@@ -1101,6 +1105,9 @@ public:
   bool
   Thaw(nsPIDOMWindowInner* aWindow);
 
+  void
+  PropagateFirstPartyStorageAccessGranted();
+
   void
   EnableDebugger();
 

extensions/cookie/nsPermissionManager.cpp

@@ -2117,9 +2117,31 @@ nsPermissionManager::CloseDB(bool aRebuildOnSuccess)
   }
 }
 
+nsresult
+nsPermissionManager::RemoveAllFromIPC()
+{
+  MOZ_ASSERT(IsChildProcess());
+
+  // Remove from memory and notify immediately. Since the in-memory
+  // database is authoritative, we do not need confirmation from the
+  // on-disk database to notify observers.
+  RemoveAllFromMemory();
+
+  return NS_OK;
+}
+
 nsresult
 nsPermissionManager::RemoveAllInternal(bool aNotifyObservers)
 {
+  ENSURE_NOT_CHILD_PROCESS;
+
+  // Let's broadcast the removeAll() to any content process.
+  nsTArray<ContentParent*> parents;
+  ContentParent::GetAll(parents);
+  for (ContentParent* parent : parents) {
+    Unused << parent->SendRemoveAllPermissions();
+  }
+
   // Remove from memory and notify immediately. Since the in-memory
   // database is authoritative, we do not need confirmation from the
   // on-disk database to notify observers.
@@ -2535,15 +2557,23 @@ NS_IMETHODIMP nsPermissionManager::GetEnumerator(nsISimpleEnumerator **aEnum)
 
 NS_IMETHODIMP nsPermissionManager::GetAllForURI(nsIURI* aURI, nsISimpleEnumerator **aEnum)
 {
-  nsCOMArray<nsIPermission> array;
-
   nsCOMPtr<nsIPrincipal> principal;
   nsresult rv = GetPrincipal(aURI, getter_AddRefs(principal));
   NS_ENSURE_SUCCESS(rv, rv);
 
-  MOZ_ASSERT(PermissionAvailable(principal, nullptr));
+  return GetAllForPrincipal(principal, aEnum);
+}
 
-  RefPtr<PermissionKey> key = PermissionKey::CreateFromPrincipal(principal, rv);
+NS_IMETHODIMP
+nsPermissionManager::GetAllForPrincipal(nsIPrincipal* aPrincipal,
+                                        nsISimpleEnumerator** aEnum)
+{
+  nsCOMArray<nsIPermission> array;
+
+  MOZ_ASSERT(PermissionAvailable(aPrincipal, nullptr));
+
+  nsresult rv;
+  RefPtr<PermissionKey> key = PermissionKey::CreateFromPrincipal(aPrincipal, rv);
   if (!key) {
     MOZ_ASSERT(NS_FAILED(rv));
     return rv;
@@ -2559,7 +2589,7 @@ NS_IMETHODIMP nsPermissionManager::GetAllForURI(nsIURI* aURI, nsISimpleEnumerato
       }
 
       nsCOMPtr<nsIPermission> permission =
-        nsPermission::Create(principal,
+        nsPermission::Create(aPrincipal,
                              mTypeArray.ElementAt(permEntry.mType),
                              permEntry.mPermission,
                              permEntry.mExpireType,

extensions/cookie/nsPermissionManager.h

@@ -284,6 +284,9 @@ public:
    */
   static nsTArray<nsCString> GetAllKeysForPrincipal(nsIPrincipal* aPrincipal);
 
+  // From ContentChild.
+  nsresult RemoveAllFromIPC();
+
 private:
   virtual ~nsPermissionManager();
 

gfx/layers/apz/src/APZUpdater.cpp

@@ -455,7 +455,7 @@ APZUpdater::RunOnControllerThread(LayersId aLayersId, already_AddRefed<Runnable>
 bool
 APZUpdater::UsingWebRenderUpdaterThread() const
 {
-  return (mIsUsingWebRender && gfxPrefs::WebRenderAsyncSceneBuild());
+  return mIsUsingWebRender;
 }
 
 /*static*/ already_AddRefed<APZUpdater>
@@ -564,8 +564,6 @@ apz_register_updater(mozilla::wr::WrWindowId aWindowId)
 void
 apz_pre_scene_swap(mozilla::wr::WrWindowId aWindowId)
 {
-  // This should never get called unless async scene building is enabled.
-  MOZ_ASSERT(gfxPrefs::WebRenderAsyncSceneBuild());
   mozilla::layers::APZUpdater::PrepareForSceneSwap(aWindowId);
 }
 
@@ -573,8 +571,6 @@ void
 apz_post_scene_swap(mozilla::wr::WrWindowId aWindowId,
                     mozilla::wr::WrPipelineInfo aInfo)
 {
-  // This should never get called unless async scene building is enabled.
-  MOZ_ASSERT(gfxPrefs::WebRenderAsyncSceneBuild());
   mozilla::layers::APZUpdater::CompleteSceneSwap(aWindowId, aInfo);
   wr_pipeline_info_delete(aInfo);
 }
@@ -582,19 +578,12 @@ apz_post_scene_swap(mozilla::wr::WrWindowId aWindowId,
 void
 apz_run_updater(mozilla::wr::WrWindowId aWindowId)
 {
-  // This should never get called unless async scene building is enabled.
-  MOZ_ASSERT(gfxPrefs::WebRenderAsyncSceneBuild());
   mozilla::layers::APZUpdater::ProcessPendingTasks(aWindowId);
 }
 
 void
 apz_deregister_updater(mozilla::wr::WrWindowId aWindowId)
 {
-  // Run anything that's still left. Note that this function gets called even
-  // if async scene building is off, but in that case we don't want to do
-  // anything (because the updater thread will be the compositor thread, and
-  // this will be called on the scene builder thread).
-  if (gfxPrefs::WebRenderAsyncSceneBuild()) {
-    mozilla::layers::APZUpdater::ProcessPendingTasks(aWindowId);
-  }
+  // Run anything that's still left.
+  mozilla::layers::APZUpdater::ProcessPendingTasks(aWindowId);
 }

gfx/layers/wr/WebRenderBridgeParent.cpp

@@ -741,11 +741,8 @@ WebRenderBridgeParent::RecvSetDisplayList(const gfx::IntSize& aSize,
 
     mApi->SendTransaction(txn);
 
-    if (!gfxPrefs::WebRenderAsyncSceneBuild()) {
-      // With async-scene-build enabled, we will trigger this after the scene
-      // build is done, so we don't need to do it here.
-      ScheduleGenerateFrame();
-    }
+    // We will schedule generating a frame after the scene
+    // build is done, so we don't need to do it here.
   }
 
   HoldPendingTransactionId(wrEpoch, aTransactionId, aRefreshStartTime, aTxnStartTime, aFwdTime);
@@ -929,29 +926,25 @@ WebRenderBridgeParent::FlushSceneBuilds()
 {
   MOZ_ASSERT(CompositorThreadHolder::IsInCompositorThread());
 
-  if (gfxPrefs::WebRenderAsyncSceneBuild()) {
-    // If we are sending transactions through the scene builder thread, we need
-    // to block until all the inflight transactions have been processed. This
-    // flush message blocks until all previously sent scenes have been built
-    // and received by the render backend thread.
-    mApi->FlushSceneBuilder();
-    // The post-swap hook for async-scene-building calls the
-    // ScheduleRenderOnCompositorThread function from the scene builder thread,
-    // which then triggers a call to ScheduleGenerateFrame() on the compositor
-    // thread. But since *this* function is running on the compositor thread,
-    // that scheduling will not happen until this call stack unwinds (or we
-    // could spin a nested event loop, but that's more messy). Instead, we
-    // simulate it ourselves by calling ScheduleGenerateFrame() directly.
-    // In the case where async scene building is disabled, the
-    // ScheduleGenerateFrame() call in RecvSetDisplayList() serves this purpose.
-    // Note also that the post-swap hook will run and do another
-    // ScheduleGenerateFrame() after we unwind here, so we will end up with an
-    // extra render/composite that is probably avoidable, but in practice we
-    // shouldn't be calling this function all that much in production so this
-    // is probably fine. If it becomes an issue we can add more state tracking
-    // machinery to optimize it away.
-    ScheduleGenerateFrame();
-  }
+  // Since we are sending transactions through the scene builder thread, we need
+  // to block until all the inflight transactions have been processed. This
+  // flush message blocks until all previously sent scenes have been built
+  // and received by the render backend thread.
+  mApi->FlushSceneBuilder();
+  // The post-swap hook for async-scene-building calls the
+  // ScheduleRenderOnCompositorThread function from the scene builder thread,
+  // which then triggers a call to ScheduleGenerateFrame() on the compositor
+  // thread. But since *this* function is running on the compositor thread,
+  // that scheduling will not happen until this call stack unwinds (or we
+  // could spin a nested event loop, but that's more messy). Instead, we
+  // simulate it ourselves by calling ScheduleGenerateFrame() directly.
+  // Note also that the post-swap hook will run and do another
+  // ScheduleGenerateFrame() after we unwind here, so we will end up with an
+  // extra render/composite that is probably avoidable, but in practice we
+  // shouldn't be calling this function all that much in production so this
+  // is probably fine. If it becomes an issue we can add more state tracking
+  // machinery to optimize it away.
+  ScheduleGenerateFrame();
 }
 
 void

gfx/thebes/gfxPrefs.h

@@ -518,7 +518,6 @@ private:
 
   DECL_GFX_PREF(Once, "gfx.webrender.all",                     WebRenderAll, bool, false);
   DECL_GFX_PREF(Once, "gfx.webrender.all.qualified",           WebRenderAllQualified, bool, false);
-  DECL_GFX_PREF(Once, "gfx.webrender.async-scene-build",       WebRenderAsyncSceneBuild, bool, true);
   DECL_GFX_PREF(Live, "gfx.webrender.blob-images",             WebRenderBlobImages, bool, true);
   DECL_GFX_PREF(Live, "gfx.webrender.blob.invalidation",       WebRenderBlobInvalidation, bool, false);
   DECL_GFX_PREF(Once, "gfx.webrender.enabled",                 WebRenderEnabledDoNotUseDirectly, bool, false);

gfx/webrender_bindings/RenderThread.cpp

@@ -508,8 +508,8 @@ RenderThread::RegisterExternalImage(uint64_t aExternalImageId, already_AddRefed<
   if (mHasShutdown) {
     return;
   }
-  MOZ_ASSERT(!mRenderTextures.GetWeak(aExternalImageId));
-  mRenderTextures.Put(aExternalImageId, std::move(aTexture));
+  MOZ_ASSERT(mRenderTextures.find(aExternalImageId) == mRenderTextures.end());
+  mRenderTextures.emplace(aExternalImageId, std::move(aTexture));
 }
 
 void
@@ -519,7 +519,8 @@ RenderThread::UnregisterExternalImage(uint64_t aExternalImageId)
   if (mHasShutdown) {
     return;
   }
-  MOZ_ASSERT(mRenderTextures.GetWeak(aExternalImageId));
+  auto it = mRenderTextures.find(aExternalImageId);
+  MOZ_ASSERT(it != mRenderTextures.end());
   if (!IsInRenderThread()) {
     // The RenderTextureHost should be released in render thread. So, post the
     // deletion task here.
@@ -528,15 +529,15 @@ RenderThread::UnregisterExternalImage(uint64_t aExternalImageId)
     // deletion. Then the buffer in RenderTextureHost becomes invalid. It's fine
     // for this situation. Gecko will only release the buffer if WR doesn't need
     // it. So, no one will access the invalid buffer in RenderTextureHost.
-    RefPtr<RenderTextureHost> texture;
-    mRenderTextures.Remove(aExternalImageId, getter_AddRefs(texture));
+    RefPtr<RenderTextureHost> texture = it->second;
+    mRenderTextures.erase(it);
     mRenderTexturesDeferred.emplace_back(std::move(texture));
     Loop()->PostTask(NewRunnableMethod(
       "RenderThread::DeferredRenderTextureHostDestroy",
       this, &RenderThread::DeferredRenderTextureHostDestroy
     ));
   } else {
-    mRenderTextures.Remove(aExternalImageId);
+    mRenderTextures.erase(it);
   }
 }
 
@@ -546,8 +547,8 @@ RenderThread::UnregisterExternalImageDuringShutdown(uint64_t aExternalImageId)
   MOZ_ASSERT(IsInRenderThread());
   MutexAutoLock lock(mRenderTextureMapLock);
   MOZ_ASSERT(mHasShutdown);
-  MOZ_ASSERT(mRenderTextures.GetWeak(aExternalImageId));
-  mRenderTextures.Remove(aExternalImageId);
+  MOZ_ASSERT(mRenderTextures.find(aExternalImageId) != mRenderTextures.end());
+  mRenderTextures.erase(aExternalImageId);
 }
 
 void
@@ -563,8 +564,12 @@ RenderThread::GetRenderTexture(wr::WrExternalImageId aExternalImageId)
   MOZ_ASSERT(IsInRenderThread());
 
   MutexAutoLock lock(mRenderTextureMapLock);
-  MOZ_ASSERT(mRenderTextures.GetWeak(aExternalImageId.mHandle));
-  return mRenderTextures.GetWeak(aExternalImageId.mHandle);
+  auto it = mRenderTextures.find(aExternalImageId.mHandle);
+  MOZ_ASSERT(it != mRenderTextures.end());
+  if (it == mRenderTextures.end()) {
+    return nullptr;
+  }
+  return it->second;
 }
 
 void
@@ -592,8 +597,8 @@ RenderThread::HandleDeviceReset(const char* aWhere, bool aNotify)
   {
     MutexAutoLock lock(mRenderTextureMapLock);
     mRenderTexturesDeferred.clear();
-    for (auto iter = mRenderTextures.Iter(); !iter.Done(); iter.Next()) {
-      iter.UserData()->ClearCachedResources();
+    for (const auto& entry : mRenderTextures) {
+      entry.second->ClearCachedResources();
     }
   }
 

gfx/webrender_bindings/RenderThread.h

@@ -207,7 +207,7 @@ private:
   std::unordered_map<uint64_t, WindowInfo*> mWindowInfos;
 
   Mutex mRenderTextureMapLock;
-  nsRefPtrHashtable<nsUint64HashKey, RenderTextureHost> mRenderTextures;
+  std::unordered_map<uint64_t, RefPtr<RenderTextureHost>> mRenderTextures;
   // Used to remove all RenderTextureHost that are going to be removed by
   // a deferred callback and remove them right away without waiting for the callback.
   // On device reset we have to remove all GL related resources right away.

gfx/webrender_bindings/WebRenderAPI.cpp

@@ -135,7 +135,7 @@ private:
 
 
 TransactionBuilder::TransactionBuilder(bool aUseSceneBuilderThread)
-  : mUseSceneBuilderThread(gfxPrefs::WebRenderAsyncSceneBuild() && aUseSceneBuilderThread)
+  : mUseSceneBuilderThread(aUseSceneBuilderThread)
 {
   mTxn = wr_transaction_new(mUseSceneBuilderThread);
 }

image/ImageCacheKey.cpp

@@ -142,9 +142,8 @@ ImageCacheKey::GetSpecialCaseDocumentToken(nsIDocument* aDocument, nsIURI* aURI)
   // If this document has been marked as tracker, let's use its address to make
   // a unique cache key.
   if (!pointer && aDocument &&
-      nsContentUtils::StorageDisabledByAntiTracking(nullptr,
-                                                    aDocument->GetChannel(),
-                                                    aURI)) {
+      nsContentUtils::StorageDisabledByAntiTracking(aDocument->GetInnerWindow(),
+                                                    nullptr, aURI)) {
     pointer = aDocument;
   }
 

intl/icu-patches/bug-1466471-case-conversion-bugfix.diff

@@ -0,0 +1,41 @@
+https://ssl.icu-project.org/trac/ticket/13851
+
+Index: source/common/ucase.cpp
+===================================================================
+--- a/intl/icu/source/common/ucase.cpp	(Revision 41549)
++++ b/intl/icu/source/common/ucase.cpp	(Revision 41550)
+@@ -270,6 +270,7 @@
+             }
+         }
+         if(HAS_SLOT(excWord, UCASE_EXC_DELTA)) {
++            pe=pe0;
+             int32_t delta;
+             GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+             sa->add(sa->set, (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta);
+@@ -1167,7 +1168,7 @@
+ 
+         if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+             int32_t delta;
+-            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
++            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+             return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+         }
+         if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
+@@ -1261,7 +1262,7 @@
+ 
+         if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+             int32_t delta;
+-            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
++            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+             return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+         }
+         if(!upperNotTitle && HAS_SLOT(excWord, UCASE_EXC_TITLE)) {
+@@ -1469,7 +1470,7 @@
+         }
+         if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+             int32_t delta;
+-            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
++            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+             return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+         }
+         if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) {

intl/icu-patches/suppress-warnings.diff

@@ -1,11 +1,11 @@
 diff --git a/intl/icu/source/acinclude.m4 b/intl/icu/source/acinclude.m4
 --- a/intl/icu/source/acinclude.m4
 +++ b/intl/icu/source/acinclude.m4
-@@ -473,30 +473,36 @@ AC_DEFUN([AC_CHECK_STRICT_COMPILE],
+@@ -469,30 +469,36 @@ AC_DEFUN([AC_CHECK_STRICT_COMPILE],
              *)
                  # Do not use -ansi. It limits us to C90, and it breaks some platforms.
-                 # We use -std=c99 to disable the gnu99 defaults and its associated warnings
-                 CFLAGS="$CFLAGS -std=c99"
+                 # We use -std=c11 to disable the gnu99 defaults and its associated warnings
+                 CFLAGS="$CFLAGS -std=c11"
                  ;;
              esac
              
@@ -41,11 +41,11 @@ diff --git a/intl/icu/source/acinclude.m4 b/intl/icu/source/acinclude.m4
 diff --git a/intl/icu/source/configure b/intl/icu/source/configure
 --- a/intl/icu/source/configure
 +++ b/intl/icu/source/configure
-@@ -4319,30 +4319,36 @@ fi
+@@ -4359,30 +4359,36 @@ fi
              *)
                  # Do not use -ansi. It limits us to C90, and it breaks some platforms.
-                 # We use -std=c99 to disable the gnu99 defaults and its associated warnings
-                 CFLAGS="$CFLAGS -std=c99"
+                 # We use -std=c11 to disable the gnu99 defaults and its associated warnings
+                 CFLAGS="$CFLAGS -std=c11"
                  ;;
              esac
  

intl/icu/SVN-INFO

@@ -1,10 +1,10 @@
 Path: icu4c
-URL: https://ssl.icu-project.org/repos/icu/tags/release-61-1/icu4c
-Relative URL: ^/tags/release-61-1/icu4c
+URL: https://ssl.icu-project.org/repos/icu/tags/release-62-1/icu4c
+Relative URL: ^/tags/release-62-1/icu4c
 Repository Root: https://ssl.icu-project.org/repos/icu
 Repository UUID: 251d0590-4201-4cf1-90de-194747b24ca1
 Node Kind: directory
-Last Changed Author: heninger
-Last Changed Rev: 41146
-Last Changed Date: 2018-03-23 22:14:04 +0000 (Fri, 23 Mar 2018)
+Last Changed Author: yoshito
+Last Changed Rev: 41542
+Last Changed Date: 2018-06-20 05:34:56 +0000 (Wed, 20 Jun 2018)
 

intl/icu/source/acinclude.m4

@@ -464,12 +464,12 @@ AC_DEFUN([AC_CHECK_STRICT_COMPILE],
         then
             case "${host}" in
             *-*-solaris*)
-                # Don't use -std=c99 on Solaris because of timezone check fails
+                # Don't use -std=c11 on Solaris because of timezone check fails
                 ;;
             *)
                 # Do not use -ansi. It limits us to C90, and it breaks some platforms.
-                # We use -std=c99 to disable the gnu99 defaults and its associated warnings
-                CFLAGS="$CFLAGS -std=c99"
+                # We use -std=c11 to disable the gnu99 defaults and its associated warnings
+                CFLAGS="$CFLAGS -std=c11"
                 ;;
             esac
             

intl/icu/source/allinone/allinone.sln

@@ -116,9 +116,9 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "toolutil", "..\tools\toolut
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "uconv", "..\extra\uconv\uconv.vcxproj", "{DBA4088D-F6F9-4F8F-8820-082A4765C16C}"
 	ProjectSection(ProjectDependencies) = postProject
+		{97521D06-EC47-45D4-8BD0-9E16B3F93B2A} = {97521D06-EC47-45D4-8BD0-9E16B3F93B2A}
 		{0178B127-6269-407D-B112-93877BB62776} = {0178B127-6269-407D-B112-93877BB62776}
 		{73C0A65B-D1F2-4DE1-B3A6-15DAD2C23F3D} = {73C0A65B-D1F2-4DE1-B3A6-15DAD2C23F3D}
-		{D3065ADB-8820-4CC7-9B6C-9510833961A3} = {D3065ADB-8820-4CC7-9B6C-9510833961A3}
 		{4C8454FE-81D3-4CA3-9927-29BA96F03DAC} = {4C8454FE-81D3-4CA3-9927-29BA96F03DAC}
 	EndProjectSection
 EndProject

intl/icu/source/common/Makefile.in

@@ -111,7 +111,8 @@ util.o util_props.o parsepos.o locbased.o cwchar.o wintz.o dtintrv.o ucnvsel.o p
 ulist.o uloc_tag.o icudataver.o icuplug.o listformatter.o ulistformatter.o \
 sharedobject.o simpleformatter.o unifiedcache.o uloc_keytype.o \
 ubiditransform.o \
-pluralmap.o
+pluralmap.o \
+static_unicode_sets.o
 
 ## Header files to install
 HEADERS = $(srcdir)/unicode/*.h

intl/icu/source/common/charstr.cpp

@@ -23,6 +23,18 @@
 
 U_NAMESPACE_BEGIN
 
+CharString::CharString(CharString&& src) U_NOEXCEPT
+        : buffer(std::move(src.buffer)), len(src.len) {
+    src.len = 0;  // not strictly necessary because we make no guarantees on the source string
+}
+
+CharString& CharString::operator=(CharString&& src) U_NOEXCEPT {
+    buffer = std::move(src.buffer);
+    len = src.len;
+    src.len = 0;  // not strictly necessary because we make no guarantees on the source string
+    return *this;
+}
+
 CharString &CharString::copyFrom(const CharString &s, UErrorCode &errorCode) {
     if(U_SUCCESS(errorCode) && this!=&s && ensureCapacity(s.len+1, 0, errorCode)) {
         len=s.len;

intl/icu/source/common/charstr.h

@@ -55,6 +55,18 @@ public:
     }
     ~CharString() {}
 
+    /**
+     * Move constructor; might leave src in an undefined state.
+     * This string will have the same contents and state that the source string had.
+     */
+    CharString(CharString &&src) U_NOEXCEPT;
+    /**
+     * Move assignment operator; might leave src in an undefined state.
+     * This string will have the same contents and state that the source string had.
+     * The behavior is undefined if *this and src are the same object.
+     */
+    CharString &operator=(CharString &&src) U_NOEXCEPT;
+
     /**
      * Replaces this string's contents with the other string's contents.
      * CharString does not support the standard copy constructor nor

intl/icu/source/common/cmemory.h

@@ -299,6 +299,14 @@ public:
      * Destructor deletes the array (if owned).
      */
     ~MaybeStackArray() { releaseArray(); }
+    /**
+     * Move constructor: transfers ownership or copies the stack array.
+     */
+    MaybeStackArray(MaybeStackArray<T, stackCapacity> &&src) U_NOEXCEPT;
+    /**
+     * Move assignment: transfers ownership or copies the stack array.
+     */
+    MaybeStackArray<T, stackCapacity> &operator=(MaybeStackArray<T, stackCapacity> &&src) U_NOEXCEPT;
     /**
      * Returns the array capacity (number of T items).
      * @return array capacity
@@ -376,6 +384,11 @@ private:
             uprv_free(ptr);
         }
     }
+    void resetToStackArray() {
+        ptr=stackArray;
+        capacity=stackCapacity;
+        needToRelease=FALSE;
+    }
     /* No comparison operators with other MaybeStackArray's. */
     bool operator==(const MaybeStackArray & /*other*/) {return FALSE;}
     bool operator!=(const MaybeStackArray & /*other*/) {return TRUE;}
@@ -398,6 +411,34 @@ private:
 #endif
 };
 
+template<typename T, int32_t stackCapacity>
+icu::MaybeStackArray<T, stackCapacity>::MaybeStackArray(
+        MaybeStackArray <T, stackCapacity>&& src) U_NOEXCEPT
+        : ptr(src.ptr), capacity(src.capacity), needToRelease(src.needToRelease) {
+    if (src.ptr == src.stackArray) {
+        ptr = stackArray;
+        uprv_memcpy(stackArray, src.stackArray, sizeof(T) * src.capacity);
+    } else {
+        src.resetToStackArray();  // take ownership away from src
+    }
+}
+
+template<typename T, int32_t stackCapacity>
+inline MaybeStackArray <T, stackCapacity>&
+MaybeStackArray<T, stackCapacity>::operator=(MaybeStackArray <T, stackCapacity>&& src) U_NOEXCEPT {
+    releaseArray();  // in case this instance had its own memory allocated
+    capacity = src.capacity;
+    needToRelease = src.needToRelease;
+    if (src.ptr == src.stackArray) {
+        ptr = stackArray;
+        uprv_memcpy(stackArray, src.stackArray, sizeof(T) * src.capacity);
+    } else {
+        ptr = src.ptr;
+        src.resetToStackArray();  // take ownership away from src
+    }
+    return *this;
+}
+
 template<typename T, int32_t stackCapacity>
 inline T *MaybeStackArray<T, stackCapacity>::resize(int32_t newCapacity, int32_t length) {
     if(newCapacity>0) {
@@ -447,9 +488,7 @@ inline T *MaybeStackArray<T, stackCapacity>::orphanOrClone(int32_t length, int32
         uprv_memcpy(p, ptr, (size_t)length*sizeof(T));
     }
     resultCapacity=length;
-    ptr=stackArray;
-    capacity=stackCapacity;
-    needToRelease=FALSE;
+    resetToStackArray();
     return p;
 }
 

intl/icu/source/common/common.vcxproj

@@ -85,7 +85,7 @@
       <ProgramDataBaseFileName>.\x86\Release/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin\icuuc61.dll</OutputFile>
+      <OutputFile>..\..\bin\icuuc62.dll</OutputFile>
       <AdditionalLibraryDirectories>.\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <ProgramDatabaseFile>.\..\..\lib\icuuc.pdb</ProgramDatabaseFile>
       <DataExecutionPrevention>
@@ -105,7 +105,7 @@
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin\icuuc61d.dll</OutputFile>
+      <OutputFile>..\..\bin\icuuc62d.dll</OutputFile>
       <AdditionalLibraryDirectories>.\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <ProgramDatabaseFile>.\..\..\lib\icuucd.pdb</ProgramDatabaseFile>
       <DataExecutionPrevention>
@@ -124,7 +124,7 @@
       <ProgramDataBaseFileName>.\x64\Release/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin64\icuuc61.dll</OutputFile>
+      <OutputFile>..\..\bin64\icuuc62.dll</OutputFile>
       <AdditionalLibraryDirectories>.\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <ProgramDatabaseFile>.\..\..\lib64\icuuc.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib64\icuuc.lib</ImportLibrary>
@@ -142,7 +142,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin64\icuuc61d.dll</OutputFile>
+      <OutputFile>..\..\bin64\icuuc62d.dll</OutputFile>
       <AdditionalLibraryDirectories>.\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <ProgramDatabaseFile>.\..\..\lib64\icuucd.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib64\icuucd.lib</ImportLibrary>
@@ -334,6 +334,7 @@
     <ClCompile Include="utf_impl.cpp" />
     <ClCompile Include="listformatter.cpp" />
     <ClCompile Include="ulistformatter.cpp" />
+    <ClCompile Include="static_unicode_sets.cpp" />
     <ClInclude Include="localsvc.h" />
     <ClInclude Include="msvcres.h" />
     <ClInclude Include="pluralmap.h" />
@@ -439,6 +440,7 @@
     <ClInclude Include="uinvchar.h" />
     <ClInclude Include="ustr_cnv.h" />
     <ClInclude Include="ustr_imp.h" />
+    <ClInclude Include="static_unicode_sets.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="common.rc" />
@@ -448,4 +450,4 @@
   <Import Project="$(SolutionDir)\Windows.CopyUnicodeHeaderFiles.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

intl/icu/source/common/common.vcxproj.filters

@@ -607,6 +607,9 @@
     <ClCompile Include="ubiditransform.cpp">
       <Filter>bidi</Filter>
     </ClCompile>
+    <ClCompile Include="static_unicode_sets.cpp">
+      <Filter>formatting</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="ubidi_props.h">
@@ -936,6 +939,9 @@
     <ClInclude Include="unicode\ubiditransform.h">
       <Filter>bidi</Filter>
     </ClInclude>
+    <ClInclude Include="static_unicode_sets.h">
+      <Filter>formatting</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="common.rc">

intl/icu/source/common/common_uwp.vcxproj

@@ -184,7 +184,7 @@
       <ProgramDataBaseFileName>.\x86\ReleaseUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin32uwp\icuuc61.dll</OutputFile>
+      <OutputFile>..\..\bin32uwp\icuuc62.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\lib32uwp\icuuc.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib32uwp\icuuc.lib</ImportLibrary>
     </Link>
@@ -200,7 +200,7 @@
       <ProgramDataBaseFileName>.\x86\DebugUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin32uwp\icuuc61d.dll</OutputFile>
+      <OutputFile>..\..\bin32uwp\icuuc62d.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\lib32uwp\icuucd.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib32uwp\icuucd.lib</ImportLibrary>
     </Link>
@@ -216,7 +216,7 @@
       <ProgramDataBaseFileName>.\x64\ReleaseUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin64uwp\icuuc61.dll</OutputFile>
+      <OutputFile>..\..\bin64uwp\icuuc62.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\lib64uwp\icuuc.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib64uwp\icuuc.lib</ImportLibrary>
     </Link>
@@ -232,7 +232,7 @@
       <ProgramDataBaseFileName>.\x64\DebugUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\bin64uwp\icuuc61d.dll</OutputFile>
+      <OutputFile>..\..\bin64uwp\icuuc62d.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\lib64uwp\icuucd.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\lib64uwp\icuucd.lib</ImportLibrary>
     </Link>
@@ -248,7 +248,7 @@
       <ProgramDataBaseFileName>.\ARM\ReleaseUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\binARMuwp\icuuc61.dll</OutputFile>
+      <OutputFile>..\..\binARMuwp\icuuc62.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\libARMuwp\icuuc.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\libARMuwp\icuuc.lib</ImportLibrary>
     </Link>
@@ -264,7 +264,7 @@
       <ProgramDataBaseFileName>.\ARM\DebugUWP/</ProgramDataBaseFileName>
     </ClCompile>
     <Link>
-      <OutputFile>..\..\binARMuwp\icuuc61d.dll</OutputFile>
+      <OutputFile>..\..\binARMuwp\icuuc62d.dll</OutputFile>
       <ProgramDatabaseFile>.\..\..\libARMuwp\icuucd.pdb</ProgramDatabaseFile>
       <ImportLibrary>..\..\libARMuwp\icuucd.lib</ImportLibrary>
     </Link>
@@ -459,6 +459,7 @@
     <ClCompile Include="utf_impl.cpp" />
     <ClCompile Include="listformatter.cpp" />
     <ClCompile Include="ulistformatter.cpp" />
+    <ClCompile Include="static_unicode_sets.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="localsvc.h" />
@@ -565,6 +566,7 @@
     <ClInclude Include="uinvchar.h" />
     <ClInclude Include="ustr_cnv.h" />
     <ClInclude Include="ustr_imp.h" />
+    <ClInclude Include="static_unicode_sets.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="common.rc" />
@@ -572,4 +574,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

intl/icu/source/common/edits.cpp

@@ -4,10 +4,12 @@
 // edits.cpp
 // created: 2017feb08 Markus W. Scherer
 
-#include "unicode/utypes.h"
 #include "unicode/edits.h"
+#include "unicode/unistr.h"
+#include "unicode/utypes.h"
 #include "cmemory.h"
 #include "uassert.h"
+#include "util.h"
 
 U_NAMESPACE_BEGIN
 
@@ -773,4 +775,29 @@ int32_t Edits::Iterator::sourceIndexFromDestinationIndex(int32_t i, UErrorCode &
     }
 }
 
+UnicodeString& Edits::Iterator::toString(UnicodeString& sb) const {
+    sb.append(u"{ src[", -1);
+    ICU_Utility::appendNumber(sb, srcIndex);
+    sb.append(u"..", -1);
+    ICU_Utility::appendNumber(sb, srcIndex + oldLength_);
+    if (changed) {
+        sb.append(u"] ⇝ dest[", -1);
+    } else {
+        sb.append(u"] ≡ dest[", -1);
+    }
+    ICU_Utility::appendNumber(sb, destIndex);
+    sb.append(u"..", -1);
+    ICU_Utility::appendNumber(sb, destIndex + newLength_);
+    if (changed) {
+        sb.append(u"], repl[", -1);
+        ICU_Utility::appendNumber(sb, replIndex);
+        sb.append(u"..", -1);
+        ICU_Utility::appendNumber(sb, replIndex + newLength_);
+        sb.append(u"] }", -1);
+    } else {
+        sb.append(u"] (no-change) }", -1);
+    }
+    return sb;
+}
+
 U_NAMESPACE_END

intl/icu/source/common/locmap.cpp

@@ -1015,7 +1015,7 @@ static const char*
 getPosixID(const ILcidPosixMap *this_0, uint32_t hostID)
 {
     uint32_t i;
-    for (i = 0; i <= this_0->numRegions; i++)
+    for (i = 0; i < this_0->numRegions; i++)
     {
         if (this_0->regionMaps[i].hostID == hostID)
         {

intl/icu/source/common/putil.cpp

@@ -533,6 +533,28 @@ uprv_fmin(double x, double y)
     return (x > y ? y : x);
 }
 
+U_CAPI UBool U_EXPORT2
+uprv_add32_overflow(int32_t a, int32_t b, int32_t* res) {
+    // NOTE: Some compilers (GCC, Clang) have primitives available, like __builtin_add_overflow.
+    // This function could be optimized by calling one of those primitives.
+    auto a64 = static_cast<int64_t>(a);
+    auto b64 = static_cast<int64_t>(b);
+    int64_t res64 = a64 + b64;
+    *res = static_cast<int32_t>(res64);
+    return res64 != *res;
+}
+
+U_CAPI UBool U_EXPORT2
+uprv_mul32_overflow(int32_t a, int32_t b, int32_t* res) {
+    // NOTE: Some compilers (GCC, Clang) have primitives available, like __builtin_mul_overflow.
+    // This function could be optimized by calling one of those primitives.
+    auto a64 = static_cast<int64_t>(a);
+    auto b64 = static_cast<int64_t>(b);
+    int64_t res64 = a64 * b64;
+    *res = static_cast<int32_t>(res64);
+    return res64 != *res;
+}
+
 /**
  * Truncates the given double.
  * trunc(3.3) = 3.0, trunc (-3.3) = -3.0

intl/icu/source/common/putilimp.h

@@ -391,6 +391,32 @@ U_INTERNAL double  U_EXPORT2 uprv_log(double d);
  */
 U_INTERNAL double  U_EXPORT2 uprv_round(double x);
 
+/**
+ * Adds the signed integers a and b, storing the result in res.
+ * Checks for signed integer overflow.
+ * Similar to the GCC/Clang extension __builtin_add_overflow
+ *
+ * @param a The first operand.
+ * @param b The second operand.
+ * @param res a + b
+ * @return true if overflow occurred; false if no overflow occurred.
+ * @internal
+ */
+U_INTERNAL UBool U_EXPORT2 uprv_add32_overflow(int32_t a, int32_t b, int32_t* res);
+
+/**
+ * Multiplies the signed integers a and b, storing the result in res.
+ * Checks for signed integer overflow.
+ * Similar to the GCC/Clang extension __builtin_mul_overflow
+ *
+ * @param a The first multiplicand.
+ * @param b The second multiplicand.
+ * @param res a * b
+ * @return true if overflow occurred; false if no overflow occurred.
+ * @internal
+ */
+U_INTERNAL UBool U_EXPORT2 uprv_mul32_overflow(int32_t a, int32_t b, int32_t* res);
+
 #if 0
 /**
  * Returns the number of digits after the decimal point in a double number x.

intl/icu/source/common/rbbi.cpp

@@ -651,7 +651,7 @@ UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
     }
 
     // Adjust offset to be on a code point boundary and not beyond the end of the text.
-    // Note that isBoundary() is always be false for offsets that are not on code point boundaries.
+    // Note that isBoundary() is always false for offsets that are not on code point boundaries.
     // But we still need the side effect of leaving iteration at the following boundary.
 
     utext_setNativeIndex(&fText, offset);
@@ -937,26 +937,23 @@ int32_t RuleBasedBreakIterator::handleNext() {
 }
 
 
-
 //-----------------------------------------------------------------------------------
 //
-//  handlePrevious()
+//  handleSafePrevious()
 //
 //      Iterate backwards using the safe reverse rules.
-//      The logic of this function is very similar to handleNext(), above.
+//      The logic of this function is similar to handleNext(), but simpler
+//      because the safe table does not require as many options.
 //
 //-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
+int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
     int32_t             state;
     uint16_t            category        = 0;
-    RBBIRunMode         mode;
     RBBIStateTableRow  *row;
     UChar32             c;
-    LookAheadResults    lookAheadMatches;
     int32_t             result          = 0;
-    int32_t             initialPosition = 0;
 
-    const RBBIStateTable *stateTable = fData->fSafeRevTable;
+    const RBBIStateTable *stateTable = fData->fReverseTable;
     UTEXT_SETNATIVEINDEX(&fText, fromPosition);
     #ifdef RBBI_DEBUG
         if (gTrace) {
@@ -969,54 +966,24 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
         return BreakIterator::DONE;
     }
 
-    //  Set up the starting char.
-    initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
-    result          = initialPosition;
-    c               = UTEXT_PREVIOUS32(&fText);
-
     //  Set the initial state for the state machine
+    c = UTEXT_PREVIOUS32(&fText);
     state = START_STATE;
     row = (RBBIStateTableRow *)
             (stateTable->fTableData + (stateTable->fRowLen * state));
-    category = 3;
-    mode     = RBBI_RUN;
-    if (stateTable->fFlags & RBBI_BOF_REQUIRED) {
-        category = 2;
-        mode     = RBBI_START;
-    }
-
 
     // loop until we reach the start of the text or transition to state 0
     //
-    for (;;) {
-        if (c == U_SENTINEL) {
-            // Reached end of input string.
-            if (mode == RBBI_END) {
-                // We have already run the loop one last time with the
-                //   character set to the psueudo {eof} value.  Now it is time
-                //   to unconditionally bail out.
-                break;
-            }
-            // Run the loop one last time with the fake end-of-input character category.
-            mode = RBBI_END;
-            category = 1;
-        }
+    for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) {
 
+        // look up the current character's character category, which tells us
+        // which column in the state table to look at.
+        // Note:  the 16 in UTRIE_GET16 refers to the size of the data being returned,
+        //        not the size of the character going in, which is a UChar32.
         //
-        // Get the char category.  An incoming category of 1 or 2 means that
-        //      we are preset for doing the beginning or end of input, and
-        //      that we shouldn't get a category from an actual text input character.
-        //
-        if (mode == RBBI_RUN) {
-            // look up the current character's character category, which tells us
-            // which column in the state table to look at.
-            // Note:  the 16 in UTRIE_GET16 refers to the size of the data being returned,
-            //        not the size of the character going in, which is a UChar32.
-            //
-            //  And off the dictionary flag bit. For reverse iteration it is not used.
-            category = UTRIE2_GET16(fData->fTrie, c);
-            category &= ~0x4000;
-        }
+        //  And off the dictionary flag bit. For reverse iteration it is not used.
+        category = UTRIE2_GET16(fData->fTrie, c);
+        category &= ~0x4000;
 
         #ifdef RBBI_DEBUG
             if (gTrace) {
@@ -1032,65 +999,21 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
 
         // State Transition - move machine to its next state
         //
-
         // fNextState is a variable-length array.
         U_ASSERT(category<fData->fHeader->fCatCount);
         state = row->fNextState[category];  /*Not accessing beyond memory*/
         row = (RBBIStateTableRow *)
             (stateTable->fTableData + (stateTable->fRowLen * state));
 
-        if (row->fAccepting == -1) {
-            // Match found, common case.
-            result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
-        }
-
-        int16_t completedRule = row->fAccepting;
-        if (completedRule > 0) {
-            // Lookahead match is completed.
-            int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule);
-            if (lookaheadResult >= 0) {
-                UTEXT_SETNATIVEINDEX(&fText, lookaheadResult);
-                return lookaheadResult;
-            }
-        }
-        int16_t rule = row->fLookAhead;
-        if (rule != 0) {
-            // At the position of a '/' in a look-ahead match. Record it.
-            int32_t  pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
-            lookAheadMatches.setPosition(rule, pos);
-        }
-
         if (state == STOP_STATE) {
             // This is the normal exit from the lookup state machine.
-            // We have advanced through the string until it is certain that no
-            //   longer match is possible, no matter what characters follow.
+            // Transistion to state zero means we have found a safe point.
             break;
         }
-
-        // Move (backwards) to the next character to process.
-        // If this is a beginning-of-input loop iteration, don't advance
-        //    the input position.  The next iteration will be processing the
-        //    first real input character.
-        if (mode == RBBI_RUN) {
-            c = UTEXT_PREVIOUS32(&fText);
-        } else {
-            if (mode == RBBI_START) {
-                mode = RBBI_RUN;
-            }
-        }
     }
 
     // The state machine is done.  Check whether it found a match...
-
-    // If the iterator failed to advance in the match engine, force it ahead by one.
-    //   (This really indicates a defect in the break rules.  They should always match
-    //    at least one character.)
-    if (result == initialPosition) {
-        UTEXT_SETNATIVEINDEX(&fText, initialPosition);
-        UTEXT_PREVIOUS32(&fText);
-        result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
-    }
-
+    result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
     #ifdef RBBI_DEBUG
         if (gTrace) {
             RBBIDebugPrintf("result = %d\n\n", result);
@@ -1099,7 +1022,6 @@ int32_t RuleBasedBreakIterator::handlePrevious(int32_t fromPosition) {
     return result;
 }
 
-
 //-------------------------------------------------------------------------------
 //
 //   getRuleStatus()   Return the break rule tag associated with the current

intl/icu/source/common/rbbi_cache.cpp

@@ -206,7 +206,7 @@ void RuleBasedBreakIterator::DictionaryCache::populateDictionary(int32_t startPo
  *   BreakCache implemetation
  */
 
-RuleBasedBreakIterator::BreakCache::BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status) : 
+RuleBasedBreakIterator::BreakCache::BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status) :
         fBI(bi), fSideBuffer(status) {
     reset();
 }
@@ -299,7 +299,7 @@ void RuleBasedBreakIterator::BreakCache::previous(UErrorCode &status) {
     fBI->fPosition = fTextIdx;
     fBI->fRuleStatusIndex = fStatuses[fBufIdx];
     return;
-}    
+}
 
 
 UBool RuleBasedBreakIterator::BreakCache::seek(int32_t pos) {
@@ -317,7 +317,7 @@ UBool RuleBasedBreakIterator::BreakCache::seek(int32_t pos) {
         fTextIdx = fBoundaries[fBufIdx];
         return TRUE;
     }
-    
+
     int32_t min = fStartBufIdx;
     int32_t max = fEndBufIdx;
     while (min != max) {
@@ -354,16 +354,33 @@ UBool RuleBasedBreakIterator::BreakCache::populateNear(int32_t position, UErrorC
     if ((position < fBoundaries[fStartBufIdx] - 15) || position > (fBoundaries[fEndBufIdx] + 15)) {
         int32_t aBoundary = 0;
         int32_t ruleStatusIndex = 0;
-        // TODO: check for position == length of text. Although may still need to back up to get rule status.
         if (position > 20) {
-            int32_t backupPos = fBI->handlePrevious(position);
-            fBI->fPosition = backupPos;
-            aBoundary = fBI->handleNext();                // Ignore dictionary, just finding a rule based boundary.
-            ruleStatusIndex = fBI->fRuleStatusIndex;
+            int32_t backupPos = fBI->handleSafePrevious(position);
+
+            if (backupPos > 0) {
+                // Advance to the boundary following the backup position.
+                // There is a complication: the safe reverse rules identify pairs of code points
+                // that are safe. If advancing from the safe point moves forwards by less than
+                // two code points, we need to advance one more time to ensure that the boundary
+                // is good, including a correct rules status value.
+                //
+                fBI->fPosition = backupPos;
+                aBoundary = fBI->handleNext();
+                if (aBoundary <= backupPos + 4) {
+                    // +4 is a quick test for possibly having advanced only one codepoint.
+                    // Four being the length of the longest potential code point, a supplementary in UTF-8
+                    utext_setNativeIndex(&fBI->fText, aBoundary);
+                    if (backupPos == utext_getPreviousNativeIndex(&fBI->fText)) {
+                        // The initial handleNext() only advanced by a single code point. Go again.
+                        aBoundary = fBI->handleNext();   // Safe rules identify safe pairs.
+                    }
+                }
+                ruleStatusIndex = fBI->fRuleStatusIndex;
+            }
         }
-        reset(aBoundary, ruleStatusIndex);               // Reset cache to hold aBoundary as a single starting point.
+        reset(aBoundary, ruleStatusIndex);        // Reset cache to hold aBoundary as a single starting point.
     }
-    
+
     // Fill in boundaries between existing cache content and the new requested position.
 
     if (fBoundaries[fEndBufIdx] < position) {
@@ -485,16 +502,30 @@ UBool RuleBasedBreakIterator::BreakCache::populatePreceding(UErrorCode &status)
         if (backupPosition <= 0) {
             backupPosition = 0;
         } else {
-            backupPosition = fBI->handlePrevious(backupPosition);
+            backupPosition = fBI->handleSafePrevious(backupPosition);
         }
         if (backupPosition == UBRK_DONE || backupPosition == 0) {
             position = 0;
             positionStatusIdx = 0;
         } else {
-            fBI->fPosition = backupPosition;  // TODO: pass starting position in a clearer way.
+            // Advance to the boundary following the backup position.
+            // There is a complication: the safe reverse rules identify pairs of code points
+            // that are safe. If advancing from the safe point moves forwards by less than
+            // two code points, we need to advance one more time to ensure that the boundary
+            // is good, including a correct rules status value.
+            //
+            fBI->fPosition = backupPosition;
             position = fBI->handleNext();
+            if (position <= backupPosition + 4) {
+                // +4 is a quick test for possibly having advanced only one codepoint.
+                // Four being the length of the longest potential code point, a supplementary in UTF-8
+                utext_setNativeIndex(&fBI->fText, position);
+                if (backupPosition == utext_getPreviousNativeIndex(&fBI->fText)) {
+                    // The initial handleNext() only advanced by a single code point. Go again.
+                    position = fBI->handleNext();   // Safe rules identify safe pairs.
+                }
+            };
             positionStatusIdx = fBI->fRuleStatusIndex;
-
         }
     } while (position >= fromPosition);
 
@@ -533,7 +564,7 @@ UBool RuleBasedBreakIterator::BreakCache::populatePreceding(UErrorCode &status)
             }
             U_ASSERT(position==dictSegEndPosition || position>=fromPosition);
         }
-            
+
         if (!segmentHandledByDictionary && position < fromPosition) {
             fSideBuffer.addElement(position, status);
             fSideBuffer.addElement(positionStatusIdx, status);
@@ -559,7 +590,7 @@ UBool RuleBasedBreakIterator::BreakCache::populatePreceding(UErrorCode &status)
             break;
         }
     }
-      
+
     return success;
 }
 

intl/icu/source/common/rbbidata.cpp

@@ -81,8 +81,6 @@ void RBBIDataWrapper::init0() {
     fHeader = NULL;
     fForwardTable = NULL;
     fReverseTable = NULL;
-    fSafeFwdTable = NULL;
-    fSafeRevTable = NULL;
     fRuleSource   = NULL;
     fRuleStatusTable = NULL;
     fTrie         = NULL;
@@ -111,21 +109,6 @@ void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
     if (data->fRTableLen != 0) {
         fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable);
     }
-    if (data->fSFTableLen != 0) {
-        fSafeFwdTable = (RBBIStateTable *)((char *)data + fHeader->fSFTable);
-    }
-    if (data->fSRTableLen != 0) {
-        fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable);
-    }
-
-    // Rule Compatibility Hacks
-    //    If a rule set includes reverse rules but does not explicitly include safe reverse rules,
-    //    the reverse rules are to be treated as safe reverse rules.
-
-    if (fSafeRevTable == NULL && fReverseTable != NULL) {
-        fSafeRevTable = fReverseTable;
-        fReverseTable = NULL;
-    }
 
     fTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
                                       (uint8_t *)data + fHeader->fTrie,
@@ -277,8 +260,6 @@ void  RBBIDataWrapper::printData() {
 
     printTable("Forward State Transition Table", fForwardTable);
     printTable("Reverse State Transition Table", fReverseTable);
-    printTable("Safe Forward State Transition Table", fSafeFwdTable);
-    printTable("Safe Reverse State Transition Table", fSafeRevTable);
 
     RBBIDebugPrintf("\nOrignal Rules source:\n");
     for (int32_t c=0; fRuleSource[c] != 0; c++) {
@@ -418,28 +399,6 @@ ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outD
                             outBytes+tableStartOffset+topSize, status);
     }
 
-    // Safe Forward state table.  Same layout as forward table, above.
-    tableStartOffset = ds->readUInt32(rbbiDH->fSFTable);
-    tableLength      = ds->readUInt32(rbbiDH->fSFTableLen);
-
-    if (tableLength > 0) {
-        ds->swapArray32(ds, inBytes+tableStartOffset, topSize, 
-                            outBytes+tableStartOffset, status);
-        ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
-                            outBytes+tableStartOffset+topSize, status);
-    }
-
-    // Safe Reverse state table.  Same layout as forward table, above.
-    tableStartOffset = ds->readUInt32(rbbiDH->fSRTable);
-    tableLength      = ds->readUInt32(rbbiDH->fSRTableLen);
-
-    if (tableLength > 0) {
-        ds->swapArray32(ds, inBytes+tableStartOffset, topSize, 
-                            outBytes+tableStartOffset, status);
-        ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
-                            outBytes+tableStartOffset+topSize, status);
-    }
-
     // Trie table for character categories
     utrie2_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
                     outBytes+ds->readUInt32(rbbiDH->fTrie), status);

intl/icu/source/common/rbbidata.h

@@ -58,7 +58,7 @@ ubrk_swap(const UDataSwapper *ds,
 U_NAMESPACE_BEGIN
 
 // The current RBBI data format version.
-static const uint8_t RBBI_DATA_FORMAT_VERSION[] = {4, 0, 0, 0};
+static const uint8_t RBBI_DATA_FORMAT_VERSION[] = {5, 0, 0, 0};
 
 /*  
  *   The following structs map exactly onto the raw data from ICU common data file. 
@@ -81,10 +81,6 @@ struct RBBIDataHeader {
     uint32_t         fFTableLen;
     uint32_t         fRTable;         /*  Offset to the reverse state transition table. */
     uint32_t         fRTableLen;
-    uint32_t         fSFTable;        /*  safe point forward transition table */
-    uint32_t         fSFTableLen;
-    uint32_t         fSRTable;        /*  safe point reverse transition table */
-    uint32_t         fSRTableLen;
     uint32_t         fTrie;           /*  Offset to Trie data for character categories */
     uint32_t         fTrieLen;
     uint32_t         fRuleSource;     /*  Offset to the source for for the break */
@@ -174,8 +170,6 @@ public:
     const RBBIDataHeader     *fHeader;
     const RBBIStateTable     *fForwardTable;
     const RBBIStateTable     *fReverseTable;
-    const RBBIStateTable     *fSafeFwdTable;
-    const RBBIStateTable     *fSafeRevTable;
     const UChar              *fRuleSource;
     const int32_t            *fRuleStatusTable; 
 

intl/icu/source/common/rbbirb.cpp

@@ -62,10 +62,7 @@ RBBIRuleBuilder::RBBIRuleBuilder(const UnicodeString   &rules,
     fSafeFwdTree        = NULL;
     fSafeRevTree        = NULL;
     fDefaultTree        = &fForwardTree;
-    fForwardTables      = NULL;
-    fReverseTables      = NULL;
-    fSafeFwdTables      = NULL;
-    fSafeRevTables      = NULL;
+    fForwardTable       = NULL;
     fRuleStatusVals     = NULL;
     fChainRules         = FALSE;
     fLBCMNoChain        = FALSE;
@@ -114,11 +111,7 @@ RBBIRuleBuilder::~RBBIRuleBuilder() {
 
     delete fUSetNodes;
     delete fSetBuilder;
-    delete fForwardTables;
-    delete fReverseTables;
-    delete fSafeFwdTables;
-    delete fSafeRevTables;
-
+    delete fForwardTable;
     delete fForwardTree;
     delete fReverseTree;
     delete fSafeFwdTree;
@@ -157,21 +150,15 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
     //     without the padding.
     //
     int32_t headerSize        = align8(sizeof(RBBIDataHeader));
-    int32_t forwardTableSize  = align8(fForwardTables->getTableSize());
-    int32_t reverseTableSize  = align8(fReverseTables->getTableSize());
-    int32_t safeFwdTableSize  = align8(fSafeFwdTables->getTableSize());
-    int32_t safeRevTableSize  = align8(fSafeRevTables->getTableSize());
+    int32_t forwardTableSize  = align8(fForwardTable->getTableSize());
+    int32_t reverseTableSize  = align8(fForwardTable->getSafeTableSize());
     int32_t trieSize          = align8(fSetBuilder->getTrieSize());
     int32_t statusTableSize   = align8(fRuleStatusVals->size() * sizeof(int32_t));
     int32_t rulesSize         = align8((fStrippedRules.length()+1) * sizeof(UChar));
 
-    (void)safeFwdTableSize;
-
     int32_t         totalSize = headerSize
-                                + forwardTableSize 
-                                + /* reverseTableSize */ 0
-                                + /* safeFwdTableSize */ 0
-                                + (safeRevTableSize ? safeRevTableSize : reverseTableSize)
+                                + forwardTableSize
+                                + reverseTableSize
                                 + statusTableSize + trieSize + rulesSize;
 
     RBBIDataHeader  *data     = (RBBIDataHeader *)uprv_malloc(totalSize);
@@ -190,38 +177,13 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
     data->fLength           = totalSize;
     data->fCatCount         = fSetBuilder->getNumCharCategories();
 
-    // Only save the forward table and the safe reverse table,
-    // because these are the only ones used at run-time.
-    //
-    // For the moment, we still build the other tables if they are present in the rule source files,
-    // for backwards compatibility. Old rule files need to work, and this is the simplest approach.
-    //
-    // Additional backwards compatibility consideration: if no safe rules are provided, consider the
-    // reverse rules to actually be the safe reverse rules.
-
     data->fFTable        = headerSize;
     data->fFTableLen     = forwardTableSize;
 
-    // Do not save Reverse Table.
-    data->fRTable        = data->fFTable  + forwardTableSize;
-    data->fRTableLen     = 0;
+    data->fRTable        = data->fFTable  + data->fFTableLen;
+    data->fRTableLen     = reverseTableSize;
 
-    // Do not save the Safe Forward table.
-    data->fSFTable       = data->fRTable + 0;
-    data->fSFTableLen    = 0;
-
-    data->fSRTable       = data->fSFTable + 0;
-    if (safeRevTableSize > 0) {
-        data->fSRTableLen    = safeRevTableSize;
-    } else if (reverseTableSize > 0) {
-        data->fSRTableLen    = reverseTableSize;
-    } else {
-        U_ASSERT(FALSE);    // Rule build should have failed for lack of a reverse table
-                            // before reaching this point.
-    }
-        
-
-    data->fTrie          = data->fSRTable + data->fSRTableLen;
+    data->fTrie          = data->fRTable + data->fRTableLen;
     data->fTrieLen       = fSetBuilder->getTrieSize();
     data->fStatusTable   = data->fTrie    + trieSize;
     data->fStatusTableLen= statusTableSize;
@@ -230,15 +192,8 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
 
     uprv_memset(data->fReserved, 0, sizeof(data->fReserved));
 
-    fForwardTables->exportTable((uint8_t *)data + data->fFTable);
-    // fReverseTables->exportTable((uint8_t *)data + data->fRTable);
-    // fSafeFwdTables->exportTable((uint8_t *)data + data->fSFTable);
-    if (safeRevTableSize > 0) {
-        fSafeRevTables->exportTable((uint8_t *)data + data->fSRTable);
-    } else {
-        fReverseTables->exportTable((uint8_t *)data + data->fSRTable);
-    }
-
+    fForwardTable->exportTable((uint8_t *)data + data->fFTable);
+    fForwardTable->exportSafeTable((uint8_t *)data + data->fRTable);
     fSetBuilder->serializeTrie ((uint8_t *)data + data->fTrie);
 
     int32_t *ruleStatusTable = (int32_t *)((uint8_t *)data + data->fStatusTable);
@@ -252,10 +207,6 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
 }
 
 
-
-
-
-
 //----------------------------------------------------------------------------------------
 //
 //  createRuleBasedBreakIterator    construct from source rules that are passed in
@@ -267,8 +218,6 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString    &rules,
                                     UParseError      *parseError,
                                     UErrorCode       &status)
 {
-    // status checked below
-
     //
     // Read the input rules, generate a parse tree, symbol table,
     // and list of all Unicode Sets referenced by the rules.
@@ -277,66 +226,13 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString    &rules,
     if (U_FAILURE(status)) { // status checked here bcos build below doesn't
         return NULL;
     }
-    builder.fScanner->parse();
 
-    //
-    // UnicodeSet processing.
-    //    Munge the Unicode Sets to create a set of character categories.
-    //    Generate the mapping tables (TRIE) from input code points to
-    //    the character categories.
-    //
-    builder.fSetBuilder->buildRanges();
+    RBBIDataHeader *data = builder.build(status);
 
-
-    //
-    //   Generate the DFA state transition table.
-    //
-    builder.fForwardTables = new RBBITableBuilder(&builder, &builder.fForwardTree);
-    builder.fReverseTables = new RBBITableBuilder(&builder, &builder.fReverseTree);
-    builder.fSafeFwdTables = new RBBITableBuilder(&builder, &builder.fSafeFwdTree);
-    builder.fSafeRevTables = new RBBITableBuilder(&builder, &builder.fSafeRevTree);
-    if (builder.fForwardTables == NULL || builder.fReverseTables == NULL ||
-        builder.fSafeFwdTables == NULL || builder.fSafeRevTables == NULL)
-    {
-        status = U_MEMORY_ALLOCATION_ERROR;
-        delete builder.fForwardTables; builder.fForwardTables = NULL;
-        delete builder.fReverseTables; builder.fReverseTables = NULL;
-        delete builder.fSafeFwdTables; builder.fSafeFwdTables = NULL;
-        delete builder.fSafeRevTables; builder.fSafeRevTables = NULL;
-        return NULL;
+    if (U_FAILURE(status)) {
+        return nullptr;
     }
 
-    builder.fForwardTables->build();
-    builder.fReverseTables->build();
-    builder.fSafeFwdTables->build();
-    builder.fSafeRevTables->build();
-
-#ifdef RBBI_DEBUG
-    if (builder.fDebugEnv && uprv_strstr(builder.fDebugEnv, "states")) {
-        builder.fForwardTables->printRuleStatusTable();
-    }
-#endif
-
-    builder.optimizeTables();
-    builder.fSetBuilder->buildTrie();
-
-
-
-    //
-    //   Package up the compiled data into a memory image
-    //      in the run-time format.
-    //
-    RBBIDataHeader *data = builder.flattenData(); // returns NULL if error
-    if (U_FAILURE(*builder.fStatus)) {
-        return NULL;
-    }
-
-
-    //
-    //  Clean up the compiler related stuff
-    //
-
-
     //
     //  Create a break iterator from the compiled rules.
     //     (Identical to creation from stored pre-compiled rules)
@@ -353,27 +249,71 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString    &rules,
     return This;
 }
 
-void RBBIRuleBuilder::optimizeTables() {
-    int32_t leftClass;
-    int32_t rightClass;
-
-    leftClass = 3;
-    rightClass = 0;
-    while (fForwardTables->findDuplCharClassFrom(leftClass, rightClass)) {
-        fSetBuilder->mergeCategories(leftClass, rightClass);
-        fForwardTables->removeColumn(rightClass);
-        fReverseTables->removeColumn(rightClass);
-        fSafeFwdTables->removeColumn(rightClass);
-        fSafeRevTables->removeColumn(rightClass);
+RBBIDataHeader *RBBIRuleBuilder::build(UErrorCode &status) {
+    if (U_FAILURE(status)) {
+        return nullptr;
     }
 
-    fForwardTables->removeDuplicateStates();
-    fReverseTables->removeDuplicateStates();
-    fSafeFwdTables->removeDuplicateStates();
-    fSafeRevTables->removeDuplicateStates();
+    fScanner->parse();
+    if (U_FAILURE(status)) {
+        return nullptr;
+    }
+
+    //
+    // UnicodeSet processing.
+    //    Munge the Unicode Sets to create a set of character categories.
+    //    Generate the mapping tables (TRIE) from input code points to
+    //    the character categories.
+    //
+    fSetBuilder->buildRanges();
+
+    //
+    //   Generate the DFA state transition table.
+    //
+    fForwardTable = new RBBITableBuilder(this, &fForwardTree, status);
+    if (fForwardTable == nullptr) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        return nullptr;
+    }
+
+    fForwardTable->buildForwardTable();
+    optimizeTables();
+    fForwardTable->buildSafeReverseTable(status);
 
 
+#ifdef RBBI_DEBUG
+    if (fDebugEnv && uprv_strstr(fDebugEnv, "states")) {
+        fForwardTable->printStates();
+        fForwardTable->printRuleStatusTable();
+        fForwardTable->printReverseTable();
+    }
+#endif
 
+    fSetBuilder->buildTrie();
+
+    //
+    //   Package up the compiled data into a memory image
+    //      in the run-time format.
+    //
+    RBBIDataHeader *data = flattenData(); // returns NULL if error
+    if (U_FAILURE(status)) {
+        return nullptr;
+    }
+    return data;
+}
+
+void RBBIRuleBuilder::optimizeTables() {
+
+    // Begin looking for duplicates with char class 3.
+    // Classes 0, 1 and 2 are special; they are unused, {bof} and {eof} respectively,
+    // and should not have other categories merged into them.
+    IntPair duplPair = {3, 0};
+
+    while (fForwardTable->findDuplCharClassFrom(&duplPair)) {
+        fSetBuilder->mergeCategories(duplPair);
+        fForwardTable->removeColumn(duplPair.second);
+    }
+    fForwardTable->removeDuplicateStates();
 }
 
 U_NAMESPACE_END

intl/icu/source/common/rbbirb.h

@@ -18,6 +18,8 @@
 
 #if !UCONFIG_NO_BREAK_ITERATION
 
+#include <utility>
+
 #include "unicode/uobject.h"
 #include "unicode/rbbi.h"
 #include "unicode/uniset.h"
@@ -25,8 +27,7 @@
 #include "uhash.h"
 #include "uvector.h"
 #include "unicode/symtable.h"// For UnicodeSet parsing, is the interface that
-                          //    looks up references to $variables within a set.
-
+                             //    looks up references to $variables within a set.
 
 
 U_NAMESPACE_BEGIN
@@ -123,10 +124,16 @@ public:
     RBBIRuleBuilder(const UnicodeString  &rules,
                     UParseError          *parseErr,
                     UErrorCode           &status
-        );
+    );
 
     virtual    ~RBBIRuleBuilder();
 
+    /**
+     *  Build the state tables and char class Trie from the source rules.
+     */
+    RBBIDataHeader  *build(UErrorCode &status);
+
+
     /**
      * Fold together redundant character classes (table columns) and
      * redundant states (table rows). Done after initial table generation,
@@ -162,10 +169,7 @@ public:
     RBBISetBuilder                *fSetBuilder;      // Set and Character Category builder.
     UVector                       *fUSetNodes;       // Vector of all uset nodes.
 
-    RBBITableBuilder              *fForwardTables;   // State transition tables
-    RBBITableBuilder              *fReverseTables;
-    RBBITableBuilder              *fSafeFwdTables;
-    RBBITableBuilder              *fSafeRevTables;
+    RBBITableBuilder              *fForwardTable;    // State transition table, build time form.
 
     UVector                       *fRuleStatusVals;  // The values that can be returned
                                                      //   from getRuleStatus().
@@ -200,6 +204,11 @@ struct RBBISetTableEl {
     RBBINode      *val;
 };
 
+/**
+ *   A pair of ints, used to bundle pairs of states or pairs of character classes.
+ */
+typedef std::pair<int32_t, int32_t> IntPair;
+
 
 //----------------------------------------------------------------------------
 //

intl/icu/source/common/rbbiscan.cpp

@@ -372,7 +372,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
         //  (forward, reverse, safe_forward, safe_reverse)
         //  OR this rule into the appropriate group of them.
         //
-        RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree);
+        RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree);
 
         if (*destRules != NULL) {
             // This is not the first rule encounted.
@@ -1122,22 +1122,6 @@ void RBBIRuleScanner::parse() {
         return;
     }
 
-    //
-    // If there were NO user specified reverse rules, set up the equivalent of ".*;"
-    //
-    if (fRB->fReverseTree == NULL) {
-        fRB->fReverseTree  = pushNewNode(RBBINode::opStar);
-        RBBINode  *operand = pushNewNode(RBBINode::setRef);
-        if (U_FAILURE(*fRB->fStatus)) {
-            return;
-        }
-        findSetFor(UnicodeString(TRUE, kAny, 3), operand);
-        fRB->fReverseTree->fLeftChild = operand;
-        operand->fParent              = fRB->fReverseTree;
-        fNodeStackPtr -= 2;
-    }
-
-
     //
     // Parsing of the input RBBI rules is complete.
     // We now have a parse tree for the rule expressions

intl/icu/source/common/rbbisetb.cpp

@@ -270,15 +270,15 @@ void RBBISetBuilder::buildTrie() {
 }
 
 
-void RBBISetBuilder::mergeCategories(int32_t left, int32_t right) {
-    U_ASSERT(left >= 1);
-    U_ASSERT(right > left);
+void RBBISetBuilder::mergeCategories(IntPair categories) {
+    U_ASSERT(categories.first >= 1);
+    U_ASSERT(categories.second > categories.first);
     for (RangeDescriptor *rd = fRangeList; rd != nullptr; rd = rd->fNext) {
         int32_t rangeNum = rd->fNum & ~DICT_BIT;
         int32_t rangeDict = rd->fNum & DICT_BIT;
-        if (rangeNum == right) {
-            rd->fNum = left | rangeDict;
-        } else if (rangeNum > right) {
+        if (rangeNum == categories.second) {
+            rd->fNum = categories.first | rangeDict;
+        } else if (rangeNum > categories.second) {
             rd->fNum--;
         }
     }

intl/icu/source/common/rbbisetb.h

@@ -94,10 +94,12 @@ public:
     UChar32  getFirstChar(int32_t  val) const;
     UBool    sawBOF() const;                 // Indicate whether any references to the {bof} pseudo
                                              //   character were encountered.
-    /** merge two character categories that have been identified as having equivalent behavior.
-     *  The ranges belonging to the right category (table column) will be added to the left.
+    /**
+     * Merge two character categories that have been identified as having equivalent behavior.
+     * The ranges belonging to the second category (table column) will be added to the first.
+     * @param categories the pair of categories to be merged.
      */
-    void     mergeCategories(int32_t left, int32_t right);
+    void     mergeCategories(IntPair categories);
 
     static constexpr int32_t DICT_BIT = 0x4000;
 

intl/icu/source/common/rbbitblb.cpp

@@ -27,21 +27,19 @@
 
 U_NAMESPACE_BEGIN
 
-RBBITableBuilder::RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode) :
- fTree(*rootNode) {
-    fRB                 = rb;
-    fStatus             = fRB->fStatus;
-    UErrorCode status   = U_ZERO_ERROR;
-    fDStates            = new UVector(status);
-    if (U_FAILURE(*fStatus)) {
-        return;
-    }
+RBBITableBuilder::RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode, UErrorCode &status) :
+        fRB(rb),
+        fTree(*rootNode),
+        fStatus(&status),
+        fDStates(nullptr),
+        fSafeTable(nullptr) {
     if (U_FAILURE(status)) {
-        *fStatus = status;
         return;
     }
-    if (fDStates == NULL) {
-        *fStatus = U_MEMORY_ALLOCATION_ERROR;;
+    // fDStates is UVector<RBBIStateDescriptor *>
+    fDStates = new UVector(status);
+    if (U_SUCCESS(status) && fDStates == nullptr ) {
+        status = U_MEMORY_ALLOCATION_ERROR;
     }
 }
 
@@ -52,17 +50,18 @@ RBBITableBuilder::~RBBITableBuilder() {
     for (i=0; i<fDStates->size(); i++) {
         delete (RBBIStateDescriptor *)fDStates->elementAt(i);
     }
-    delete   fDStates;
+    delete fDStates;
+    delete fSafeTable;
 }
 
 
 //-----------------------------------------------------------------------------
 //
-//   RBBITableBuilder::build  -  This is the main function for building the DFA state transtion
-//                               table from the RBBI rules parse tree.
+//   RBBITableBuilder::buildForwardTable  -  This is the main function for building
+//                               the DFA state transition table from the RBBI rules parse tree.
 //
 //-----------------------------------------------------------------------------
-void  RBBITableBuilder::build() {
+void  RBBITableBuilder::buildForwardTable() {
 
     if (U_FAILURE(*fStatus)) {
         return;
@@ -189,8 +188,6 @@ void  RBBITableBuilder::build() {
     //    for all tables.  Merge the ones from this table into the global set.
     //
     mergeRuleStatusVals();
-
-    if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "states")) {printStates();};
 }
 
 
@@ -1081,18 +1078,18 @@ void RBBITableBuilder::printPosSets(RBBINode *n) {
 //
 //    findDuplCharClassFrom()
 //
-bool RBBITableBuilder::findDuplCharClassFrom(int32_t &baseCategory, int32_t &duplCategory) {
+bool RBBITableBuilder::findDuplCharClassFrom(IntPair *categories) {
     int32_t numStates = fDStates->size();
     int32_t numCols = fRB->fSetBuilder->getNumCharCategories();
 
     uint16_t table_base;
     uint16_t table_dupl;
-    for (; baseCategory < numCols-1; ++baseCategory) {
-        for (duplCategory=baseCategory+1; duplCategory < numCols; ++duplCategory) {
+    for (; categories->first < numCols-1; categories->first++) {
+        for (categories->second=categories->first+1; categories->second < numCols; categories->second++) {
              for (int32_t state=0; state<numStates; state++) {
                  RBBIStateDescriptor *sd = (RBBIStateDescriptor *)fDStates->elementAt(state);
-                 table_base = (uint16_t)sd->fDtran->elementAti(baseCategory);
-                 table_dupl = (uint16_t)sd->fDtran->elementAti(duplCategory);
+                 table_base = (uint16_t)sd->fDtran->elementAti(categories->first);
+                 table_dupl = (uint16_t)sd->fDtran->elementAti(categories->second);
                  if (table_base != table_dupl) {
                      break;
                  }
@@ -1121,14 +1118,14 @@ void RBBITableBuilder::removeColumn(int32_t column) {
 /*
  * findDuplicateState
  */
-bool RBBITableBuilder::findDuplicateState(int32_t &firstState, int32_t &duplState) {
+bool RBBITableBuilder::findDuplicateState(IntPair *states) {
     int32_t numStates = fDStates->size();
     int32_t numCols = fRB->fSetBuilder->getNumCharCategories();
 
-    for (; firstState<numStates-1; ++firstState) {
-        RBBIStateDescriptor *firstSD = (RBBIStateDescriptor *)fDStates->elementAt(firstState);
-        for (duplState=firstState+1; duplState<numStates; ++duplState) {
-            RBBIStateDescriptor *duplSD = (RBBIStateDescriptor *)fDStates->elementAt(duplState);
+    for (; states->first<numStates-1; states->first++) {
+        RBBIStateDescriptor *firstSD = (RBBIStateDescriptor *)fDStates->elementAt(states->first);
+        for (states->second=states->first+1; states->second<numStates; states->second++) {
+            RBBIStateDescriptor *duplSD = (RBBIStateDescriptor *)fDStates->elementAt(states->second);
             if (firstSD->fAccepting != duplSD->fAccepting ||
                 firstSD->fLookAhead != duplSD->fLookAhead ||
                 firstSD->fTagsIdx   != duplSD->fTagsIdx) {
@@ -1139,8 +1136,8 @@ bool RBBITableBuilder::findDuplicateState(int32_t &firstState, int32_t &duplStat
                 int32_t firstVal = firstSD->fDtran->elementAti(col);
                 int32_t duplVal = duplSD->fDtran->elementAti(col);
                 if (!((firstVal == duplVal) ||
-                        ((firstVal == firstState || firstVal == duplState) &&
-                        (duplVal  == firstState || duplVal  == duplState)))) {
+                        ((firstVal == states->first || firstVal == states->second) &&
+                        (duplVal  == states->first || duplVal  == states->second)))) {
                     rowsMatch = false;
                     break;
                 }
@@ -1153,7 +1150,38 @@ bool RBBITableBuilder::findDuplicateState(int32_t &firstState, int32_t &duplStat
     return false;
 }
 
-void RBBITableBuilder::removeState(int32_t keepState, int32_t duplState) {
+
+bool RBBITableBuilder::findDuplicateSafeState(IntPair *states) {
+    int32_t numStates = fSafeTable->size();
+
+    for (; states->first<numStates-1; states->first++) {
+        UnicodeString *firstRow = static_cast<UnicodeString *>(fSafeTable->elementAt(states->first));
+        for (states->second=states->first+1; states->second<numStates; states->second++) {
+            UnicodeString *duplRow = static_cast<UnicodeString *>(fSafeTable->elementAt(states->second));
+            bool rowsMatch = true;
+            int32_t numCols = firstRow->length();
+            for (int32_t col=0; col < numCols; ++col) {
+                int32_t firstVal = firstRow->charAt(col);
+                int32_t duplVal = duplRow->charAt(col);
+                if (!((firstVal == duplVal) ||
+                        ((firstVal == states->first || firstVal == states->second) &&
+                        (duplVal  == states->first || duplVal  == states->second)))) {
+                    rowsMatch = false;
+                    break;
+                }
+            }
+            if (rowsMatch) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+
+void RBBITableBuilder::removeState(IntPair duplStates) {
+    const int32_t keepState = duplStates.first;
+    const int32_t duplState = duplStates.second;
     U_ASSERT(keepState < duplState);
     U_ASSERT(duplState < fDStates->size());
 
@@ -1188,19 +1216,44 @@ void RBBITableBuilder::removeState(int32_t keepState, int32_t duplState) {
     }
 }
 
+void RBBITableBuilder::removeSafeState(IntPair duplStates) {
+    const int32_t keepState = duplStates.first;
+    const int32_t duplState = duplStates.second;
+    U_ASSERT(keepState < duplState);
+    U_ASSERT(duplState < fSafeTable->size());
+
+    fSafeTable->removeElementAt(duplState);   // Note that fSafeTable has a deleter function
+                                              // and will auto-delete the removed element.
+    int32_t numStates = fSafeTable->size();
+    for (int32_t state=0; state<numStates; ++state) {
+        UnicodeString *sd = (UnicodeString *)fSafeTable->elementAt(state);
+        int32_t numCols = sd->length();
+        for (int32_t col=0; col<numCols; col++) {
+            int32_t existingVal = sd->charAt(col);
+            int32_t newVal = existingVal;
+            if (existingVal == duplState) {
+                newVal = keepState;
+            } else if (existingVal > duplState) {
+                newVal = existingVal - 1;
+            }
+            sd->setCharAt(col, newVal);
+        }
+    }
+}
+
 
 /*
  * RemoveDuplicateStates
  */
 void RBBITableBuilder::removeDuplicateStates() {
-    int32_t firstState = 3;
-    int32_t duplicateState = 0;
-    while (findDuplicateState(firstState, duplicateState)) {
-        // printf("Removing duplicate states (%d, %d)\n", firstState, duplicateState);
-        removeState(firstState, duplicateState);
+    IntPair dupls = {3, 0};
+    while (findDuplicateState(&dupls)) {
+        // printf("Removing duplicate states (%d, %d)\n", dupls.first, dupls.second);
+        removeState(dupls);
     }
 }
 
+
 //-----------------------------------------------------------------------------
 //
 //   getTableSize()    Calculate the size of the runtime form of this
@@ -1277,6 +1330,185 @@ void RBBITableBuilder::exportTable(void *where) {
 }
 
 
+/**
+ *   Synthesize a safe state table from the main state table.
+ */
+void RBBITableBuilder::buildSafeReverseTable(UErrorCode &status) {
+    // The safe table creation has three steps:
+
+    // 1. Identifiy pairs of character classes that are "safe." Safe means that boundaries
+    // following the pair do not depend on context or state before the pair. To test
+    // whether a pair is safe, run it through the main forward state table, starting
+    // from each state. If the the final state is the same, no matter what the starting state,
+    // the pair is safe.
+    //
+    // 2. Build a state table that recognizes the safe pairs. It's similar to their
+    // forward table, with a column for each input character [class], and a row for
+    // each state. Row 1 is the start state, and row 0 is the stop state. Initially
+    // create an additional state for each input character category; being in
+    // one of these states means that the character has been seen, and is potentially
+    // the first of a pair. In each of these rows, the entry for the second character
+    // of a safe pair is set to the stop state (0), indicating that a match was found.
+    // All other table entries are set to the state corresponding the current input
+    // character, allowing that charcter to be the of a start following pair.
+    //
+    // Because the safe rules are to be run in reverse, moving backwards in the text,
+    // the first and second pair categories are swapped when building the table.
+    //
+    // 3. Compress the table. There are typically many rows (states) that are
+    // equivalent - that have zeroes (match completed) in the same columns -
+    // and can be folded together.
+
+    // Each safe pair is stored as two UChars in the safePair string.
+    UnicodeString safePairs;
+
+    int32_t numCharClasses = fRB->fSetBuilder->getNumCharCategories();
+    int32_t numStates = fDStates->size();
+
+    for (int32_t c1=0; c1<numCharClasses; ++c1) {
+        for (int32_t c2=0; c2 < numCharClasses; ++c2) {
+            int32_t wantedEndState = -1;
+            int32_t endState = 0;
+            for (int32_t startState = 1; startState < numStates; ++startState) {
+                RBBIStateDescriptor *startStateD = static_cast<RBBIStateDescriptor *>(fDStates->elementAt(startState));
+                int32_t s2 = startStateD->fDtran->elementAti(c1);
+                RBBIStateDescriptor *s2StateD = static_cast<RBBIStateDescriptor *>(fDStates->elementAt(s2));
+                endState = s2StateD->fDtran->elementAti(c2);
+                if (wantedEndState < 0) {
+                    wantedEndState = endState;
+                } else {
+                    if (wantedEndState != endState) {
+                        break;
+                    }
+                }
+            }
+            if (wantedEndState == endState) {
+                safePairs.append((char16_t)c1);
+                safePairs.append((char16_t)c2);
+                // printf("(%d, %d) ", c1, c2);
+            }
+        }
+        // printf("\n");
+    }
+
+    // Populate the initial safe table.
+    // The table as a whole is UVector<UnicodeString>
+    // Each row is represented by a UnicodeString, being used as a Vector<int16>.
+    // Row 0 is the stop state.
+    // Row 1 is the start sate.
+    // Row 2 and beyond are other states, initially one per char class, but
+    //   after initial construction, many of the states will be combined, compacting the table.
+    // The String holds the nextState data only. The four leading fields of a row, fAccepting,
+    // fLookAhead, etc. are not needed for the safe table, and are omitted at this stage of building.
+
+    U_ASSERT(fSafeTable == nullptr);
+    fSafeTable = new UVector(uprv_deleteUObject, uhash_compareUnicodeString, numCharClasses + 2, status);
+    for (int32_t row=0; row<numCharClasses + 2; ++row) {
+        fSafeTable->addElement(new UnicodeString(numCharClasses, 0, numCharClasses+4), status);
+    }
+
+    // From the start state, each input char class transitions to the state for that input.
+    UnicodeString &startState = *static_cast<UnicodeString *>(fSafeTable->elementAt(1));
+    for (int32_t charClass=0; charClass < numCharClasses; ++charClass) {
+        // Note: +2 for the start & stop state.
+        startState.setCharAt(charClass, charClass+2);
+    }
+
+    // Initially make every other state table row look like the start state row,
+    for (int32_t row=2; row<numCharClasses+2; ++row) {
+        UnicodeString &rowState = *static_cast<UnicodeString *>(fSafeTable->elementAt(row));
+        rowState = startState;   // UnicodeString assignment, copies contents.
+    }
+
+    // Run through the safe pairs, set the next state to zero when pair has been seen.
+    // Zero being the stop state, meaning we found a safe point.
+    for (int32_t pairIdx=0; pairIdx<safePairs.length(); pairIdx+=2) {
+        int32_t c1 = safePairs.charAt(pairIdx);
+        int32_t c2 = safePairs.charAt(pairIdx + 1);
+
+        UnicodeString &rowState = *static_cast<UnicodeString *>(fSafeTable->elementAt(c2 + 2));
+        rowState.setCharAt(c1, 0);
+    }
+
+    // Remove duplicate or redundant rows from the table.
+    IntPair states = {1, 0};
+    while (findDuplicateSafeState(&states)) {
+        // printf("Removing duplicate safe states (%d, %d)\n", states.first, states.second);
+        removeSafeState(states);
+    }
+}
+
+
+//-----------------------------------------------------------------------------
+//
+//   getSafeTableSize()    Calculate the size of the runtime form of this
+//                         safe state table.
+//
+//-----------------------------------------------------------------------------
+int32_t  RBBITableBuilder::getSafeTableSize() const {
+    int32_t    size = 0;
+    int32_t    numRows;
+    int32_t    numCols;
+    int32_t    rowSize;
+
+    if (fSafeTable == nullptr) {
+        return 0;
+    }
+
+    size    = offsetof(RBBIStateTable, fTableData);    // The header, with no rows to the table.
+
+    numRows = fSafeTable->size();
+    numCols = fRB->fSetBuilder->getNumCharCategories();
+
+    rowSize = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t)*numCols;
+    size   += numRows * rowSize;
+    return size;
+}
+
+
+//-----------------------------------------------------------------------------
+//
+//   exportSafeTable()   export the state transition table in the format required
+//                       by the runtime engine.  getTableSize() bytes of memory
+//                       must be available at the output address "where".
+//
+//-----------------------------------------------------------------------------
+void RBBITableBuilder::exportSafeTable(void *where) {
+    RBBIStateTable    *table = (RBBIStateTable *)where;
+    uint32_t           state;
+    int                col;
+
+    if (U_FAILURE(*fStatus) || fSafeTable == nullptr) {
+        return;
+    }
+
+    int32_t catCount = fRB->fSetBuilder->getNumCharCategories();
+    if (catCount > 0x7fff ||
+            fSafeTable->size() > 0x7fff) {
+        *fStatus = U_BRK_INTERNAL_ERROR;
+        return;
+    }
+
+    table->fRowLen    = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t) * catCount;
+    table->fNumStates = fSafeTable->size();
+    table->fFlags     = 0;
+    table->fReserved  = 0;
+
+    for (state=0; state<table->fNumStates; state++) {
+        UnicodeString *rowString = (UnicodeString *)fSafeTable->elementAt(state);
+        RBBIStateTableRow   *row = (RBBIStateTableRow *)(table->fTableData + state*table->fRowLen);
+        row->fAccepting = 0;
+        row->fLookAhead = 0;
+        row->fTagIdx    = 0;
+        row->fReserved  = 0;
+        for (col=0; col<catCount; col++) {
+            row->fNextState[col] = rowString->charAt(col);
+        }
+    }
+}
+
+
+
 
 //-----------------------------------------------------------------------------
 //
@@ -1331,6 +1563,47 @@ void RBBITableBuilder::printStates() {
 #endif
 
 
+//-----------------------------------------------------------------------------
+//
+//   printSafeTable    Debug Function.  Dump the fully constructed safe table.
+//
+//-----------------------------------------------------------------------------
+#ifdef RBBI_DEBUG
+void RBBITableBuilder::printReverseTable() {
+    int     c;    // input "character"
+    int     n;    // state number
+
+    RBBIDebugPrintf("    Safe Reverse Table \n");
+    if (fSafeTable == nullptr) {
+        RBBIDebugPrintf("   --- nullptr ---\n");
+        return;
+    }
+    RBBIDebugPrintf("state |           i n p u t     s y m b o l s \n");
+    RBBIDebugPrintf("      | Acc  LA    Tag");
+    for (c=0; c<fRB->fSetBuilder->getNumCharCategories(); c++) {
+        RBBIDebugPrintf(" %2d", c);
+    }
+    RBBIDebugPrintf("\n");
+    RBBIDebugPrintf("      |---------------");
+    for (c=0; c<fRB->fSetBuilder->getNumCharCategories(); c++) {
+        RBBIDebugPrintf("---");
+    }
+    RBBIDebugPrintf("\n");
+
+    for (n=0; n<fSafeTable->size(); n++) {
+        UnicodeString *rowString = (UnicodeString *)fSafeTable->elementAt(n);
+        RBBIDebugPrintf("  %3d | " , n);
+        RBBIDebugPrintf("%3d %3d %5d ", 0, 0, 0);  // Accepting, LookAhead, Tags
+        for (c=0; c<fRB->fSetBuilder->getNumCharCategories(); c++) {
+            RBBIDebugPrintf(" %2d", rowString->charAt(c));
+        }
+        RBBIDebugPrintf("\n");
+    }
+    RBBIDebugPrintf("\n\n");
+}
+#endif
+
+
 
 //-----------------------------------------------------------------------------
 //

intl/icu/source/common/rbbitblb.h

@@ -17,6 +17,7 @@
 #include "unicode/utypes.h"
 #include "unicode/uobject.h"
 #include "unicode/rbbi.h"
+#include "rbbirb.h"
 #include "rbbinode.h"
 
 
@@ -37,22 +38,28 @@ class UVector32;
 
 class RBBITableBuilder : public UMemory {
 public:
-    RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode);
+    RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode, UErrorCode &status);
     ~RBBITableBuilder();
 
-    void     build();
-    int32_t  getTableSize() const;      // Return the runtime size in bytes of
-                                        //     the built state table
+    void     buildForwardTable();
+
+    /** Return the runtime size in bytes of the built state table.  */
+    int32_t  getTableSize() const;
 
     /** Fill in the runtime state table. Sufficient memory must exist at the specified location.
      */
     void     exportTable(void *where);
 
-    /** Find duplicate (redundant) character classes, beginning after the specifed
-     *  pair, within this state table. This is an iterator-like function, used to
-     *  identify char classes (state table columns) that can be eliminated.
+    /**
+     *  Find duplicate (redundant) character classes. Begin looking with categories.first.
+     *  Duplicate, if found are returned in the categories parameter.
+     *  This is an iterator-like function, used to identify character classes
+     *  (state table columns) that can be eliminated.
+     *  @param categories in/out parameter, specifies where to start looking for duplicates,
+     *                and returns the first pair of duplicates found, if any.
+     *  @return true if duplicate char classes were found, false otherwise.
      */
-    bool     findDuplCharClassFrom(int &baseClass, int &duplClass);
+    bool     findDuplCharClassFrom(IntPair *categories);
 
     /** Remove a column from the state table. Used when two character categories
      *  have been found equivalent, and merged together, to eliminate the uneeded table column.
@@ -62,6 +69,16 @@ public:
     /** Check for, and remove dupicate states (table rows). */
     void     removeDuplicateStates();
 
+    /** Build the safe reverse table from the already-constructed forward table. */
+    void     buildSafeReverseTable(UErrorCode &status);
+
+    /** Return the runtime size in bytes of the built safe reverse state table. */
+    int32_t  getSafeTableSize() const;
+
+    /** Fill in the runtime safe state table. Sufficient memory must exist at the specified location.
+     */
+    void     exportSafeTable(void *where);
+
 
 private:
     void     calcNullable(RBBINode *n);
@@ -84,20 +101,36 @@ private:
 
     void     addRuleRootNodes(UVector *dest, RBBINode *node);
 
-    /** Find the next duplicate state. An iterator function.
-     * @param firstState (in/out) begin looking at this state, return the first of the
-     *                   pair of duplicates.
-     * @param duplicateState returns the duplicate state of fistState
-     * @return true if a duplicate pair of states was found.
+    /**
+     *  Find duplicate (redundant) states, beginning at the specified pair,
+     *  within this state table. This is an iterator-like function, used to
+     *  identify states (state table rows) that can be eliminated.
+     *  @param states in/out parameter, specifies where to start looking for duplicates,
+     *                and returns the first pair of duplicates found, if any.
+     *  @return true if duplicate states were found, false otherwise.
      */
-    bool findDuplicateState(int32_t &firstState, int32_t &duplicateState);
+    bool findDuplicateState(IntPair *states);
 
-    /** Remove a duplicate state/
-     * @param keepState First of the duplicate pair. Keep it.
-     * @param duplState Duplicate state. Remove it. Redirect all references to the duplicate state
-     *                  to refer to keepState instead.
+    /** Remove a duplicate state.
+     * @param duplStates The duplicate states. The first is kept, the second is removed.
+     *                   All references to the second in the state table are retargeted
+     *                   to the first.
      */
-    void removeState(int32_t keepState, int32_t duplState);
+    void removeState(IntPair duplStates);
+
+    /** Find the next duplicate state in the safe reverse table. An iterator function.
+     *  @param states in/out parameter, specifies where to start looking for duplicates,
+     *                and returns the first pair of duplicates found, if any.
+     *  @return true if a duplicate pair of states was found.
+     */
+    bool findDuplicateSafeState(IntPair *states);
+
+    /** Remove a duplicate state from the safe table.
+     * @param duplStates The duplicate states. The first is kept, the second is removed.
+     *                   All references to the second in the state table are retargeted
+     *                   to the first.
+     */
+    void removeSafeState(IntPair duplStates);
 
     // Set functions for UVector.
     //   TODO:  make a USet subclass of UVector
@@ -113,11 +146,13 @@ public:
     void     printPosSets(RBBINode *n /* = NULL*/);
     void     printStates();
     void     printRuleStatusTable();
+    void     printReverseTable();
 #else
     #define  printSet(s)
     #define  printPosSets(n)
     #define  printStates()
     #define  printRuleStatusTable()
+    #define  printReverseTable()
 #endif
 
 private:
@@ -126,10 +161,14 @@ private:
                                            //   table for.
     UErrorCode       *fStatus;
 
+    /** State Descriptors, UVector<RBBIStateDescriptor> */
     UVector          *fDStates;            //  D states (Aho's terminology)
                                            //  Index is state number
                                            //  Contents are RBBIStateDescriptor pointers.
 
+    /** Synthesized safe table, UVector of UnicodeString, one string per table row.   */
+    UVector          *fSafeTable;
+
 
     RBBITableBuilder(const RBBITableBuilder &other); // forbid copying of this class
     RBBITableBuilder &operator=(const RBBITableBuilder &other); // forbid copying of this class

intl/icu/source/common/serv.cpp

@@ -547,16 +547,15 @@ outerEnd:
             if (putInCache && cacheResult) {
                 serviceCache->put(result->actualDescriptor, result, status);
                 if (U_FAILURE(status)) {
-                    delete result;
                     return NULL;
                 }
 
                 if (cacheDescriptorList._obj != NULL) {
                     for (int32_t i = cacheDescriptorList._obj->size(); --i >= 0;) {
                         UnicodeString* desc = (UnicodeString*)cacheDescriptorList._obj->elementAt(i);
+
                         serviceCache->put(*desc, result, status);
                         if (U_FAILURE(status)) {
-                            delete result;
                             return NULL;
                         }
 

intl/icu/source/common/static_unicode_sets.cpp

@@ -0,0 +1,222 @@
+// © 2018 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_FORMATTING
+
+// Allow implicit conversion from char16_t* to UnicodeString for this file:
+// Helpful in toString methods and elsewhere.
+#define UNISTR_FROM_STRING_EXPLICIT
+
+#include "static_unicode_sets.h"
+#include "umutex.h"
+#include "ucln_cmn.h"
+#include "unicode/uniset.h"
+#include "uresimp.h"
+#include "cstring.h"
+#include "uassert.h"
+
+using namespace icu;
+using namespace icu::unisets;
+
+
+namespace {
+
+UnicodeSet* gUnicodeSets[COUNT] = {};
+
+// Save the empty instance in static memory to have well-defined behavior if a
+// regular UnicodeSet cannot be allocated.
+char gEmptyUnicodeSet[sizeof(UnicodeSet)];
+
+// Whether the gEmptyUnicodeSet is initialized and ready to use.
+UBool gEmptyUnicodeSetInitialized = FALSE;
+
+inline UnicodeSet* getImpl(Key key) {
+    UnicodeSet* candidate = gUnicodeSets[key];
+    if (candidate == nullptr) {
+        return reinterpret_cast<UnicodeSet*>(gEmptyUnicodeSet);
+    }
+    return candidate;
+}
+
+UnicodeSet* computeUnion(Key k1, Key k2) {
+    UnicodeSet* result = new UnicodeSet();
+    if (result == nullptr) {
+        return nullptr;
+    }
+    result->addAll(*getImpl(k1));
+    result->addAll(*getImpl(k2));
+    result->freeze();
+    return result;
+}
+
+UnicodeSet* computeUnion(Key k1, Key k2, Key k3) {
+    UnicodeSet* result = new UnicodeSet();
+    if (result == nullptr) {
+        return nullptr;
+    }
+    result->addAll(*getImpl(k1));
+    result->addAll(*getImpl(k2));
+    result->addAll(*getImpl(k3));
+    result->freeze();
+    return result;
+}
+
+
+void saveSet(Key key, const UnicodeString& unicodeSetPattern, UErrorCode& status) {
+    // assert unicodeSets.get(key) == null;
+    gUnicodeSets[key] = new UnicodeSet(unicodeSetPattern, status);
+}
+
+class ParseDataSink : public ResourceSink {
+  public:
+    void put(const char* key, ResourceValue& value, UBool /*noFallback*/, UErrorCode& status) U_OVERRIDE {
+        ResourceTable contextsTable = value.getTable(status);
+        if (U_FAILURE(status)) { return; }
+        for (int i = 0; contextsTable.getKeyAndValue(i, key, value); i++) {
+            if (uprv_strcmp(key, "date") == 0) {
+                // ignore
+            } else {
+                ResourceTable strictnessTable = value.getTable(status);
+                if (U_FAILURE(status)) { return; }
+                for (int j = 0; strictnessTable.getKeyAndValue(j, key, value); j++) {
+                    bool isLenient = (uprv_strcmp(key, "lenient") == 0);
+                    ResourceArray array = value.getArray(status);
+                    if (U_FAILURE(status)) { return; }
+                    for (int k = 0; k < array.getSize(); k++) {
+                        array.getValue(k, value);
+                        UnicodeString str = value.getUnicodeString(status);
+                        if (U_FAILURE(status)) { return; }
+                        // There is both lenient and strict data for comma/period,
+                        // but not for any of the other symbols.
+                        if (str.indexOf(u'.') != -1) {
+                            saveSet(isLenient ? PERIOD : STRICT_PERIOD, str, status);
+                        } else if (str.indexOf(u',') != -1) {
+                            saveSet(isLenient ? COMMA : STRICT_COMMA, str, status);
+                        } else if (str.indexOf(u'+') != -1) {
+                            saveSet(PLUS_SIGN, str, status);
+                        } else if (str.indexOf(u'‒') != -1) {
+                            saveSet(MINUS_SIGN, str, status);
+                        } else if (str.indexOf(u'$') != -1) {
+                            saveSet(DOLLAR_SIGN, str, status);
+                        } else if (str.indexOf(u'£') != -1) {
+                            saveSet(POUND_SIGN, str, status);
+                        } else if (str.indexOf(u'₨') != -1) {
+                            saveSet(RUPEE_SIGN, str, status);
+                        }
+                        if (U_FAILURE(status)) { return; }
+                    }
+                }
+            }
+        }
+    }
+};
+
+
+icu::UInitOnce gNumberParseUniSetsInitOnce = U_INITONCE_INITIALIZER;
+
+UBool U_CALLCONV cleanupNumberParseUniSets() {
+    if (gEmptyUnicodeSetInitialized) {
+        reinterpret_cast<UnicodeSet*>(gEmptyUnicodeSet)->~UnicodeSet();
+        gEmptyUnicodeSetInitialized = FALSE;
+    }
+    for (int32_t i = 0; i < COUNT; i++) {
+        delete gUnicodeSets[i];
+        gUnicodeSets[i] = nullptr;
+    }
+    gNumberParseUniSetsInitOnce.reset();
+    return TRUE;
+}
+
+void U_CALLCONV initNumberParseUniSets(UErrorCode& status) {
+    ucln_common_registerCleanup(UCLN_COMMON_NUMPARSE_UNISETS, cleanupNumberParseUniSets);
+
+    // Initialize the empty instance for well-defined fallback behavior
+    new(gEmptyUnicodeSet) UnicodeSet();
+    reinterpret_cast<UnicodeSet*>(gEmptyUnicodeSet)->freeze();
+    gEmptyUnicodeSetInitialized = TRUE;
+
+    // These sets were decided after discussion with icu-design@. See tickets #13084 and #13309.
+    // Zs+TAB is "horizontal whitespace" according to UTS #18 (blank property).
+    gUnicodeSets[DEFAULT_IGNORABLES] = new UnicodeSet(
+            u"[[:Zs:][\\u0009][:Bidi_Control:][:Variation_Selector:]]", status);
+    gUnicodeSets[STRICT_IGNORABLES] = new UnicodeSet(u"[[:Bidi_Control:]]", status);
+
+    LocalUResourceBundlePointer rb(ures_open(nullptr, "root", &status));
+    if (U_FAILURE(status)) { return; }
+    ParseDataSink sink;
+    ures_getAllItemsWithFallback(rb.getAlias(), "parse", sink, status);
+    if (U_FAILURE(status)) { return; }
+
+    // NOTE: It is OK for these assertions to fail if there was a no-data build.
+    U_ASSERT(gUnicodeSets[COMMA] != nullptr);
+    U_ASSERT(gUnicodeSets[STRICT_COMMA] != nullptr);
+    U_ASSERT(gUnicodeSets[PERIOD] != nullptr);
+    U_ASSERT(gUnicodeSets[STRICT_PERIOD] != nullptr);
+
+    gUnicodeSets[OTHER_GROUPING_SEPARATORS] = new UnicodeSet(
+            u"['٬‘’＇\\u0020\\u00A0\\u2000-\\u200A\\u202F\\u205F\\u3000]", status);
+    gUnicodeSets[ALL_SEPARATORS] = computeUnion(COMMA, PERIOD, OTHER_GROUPING_SEPARATORS);
+    gUnicodeSets[STRICT_ALL_SEPARATORS] = computeUnion(
+            STRICT_COMMA, STRICT_PERIOD, OTHER_GROUPING_SEPARATORS);
+
+    U_ASSERT(gUnicodeSets[MINUS_SIGN] != nullptr);
+    U_ASSERT(gUnicodeSets[PLUS_SIGN] != nullptr);
+
+    gUnicodeSets[PERCENT_SIGN] = new UnicodeSet(u"[%٪]", status);
+    gUnicodeSets[PERMILLE_SIGN] = new UnicodeSet(u"[‰؉]", status);
+    gUnicodeSets[INFINITY_KEY] = new UnicodeSet(u"[∞]", status);
+
+    U_ASSERT(gUnicodeSets[DOLLAR_SIGN] != nullptr);
+    U_ASSERT(gUnicodeSets[POUND_SIGN] != nullptr);
+    U_ASSERT(gUnicodeSets[RUPEE_SIGN] != nullptr);
+    gUnicodeSets[YEN_SIGN] = new UnicodeSet(u"[¥\\uffe5]", status);
+
+    gUnicodeSets[DIGITS] = new UnicodeSet(u"[:digit:]", status);
+
+    gUnicodeSets[DIGITS_OR_ALL_SEPARATORS] = computeUnion(DIGITS, ALL_SEPARATORS);
+    gUnicodeSets[DIGITS_OR_STRICT_ALL_SEPARATORS] = computeUnion(DIGITS, STRICT_ALL_SEPARATORS);
+
+    for (auto* uniset : gUnicodeSets) {
+        if (uniset != nullptr) {
+            uniset->freeze();
+        }
+    }
+}
+
+}
+
+const UnicodeSet* unisets::get(Key key) {
+    UErrorCode localStatus = U_ZERO_ERROR;
+    umtx_initOnce(gNumberParseUniSetsInitOnce, &initNumberParseUniSets, localStatus);
+    if (U_FAILURE(localStatus)) {
+        return reinterpret_cast<UnicodeSet*>(gEmptyUnicodeSet);
+    }
+    return getImpl(key);
+}
+
+Key unisets::chooseFrom(UnicodeString str, Key key1) {
+    return get(key1)->contains(str) ? key1 : NONE;
+}
+
+Key unisets::chooseFrom(UnicodeString str, Key key1, Key key2) {
+    return get(key1)->contains(str) ? key1 : chooseFrom(str, key2);
+}
+
+//Key unisets::chooseCurrency(UnicodeString str) {
+//    if (get(DOLLAR_SIGN)->contains(str)) {
+//        return DOLLAR_SIGN;
+//    } else if (get(POUND_SIGN)->contains(str)) {
+//        return POUND_SIGN;
+//    } else if (get(RUPEE_SIGN)->contains(str)) {
+//        return RUPEE_SIGN;
+//    } else if (get(YEN_SIGN)->contains(str)) {
+//        return YEN_SIGN;
+//    } else {
+//        return NONE;
+//    }
+//}
+
+
+#endif /* #if !UCONFIG_NO_FORMATTING */

intl/icu/source/common/static_unicode_sets.h

@@ -0,0 +1,119 @@
+// © 2018 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// This file is in common instead of i18n because it is needed by ucurr.cpp.
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_FORMATTING
+#ifndef __STATIC_UNICODE_SETS_H__
+#define __STATIC_UNICODE_SETS_H__
+
+#include "unicode/uniset.h"
+#include "unicode/unistr.h"
+
+U_NAMESPACE_BEGIN
+namespace unisets {
+
+enum Key {
+    // NONE is used to indicate null in chooseFrom().
+    // EMPTY is used to get an empty UnicodeSet.
+    NONE = -1,
+    EMPTY = 0,
+
+    // Ignorables
+    DEFAULT_IGNORABLES,
+    STRICT_IGNORABLES,
+
+    // Separators
+    // Notes:
+    // - COMMA is a superset of STRICT_COMMA
+    // - PERIOD is a superset of SCRICT_PERIOD
+    // - ALL_SEPARATORS is the union of COMMA, PERIOD, and OTHER_GROUPING_SEPARATORS
+    // - STRICT_ALL_SEPARATORS is the union of STRICT_COMMA, STRICT_PERIOD, and OTHER_GRP_SEPARATORS
+    COMMA,
+    PERIOD,
+    STRICT_COMMA,
+    STRICT_PERIOD,
+    OTHER_GROUPING_SEPARATORS,
+    ALL_SEPARATORS,
+    STRICT_ALL_SEPARATORS,
+
+    // Symbols
+    MINUS_SIGN,
+    PLUS_SIGN,
+    PERCENT_SIGN,
+    PERMILLE_SIGN,
+    INFINITY_KEY, // INFINITY is defined in cmath
+
+    // Currency Symbols
+    DOLLAR_SIGN,
+    POUND_SIGN,
+    RUPEE_SIGN,
+    YEN_SIGN, // not in CLDR data, but Currency.java wants it
+
+    // Other
+    DIGITS,
+
+    // Combined Separators with Digits (for lead code points)
+    DIGITS_OR_ALL_SEPARATORS,
+    DIGITS_OR_STRICT_ALL_SEPARATORS,
+
+    // The number of elements in the enum.
+    COUNT
+};
+
+/**
+ * Gets the static-allocated UnicodeSet according to the provided key. The
+ * pointer will be deleted during u_cleanup(); the caller should NOT delete it.
+ *
+ * Exported as U_COMMON_API for ucurr.cpp
+ *
+ * @param key The desired UnicodeSet according to the enum in this file.
+ * @return The requested UnicodeSet. Guaranteed to be frozen and non-null, but
+ *         may be empty if an error occurred during data loading.
+ */
+U_COMMON_API const UnicodeSet* get(Key key);
+
+/**
+ * Checks if the UnicodeSet given by key1 contains the given string.
+ *
+ * Exported as U_COMMON_API for numparse_decimal.cpp
+ *
+ * @param str The string to check.
+ * @param key1 The set to check.
+ * @return key1 if the set contains str, or NONE if not.
+ */
+U_COMMON_API Key chooseFrom(UnicodeString str, Key key1);
+
+/**
+ * Checks if the UnicodeSet given by either key1 or key2 contains the string.
+ *
+ * Exported as U_COMMON_API for numparse_decimal.cpp
+ *
+ * @param str The string to check.
+ * @param key1 The first set to check.
+ * @param key2 The second set to check.
+ * @return key1 if that set contains str; key2 if that set contains str; or
+ *         NONE if neither set contains str.
+ */
+U_COMMON_API Key chooseFrom(UnicodeString str, Key key1, Key key2);
+
+// Unused in C++:
+// Key chooseCurrency(UnicodeString str);
+// Used instead:
+static const struct {
+    Key key;
+    UChar32 exemplar;
+} kCurrencyEntries[] = {
+    {DOLLAR_SIGN, u'$'},
+    {POUND_SIGN, u'£'},
+    {RUPEE_SIGN, u'₨'},
+    {YEN_SIGN, u'¥'},
+};
+
+} // namespace unisets
+U_NAMESPACE_END
+
+#endif //__STATIC_UNICODE_SETS_H__
+#endif /* #if !UCONFIG_NO_FORMATTING */

intl/icu/source/common/ucase.cpp

@@ -138,6 +138,11 @@ ucase_tolower(UChar32 c) {
     } else {
         const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
         uint16_t excWord=*pe++;
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
             GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe, c);
         }
@@ -155,6 +160,11 @@ ucase_toupper(UChar32 c) {
     } else {
         const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
         uint16_t excWord=*pe++;
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) {
             GET_SLOT_VALUE(excWord, UCASE_EXC_UPPER, pe, c);
         }
@@ -172,6 +182,11 @@ ucase_totitle(UChar32 c) {
     } else {
         const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
         uint16_t excWord=*pe++;
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         int32_t idx;
         if(HAS_SLOT(excWord, UCASE_EXC_TITLE)) {
             idx=UCASE_EXC_TITLE;
@@ -254,6 +269,12 @@ ucase_addCaseClosure(UChar32 c, const USetAdder *sa) {
                 sa->add(sa->set, c);
             }
         }
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA)) {
+            pe=pe0;
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+            sa->add(sa->set, (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta);
+        }
 
         /* get the closure string pointer & length */
         if(HAS_SLOT(excWord, UCASE_EXC_CLOSURE)) {
@@ -590,7 +611,12 @@ ucase_isSoftDotted(UChar32 c) {
 U_CAPI UBool U_EXPORT2
 ucase_isCaseSensitive(UChar32 c) {
     uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
-    return (UBool)((props&UCASE_SENSITIVE)!=0);
+    if(!UCASE_HAS_EXCEPTION(props)) {
+        return (UBool)((props&UCASE_SENSITIVE)!=0);
+    } else {
+        const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
+        return (UBool)((*pe&UCASE_EXC_SENSITIVE)!=0);
+    }
 }
 
 /* string casing ------------------------------------------------------------ */
@@ -1140,6 +1166,11 @@ ucase_toFullLower(UChar32 c,
             }
         }
 
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
             GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe2, result);
         }
@@ -1229,6 +1260,11 @@ toUpperOrTitle(UChar32 c,
             }
         }
 
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(!upperNotTitle && HAS_SLOT(excWord, UCASE_EXC_TITLE)) {
             idx=UCASE_EXC_TITLE;
         } else if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) {
@@ -1334,6 +1370,14 @@ ucase_fold(UChar32 c, uint32_t options) {
                 }
             }
         }
+        if((excWord&UCASE_EXC_NO_SIMPLE_CASE_FOLDING)!=0) {
+            return c;
+        }
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) {
             idx=UCASE_EXC_FOLD;
         } else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
@@ -1421,6 +1465,14 @@ ucase_toFullFolding(UChar32 c,
             }
         }
 
+        if((excWord&UCASE_EXC_NO_SIMPLE_CASE_FOLDING)!=0) {
+            return ~c;
+        }
+        if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+            int32_t delta;
+            GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+            return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+        }
         if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) {
             idx=UCASE_EXC_FOLD;
         } else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {

intl/icu/source/common/ucase.h

@@ -354,8 +354,8 @@ enum {
 #define UCASE_IS_UPPER_OR_TITLE(props) ((props)&2)
 
 #define UCASE_IGNORABLE         4
-#define UCASE_SENSITIVE         8
-#define UCASE_EXCEPTION         0x10
+#define UCASE_EXCEPTION         8
+#define UCASE_SENSITIVE         0x10
 
 #define UCASE_HAS_EXCEPTION(props) ((props)&UCASE_EXCEPTION)
 
@@ -379,9 +379,9 @@ enum {
 #   define UCASE_GET_DELTA(props) (int16_t)(((props)&0x8000) ? (((props)>>UCASE_DELTA_SHIFT)|0xfe00) : ((uint16_t)(props)>>UCASE_DELTA_SHIFT))
 #endif
 
-/* exception: bits 15..5 are an unsigned 11-bit index into the exceptions array */
-#define UCASE_EXC_SHIFT     5
-#define UCASE_EXC_MASK      0xffe0
+/* exception: bits 15..4 are an unsigned 12-bit index into the exceptions array */
+#define UCASE_EXC_SHIFT     4
+#define UCASE_EXC_MASK      0xfff0
 #define UCASE_MAX_EXCEPTIONS ((UCASE_EXC_MASK>>UCASE_EXC_SHIFT)+1)
 
 /* definitions for 16-bit main exceptions word ------------------------------ */
@@ -392,7 +392,7 @@ enum {
     UCASE_EXC_FOLD,
     UCASE_EXC_UPPER,
     UCASE_EXC_TITLE,
-    UCASE_EXC_4,            /* reserved */
+    UCASE_EXC_DELTA,
     UCASE_EXC_5,            /* reserved */
     UCASE_EXC_CLOSURE,
     UCASE_EXC_FULL_MAPPINGS,
@@ -402,7 +402,11 @@ enum {
 /* each slot is 2 uint16_t instead of 1 */
 #define UCASE_EXC_DOUBLE_SLOTS      0x100
 
-/* reserved: exception bits 11..9 */
+enum {
+    UCASE_EXC_NO_SIMPLE_CASE_FOLDING=0x200,
+    UCASE_EXC_DELTA_IS_NEGATIVE=0x400,
+    UCASE_EXC_SENSITIVE=0x800
+};
 
 /* UCASE_EXC_DOT_MASK=UCASE_DOT_MASK<<UCASE_EXC_DOT_SHIFT */
 #define UCASE_EXC_DOT_SHIFT     7

intl/icu/source/common/ucln_cmn.h

@@ -33,6 +33,7 @@ Please keep the order of enums declared in same order
 as the cleanup functions are suppose to be called. */
 typedef enum ECleanupCommonType {
     UCLN_COMMON_START = -1,
+    UCLN_COMMON_NUMPARSE_UNISETS,
     UCLN_COMMON_USPREP,
     UCLN_COMMON_BREAKITERATOR,
     UCLN_COMMON_RBBI,

intl/icu/source/common/ucnv_bld.cpp

@@ -261,6 +261,11 @@ static UBool U_CALLCONV ucnv_cleanup(void) {
     return (SHARED_DATA_HASHTABLE == NULL);
 }
 
+U_CAPI void U_EXPORT2
+ucnv_enableCleanup() {
+    ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup);
+}
+
 static UBool U_CALLCONV
 isCnvAcceptable(void * /*context*/,
                 const char * /*type*/, const char * /*name*/,
@@ -439,7 +444,7 @@ ucnv_shareConverterData(UConverterSharedData * data)
         SHARED_DATA_HASHTABLE = uhash_openSize(uhash_hashChars, uhash_compareChars, NULL,
                             ucnv_io_countKnownConverters(&err)*UCNV_CACHE_LOAD_FACTOR,
                             &err);
-        ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup);
+        ucnv_enableCleanup();
 
         if (U_FAILURE(err))
             return;
@@ -1099,7 +1104,7 @@ static void U_CALLCONV initAvailableConvertersList(UErrorCode &errCode) {
     U_ASSERT(gAvailableConverterCount == 0);
     U_ASSERT(gAvailableConverters == NULL);
 
-    ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup);
+    ucnv_enableCleanup();
     UEnumeration *allConvEnum = ucnv_openAllNames(&errCode);
     int32_t allConverterCount = uenum_count(allConvEnum, &errCode);
     if (U_FAILURE(errCode)) {
@@ -1205,7 +1210,7 @@ internalSetName(const char *name, UErrorCode *status) {
     //             -- Andy
     gDefaultConverterName = gDefaultConverterNameBuffer;
 
-    ucln_common_registerCleanup(UCLN_COMMON_UCNV, ucnv_cleanup);
+    ucnv_enableCleanup();
 
     umtx_unlock(&cnvCacheMutex);
 }

intl/icu/source/common/ucnv_bld.h

@@ -288,6 +288,9 @@ ucnv_swap(const UDataSwapper *ds,
           const void *inData, int32_t length, void *outData,
           UErrorCode *pErrorCode);
 
+U_CAPI void U_EXPORT2
+ucnv_enableCleanup();
+
 #endif
 
 #endif /* _UCNV_BLD */

intl/icu/source/common/ucurr.cpp

@@ -16,10 +16,14 @@
 #include "unicode/ures.h"
 #include "unicode/ustring.h"
 #include "unicode/parsepos.h"
+#include "unicode/uniset.h"
+#include "unicode/usetiter.h"
+#include "unicode/utf16.h"
 #include "ustr_imp.h"
 #include "charstr.h"
 #include "cmemory.h"
 #include "cstring.h"
+#include "static_unicode_sets.h"
 #include "uassert.h"
 #include "umutex.h"
 #include "ucln_cmn.h"
@@ -65,14 +69,6 @@ static const int32_t POW10[] = { 1, 10, 100, 1000, 10000, 100000,
 
 static const int32_t MAX_POW10 = UPRV_LENGTHOF(POW10) - 1;
 
-// Defines equivalent currency symbols.
-static const char *EQUIV_CURRENCY_SYMBOLS[][2] = {
-    {"\\u00a5", "\\uffe5"},
-    {"$", "\\ufe69"},
-    {"$", "\\uff04"},
-    {"\\u20a8", "\\u20b9"},
-    {"\\u00a3", "\\u20a4"}};
-
 #define ISO_CURRENCY_CODE_LENGTH 3
 
 //------------------------------------------------------------
@@ -1287,17 +1283,28 @@ static void
 linearSearch(const CurrencyNameStruct* currencyNames, 
              int32_t begin, int32_t end,
              const UChar* text, int32_t textLen,
+             int32_t *partialMatchLen,
              int32_t *maxMatchLen, int32_t* maxMatchIndex) {
+    int32_t initialPartialMatchLen = *partialMatchLen;
     for (int32_t index = begin; index <= end; ++index) {
         int32_t len = currencyNames[index].currencyNameLen;
         if (len > *maxMatchLen && len <= textLen &&
             uprv_memcmp(currencyNames[index].currencyName, text, len * sizeof(UChar)) == 0) {
+            *partialMatchLen = MAX(*partialMatchLen, len);
             *maxMatchIndex = index;
             *maxMatchLen = len;
 #ifdef UCURR_DEBUG
             printf("maxMatchIndex = %d, maxMatchLen = %d\n",
                    *maxMatchIndex, *maxMatchLen);
 #endif
+        } else {
+            // Check for partial matches.
+            for (int32_t i=initialPartialMatchLen; i<MIN(len, textLen); i++) {
+                if (currencyNames[index].currencyName[i] != text[i]) {
+                    break;
+                }
+                *partialMatchLen = MAX(*partialMatchLen, i + 1);
+            }
         }
     }
 }
@@ -1314,7 +1321,8 @@ linearSearch(const CurrencyNameStruct* currencyNames,
 static void
 searchCurrencyName(const CurrencyNameStruct* currencyNames, 
                    int32_t total_currency_count,
-                   const UChar* text, int32_t textLen, 
+                   const UChar* text, int32_t textLen,
+                   int32_t *partialMatchLen,
                    int32_t* maxMatchLen, int32_t* maxMatchIndex) {
     *maxMatchIndex = -1;
     *maxMatchLen = 0;
@@ -1344,6 +1352,7 @@ searchCurrencyName(const CurrencyNameStruct* currencyNames,
         if (binarySearchBegin == -1) { // did not find the range
             break;
         }
+        *partialMatchLen = MAX(*partialMatchLen, index + 1);
         if (matchIndex != -1) { 
             // find an exact match for text from text[0] to text[index] 
             // in currencyNames array.
@@ -1354,6 +1363,7 @@ searchCurrencyName(const CurrencyNameStruct* currencyNames,
             // linear search if within threshold.
             linearSearch(currencyNames, binarySearchBegin, binarySearchEnd,
                          text, textLen,
+                         partialMatchLen,
                          maxMatchLen, maxMatchIndex);
             break;
         }
@@ -1422,19 +1432,13 @@ currency_cache_cleanup(void) {
 }
 
 
-U_CAPI void
-uprv_parseCurrency(const char* locale,
-                   const icu::UnicodeString& text,
-                   icu::ParsePosition& pos,
-                   int8_t type,
-                   UChar* result,
-                   UErrorCode& ec)
-{
-    U_NAMESPACE_USE
-
-    if (U_FAILURE(ec)) {
-        return;
-    }
+/**
+ * Loads the currency name data from the cache, or from resource bundles if necessary.
+ * The refCount is automatically incremented.  It is the caller's responsibility
+ * to decrement it when done!
+ */
+static CurrencyNameCacheEntry*
+getCacheEntry(const char* locale, UErrorCode& ec) {
 
     int32_t total_currency_name_count = 0;
     CurrencyNameStruct* currencyNames = NULL;
@@ -1455,17 +1459,13 @@ uprv_parseCurrency(const char* locale,
     }
     if (found != -1) {
         cacheEntry = currCache[found];
-        currencyNames = cacheEntry->currencyNames;
-        total_currency_name_count = cacheEntry->totalCurrencyNameCount;
-        currencySymbols = cacheEntry->currencySymbols;
-        total_currency_symbol_count = cacheEntry->totalCurrencySymbolCount;
         ++(cacheEntry->refCount);
     }
     umtx_unlock(&gCurrencyCacheMutex);
     if (found == -1) {
         collectCurrencyNames(locale, &currencyNames, &total_currency_name_count, &currencySymbols, &total_currency_symbol_count, ec);
         if (U_FAILURE(ec)) {
-            return;
+            return NULL;
         }
         umtx_lock(&gCurrencyCacheMutex);
         // check again.
@@ -1500,20 +1500,50 @@ uprv_parseCurrency(const char* locale,
             cacheEntry->totalCurrencySymbolCount = total_currency_symbol_count;
             cacheEntry->refCount = 2; // one for cache, one for reference
             currentCacheEntryIndex = (currentCacheEntryIndex + 1) % CURRENCY_NAME_CACHE_NUM;
-            ucln_common_registerCleanup(UCLN_COMMON_CURRENCY, currency_cache_cleanup);
+            ucln_common_registerCleanup(UCLN_COMMON_CURRENCY, currency_cleanup);
         } else {
             deleteCurrencyNames(currencyNames, total_currency_name_count);
             deleteCurrencyNames(currencySymbols, total_currency_symbol_count);
             cacheEntry = currCache[found];
-            currencyNames = cacheEntry->currencyNames;
-            total_currency_name_count = cacheEntry->totalCurrencyNameCount;
-            currencySymbols = cacheEntry->currencySymbols;
-            total_currency_symbol_count = cacheEntry->totalCurrencySymbolCount;
             ++(cacheEntry->refCount);
         }
         umtx_unlock(&gCurrencyCacheMutex);
     }
 
+    return cacheEntry;
+}
+
+static void releaseCacheEntry(CurrencyNameCacheEntry* cacheEntry) {
+    umtx_lock(&gCurrencyCacheMutex);
+    --(cacheEntry->refCount);
+    if (cacheEntry->refCount == 0) {  // remove
+        deleteCacheEntry(cacheEntry);
+    }
+    umtx_unlock(&gCurrencyCacheMutex);
+}
+
+U_CAPI void
+uprv_parseCurrency(const char* locale,
+                   const icu::UnicodeString& text,
+                   icu::ParsePosition& pos,
+                   int8_t type,
+                   int32_t* partialMatchLen,
+                   UChar* result,
+                   UErrorCode& ec) {
+    U_NAMESPACE_USE
+    if (U_FAILURE(ec)) {
+        return;
+    }
+    CurrencyNameCacheEntry* cacheEntry = getCacheEntry(locale, ec);
+    if (U_FAILURE(ec)) {
+        return;
+    }
+
+    int32_t total_currency_name_count = cacheEntry->totalCurrencyNameCount;
+    CurrencyNameStruct* currencyNames = cacheEntry->currencyNames;
+    int32_t total_currency_symbol_count = cacheEntry->totalCurrencySymbolCount;
+    CurrencyNameStruct* currencySymbols = cacheEntry->currencySymbols;
+
     int32_t start = pos.getIndex();
 
     UChar inputText[MAX_CURRENCY_NAME_LEN];  
@@ -1523,11 +1553,14 @@ uprv_parseCurrency(const char* locale,
     UErrorCode ec1 = U_ZERO_ERROR;
     textLen = u_strToUpper(upperText, MAX_CURRENCY_NAME_LEN, inputText, textLen, locale, &ec1);
 
+    // Make sure partialMatchLen is initialized
+    *partialMatchLen = 0;
+
     int32_t max = 0;
     int32_t matchIndex = -1;
     // case in-sensitive comparision against currency names
     searchCurrencyName(currencyNames, total_currency_name_count, 
-                       upperText, textLen, &max, &matchIndex);
+                       upperText, textLen, partialMatchLen, &max, &matchIndex);
 
 #ifdef UCURR_DEBUG
     printf("search in names, max = %d, matchIndex = %d\n", max, matchIndex);
@@ -1538,7 +1571,8 @@ uprv_parseCurrency(const char* locale,
     if (type != UCURR_LONG_NAME) {  // not name only
         // case sensitive comparison against currency symbols and ISO code.
         searchCurrencyName(currencySymbols, total_currency_symbol_count, 
-                           inputText, textLen, 
+                           inputText, textLen,
+                           partialMatchLen,
                            &maxInSymbol, &matchIndexInSymbol);
     }
 
@@ -1555,15 +1589,38 @@ uprv_parseCurrency(const char* locale,
     } else if (maxInSymbol >= max && matchIndexInSymbol != -1) {
         u_charsToUChars(currencySymbols[matchIndexInSymbol].IsoCode, result, 4);
         pos.setIndex(start + maxInSymbol);
-    } 
+    }
 
     // decrease reference count
-    umtx_lock(&gCurrencyCacheMutex);
-    --(cacheEntry->refCount);
-    if (cacheEntry->refCount == 0) {  // remove 
-        deleteCacheEntry(cacheEntry);
+    releaseCacheEntry(cacheEntry);
+}
+
+void uprv_currencyLeads(const char* locale, icu::UnicodeSet& result, UErrorCode& ec) {
+    U_NAMESPACE_USE
+    if (U_FAILURE(ec)) {
+        return;
     }
-    umtx_unlock(&gCurrencyCacheMutex);
+    CurrencyNameCacheEntry* cacheEntry = getCacheEntry(locale, ec);
+    if (U_FAILURE(ec)) {
+        return;
+    }
+
+    for (int32_t i=0; i<cacheEntry->totalCurrencySymbolCount; i++) {
+        const CurrencyNameStruct& info = cacheEntry->currencySymbols[i];
+        UChar32 cp;
+        U16_GET(info.currencyName, 0, 0, info.currencyNameLen, cp);
+        result.add(cp);
+    }
+
+    for (int32_t i=0; i<cacheEntry->totalCurrencyNameCount; i++) {
+        const CurrencyNameStruct& info = cacheEntry->currencyNames[i];
+        UChar32 cp;
+        U16_GET(info.currencyName, 0, 0, info.currencyNameLen, cp);
+        result.add(cp);
+    }
+
+    // decrease reference count
+    releaseCacheEntry(cacheEntry);
 }
 
 
@@ -1729,7 +1786,8 @@ static const struct CurrencyList {
     {"BUK", UCURR_COMMON|UCURR_DEPRECATED},
     {"BWP", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"BYB", UCURR_COMMON|UCURR_DEPRECATED},
-    {"BYR", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"BYN", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"BYR", UCURR_COMMON|UCURR_DEPRECATED},
     {"BZD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"CAD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"CDF", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1739,6 +1797,7 @@ static const struct CurrencyList {
     {"CLE", UCURR_COMMON|UCURR_DEPRECATED},
     {"CLF", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"CLP", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"CNH", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"CNX", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"CNY", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"COP", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1761,7 +1820,7 @@ static const struct CurrencyList {
     {"ECV", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"EEK", UCURR_COMMON|UCURR_DEPRECATED},
     {"EGP", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"EQE", UCURR_COMMON|UCURR_DEPRECATED},
+    {"EQE", UCURR_COMMON|UCURR_DEPRECATED}, // questionable, remove?
     {"ERN", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"ESA", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"ESB", UCURR_UNCOMMON|UCURR_DEPRECATED},
@@ -1785,7 +1844,7 @@ static const struct CurrencyList {
     {"GRD", UCURR_COMMON|UCURR_DEPRECATED},
     {"GTQ", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"GWE", UCURR_COMMON|UCURR_DEPRECATED},
-    {"GWP", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"GWP", UCURR_COMMON|UCURR_DEPRECATED},
     {"GYD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"HKD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"HNL", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1823,13 +1882,13 @@ static const struct CurrencyList {
     {"LKR", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"LRD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"LSL", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"LSM", UCURR_COMMON|UCURR_DEPRECATED},
-    {"LTL", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"LSM", UCURR_COMMON|UCURR_DEPRECATED}, // questionable, remove?
+    {"LTL", UCURR_COMMON|UCURR_DEPRECATED},
     {"LTT", UCURR_COMMON|UCURR_DEPRECATED},
     {"LUC", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"LUF", UCURR_COMMON|UCURR_DEPRECATED},
     {"LUL", UCURR_UNCOMMON|UCURR_DEPRECATED},
-    {"LVL", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"LVL", UCURR_COMMON|UCURR_DEPRECATED},
     {"LVR", UCURR_COMMON|UCURR_DEPRECATED},
     {"LYD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MAD", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1845,18 +1904,19 @@ static const struct CurrencyList {
     {"MMK", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MNT", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MOP", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"MRO", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"MRO", UCURR_COMMON|UCURR_DEPRECATED},
+    {"MRU", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MTL", UCURR_COMMON|UCURR_DEPRECATED},
     {"MTP", UCURR_COMMON|UCURR_DEPRECATED},
     {"MUR", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"MVP", UCURR_COMMON|UCURR_DEPRECATED},
+    {"MVP", UCURR_COMMON|UCURR_DEPRECATED}, // questionable, remove?
     {"MVR", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MWK", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MXN", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"MXP", UCURR_COMMON|UCURR_DEPRECATED},
     {"MXV", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"MYR", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"MZE", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"MZE", UCURR_COMMON|UCURR_DEPRECATED},
     {"MZM", UCURR_COMMON|UCURR_DEPRECATED},
     {"MZN", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"NAD", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1897,15 +1957,16 @@ static const struct CurrencyList {
     {"SGD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SHP", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SIT", UCURR_COMMON|UCURR_DEPRECATED},
-    {"SKK", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"SKK", UCURR_COMMON|UCURR_DEPRECATED},
     {"SLL", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SOS", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SRD", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SRG", UCURR_COMMON|UCURR_DEPRECATED},
     {"SSP", UCURR_COMMON|UCURR_NON_DEPRECATED},
-    {"STD", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"STD", UCURR_COMMON|UCURR_DEPRECATED},
+    {"STN", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SUR", UCURR_COMMON|UCURR_DEPRECATED},
-    {"SVC", UCURR_COMMON|UCURR_NON_DEPRECATED},
+    {"SVC", UCURR_COMMON|UCURR_DEPRECATED},
     {"SYP", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"SZL", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"THB", UCURR_COMMON|UCURR_NON_DEPRECATED},
@@ -1954,7 +2015,7 @@ static const struct CurrencyList {
     {"XPD", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"XPF", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"XPT", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
-    {"XRE", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
+    {"XRE", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"XSU", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"XTS", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
     {"XUA", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
@@ -1965,15 +2026,15 @@ static const struct CurrencyList {
     {"YUM", UCURR_COMMON|UCURR_DEPRECATED},
     {"YUN", UCURR_COMMON|UCURR_DEPRECATED},
     {"YUR", UCURR_COMMON|UCURR_DEPRECATED},
-    {"ZAL", UCURR_UNCOMMON|UCURR_NON_DEPRECATED},
+    {"ZAL", UCURR_UNCOMMON|UCURR_DEPRECATED},
     {"ZAR", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"ZMK", UCURR_COMMON|UCURR_DEPRECATED},
     {"ZMW", UCURR_COMMON|UCURR_NON_DEPRECATED},
     {"ZRN", UCURR_COMMON|UCURR_DEPRECATED},
     {"ZRZ", UCURR_COMMON|UCURR_DEPRECATED},
+    {"ZWD", UCURR_COMMON|UCURR_DEPRECATED},
     {"ZWL", UCURR_COMMON|UCURR_DEPRECATED},
     {"ZWR", UCURR_COMMON|UCURR_DEPRECATED},
-    {"ZWD", UCURR_COMMON|UCURR_DEPRECATED},
     { NULL, 0 } // Leave here to denote the end of the list.
 };
 
@@ -2144,16 +2205,20 @@ static void U_CALLCONV initIsoCodes(UErrorCode &status) {
 }
 
 static void populateCurrSymbolsEquiv(icu::Hashtable *hash, UErrorCode &status) {
-    if (U_FAILURE(status)) {
-        return;
-    }
-    int32_t length = UPRV_LENGTHOF(EQUIV_CURRENCY_SYMBOLS);
-    for (int32_t i = 0; i < length; ++i) {
-        icu::UnicodeString lhs(EQUIV_CURRENCY_SYMBOLS[i][0], -1, US_INV);
-        icu::UnicodeString rhs(EQUIV_CURRENCY_SYMBOLS[i][1], -1, US_INV);
-        makeEquivalent(lhs.unescape(), rhs.unescape(), hash, status);
-        if (U_FAILURE(status)) {
-            return;
+    if (U_FAILURE(status)) { return; }
+    for (auto& entry : unisets::kCurrencyEntries) {
+        UnicodeString exemplar(entry.exemplar);
+        const UnicodeSet* set = unisets::get(entry.key);
+        if (set == nullptr) { return; }
+        UnicodeSetIterator it(*set);
+        while (it.next()) {
+            UnicodeString value = it.getString();
+            if (value == exemplar) {
+                // No need to mark the exemplar character as an equivalent
+                continue;
+            }
+            makeEquivalent(exemplar, value, hash, status);
+            if (U_FAILURE(status)) { return; }
         }
     }
 }

intl/icu/source/common/ucurrimp.h

@@ -13,6 +13,7 @@
 #include "unicode/utypes.h"
 #include "unicode/unistr.h"
 #include "unicode/parsepos.h"
+#include "unicode/uniset.h"
 
 /**
  * Internal method.  Given a currency ISO code and a locale, return
@@ -36,6 +37,8 @@ uprv_getStaticCurrencyName(const UChar* iso, const char* loc,
  * match, then the display name is preferred, unless it's length
  * is less than 3.
  *
+ * The parameters must not be NULL.
+ *
  * @param locale the locale of the display names to match
  * @param text the text to parse
  * @param pos input-output position; on input, the position within
@@ -43,6 +46,8 @@ uprv_getStaticCurrencyName(const UChar* iso, const char* loc,
  * on output, the position after the last matched character. If
  * the parse fails, the position in unchanged upon output.
  * @param type currency type to parse against, LONG_NAME only or not
+ * @param partialMatchLen The length of the longest matching prefix;
+ * this may be nonzero even if no full currency was matched.
  * @return the ISO 4217 code, as a string, of the best match, or
  * null if there is no match
  *
@@ -53,9 +58,21 @@ uprv_parseCurrency(const char* locale,
                    const icu::UnicodeString& text,
                    icu::ParsePosition& pos,
                    int8_t type,
+                   int32_t* partialMatchLen,
                    UChar* result,
                    UErrorCode& ec);
 
+/**
+ * Puts all possible first-characters of a currency into the
+ * specified UnicodeSet.
+ *
+ * @param locale the locale of the display names of interest
+ * @param result the UnicodeSet to which to add the starting characters
+ */
+void uprv_currencyLeads(const char* locale, icu::UnicodeSet& result, UErrorCode& ec);
+
+
+
 #endif /* #ifndef _UCURR_IMP_H_ */
 
 //eof

intl/icu/source/common/unicode/brkiter.h

@@ -638,7 +638,7 @@ protected:
 
 private:
 
-    /** @internal */
+    /** @internal (private) */
     char actualLocale[ULOC_FULLNAME_CAPACITY];
     char validLocale[ULOC_FULLNAME_CAPACITY];
 };

intl/icu/source/common/unicode/bytestriebuilder.h

@@ -143,7 +143,7 @@ private:
     virtual int32_t getMaxLinearMatchLength() const { return BytesTrie::kMaxLinearMatchLength; }
 
     /**
-     * @internal
+     * @internal (private)
      */
     class BTLinearMatchNode : public LinearMatchNode {
     public:

intl/icu/source/common/unicode/caniter.h

@@ -153,13 +153,13 @@ private:
 
     /**
      * Copy constructor. Private for now.
-     * @internal
+     * @internal (private)
      */
     CanonicalIterator(const CanonicalIterator& other);
 
     /**
      * Assignment operator. Private for now.
-     * @internal
+     * @internal (private)
      */
     CanonicalIterator& operator=(const CanonicalIterator& other);
 

intl/icu/source/common/unicode/docmain.h

@@ -139,7 +139,7 @@
  *   </tr>
  *   <tr>
  *     <td>Number Formatting</td>
- *     <td>unum.h</td>
+ *     <td>unumberformatter.h, unum.h</td>
  *     <td>icu::number::NumberFormatter (ICU 60+) or icu::NumberFormat (older versions)</td>
  *   </tr>
  *   <tr>

intl/icu/source/common/unicode/edits.h

@@ -17,10 +17,57 @@
 
 U_NAMESPACE_BEGIN
 
+class UnicodeString;
+
 /**
- * Records lengths of string edits but not replacement text.
- * Supports replacements, insertions, deletions in linear progression.
- * Does not support moving/reordering of text.
+ * Records lengths of string edits but not replacement text. Supports replacements, insertions, deletions
+ * in linear progression. Does not support moving/reordering of text.
+ *
+ * There are two types of edits: <em>change edits</em> and <em>no-change edits</em>. Add edits to
+ * instances of this class using {@link #addReplace(int, int)} (for change edits) and
+ * {@link #addUnchanged(int)} (for no-change edits). Change edits are retained with full granularity,
+ * whereas adjacent no-change edits are always merged together. In no-change edits, there is a one-to-one
+ * mapping between code points in the source and destination strings.
+ *
+ * After all edits have been added, instances of this class should be considered immutable, and an
+ * {@link Edits::Iterator} can be used for queries.
+ *
+ * There are four flavors of Edits::Iterator:
+ *
+ * <ul>
+ * <li>{@link #getFineIterator()} retains full granularity of change edits.
+ * <li>{@link #getFineChangesIterator()} retains full granularity of change edits, and when calling
+ * next() on the iterator, skips over no-change edits (unchanged regions).
+ * <li>{@link #getCoarseIterator()} treats adjacent change edits as a single edit. (Adjacent no-change
+ * edits are automatically merged during the construction phase.)
+ * <li>{@link #getCoarseChangesIterator()} treats adjacent change edits as a single edit, and when
+ * calling next() on the iterator, skips over no-change edits (unchanged regions).
+ * </ul>
+ *
+ * For example, consider the string "abcßDeF", which case-folds to "abcssdef". This string has the
+ * following fine edits:
+ * <ul>
+ * <li>abc ⇨ abc (no-change)
+ * <li>ß ⇨ ss (change)
+ * <li>D ⇨ d (change)
+ * <li>e ⇨ e (no-change)
+ * <li>F ⇨ f (change)
+ * </ul>
+ * and the following coarse edits (note how adjacent change edits get merged together):
+ * <ul>
+ * <li>abc ⇨ abc (no-change)
+ * <li>ßD ⇨ ssd (change)
+ * <li>e ⇨ e (no-change)
+ * <li>F ⇨ f (change)
+ * </ul>
+ *
+ * The "fine changes" and "coarse changes" iterators will step through only the change edits when their
+ * {@link Edits::Iterator#next()} methods are called. They are identical to the non-change iterators when
+ * their {@link Edits::Iterator#findSourceIndex(int)} or {@link Edits::Iterator#findDestinationIndex(int)}
+ * methods are used to walk through the string.
+ *
+ * For examples of how to use this class, see the test <code>TestCaseMapEditsIteratorDocs</code> in
+ * UCharacterCaseTest.java.
  *
  * An Edits object tracks a separate UErrorCode, but ICU string transformation functions
  * (e.g., case mapping functions) merge any such errors into their API's UErrorCode.
@@ -91,13 +138,13 @@ public:
     void reset() U_NOEXCEPT;
 
     /**
-     * Adds a record for an unchanged segment of text.
+     * Adds a no-change edit: a record for an unchanged segment of text.
      * Normally called from inside ICU string transformation functions, not user code.
      * @stable ICU 59
      */
     void addUnchanged(int32_t unchangedLength);
     /**
-     * Adds a record for a text replacement/insertion/deletion.
+     * Adds a change edit: a record for a text replacement/insertion/deletion.
      * Normally called from inside ICU string transformation functions, not user code.
      * @stable ICU 59
      */
@@ -136,6 +183,18 @@ public:
 
     /**
      * Access to the list of edits.
+     *
+     * At any moment in time, an instance of this class points to a single edit: a "window" into a span
+     * of the source string and the corresponding span of the destination string. The source string span
+     * starts at {@link #sourceIndex()} and runs for {@link #oldLength()} chars; the destination string
+     * span starts at {@link #destinationIndex()} and runs for {@link #newLength()} chars.
+     *
+     * The iterator can be moved between edits using the {@link #next()}, {@link #findSourceIndex(int)},
+     * and {@link #findDestinationIndex(int)} methods. Calling any of these methods mutates the iterator
+     * to make it point to the corresponding edit.
+     *
+     * For more information, see the documentation for {@link Edits}.
+     *
      * @see getCoarseIterator
      * @see getFineIterator
      * @stable ICU 59
@@ -162,7 +221,7 @@ public:
         Iterator &operator=(const Iterator &other) = default;
 
         /**
-         * Advances to the next edit.
+         * Advances the iterator to the next edit.
          * @param errorCode ICU error code. Its input value must pass the U_SUCCESS() test,
          *                  or else the function returns immediately. Check for U_FAILURE()
          *                  on output or use with function chaining. (See User Guide for details.)
@@ -172,9 +231,9 @@ public:
         UBool next(UErrorCode &errorCode) { return next(onlyChanges_, errorCode); }
 
         /**
-         * Finds the edit that contains the source index.
-         * The source index may be found in a non-change
-         * even if normal iteration would skip non-changes.
+         * Moves the iterator to the edit that contains the source index.
+         * The source index may be found in a no-change edit
+         * even if normal iteration would skip no-change edits.
          * Normal iteration can continue from a found edit.
          *
          * The iterator state before this search logically does not matter.
@@ -196,9 +255,9 @@ public:
 
 #ifndef U_HIDE_DRAFT_API
         /**
-         * Finds the edit that contains the destination index.
-         * The destination index may be found in a non-change
-         * even if normal iteration would skip non-changes.
+         * Moves the iterator to the edit that contains the destination index.
+         * The destination index may be found in a no-change edit
+         * even if normal iteration would skip no-change edits.
          * Normal iteration can continue from a found edit.
          *
          * The iterator state before this search logically does not matter.
@@ -219,7 +278,7 @@ public:
         }
 
         /**
-         * Returns the destination index corresponding to the given source index.
+         * Computes the destination index corresponding to the given source index.
          * If the source index is inside a change edit (not at its start),
          * then the destination index at the end of that edit is returned,
          * since there is no information about index mapping inside a change edit.
@@ -243,7 +302,7 @@ public:
         int32_t destinationIndexFromSourceIndex(int32_t i, UErrorCode &errorCode);
 
         /**
-         * Returns the source index corresponding to the given destination index.
+         * Computes the source index corresponding to the given destination index.
          * If the destination index is inside a change edit (not at its start),
          * then the source index at the end of that edit is returned,
          * since there is no information about index mapping inside a change edit.
@@ -268,17 +327,27 @@ public:
 #endif  // U_HIDE_DRAFT_API
 
         /**
+         * Returns whether the edit currently represented by the iterator is a change edit.
+         *
          * @return TRUE if this edit replaces oldLength() units with newLength() different ones.
          *         FALSE if oldLength units remain unchanged.
          * @stable ICU 59
          */
         UBool hasChange() const { return changed; }
+
         /**
+         * The length of the current span in the source string, which starts at {@link #sourceIndex}.
+         *
          * @return the number of units in the original string which are replaced or remain unchanged.
          * @stable ICU 59
          */
         int32_t oldLength() const { return oldLength_; }
+
         /**
+         * The length of the current span in the destination string, which starts at
+         * {@link #destinationIndex}, or in the replacement string, which starts at
+         * {@link #replacementIndex}.
+         *
          * @return the number of units in the modified string, if hasChange() is TRUE.
          *         Same as oldLength if hasChange() is FALSE.
          * @stable ICU 59
@@ -286,22 +355,52 @@ public:
         int32_t newLength() const { return newLength_; }
 
         /**
+         * The start index of the current span in the source string; the span has length
+         * {@link #oldLength}.
+         *
          * @return the current index into the source string
          * @stable ICU 59
          */
         int32_t sourceIndex() const { return srcIndex; }
+
         /**
+         * The start index of the current span in the replacement string; the span has length
+         * {@link #newLength}. Well-defined only if the current edit is a change edit.
+         * <p>
+         * The <em>replacement string</em> is the concatenation of all substrings of the destination
+         * string corresponding to change edits.
+         * <p>
+         * This method is intended to be used together with operations that write only replacement
+         * characters (e.g., {@link CaseMap#omitUnchangedText()}). The source string can then be modified
+         * in-place.
+         *
          * @return the current index into the replacement-characters-only string,
          *         not counting unchanged spans
          * @stable ICU 59
          */
-        int32_t replacementIndex() const { return replIndex; }
+        int32_t replacementIndex() const {
+            // TODO: Throw an exception if we aren't in a change edit?
+            return replIndex;
+        }
+
         /**
+         * The start index of the current span in the destination string; the span has length
+         * {@link #newLength}.
+         *
          * @return the current index into the full destination string
          * @stable ICU 59
          */
         int32_t destinationIndex() const { return destIndex; }
 
+#ifndef U_HIDE_INTERNAL_API
+        /**
+         * A string representation of the current edit represented by the iterator for debugging. You
+         * should not depend on the contents of the return string.
+         * @internal
+         */
+        UnicodeString& toString(UnicodeString& appendTo) const;
+#endif  // U_HIDE_INTERNAL_API
+
     private:
         friend class Edits;
 
@@ -330,8 +429,10 @@ public:
     };
 
     /**
-     * Returns an Iterator for coarse-grained changes for simple string updates.
-     * Skips non-changes.
+     * Returns an Iterator for coarse-grained change edits
+     * (adjacent change edits are treated as one).
+     * Can be used to perform simple string updates.
+     * Skips no-change edits.
      * @return an Iterator that merges adjacent changes.
      * @stable ICU 59
      */
@@ -340,7 +441,10 @@ public:
     }
 
     /**
-     * Returns an Iterator for coarse-grained changes and non-changes for simple string updates.
+     * Returns an Iterator for coarse-grained change and no-change edits
+     * (adjacent change edits are treated as one).
+     * Can be used to perform simple string updates.
+     * Adjacent change edits are treated as one edit.
      * @return an Iterator that merges adjacent changes.
      * @stable ICU 59
      */
@@ -349,8 +453,10 @@ public:
     }
 
     /**
-     * Returns an Iterator for fine-grained changes for modifying styled text.
-     * Skips non-changes.
+     * Returns an Iterator for fine-grained change edits
+     * (full granularity of change edits is retained).
+     * Can be used for modifying styled text.
+     * Skips no-change edits.
      * @return an Iterator that separates adjacent changes.
      * @stable ICU 59
      */
@@ -359,7 +465,9 @@ public:
     }
 
     /**
-     * Returns an Iterator for fine-grained changes and non-changes for modifying styled text.
+     * Returns an Iterator for fine-grained change and no-change edits
+     * (full granularity of change edits is retained).
+     * Can be used for modifying styled text.
      * @return an Iterator that separates adjacent changes.
      * @stable ICU 59
      */

intl/icu/source/common/unicode/platform.h

@@ -196,20 +196,6 @@
 #   define U_PLATFORM U_PF_UNKNOWN
 #endif
 
-/**
- * \def UPRV_INCOMPLETE_CPP11_SUPPORT
- * This switch turns off ICU 60 NumberFormatter code.
- * By default, this switch is enabled on AIX and z/OS,
- * which have poor C++11 support.
- *
- * NOTE: This switch is intended to be temporary; see #13393.
- *
- * @internal
- */
-#ifndef UPRV_INCOMPLETE_CPP11_SUPPORT
-#   define UPRV_INCOMPLETE_CPP11_SUPPORT (U_PLATFORM == U_PF_AIX || U_PLATFORM == U_PF_OS390 || U_PLATFORM == U_PF_SOLARIS )
-#endif
-
 /**
  * \def CYGWINMSVC
  * Defined if this is Windows with Cygwin, but using MSVC rather than gcc.

intl/icu/source/common/unicode/rbbi.h

@@ -55,7 +55,7 @@ class U_COMMON_API RuleBasedBreakIterator /*U_FINAL*/ : public BreakIterator {
 private:
     /**
      * The UText through which this BreakIterator accesses the text
-     * @internal
+     * @internal (private)
      */
     UText  fText;
 
@@ -70,13 +70,6 @@ public:
     RBBIDataWrapper    *fData;
 private:
 
-    /** 
-     *  The iteration state - current position, rule status for the current position,
-     *                        and whether the iterator ran off the end, yielding UBRK_DONE.
-     *                        Current position is pinned to be 0 < position <= text.length.
-     *                        Current position is always set to a boundary.
-     *  @internal
-    */
     /**
       * The current  position of the iterator. Pinned, 0 < fPosition <= text.length.
       * Never has the value UBRK_DONE (-1).
@@ -628,25 +621,26 @@ private:
     /**
      * Dumps caches and performs other actions associated with a complete change
      * in text or iteration position.
-     * @internal
+     * @internal (private)
      */
     void reset(void);
 
     /**
       * Common initialization function, used by constructors and bufferClone.
-      * @internal
+      * @internal (private)
       */
     void init(UErrorCode &status);
 
     /**
-     * Iterate backwards from an arbitrary position in the input text using the Safe Reverse rules.
+     * Iterate backwards from an arbitrary position in the input text using the
+     * synthesized Safe Reverse rules.
      * This locates a "Safe Position" from which the forward break rules
      * will operate correctly. A Safe Position is not necessarily a boundary itself.
      *
      * @param fromPosition the position in the input text to begin the iteration.
-     * @internal
+     * @internal (private)
      */
-    int32_t handlePrevious(int32_t fromPosition);
+    int32_t handleSafePrevious(int32_t fromPosition);
 
     /**
      * Find a rule-based boundary by running the state machine.
@@ -658,7 +652,7 @@ private:
      *                         If > 0, the segment will be further subdivided
      *    fRuleStatusIndex     Info from the state table indicating which rules caused the boundary.
      *
-     * @internal
+     * @internal (private)
      */
     int32_t handleNext();
 
@@ -667,7 +661,7 @@ private:
      * This function returns the appropriate LanguageBreakEngine for a
      * given character c.
      * @param c         A character in the dictionary set
-     * @internal
+     * @internal (private)
      */
     const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
 

intl/icu/source/common/unicode/uchar.h

@@ -42,7 +42,7 @@ U_CDECL_BEGIN
  * @see u_getUnicodeVersion
  * @stable ICU 2.0
  */
-#define U_UNICODE_VERSION "10.0"
+#define U_UNICODE_VERSION "11.0"
 
 /**
  * \file
@@ -446,6 +446,13 @@ typedef enum UProperty {
      * @stable ICU 60
      */
     UCHAR_PREPENDED_CONCATENATION_MARK=63,
+    /**
+     * Binary property Extended_Pictographic.
+     * See http://www.unicode.org/reports/tr51/#Emoji_Properties
+     *
+     * @stable ICU 62
+     */
+    UCHAR_EXTENDED_PICTOGRAPHIC=64,
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the last constant for binary Unicode properties.
@@ -1683,6 +1690,31 @@ enum UBlockCode {
     /** @stable ICU 60 */
     UBLOCK_ZANABAZAR_SQUARE = 280, /*[11A00]*/
 
+    // New blocks in Unicode 11.0
+
+    /** @stable ICU 62 */
+    UBLOCK_CHESS_SYMBOLS = 281, /*[1FA00]*/
+    /** @stable ICU 62 */
+    UBLOCK_DOGRA = 282, /*[11800]*/
+    /** @stable ICU 62 */
+    UBLOCK_GEORGIAN_EXTENDED = 283, /*[1C90]*/
+    /** @stable ICU 62 */
+    UBLOCK_GUNJALA_GONDI = 284, /*[11D60]*/
+    /** @stable ICU 62 */
+    UBLOCK_HANIFI_ROHINGYA = 285, /*[10D00]*/
+    /** @stable ICU 62 */
+    UBLOCK_INDIC_SIYAQ_NUMBERS = 286, /*[1EC70]*/
+    /** @stable ICU 62 */
+    UBLOCK_MAKASAR = 287, /*[11EE0]*/
+    /** @stable ICU 62 */
+    UBLOCK_MAYAN_NUMERALS = 288, /*[1D2E0]*/
+    /** @stable ICU 62 */
+    UBLOCK_MEDEFAIDRIN = 289, /*[16E40]*/
+    /** @stable ICU 62 */
+    UBLOCK_OLD_SOGDIAN = 290, /*[10F00]*/
+    /** @stable ICU 62 */
+    UBLOCK_SOGDIAN = 291, /*[10F30]*/
+
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal UBlockCode value.
@@ -1690,7 +1722,7 @@ enum UBlockCode {
      *
      * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
      */
-    UBLOCK_COUNT = 281,
+    UBLOCK_COUNT = 292,
 #endif  // U_HIDE_DEPRECATED_API
 
     /** @stable ICU 2.0 */
@@ -1979,6 +2011,9 @@ typedef enum UJoiningGroup {
     U_JG_MALAYALAM_SSA,  /**< @stable ICU 60 */
     U_JG_MALAYALAM_TTA,  /**< @stable ICU 60 */
 
+    U_JG_HANIFI_ROHINGYA_KINNA_YA,  /**< @stable ICU 62 */
+    U_JG_HANIFI_ROHINGYA_PA,  /**< @stable ICU 62 */
+
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal UJoiningGroup value.
@@ -2029,6 +2064,7 @@ typedef enum UGraphemeClusterBreak {
     U_GCB_GLUE_AFTER_ZWJ = 16,  /*[GAZ]*/
     /** @stable ICU 58 */
     U_GCB_ZWJ = 17,             /*[ZWJ]*/
+
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal UGraphemeClusterBreak value.
@@ -2090,6 +2126,9 @@ typedef enum UWordBreakValues {
     U_WB_GLUE_AFTER_ZWJ = 20,   /*[GAZ]*/
     /** @stable ICU 58 */
     U_WB_ZWJ = 21,              /*[ZWJ]*/
+    /** @stable ICU 62 */
+    U_WB_WSEGSPACE = 22,        /*[WSEGSPACE]*/
+
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal UWordBreakValues value.
@@ -2097,7 +2136,7 @@ typedef enum UWordBreakValues {
      *
      * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
      */
-    U_WB_COUNT = 22
+    U_WB_COUNT = 23
 #endif  // U_HIDE_DEPRECATED_API
 } UWordBreakValues;
 

intl/icu/source/common/unicode/unistr.h

@@ -1892,7 +1892,7 @@ public:
   UnicodeString &fastCopyFrom(const UnicodeString &src);
 
   /**
-   * Move assignment operator, might leave src in bogus state.
+   * Move assignment operator; might leave src in bogus state.
    * This string will have the same contents and state that the source string had.
    * The behavior is undefined if *this and src are the same object.
    * @param src source string
@@ -1905,7 +1905,7 @@ public:
 
   // do not use #ifndef U_HIDE_DRAFT_API for moveFrom, needed by non-draft API
   /**
-   * Move assignment, might leave src in bogus state.
+   * Move assignment; might leave src in bogus state.
    * This string will have the same contents and state that the source string had.
    * The behavior is undefined if *this and src are the same object.
    *
@@ -3314,7 +3314,7 @@ public:
   UnicodeString(const UnicodeString& that);
 
   /**
-   * Move constructor, might leave src in bogus state.
+   * Move constructor; might leave src in bogus state.
    * This string will have the same contents and state that the source string had.
    * @param src source string
    * @stable ICU 56

intl/icu/source/common/unicode/urename.h

@@ -613,6 +613,7 @@
 #define ucnv_createConverterFromPackage U_ICU_ENTRY_POINT_RENAME(ucnv_createConverterFromPackage)
 #define ucnv_createConverterFromSharedData U_ICU_ENTRY_POINT_RENAME(ucnv_createConverterFromSharedData)
 #define ucnv_detectUnicodeSignature U_ICU_ENTRY_POINT_RENAME(ucnv_detectUnicodeSignature)
+#define ucnv_enableCleanup U_ICU_ENTRY_POINT_RENAME(ucnv_enableCleanup)
 #define ucnv_extContinueMatchFromU U_ICU_ENTRY_POINT_RENAME(ucnv_extContinueMatchFromU)
 #define ucnv_extContinueMatchToU U_ICU_ENTRY_POINT_RENAME(ucnv_extContinueMatchToU)
 #define ucnv_extGetUnicodeSet U_ICU_ENTRY_POINT_RENAME(ucnv_extGetUnicodeSet)
@@ -1170,6 +1171,16 @@
 #define unum_setSymbol U_ICU_ENTRY_POINT_RENAME(unum_setSymbol)
 #define unum_setTextAttribute U_ICU_ENTRY_POINT_RENAME(unum_setTextAttribute)
 #define unum_toPattern U_ICU_ENTRY_POINT_RENAME(unum_toPattern)
+#define unumf_close U_ICU_ENTRY_POINT_RENAME(unumf_close)
+#define unumf_closeResult U_ICU_ENTRY_POINT_RENAME(unumf_closeResult)
+#define unumf_formatDecimal U_ICU_ENTRY_POINT_RENAME(unumf_formatDecimal)
+#define unumf_formatDouble U_ICU_ENTRY_POINT_RENAME(unumf_formatDouble)
+#define unumf_formatInt U_ICU_ENTRY_POINT_RENAME(unumf_formatInt)
+#define unumf_openForSkeletonAndLocale U_ICU_ENTRY_POINT_RENAME(unumf_openForSkeletonAndLocale)
+#define unumf_openResult U_ICU_ENTRY_POINT_RENAME(unumf_openResult)
+#define unumf_resultGetAllFieldPositions U_ICU_ENTRY_POINT_RENAME(unumf_resultGetAllFieldPositions)
+#define unumf_resultNextFieldPosition U_ICU_ENTRY_POINT_RENAME(unumf_resultNextFieldPosition)
+#define unumf_resultToString U_ICU_ENTRY_POINT_RENAME(unumf_resultToString)
 #define unumsys_close U_ICU_ENTRY_POINT_RENAME(unumsys_close)
 #define unumsys_getDescription U_ICU_ENTRY_POINT_RENAME(unumsys_getDescription)
 #define unumsys_getName U_ICU_ENTRY_POINT_RENAME(unumsys_getName)
@@ -1209,6 +1220,7 @@
 #define uplug_setPlugNoUnload U_ICU_ENTRY_POINT_RENAME(uplug_setPlugNoUnload)
 #define uprops_getSource U_ICU_ENTRY_POINT_RENAME(uprops_getSource)
 #define upropsvec_addPropertyStarts U_ICU_ENTRY_POINT_RENAME(upropsvec_addPropertyStarts)
+#define uprv_add32_overflow U_ICU_ENTRY_POINT_RENAME(uprv_add32_overflow)
 #define uprv_aestrncpy U_ICU_ENTRY_POINT_RENAME(uprv_aestrncpy)
 #define uprv_asciiFromEbcdic U_ICU_ENTRY_POINT_RENAME(uprv_asciiFromEbcdic)
 #define uprv_asciitolower U_ICU_ENTRY_POINT_RENAME(uprv_asciitolower)
@@ -1343,6 +1355,7 @@
 #define uprv_maximumPtr U_ICU_ENTRY_POINT_RENAME(uprv_maximumPtr)
 #define uprv_min U_ICU_ENTRY_POINT_RENAME(uprv_min)
 #define uprv_modf U_ICU_ENTRY_POINT_RENAME(uprv_modf)
+#define uprv_mul32_overflow U_ICU_ENTRY_POINT_RENAME(uprv_mul32_overflow)
 #define uprv_parseCurrency U_ICU_ENTRY_POINT_RENAME(uprv_parseCurrency)
 #define uprv_pathIsAbsolute U_ICU_ENTRY_POINT_RENAME(uprv_pathIsAbsolute)
 #define uprv_pow U_ICU_ENTRY_POINT_RENAME(uprv_pow)

intl/icu/source/common/unicode/uscript.h

@@ -451,6 +451,21 @@ typedef enum UScriptCode {
       /** @stable ICU 60 */
       USCRIPT_ZANABAZAR_SQUARE              = 177,/* Zanb */
 
+      /** @stable ICU 62 */
+      USCRIPT_DOGRA                         = 178,/* Dogr */
+      /** @stable ICU 62 */
+      USCRIPT_GUNJALA_GONDI                 = 179,/* Gong */
+      /** @stable ICU 62 */
+      USCRIPT_MAKASAR                       = 180,/* Maka */
+      /** @stable ICU 62 */
+      USCRIPT_MEDEFAIDRIN                   = 181,/* Medf */
+      /** @stable ICU 62 */
+      USCRIPT_HANIFI_ROHINGYA               = 182,/* Rohg */
+      /** @stable ICU 62 */
+      USCRIPT_SOGDIAN                       = 183,/* Sogd */
+      /** @stable ICU 62 */
+      USCRIPT_OLD_SOGDIAN                   = 184,/* Sogo */
+
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal UScriptCode value.
@@ -458,7 +473,7 @@ typedef enum UScriptCode {
      *
      * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
      */
-    USCRIPT_CODE_LIMIT    = 178
+    USCRIPT_CODE_LIMIT    = 185
 #endif  // U_HIDE_DEPRECATED_API
 } UScriptCode;
 

intl/icu/source/common/unicode/utypes.h

@@ -542,12 +542,15 @@ typedef enum UErrorCode {
 #ifndef U_HIDE_DRAFT_API
     U_NUMBER_ARG_OUTOFBOUNDS_ERROR,   /**< The argument to a NumberFormatter helper method was out of bounds; the bounds are usually 0 to 999. @draft ICU 61 */
 #endif  // U_HIDE_DRAFT_API
+#ifndef U_HIDE_DRAFT_API
+    U_NUMBER_SKELETON_SYNTAX_ERROR,   /**< The number skeleton passed to C++ NumberFormatter or C UNumberFormatter was invalid or contained a syntax error. @draft ICU 62 */
+#endif  // U_HIDE_DRAFT_API
 #ifndef U_HIDE_DEPRECATED_API
     /**
      * One more than the highest normal formatting API error code.
      * @deprecated ICU 58 The numeric value may change over time, see ICU ticket #12420.
      */
-    U_FMT_PARSE_ERROR_LIMIT = 0x10113,
+    U_FMT_PARSE_ERROR_LIMIT = 0x10114,
 #endif  // U_HIDE_DEPRECATED_API
 
     /*

intl/icu/source/common/unicode/uvernum.h

@@ -58,7 +58,7 @@
  *  This value will change in the subsequent releases of ICU
  *  @stable ICU 2.4
  */
-#define U_ICU_VERSION_MAJOR_NUM 61
+#define U_ICU_VERSION_MAJOR_NUM 62
 
 /** The current ICU minor version as an integer.
  *  This value will change in the subsequent releases of ICU
@@ -84,7 +84,7 @@
  *  This value will change in the subsequent releases of ICU
  *  @stable ICU 2.6
  */
-#define U_ICU_VERSION_SUFFIX _61
+#define U_ICU_VERSION_SUFFIX _62
 
 /**
  * \def U_DEF2_ICU_ENTRY_POINT_RENAME
@@ -119,7 +119,7 @@
  *  This value will change in the subsequent releases of ICU
  *  @stable ICU 2.4
  */
-#define U_ICU_VERSION "61.1"
+#define U_ICU_VERSION "62.1"
 
 /**
  * The current ICU library major version number as a string, for library name suffixes.
@@ -132,13 +132,13 @@
  *
  * @stable ICU 2.6
  */
-#define U_ICU_VERSION_SHORT "61"
+#define U_ICU_VERSION_SHORT "62"
 
 #ifndef U_HIDE_INTERNAL_API
 /** Data version in ICU4C.
  * @internal ICU 4.4 Internal Use Only
  **/
-#define U_ICU_DATA_VERSION "61.1"
+#define U_ICU_DATA_VERSION "62.1"
 #endif  /* U_HIDE_INTERNAL_API */
 
 /*===========================================================================

intl/icu/source/common/uprops.cpp

@@ -282,6 +282,7 @@ static const BinaryProperty binProps[UCHAR_BINARY_LIMIT]={
     { 2,                U_MASK(UPROPS_2_EMOJI_COMPONENT), defaultContains },
     { 2,                0, isRegionalIndicator },
     { 1,                U_MASK(UPROPS_PREPENDED_CONCATENATION_MARK), defaultContains },
+    { 2,                U_MASK(UPROPS_2_EXTENDED_PICTOGRAPHIC), defaultContains },
 };
 
 U_CAPI UBool U_EXPORT2

intl/icu/source/common/uprops.h

@@ -196,8 +196,7 @@ enum {
 /*
  * Properties in vector word 2
  * Bits
- * 31..27   http://www.unicode.org/reports/tr51/#Emoji_Properties
- *     26   reserved
+ * 31..26   http://www.unicode.org/reports/tr51/#Emoji_Properties
  * 25..20   Line Break
  * 19..15   Sentence Break
  * 14..10   Word Break
@@ -205,7 +204,8 @@ enum {
  *  4.. 0   Decomposition Type
  */
 enum {
-    UPROPS_2_EMOJI_COMPONENT=27,
+    UPROPS_2_EXTENDED_PICTOGRAPHIC=26,
+    UPROPS_2_EMOJI_COMPONENT,
     UPROPS_2_EMOJI,
     UPROPS_2_EMOJI_PRESENTATION,
     UPROPS_2_EMOJI_MODIFIER,

intl/icu/source/common/uscript_props.cpp

@@ -71,7 +71,7 @@ const int32_t SCRIPT_PROPS[] = {
     0x0EA5 | RECOMMENDED | LB_LETTERS,  // Laoo
     0x004C | RECOMMENDED | CASED,  // Latn
     0x0D15 | RECOMMENDED,  // Mlym
-    0x1826 | LIMITED_USE,  // Mong
+    0x1826 | EXCLUSION,  // Mong
     0x1000 | RECOMMENDED | LB_LETTERS,  // Mymr
     0x168F | EXCLUSION,  // Ogam
     0x10300 | EXCLUSION,  // Ital
@@ -222,6 +222,13 @@ const int32_t SCRIPT_PROPS[] = {
     0x11D10 | EXCLUSION,  // Gonm
     0x11A5C | EXCLUSION,  // Soyo
     0x11A0B | EXCLUSION,  // Zanb
+    0x1180B | EXCLUSION,  // Dogr
+    0x11D71 | LIMITED_USE,  // Gong
+    0x11EE5 | EXCLUSION,  // Maka
+    0x16E40 | EXCLUSION | CASED,  // Medf
+    0x10D12 | LIMITED_USE | RTL,  // Rohg
+    0x10F42 | EXCLUSION | RTL,  // Sogd
+    0x10F19 | EXCLUSION | RTL,  // Sogo
     // End copy-paste from parsescriptmetadata.py
 };
 

intl/icu/source/common/ustr_cnv.cpp

@@ -28,6 +28,7 @@
 #include "cmemory.h"
 #include "umutex.h"
 #include "ustr_cnv.h"
+#include "ucnv_bld.h"
 
 /* mutexed access to a shared default converter ----------------------------- */
 
@@ -68,8 +69,8 @@ u_releaseDefaultConverter(UConverter *converter)
         if (converter != NULL) {
             ucnv_reset(converter);
         }
+        ucnv_enableCleanup();
         umtx_lock(NULL);
-
         if(gDefaultConverter == NULL) {
             gDefaultConverter = converter;
             converter = NULL;