Bug 1624128 - Update CK_GCM_PARAMS uses for PKCS11 v3.0 definition r=keeler

This patch initializes the ulIvBits member of CK_GCM_PARAMS, which is new in PKCS11 v3.

For libprio, we instead define NSS_PKCS11_2_0_COMPAT, which yields the old struct definition.

Differential Revision: https://phabricator.services.mozilla.com/D67740

--HG--
extra : moz-landing-system : lando

dom/crypto/WebCryptoTask.cpp

@@ -600,6 +600,7 @@ class AesTask : public ReturnArrayBufferViewTask, public DeferredData {
       case CKM_AES_GCM:
         gcmParams.pIv = mIv.Elements();
         gcmParams.ulIvLen = mIv.Length();
+        gcmParams.ulIvBits = gcmParams.ulIvLen * 8;
         gcmParams.pAAD = mAad.Elements();
         gcmParams.ulAADLen = mAad.Length();
         gcmParams.ulTagBits = mTagLength;

netwerk/srtp/src/crypto/cipher/aes_gcm_nss.c

@@ -271,6 +271,7 @@ static srtp_err_status_t srtp_aes_gcm_nss_do_crypto(void *cv,
 
     c->params.pIv = c->iv;
     c->params.ulIvLen = GCM_IV_LEN;
+    c->params.ulIvBits = GCM_IV_LEN * 8;
     c->params.pAAD = c->aad;
     c->params.ulAADLen = c->aad_size;
 

security/manager/ssl/OSKeyStore.cpp

@@ -663,6 +663,7 @@ nsresult AbstractOSKeyStore::DoCipher(const UniquePK11SymKey& aSymKey,
   CK_GCM_PARAMS gcm_params;
   gcm_params.pIv = const_cast<unsigned char*>(ivp);
   gcm_params.ulIvLen = mIVLength;
+  gcm_params.ulIvBits = gcm_params.ulIvLen * 8;
   gcm_params.ulTagBits = 128;
   gcm_params.pAAD = nullptr;
   gcm_params.ulAADLen = 0;

third_party/prio/moz.build

@@ -42,3 +42,8 @@ SOURCES += [
 ]
 
 FINAL_LIBRARY = 'xul'
+
+# Use PKCS11 v2 struct definitions for now, otherwise NSS requires
+# CK_GCM_PARAMS.ulIvBits to be set. This workaround is only required
+# until NSS 3.52 RTM and upstream correctly initializes the field.
+DEFINES['NSS_PKCS11_2_0_COMPAT'] = True