Bug 1859536 - Implement EncodedAudioChunk. r=chunmin,webidl,saschanaz

I thought about not copy-pasting, but it didn't result in something that was
clearer, it was in fact more complex.

Differential Revision: https://phabricator.services.mozilla.com/D192337

dom/base/StructuredCloneHolder.cpp

@@ -34,6 +34,8 @@
 #include "mozilla/dom/DOMTypes.h"
 #include "mozilla/dom/Directory.h"
 #include "mozilla/dom/DocGroup.h"
+#include "mozilla/dom/EncodedAudioChunk.h"
+#include "mozilla/dom/EncodedAudioChunkBinding.h"
 #include "mozilla/dom/EncodedVideoChunk.h"
 #include "mozilla/dom/EncodedVideoChunkBinding.h"
 #include "mozilla/dom/File.h"
@@ -400,6 +402,7 @@ void StructuredCloneHolder::Read(nsIGlobalObject* aGlobal, JSContext* aCx,
     mClonedSurfaces.Clear();
     mInputStreamArray.Clear();
     mVideoFrames.Clear();
+    mEncodedAudioChunks.Clear();
     mEncodedVideoChunks.Clear();
     Clear();
   }
@@ -1139,6 +1142,18 @@ JSObject* StructuredCloneHolder::CustomReadHandler(
     }
   }
 
+  if (StaticPrefs::dom_media_webcodecs_enabled() &&
+      aTag == SCTAG_DOM_ENCODEDAUDIOCHUNK &&
+      CloneScope() == StructuredCloneScope::SameProcess &&
+      aCloneDataPolicy.areIntraClusterClonableSharedObjectsAllowed()) {
+    JS::Rooted<JSObject*> global(aCx, mGlobal->GetGlobalJSObject());
+    if (EncodedAudioChunk_Binding::ConstructorEnabled(aCx, global)) {
+      return EncodedAudioChunk::ReadStructuredClone(
+          aCx, mGlobal, aReader, EncodedAudioChunks()[aIndex]);
+    }
+  }
+
+
   return ReadFullySerializableObjects(aCx, aReader, aTag, false);
 }
 
@@ -1269,6 +1284,18 @@ bool StructuredCloneHolder::CustomWriteHandler(
     }
   }
 
+  // See if this is a EncodedAudioChunk object.
+  if (StaticPrefs::dom_media_webcodecs_enabled()) {
+    EncodedAudioChunk* encodedAudioChunk = nullptr;
+    if (NS_SUCCEEDED(
+            UNWRAP_OBJECT(EncodedAudioChunk, &obj, encodedAudioChunk))) {
+      SameProcessScopeRequired(aSameProcessScopeRequired);
+      return CloneScope() == StructuredCloneScope::SameProcess
+                 ? encodedAudioChunk->WriteStructuredClone(aWriter, this)
+                 : false;
+    }
+  }
+
   {
     // We only care about streams, so ReflectorToISupportsStatic is fine.
     nsCOMPtr<nsISupports> base = xpc::ReflectorToISupportsStatic(aObj);

dom/base/StructuredCloneHolder.h

@@ -165,6 +165,7 @@ class StructuredCloneHolderBase {
 };
 
 class BlobImpl;
+class EncodedAudioChunkData;
 class EncodedVideoChunkData;
 class MessagePort;
 class MessagePortIdentifier;
@@ -277,6 +278,10 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
     return mEncodedVideoChunks;
   }
 
+  nsTArray<EncodedAudioChunkData>& EncodedAudioChunks() {
+    return mEncodedAudioChunks;
+  }
+
   // Implementations of the virtual methods to allow cloning of objects which
   // JS engine itself doesn't clone.
 
@@ -388,6 +393,9 @@ class StructuredCloneHolder : public StructuredCloneHolderBase {
   // Used for cloning EncodedVideoChunk in the structured cloning algorithm.
   nsTArray<EncodedVideoChunkData> mEncodedVideoChunks;
 
+  // Used for cloning EncodedAudioChunk in the structured cloning algorithm.
+  nsTArray<EncodedAudioChunkData> mEncodedAudioChunks;
+
   // This raw pointer is only set within ::Read() and is unset by the end.
   nsIGlobalObject* MOZ_NON_OWNING_REF mGlobal;
 

dom/base/StructuredCloneTags.h

@@ -159,6 +159,8 @@ enum StructuredCloneTags : uint32_t {
 
   SCTAG_DOM_AUDIODATA,
 
+  SCTAG_DOM_ENCODEDAUDIOCHUNK,
+
   // IMPORTANT: If you plan to add an new IDB tag, it _must_ be add before the
   // "less stable" tags!
 };

dom/media/webcodecs/EncodedAudioChunk.cpp

@@ -0,0 +1,260 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/EncodedAudioChunk.h"
+#include "mozilla/dom/EncodedAudioChunkBinding.h"
+
+#include <utility>
+
+#include "MediaData.h"
+#include "TimeUnits.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/Logging.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/dom/StructuredCloneHolder.h"
+#include "mozilla/dom/StructuredCloneTags.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+using mozilla::media::TimeUnit;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+#  undef LOG_INTERNAL
+#endif  // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+  MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOGW
+#  undef LOGW
+#endif  // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+#  undef LOGE
+#endif  // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+// Only needed for refcounted objects.
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedAudioChunk, mParent)
+NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedAudioChunk)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedAudioChunk)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedAudioChunk)
+  NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+  NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+EncodedAudioChunkData::EncodedAudioChunkData(
+    already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+    const EncodedAudioChunkType& aType, int64_t aTimestamp,
+    Maybe<uint64_t>&& aDuration)
+    : mBuffer(aBuffer),
+      mType(aType),
+      mTimestamp(aTimestamp),
+      mDuration(aDuration) {
+  MOZ_ASSERT(mBuffer);
+  MOZ_ASSERT(mBuffer->Length() == mBuffer->Size());
+  MOZ_ASSERT(mBuffer->Length() <=
+             static_cast<size_t>(std::numeric_limits<uint32_t>::max()));
+}
+
+UniquePtr<EncodedAudioChunkData> EncodedAudioChunkData::Clone() const {
+  if (!mBuffer) {
+    LOGE("No buffer in EncodedAudioChunkData %p to clone!", this);
+    return nullptr;
+  }
+
+  // Since EncodedAudioChunkData can be zero-sized, cloning a zero-sized chunk
+  // is allowed.
+  if (mBuffer->Size() == 0) {
+    LOGW("Cloning an empty EncodedAudioChunkData %p", this);
+  }
+
+  auto buffer =
+      MakeRefPtr<MediaAlignedByteBuffer>(mBuffer->Data(), mBuffer->Length());
+  if (!buffer || buffer->Size() != mBuffer->Size()) {
+    LOGE("OOM to copy EncodedAudioChunkData %p", this);
+    return nullptr;
+  }
+
+  return MakeUnique<EncodedAudioChunkData>(buffer.forget(), mType, mTimestamp,
+                                           Maybe<uint64_t>(mDuration));
+}
+
+already_AddRefed<MediaRawData> EncodedAudioChunkData::TakeData() {
+  if (!mBuffer || !(*mBuffer)) {
+    LOGE("EncodedAudioChunkData %p has no data!", this);
+    return nullptr;
+  }
+
+  RefPtr<MediaRawData> sample(new MediaRawData(std::move(*mBuffer)));
+  sample->mKeyframe = mType == EncodedAudioChunkType::Key;
+  sample->mTime = TimeUnit::FromMicroseconds(mTimestamp);
+  sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp);
+
+  if (mDuration) {
+    CheckedInt64 duration(*mDuration);
+    if (!duration.isValid()) {
+      LOGE("EncodedAudioChunkData %p 's duration exceeds TimeUnit's limit",
+           this);
+      return nullptr;
+    }
+    sample->mDuration = TimeUnit::FromMicroseconds(duration.value());
+  }
+
+  return sample.forget();
+}
+
+EncodedAudioChunk::EncodedAudioChunk(
+    nsIGlobalObject* aParent, already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+    const EncodedAudioChunkType& aType, int64_t aTimestamp,
+    Maybe<uint64_t>&& aDuration)
+    : EncodedAudioChunkData(std::move(aBuffer), aType, aTimestamp,
+                            std::move(aDuration)),
+      mParent(aParent) {}
+
+EncodedAudioChunk::EncodedAudioChunk(nsIGlobalObject* aParent,
+                                     const EncodedAudioChunkData& aData)
+    : EncodedAudioChunkData(aData), mParent(aParent) {}
+
+nsIGlobalObject* EncodedAudioChunk::GetParentObject() const {
+  AssertIsOnOwningThread();
+
+  return mParent.get();
+}
+
+JSObject* EncodedAudioChunk::WrapObject(JSContext* aCx,
+                                        JS::Handle<JSObject*> aGivenProto) {
+  AssertIsOnOwningThread();
+
+  return EncodedAudioChunk_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#encodedaudiochunk-constructors
+/* static */
+already_AddRefed<EncodedAudioChunk> EncodedAudioChunk::Constructor(
+    const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit,
+    ErrorResult& aRv) {
+  nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+  if (!global) {
+    aRv.Throw(NS_ERROR_FAILURE);
+    return nullptr;
+  }
+
+  auto buffer = ProcessTypedArrays(
+      aInit.mData,
+      [&](const Span<uint8_t>& aData,
+          JS::AutoCheckCannotGC&&) -> RefPtr<MediaAlignedByteBuffer> {
+        // Make sure it's in uint32_t's range.
+        CheckedUint32 byteLength(aData.Length());
+        if (!byteLength.isValid()) {
+          aRv.Throw(NS_ERROR_INVALID_ARG);
+          return nullptr;
+        }
+        if (aData.Length() == 0) {
+          LOGW("Buffer for constructing EncodedAudioChunk is empty!");
+        }
+        RefPtr<MediaAlignedByteBuffer> buf = MakeRefPtr<MediaAlignedByteBuffer>(
+            aData.Elements(), aData.Length());
+
+        // Instead of checking *buf, size comparision is used to allow
+        // constructing a zero-sized EncodedAudioChunk.
+        if (!buf || buf->Size() != aData.Length()) {
+          aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+          return nullptr;
+        }
+        return buf;
+      });
+
+  RefPtr<EncodedAudioChunk> chunk(new EncodedAudioChunk(
+      global, buffer.forget(), aInit.mType, aInit.mTimestamp,
+      OptionalToMaybe(aInit.mDuration)));
+  return aRv.Failed() ? nullptr : chunk.forget();
+}
+
+EncodedAudioChunkType EncodedAudioChunk::Type() const {
+  AssertIsOnOwningThread();
+
+  return mType;
+}
+
+int64_t EncodedAudioChunk::Timestamp() const {
+  AssertIsOnOwningThread();
+
+  return mTimestamp;
+}
+
+Nullable<uint64_t> EncodedAudioChunk::GetDuration() const {
+  AssertIsOnOwningThread();
+  return MaybeToNullable(mDuration);
+}
+
+uint32_t EncodedAudioChunk::ByteLength() const {
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mBuffer);
+
+  return static_cast<uint32_t>(mBuffer->Length());
+}
+
+// https://w3c.github.io/webcodecs/#dom-encodedaudiochunk-copyto
+void EncodedAudioChunk::CopyTo(
+    const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+    ErrorResult& aRv) {
+  AssertIsOnOwningThread();
+
+  ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) {
+    if (mBuffer->Size() > aData.size_bytes()) {
+      aRv.ThrowTypeError(
+          "Destination ArrayBuffer smaller than source EncodedAudioChunk");
+      return;
+    }
+
+    PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size());
+  });
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps
+/* static */
+JSObject* EncodedAudioChunk::ReadStructuredClone(
+    JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader,
+    const EncodedAudioChunkData& aData) {
+  JS::Rooted<JS::Value> value(aCx, JS::NullValue());
+  // To avoid a rooting hazard error from returning a raw JSObject* before
+  // running the RefPtr destructor, RefPtr needs to be destructed before
+  // returning the raw JSObject*, which is why the RefPtr<EncodedAudioChunk> is
+  // created in the scope below. Otherwise, the static analysis infers the
+  // RefPtr cannot be safely destructed while the unrooted return JSObject* is
+  // on the stack.
+  {
+    auto frame = MakeRefPtr<EncodedAudioChunk>(aGlobal, aData);
+    if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) {
+      return nullptr;
+    }
+  }
+  return value.toObjectOrNull();
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-serialization-steps
+bool EncodedAudioChunk::WriteStructuredClone(
+    JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const {
+  AssertIsOnOwningThread();
+
+  // Indexing the chunk and send the index to the receiver.
+  const uint32_t index =
+      static_cast<uint32_t>(aHolder->EncodedAudioChunks().Length());
+  // The serialization is limited to the same process scope so it's ok to
+  // serialize a reference instead of a copy.
+  aHolder->EncodedAudioChunks().AppendElement(EncodedAudioChunkData(*this));
+  return !NS_WARN_IF(
+      !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDAUDIOCHUNK, index));
+}
+
+#undef LOGW
+#undef LOGE
+#undef LOG_INTERNAL
+
+}  // namespace mozilla::dom

dom/media/webcodecs/EncodedAudioChunk.h

@@ -0,0 +1,117 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_EncodedAudioChunk_h
+#define mozilla_dom_EncodedAudioChunk_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "nsCycleCollectionParticipant.h"
+#include "nsWrapperCache.h"
+
+class nsIGlobalObject;
+
+namespace mozilla {
+
+class MediaAlignedByteBuffer;
+class MediaRawData;
+
+namespace dom {
+
+class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class StructuredCloneHolder;
+
+enum class EncodedAudioChunkType : uint8_t;
+struct EncodedAudioChunkInit;
+
+}  // namespace dom
+}  // namespace mozilla
+
+namespace mozilla::dom {
+
+class EncodedAudioChunkData {
+ public:
+  EncodedAudioChunkData(already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+                        const EncodedAudioChunkType& aType, int64_t aTimestamp,
+                        Maybe<uint64_t>&& aDuration);
+  EncodedAudioChunkData(const EncodedAudioChunkData& aData) = default;
+  ~EncodedAudioChunkData() = default;
+
+  UniquePtr<EncodedAudioChunkData> Clone() const;
+  already_AddRefed<MediaRawData> TakeData();
+
+ protected:
+  // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX.
+  RefPtr<MediaAlignedByteBuffer> mBuffer;
+  EncodedAudioChunkType mType;
+  int64_t mTimestamp;
+  Maybe<uint64_t> mDuration;
+};
+
+class EncodedAudioChunk final : public EncodedAudioChunkData,
+                                public nsISupports,
+                                public nsWrapperCache {
+ public:
+  NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+  NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedAudioChunk)
+
+ public:
+  EncodedAudioChunk(nsIGlobalObject* aParent,
+                    already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+                    const EncodedAudioChunkType& aType, int64_t aTimestamp,
+                    Maybe<uint64_t>&& aDuration);
+
+  EncodedAudioChunk(nsIGlobalObject* aParent,
+                    const EncodedAudioChunkData& aData);
+
+ protected:
+  ~EncodedAudioChunk() = default;
+
+ public:
+  nsIGlobalObject* GetParentObject() const;
+
+  JSObject* WrapObject(JSContext* aCx,
+                       JS::Handle<JSObject*> aGivenProto) override;
+
+  static already_AddRefed<EncodedAudioChunk> Constructor(
+      const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit,
+      ErrorResult& aRv);
+
+  EncodedAudioChunkType Type() const;
+
+  int64_t Timestamp() const;
+
+  Nullable<uint64_t> GetDuration() const;
+
+  uint32_t ByteLength() const;
+
+  void CopyTo(
+      const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+      ErrorResult& aRv);
+
+  // [Serializable] implementations: {Read, Write}StructuredClone
+  static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
+                                       JSStructuredCloneReader* aReader,
+                                       const EncodedAudioChunkData& aData);
+
+  bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
+                            StructuredCloneHolder* aHolder) const;
+
+ private:
+  // EncodedAudioChunk can run on either main thread or worker thread.
+  void AssertIsOnOwningThread() const {
+    NS_ASSERT_OWNINGTHREAD(EncodedAudioChunk);
+  }
+
+  nsCOMPtr<nsIGlobalObject> mParent;
+};
+
+}  // namespace mozilla::dom
+
+#endif  // mozilla_dom_EncodedAudioChunk_h

dom/media/webcodecs/moz.build

@@ -24,6 +24,7 @@ EXPORTS.mozilla.dom += [
     "AudioData.h",
     "DecoderTemplate.h",
     "DecoderTypes.h",
+    "EncodedAudioChunk.h",
     "EncodedVideoChunk.h",
     "EncoderAgent.h",
     "EncoderTemplate.h",
@@ -39,6 +40,7 @@ UNIFIED_SOURCES += [
     "AudioData.cpp",
     "DecoderAgent.cpp",
     "DecoderTemplate.cpp",
+    "EncodedAudioChunk.cpp",
     "EncodedVideoChunk.cpp",
     "EncoderAgent.cpp",
     "EncoderTemplate.cpp",

dom/webidl/EncodedAudioChunk.webidl

@@ -0,0 +1,38 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * The origin of this IDL file is
+ * https://w3c.github.io/webcodecs/#encodedaudiochunk
+ */
+
+// [Serializable] is implemented without adding attribute here.
+[Exposed=(Window,DedicatedWorker), Pref="dom.media.webcodecs.enabled"]
+interface EncodedAudioChunk {
+  [Throws]
+  constructor(EncodedAudioChunkInit init);
+  readonly attribute EncodedAudioChunkType type;
+  readonly attribute long long timestamp;          // microseconds
+  readonly attribute unsigned long long? duration; // microseconds
+  readonly attribute unsigned long byteLength;
+
+  [Throws]
+  undefined copyTo(
+      // bug 1696216: Should be `copyTo(AllowSharedBufferSource destination, ...)`
+      ([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) destination);
+};
+
+dictionary EncodedAudioChunkInit {
+  required EncodedAudioChunkType type;
+  required [EnforceRange] long long timestamp;    // microseconds
+  [EnforceRange] unsigned long long duration;     // microseconds
+  // bug 1696216: Should be AllowSharedBufferSource
+  required ([AllowShared] ArrayBufferView or [AllowShared] ArrayBuffer) data;
+  sequence<ArrayBuffer> transfer = [];
+};
+
+enum EncodedAudioChunkType {
+    "key",
+    "delta"
+};

dom/webidl/moz.build

@@ -527,6 +527,7 @@ WEBIDL_FILES = [
     "DynamicsCompressorNode.webidl",
     "Element.webidl",
     "ElementInternals.webidl",
+    "EncodedAudioChunk.webidl",
     "EncodedVideoChunk.webidl",
     "Event.webidl",
     "EventHandler.webidl",