Bug 1759201 - Add a text recognition component. r=dminor,gregtatum

This adds just the boilerplate, and doesn't implement the OS-specific
bits. It's still useful tho.

Co-Authored-By: Greg Tatum <tatum.creative@gmail.com>

Differential Revision: https://phabricator.services.mozilla.com/D140975

widget/TextRecognition.cpp

@@ -0,0 +1,66 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TextRecognition.h"
+#include "mozilla/dom/Promise.h"
+#include "imgIContainer.h"
+
+namespace mozilla::widget {
+
+NS_IMPL_ISUPPORTS(TextRecognition, nsITextRecognition);
+
+static bool ToJSValue(JSContext* aCx, widget::TextRecognitionResult& aResult,
+                      JS::MutableHandle<JS::Value> aValue) {
+  // TODO(emilio): Expose this to JS in a nicer way than just the string.
+  nsAutoString string;
+  for (const auto& quad : aResult.mQuads) {
+    string.Append(quad.mString);
+    string.Append(u'\n');
+  }
+  return dom::ToJSValue(aCx, string, aValue);
+}
+
+NS_IMETHODIMP
+TextRecognition::FindText(imgIContainer* aImage, JSContext* aCx,
+                          dom::Promise** aResultPromise) {
+  ErrorResult rv;
+  RefPtr<dom::Promise> promise =
+      dom::Promise::Create(xpc::CurrentNativeGlobal(aCx), rv);
+  if (MOZ_UNLIKELY(rv.Failed())) {
+    return rv.StealNSResult();
+  }
+
+  RefPtr<gfx::SourceSurface> surface = aImage->GetFrame(
+      imgIContainer::FRAME_CURRENT,
+      imgIContainer::FLAG_SYNC_DECODE | imgIContainer::FLAG_ASYNC_NOTIFY);
+
+  if (NS_WARN_IF(!surface)) {
+    promise->MaybeRejectWithInvalidStateError("Failed to get surface");
+    return NS_OK;
+  }
+
+  auto promiseHolder = MakeRefPtr<nsMainThreadPtrHolder<dom::Promise>>(
+      "nsTextRecognition::SpawnOSBackgroundThread", promise);
+  FindText(*surface)->Then(
+      GetCurrentSerialEventTarget(), __func__,
+      [promiseHolder = std::move(promiseHolder)](
+          NativePromise::ResolveOrRejectValue&& aValue) {
+        dom::Promise* p = promiseHolder->get();
+        if (aValue.IsReject()) {
+          p->MaybeRejectWithNotSupportedError(aValue.RejectValue());
+          return;
+        }
+        p->MaybeResolve(aValue.ResolveValue());
+      });
+
+  promise.forget(aResultPromise);
+  return NS_OK;
+}
+
+auto TextRecognition::FindText(gfx::SourceSurface&) -> RefPtr<NativePromise> {
+  return NativePromise::CreateAndReject("Text recognition not available"_ns,
+                                        __func__);
+}
+
+}  // namespace mozilla::widget

widget/TextRecognition.h

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_widget_nsTextRecognition__
+#define mozilla_widget_nsTextRecognition__
+
+#include "mozilla/MozPromise.h"
+#include "mozilla/gfx/Point.h"
+#include "nsTArray.h"
+#include "nsITextRecognition.h"
+
+class imgIContainer;
+namespace mozilla::gfx {
+class SourceSurface;
+}
+
+namespace mozilla::widget {
+
+struct TextRecognitionQuad {
+  gfx::Point mPoints[4];
+  nsString mString;
+  // Confidence from zero to one.
+  float mConfidence = 0.0f;
+};
+
+struct TextRecognitionResult {
+  nsTArray<TextRecognitionQuad> mQuads;
+};
+
+class TextRecognition final : public nsITextRecognition {
+ public:
+  using NativePromise = MozPromise<TextRecognitionResult, nsCString,
+                                   /* IsExclusive = */ true>;
+
+  NS_DECL_NSITEXTRECOGNITION
+  NS_DECL_ISUPPORTS
+
+  TextRecognition() = default;
+
+  // This should be implemented in the OS specific file.
+  static RefPtr<NativePromise> FindText(gfx::SourceSurface&);
+
+ protected:
+  ~TextRecognition() = default;
+};
+
+}  // namespace mozilla::widget
+
+#endif

widget/components.conf

@@ -0,0 +1,16 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+Classes = [
+    {
+        'js_name': 'textRecognition',
+        'cid': '{a5b8ee77-e250-4643-a02a-5458813616a5}',
+        'contract_ids': ['@mozilla/widget/textRecognition;1'],
+        'interfaces': ['nsITextRecognition'],
+        'type': 'mozilla::widget::TextRecognition',
+        'headers': ['/widget/TextRecognition.h'],
+    }
+]

widget/moz.build

@@ -127,11 +127,16 @@ XPIDL_SOURCES += [
     "nsISharePicker.idl",
     "nsISound.idl",
     "nsISystemStatusBar.idl",
+    "nsITextRecognition.idl",
     "nsITransferable.idl",
     "nsIUserIdleService.idl",
     "nsIUserIdleServiceInternal.idl",
 ]
 
+XPCOM_MANIFESTS += [
+    "components.conf",
+]
+
 XPIDL_MODULE = "widget"
 
 EXPORTS += [
@@ -196,6 +201,7 @@ EXPORTS.mozilla.widget += [
     "RemoteLookAndFeel.h",
     "Screen.h",
     "ScreenManager.h",
+    "TextRecognition.h",
     "ThemeChangeKind.h",
     "WidgetMessageUtils.h",
     "WindowOcclusionState.h",
@@ -242,6 +248,7 @@ UNIFIED_SOURCES += [
     "SharedWidgetUtils.cpp",
     "SwipeTracker.cpp",
     "TextEventDispatcher.cpp",
+    "TextRecognition.cpp",
     "Theme.cpp",
     "ThemeCocoa.cpp",
     "ThemeColors.cpp",

widget/nsITextRecognition.idl

@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+interface imgIContainer;
+
+/**
+ * An interface to query the OS's OCR (optical character recognition) service,
+ * if available.
+ */
+[scriptable, uuid(a5b8ee77-e250-4643-a02a-5458813616a5)]
+interface nsITextRecognition : nsISupports
+{
+  /**
+   * TODO - The promise contains the string of the results. This should return
+   * a list of results that includes the text, confidence, and bounding box.
+   */
+  [implicit_jscontext]
+  Promise findText(in imgIContainer aImage);
+};