Bug 916643 - ImageCapture webidl and implementation. r=roc, r=smaug

This commit is contained in:
Alfredo Yang 2013-09-16 10:50:24 +08:00
parent 1d9ecb16df
commit 6957a19320
11 changed files with 613 additions and 1 deletions

View File

@ -2004,6 +2004,7 @@ GK_ATOM(ondataavailable, "ondataavailable")
GK_ATOM(onwarning, "onwarning")
GK_ATOM(onstart, "onstart")
GK_ATOM(onstop, "onstop")
GK_ATOM(onphoto, "onphoto")
#ifdef MOZ_GAMEPAD
GK_ATOM(ongamepadbuttondown, "ongamepadbuttondown")
GK_ATOM(ongamepadbuttonup, "ongamepadbuttonup")

View File

@ -0,0 +1,188 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "CaptureTask.h"
#include "mozilla/dom/ImageCapture.h"
#include "mozilla/dom/ImageCaptureError.h"
#include "mozilla/dom/ImageEncoder.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "gfxUtils.h"
#include "nsThreadUtils.h"
namespace mozilla {
nsresult
CaptureTask::TaskComplete(already_AddRefed<dom::DOMFile> aBlob, nsresult aRv)
{
MOZ_ASSERT(NS_IsMainThread());
DetachStream();
nsresult rv;
nsRefPtr<dom::DOMFile> blob(aBlob);
if (mPrincipalChanged) {
aRv = NS_ERROR_DOM_SECURITY_ERR;
IC_LOG("MediaStream principal should not change during TakePhoto().");
}
if (NS_SUCCEEDED(aRv)) {
rv = mImageCapture->PostBlobEvent(blob);
} else {
rv = mImageCapture->PostErrorEvent(dom::ImageCaptureError::PHOTO_ERROR, aRv);
}
// Ensure ImageCapture dereference on main thread here because the TakePhoto()
// sequences stopped here.
mImageCapture = nullptr;
return rv;
}
void
CaptureTask::AttachStream()
{
MOZ_ASSERT(NS_IsMainThread());
nsRefPtr<dom::VideoStreamTrack> track = mImageCapture->GetVideoStreamTrack();
nsRefPtr<DOMMediaStream> domStream = track->GetStream();
domStream->AddPrincipalChangeObserver(this);
nsRefPtr<MediaStream> stream = domStream->GetStream();
stream->AddListener(this);
}
void
CaptureTask::DetachStream()
{
MOZ_ASSERT(NS_IsMainThread());
nsRefPtr<dom::VideoStreamTrack> track = mImageCapture->GetVideoStreamTrack();
nsRefPtr<DOMMediaStream> domStream = track->GetStream();
domStream->RemovePrincipalChangeObserver(this);
nsRefPtr<MediaStream> stream = domStream->GetStream();
stream->RemoveListener(this);
}
void
CaptureTask::PrincipalChanged(DOMMediaStream* aMediaStream)
{
MOZ_ASSERT(NS_IsMainThread());
mPrincipalChanged = true;
}
void
CaptureTask::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
TrackRate aTrackRate,
TrackTicks aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia)
{
if (mImageGrabbedOrTrackEnd) {
return;
}
if (aTrackEvents == MediaStreamListener::TRACK_EVENT_ENDED) {
PostTrackEndEvent();
return;
}
// Callback for encoding complete, it calls on main thread.
class EncodeComplete : public dom::EncodeCompleteCallback
{
public:
EncodeComplete(CaptureTask* aTask) : mTask(aTask) {}
nsresult ReceiveBlob(already_AddRefed<dom::DOMFile> aBlob) MOZ_OVERRIDE
{
nsRefPtr<dom::DOMFile> blob(aBlob);
mTask->TaskComplete(blob.forget(), NS_OK);
mTask = nullptr;
return NS_OK;
}
protected:
nsRefPtr<CaptureTask> mTask;
};
if (aQueuedMedia.GetType() == MediaSegment::VIDEO && mTrackID == aID) {
VideoSegment* video =
const_cast<VideoSegment*> (static_cast<const VideoSegment*>(&aQueuedMedia));
VideoSegment::ChunkIterator iter(*video);
while (!iter.IsEnded()) {
VideoChunk chunk = *iter;
// Extract the first valid video frame.
VideoFrame frame;
if (!chunk.IsNull()) {
nsRefPtr<layers::Image> image;
if (chunk.mFrame.GetForceBlack()) {
// Create a black image.
image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize());
} else {
image = chunk.mFrame.GetImage();
}
MOZ_ASSERT(image);
mImageGrabbedOrTrackEnd = true;
// Encode image.
nsresult rv;
nsAutoString type(NS_LITERAL_STRING("image/jpeg"));
nsAutoString options;
rv = dom::ImageEncoder::ExtractDataFromLayersImageAsync(
type,
options,
false,
image,
new EncodeComplete(this));
if (NS_FAILED(rv)) {
PostTrackEndEvent();
}
return;
}
iter.Next();
}
}
}
void
CaptureTask::NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent)
{
if (((aEvent == EVENT_FINISHED) || (aEvent == EVENT_REMOVED)) &&
!mImageGrabbedOrTrackEnd) {
PostTrackEndEvent();
}
}
void
CaptureTask::PostTrackEndEvent()
{
mImageGrabbedOrTrackEnd = true;
// Got track end or finish event, stop the task.
class TrackEndRunnable : public nsRunnable
{
public:
TrackEndRunnable(CaptureTask* aTask)
: mTask(aTask) {}
NS_IMETHOD Run()
{
mTask->TaskComplete(nullptr, NS_ERROR_FAILURE);
mTask = nullptr;
return NS_OK;
}
protected:
nsRefPtr<CaptureTask> mTask;
};
IC_LOG("Got MediaStream track removed or finished event.");
NS_DispatchToMainThread(new TrackEndRunnable(this));
}
} // namespace mozilla

View File

@ -0,0 +1,96 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef CAPTURETASK_H
#define CAPTURETASK_H
#include "DOMMediaStream.h"
#include "MediaStreamGraph.h"
namespace mozilla {
namespace dom {
class ImageCapture;
class DOMFile;
}
/**
* CaptureTask retrieves image from MediaStream and encodes the image to jpeg in
* ImageEncoder. The whole procedures start at AttachStream(), it will add this
* class into MediaStream and retrieves an image in MediaStreamGraph thread.
* Once the image is retrieved, it will be sent to ImageEncoder and the encoded
* blob will be sent out via encoder callback in main thread.
*
* CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be
* released during the period of the capturing process described above.
*/
class CaptureTask : public MediaStreamListener,
public DOMMediaStream::PrincipalChangeObserver
{
public:
// MediaStreamListener methods.
virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID,
TrackRate aTrackRate,
TrackTicks aTrackOffset,
uint32_t aTrackEvents,
const MediaSegment& aQueuedMedia) MOZ_OVERRIDE;
virtual void NotifyEvent(MediaStreamGraph* aGraph,
MediaStreamGraphEvent aEvent) MOZ_OVERRIDE;
// DOMMediaStream::PrincipalChangeObserver method.
virtual void PrincipalChanged(DOMMediaStream* aMediaStream) MOZ_OVERRIDE;
// CaptureTask methods.
// It is called when aBlob is ready to post back to script in company with
// aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script.
//
// Note:
// this function should be called on main thread.
nsresult TaskComplete(already_AddRefed<dom::DOMFile> aBlob, nsresult aRv);
// Add listeners into MediaStream and PrincipalChangeObserver. It should be on
// main thread only.
void AttachStream();
// Remove listeners from MediaStream and PrincipalChangeObserver. It should be
// on main thread only.
void DetachStream();
// CaptureTask should be created on main thread.
CaptureTask(dom::ImageCapture* aImageCapture, TrackID aTrackID)
: mImageCapture(aImageCapture)
, mTrackID(aTrackID)
, mImageGrabbedOrTrackEnd(false)
, mPrincipalChanged(false) {}
protected:
virtual ~CaptureTask() {}
// Post a runnable on main thread to end this task and call TaskComplete to post
// error event to script. It is called off-main-thread.
void PostTrackEndEvent();
// The ImageCapture associates with this task. This reference count should not
// change in this class unless it clears this reference after a blob or error
// event to script.
nsRefPtr<dom::ImageCapture> mImageCapture;
TrackID mTrackID;
// True when an image is retrieved from MediaStreamGraph or MediaStreamGraph
// sends a track finish, end, or removed event.
bool mImageGrabbedOrTrackEnd;
// True when MediaStream principal is changed in the period of taking photo
// and it causes a NS_ERROR_DOM_SECURITY_ERR error to script.
bool mPrincipalChanged;
};
} // namespace mozilla
#endif // CAPTURETASK_H

View File

@ -0,0 +1,179 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "ImageCapture.h"
#include "mozilla/dom/BlobEvent.h"
#include "mozilla/dom/DOMException.h"
#include "mozilla/dom/ImageCaptureError.h"
#include "mozilla/dom/ImageCaptureErrorEvent.h"
#include "mozilla/dom/ImageCaptureErrorEventBinding.h"
#include "mozilla/dom/VideoStreamTrack.h"
#include "nsIDocument.h"
#include "CaptureTask.h"
namespace mozilla {
#ifdef PR_LOGGING
PRLogModuleInfo* GetICLog()
{
static PRLogModuleInfo* log = nullptr;
if (!log) {
log = PR_NewLogModule("ImageCapture");
}
return log;
}
#endif
namespace dom {
NS_IMPL_CYCLE_COLLECTION_INHERITED(ImageCapture, DOMEventTargetHelper,
mVideoStreamTrack)
NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(ImageCapture)
NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
NS_IMPL_ADDREF_INHERITED(ImageCapture, DOMEventTargetHelper)
NS_IMPL_RELEASE_INHERITED(ImageCapture, DOMEventTargetHelper)
ImageCapture::ImageCapture(VideoStreamTrack* aVideoStreamTrack,
nsPIDOMWindow* aOwnerWindow)
: DOMEventTargetHelper(aOwnerWindow)
{
MOZ_ASSERT(aOwnerWindow);
MOZ_ASSERT(aVideoStreamTrack);
mVideoStreamTrack = aVideoStreamTrack;
}
ImageCapture::~ImageCapture()
{
MOZ_ASSERT(NS_IsMainThread());
}
already_AddRefed<ImageCapture>
ImageCapture::Constructor(const GlobalObject& aGlobal,
VideoStreamTrack& aTrack,
ErrorResult& aRv)
{
nsCOMPtr<nsPIDOMWindow> win = do_QueryInterface(aGlobal.GetAsSupports());
if (!win) {
aRv.Throw(NS_ERROR_FAILURE);
return nullptr;
}
nsRefPtr<ImageCapture> object = new ImageCapture(&aTrack, win);
return object.forget();
}
VideoStreamTrack*
ImageCapture::GetVideoStreamTrack() const
{
return mVideoStreamTrack;
}
void
ImageCapture::TakePhoto(ErrorResult& aResult)
{
// According to spec, VideoStreamTrack.readyState must be "live"; however
// gecko doesn't implement it yet (bug 910249). Instead of readyState, we
// check VideoStreamTrack.enable before bug 910249 is fixed.
// The error code should be INVALID_TRACK, but spec doesn't define it in
// ImageCaptureError. So it returns PHOTO_ERROR here before spec updates.
if (!mVideoStreamTrack->Enabled()) {
PostErrorEvent(ImageCaptureError::PHOTO_ERROR, NS_ERROR_FAILURE);
return;
}
nsRefPtr<CaptureTask> task =
new CaptureTask(this, mVideoStreamTrack->GetTrackID());
// It adds itself into MediaStreamGraph, so ImageCapture doesn't need to hold the
// reference.
task->AttachStream();
}
nsresult
ImageCapture::PostBlobEvent(nsIDOMBlob* aBlob)
{
MOZ_ASSERT(NS_IsMainThread());
if (!CheckPrincipal()) {
// Media is not same-origin, don't allow the data out.
return PostErrorEvent(ImageCaptureError::PHOTO_ERROR, NS_ERROR_DOM_SECURITY_ERR);
}
BlobEventInit init;
init.mBubbles = false;
init.mCancelable = false;
init.mData = aBlob;
nsRefPtr<BlobEvent> blob_event =
BlobEvent::Constructor(this, NS_LITERAL_STRING("photo"), init);
return DispatchTrustedEvent(blob_event);
}
nsresult
ImageCapture::PostErrorEvent(uint16_t aErrorCode, nsresult aReason)
{
MOZ_ASSERT(NS_IsMainThread());
nsresult rv = CheckInnerWindowCorrectness();
NS_ENSURE_SUCCESS(rv, rv);
nsString errorMsg;
if (NS_FAILED(aReason)) {
nsCString name, message;
rv = NS_GetNameAndMessageForDOMNSResult(aReason, name, message);
if (NS_SUCCEEDED(rv)) {
CopyASCIItoUTF16(message, errorMsg);
}
}
nsRefPtr<ImageCaptureError> error =
new ImageCaptureError(this, aErrorCode, errorMsg);
ImageCaptureErrorEventInit init;
init.mBubbles = false;
init.mCancelable = false;
init.mImageCaptureError = error;
nsCOMPtr<nsIDOMEvent> event =
ImageCaptureErrorEvent::Constructor(this, NS_LITERAL_STRING("error"), init);
return DispatchTrustedEvent(event);
}
bool
ImageCapture::CheckPrincipal()
{
MOZ_ASSERT(NS_IsMainThread());
nsRefPtr<DOMMediaStream> ms = mVideoStreamTrack->GetStream();
if (!ms) {
return false;
}
nsCOMPtr<nsIPrincipal> principal = ms->GetPrincipal();
if (!GetOwner()) {
return false;
}
nsCOMPtr<nsIDocument> doc = GetOwner()->GetExtantDoc();
if (!doc || !principal) {
return false;
}
bool subsumes;
if (NS_FAILED(doc->NodePrincipal()->Subsumes(principal, &subsumes))) {
return false;
}
return subsumes;
}
} // namespace dom
} // namespace mozilla

View File

@ -0,0 +1,95 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef IMAGECAPTURE_H
#define IMAGECAPTURE_H
#include "mozilla/DOMEventTargetHelper.h"
#include "mozilla/dom/ImageCaptureBinding.h"
#include "prlog.h"
class nsIDOMBlob;
namespace mozilla {
#ifdef PR_LOGGING
#ifndef IC_LOG
PRLogModuleInfo* GetICLog();
#define IC_LOG(...) PR_LOG(GetICLog(), PR_LOG_DEBUG, (__VA_ARGS__))
#endif
#else
#ifndef IC_LOG
#define IC_LOG(...)
#endif
#endif // PR_LOGGING
namespace dom {
class VideoStreamTrack;
/**
* Implementation of https://dvcs.w3.org/hg/dap/raw-file/default/media-stream-
* capture/ImageCapture.html.
* The ImageCapture accepts a VideoStreamTrack as input source. The image will
* be sent back as a JPG format via Blob event.
*
* All the functions in ImageCapture are run in main thread.
*/
class ImageCapture MOZ_FINAL : public DOMEventTargetHelper
{
public:
NS_DECL_ISUPPORTS_INHERITED
NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(ImageCapture, DOMEventTargetHelper)
IMPL_EVENT_HANDLER(photo)
IMPL_EVENT_HANDLER(error)
// WebIDL members.
void TakePhoto(ErrorResult& aResult);
// The MediaStream passed into the constructor.
VideoStreamTrack* GetVideoStreamTrack() const;
// nsWrapperCache member
virtual JSObject* WrapObject(JSContext* aCx) MOZ_OVERRIDE
{
return ImageCaptureBinding::Wrap(aCx, this);
}
// ImageCapture class members
nsPIDOMWindow* GetParentObject() { return GetOwner(); }
static already_AddRefed<ImageCapture> Constructor(const GlobalObject& aGlobal,
VideoStreamTrack& aTrack,
ErrorResult& aRv);
ImageCapture(VideoStreamTrack* aVideoStreamTrack, nsPIDOMWindow* aOwnerWindow);
// Post a Blob event to script.
nsresult PostBlobEvent(nsIDOMBlob* aBlob);
// Post an error event to script.
// aErrorCode should be one of error codes defined in ImageCaptureError.h.
// aReason is the nsresult which maps to a error string in dom/base/domerr.msg.
nsresult PostErrorEvent(uint16_t aErrorCode, nsresult aReason = NS_OK);
bool CheckPrincipal();
protected:
virtual ~ImageCapture();
nsRefPtr<VideoStreamTrack> mVideoStreamTrack;
};
} // namespace dom
} // namespace mozilla
#endif // IMAGECAPTURE_H

View File

@ -0,0 +1,18 @@
# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
EXPORTS.mozilla.dom += [
'ImageCapture.h'
]
UNIFIED_SOURCES += [
'CaptureTask.cpp',
'ImageCapture.cpp',
]
FINAL_LIBRARY = 'xul'
FAIL_ON_WARNINGS = True

View File

@ -49,7 +49,7 @@ if CONFIG['MOZ_OMX_DECODER']:
DIRS += ['omx']
DIRS += ['omx/mediaresourcemanager']
DIRS += ['webspeech']
DIRS += ['webspeech', 'imagecapture']
if CONFIG['MOZ_EME']:
DIRS += ['eme']

View File

@ -707,6 +707,10 @@ DOMInterfaces = {
'workers': True,
}],
'ImageCapture': {
'binaryNames': { 'videoStreamTrack': 'GetVideoStreamTrack' }
},
'ImageData': {
'wrapperCache': False,
},

View File

@ -595,6 +595,8 @@ var interfaceNamesInGlobalScope =
"IDBVersionChangeEvent",
// IMPORTANT: Do not change this list without review from a DOM peer!
"Image",
// IMPORTANT: Do not change this list without review from a DOM peer!
"ImageCapture",
// IMPORTANT: Do not change this list without review from a DOM peer!
"ImageCaptureErrorEvent",
// IMPORTANT: Do not change this list without review from a DOM peer!

View File

@ -0,0 +1,28 @@
/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/.
*
* The origin of this IDL file is
* https://dvcs.w3.org/hg/dap/raw-file/default/media-stream-capture/ImageCapture.html
*
* Copyright © 2012-2014 W3C® (MIT, ERCIM, Keio, Beihang), All Rights Reserved.
* W3C liability, trademark and document use rules apply.
*/
[Constructor(VideoStreamTrack track)]
interface ImageCapture : EventTarget {
// readonly attribute PhotoSettingsOptions photoSettingsOptions;
readonly attribute VideoStreamTrack videoStreamTrack;
attribute EventHandler onphoto;
attribute EventHandler onerror;
// attribute EventHandler onphotosettingschange;
// attribute EventHandler onframegrab;
// [Throws]
// void setOptions (PhotoSettings? photoSettings);
[Throws]
void takePhoto();
// [Throws]
// void getFrame();
};

View File

@ -225,6 +225,7 @@ WEBIDL_FILES = [
'IDBRequest.webidl',
'IDBTransaction.webidl',
'IDBVersionChangeEvent.webidl',
'ImageCapture.webidl',
'ImageData.webidl',
'ImageDocument.webidl',
'InputEvent.webidl',