diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in index 7c76df7fad28..0acdf09c99f8 100644 --- a/browser/installer/package-manifest.in +++ b/browser/installer/package-manifest.in @@ -442,7 +442,9 @@ @BINPATH@/components/nsPrompter.js #ifdef MOZ_SERVICES_SYNC @BINPATH@/components/SyncComponents.manifest +@BINPATH@/components/AitcComponents.manifest @BINPATH@/components/Weave.js +@BINPATH@/components/Aitc.js #endif @BINPATH@/components/TelemetryPing.js @BINPATH@/components/TelemetryPing.manifest @@ -517,6 +519,7 @@ @BINPATH@/@PREF_DIR@/firefox-branding.js #ifdef MOZ_SERVICES_SYNC @BINPATH@/@PREF_DIR@/services-sync.js +@BINPATH@/@PREF_DIR@/services-aitc.js #endif @BINPATH@/greprefs.js @BINPATH@/defaults/autoconfig/platform.js diff --git a/build/pylib/blessings/blessings.egg-info/PKG-INFO b/build/pylib/blessings/blessings.egg-info/PKG-INFO deleted file mode 100644 index c52ca3cf917e..000000000000 --- a/build/pylib/blessings/blessings.egg-info/PKG-INFO +++ /dev/null @@ -1,426 +0,0 @@ -Metadata-Version: 1.0 -Name: blessings -Version: 1.3 -Summary: A thin, practical wrapper around terminal formatting, positioning, and more -Home-page: https://github.com/erikrose/blessings -Author: Erik Rose -Author-email: erikrose@grinchcentral.com -License: MIT -Description: ========= - Blessings - ========= - - Coding with Blessings looks like this... :: - - from blessings import Terminal - - t = Terminal() - - print t.bold('Hi there!') - print t.bold_red_on_bright_green('It hurts my eyes!') - - with t.location(0, t.height - 1): - print 'This is at the bottom.' - - Or, for byte-level control, you can drop down and play with raw terminal - capabilities:: - - print '{t.bold}All your {t.red}bold and red base{t.normal}'.format(t=t) - print t.wingo(2) - - The Pitch - ========= - - Blessings lifts several of curses_' limiting assumptions, and it makes your - code pretty, too: - - * Use styles, color, and maybe a little positioning without clearing the whole - screen first. - * Leave more than one screenful of scrollback in the buffer after your program - exits, like a well-behaved command-line app should. - * Get rid of all those noisy, C-like calls to ``tigetstr`` and ``tparm``, so - your code doesn't get crowded out by terminal bookkeeping. - * Act intelligently when somebody redirects your output to a file, omitting the - terminal control codes the user doesn't want to see (optional). - - .. _curses: http://docs.python.org/library/curses.html - - Before And After - ---------------- - - Without Blessings, this is how you'd print some underlined text at the bottom - of the screen:: - - from curses import tigetstr, setupterm, tparm - from fcntl import ioctl - from os import isatty - import struct - import sys - from termios import TIOCGWINSZ - - # If we want to tolerate having our output piped to other commands or - # files without crashing, we need to do all this branching: - if hasattr(sys.stdout, 'fileno') and isatty(sys.stdout.fileno()): - setupterm() - sc = tigetstr('sc') - cup = tigetstr('cup') - rc = tigetstr('rc') - underline = tigetstr('smul') - normal = tigetstr('sgr0') - else: - sc = cup = rc = underline = normal = '' - print sc # Save cursor position. - if cup: - # tigetnum('lines') doesn't always update promptly, hence this: - height = struct.unpack('hhhh', ioctl(0, TIOCGWINSZ, '\000' * 8))[0] - print tparm(cup, height - 1, 0) # Move cursor to bottom. - print 'This is {under}underlined{normal}!'.format(under=underline, - normal=normal) - print rc # Restore cursor position. - - Phew! That was long and full of incomprehensible trash! Let's try it again, - this time with Blessings:: - - from blessings import Terminal - - term = Terminal() - with term.location(0, term.height - 1): - print 'This is', term.underline('pretty!') - - Much better. - - What It Provides - ================ - - Blessings provides just one top-level object: ``Terminal``. Instantiating a - ``Terminal`` figures out whether you're on a terminal at all and, if so, does - any necessary terminal setup. After that, you can proceed to ask it all sorts - of things about the terminal. Terminal terminal terminal. - - Simple Formatting - ----------------- - - Lots of handy formatting codes ("capabilities" in low-level parlance) are - available as attributes on a ``Terminal``. For example:: - - from blessings import Terminal - - term = Terminal() - print 'I am ' + term.bold + 'bold' + term.normal + '!' - - You can also use them as wrappers so you don't have to say ``normal`` - afterward:: - - print 'I am', term.bold('bold') + '!' - - Or, if you want fine-grained control while maintaining some semblance of - brevity, you can combine it with Python's string formatting, which makes - attributes easy to access:: - - print 'All your {t.red}base {t.underline}are belong to us{t.normal}'.format(t=term) - - Simple capabilities of interest include... - - * ``bold`` - * ``reverse`` - * ``underline`` - * ``no_underline`` (which turns off underlining) - * ``blink`` - * ``normal`` (which turns off everything, even colors) - * ``clear_eol`` (clear to the end of the line) - * ``clear_bol`` (clear to beginning of line) - * ``clear_eos`` (clear to end of screen) - - Here are a few more which are less likely to work on all terminals: - - * ``dim`` - * ``italic`` and ``no_italic`` - * ``shadow`` and ``no_shadow`` - * ``standout`` and ``no_standout`` - * ``subscript`` and ``no_subscript`` - * ``superscript`` and ``no_superscript`` - * ``flash`` (which flashes the screen once) - - Note that, while the inverse of ``underline`` is ``no_underline``, the only way - to turn off ``bold`` or ``reverse`` is ``normal``, which also cancels any - custom colors. This is because there's no way to tell the terminal to undo - certain pieces of formatting, even at the lowest level. - - You might notice that the above aren't the typical incomprehensible terminfo - capability names; we alias a few of the harder-to-remember ones for - readability. However, you aren't limited to these: you can reference any - string-returning capability listed on the `terminfo man page`_ by the name - under the "Cap-name" column: for example, ``term.rum``. - - .. _`terminfo man page`: http://www.manpagez.com/man/5/terminfo/ - - Color - ----- - - 16 colors, both foreground and background, are available as easy-to-remember - attributes:: - - from blessings import Terminal - - term = Terminal() - print term.red + term.on_green + 'Red on green? Ick!' + term.normal - print term.bright_red + term.on_bright_blue + 'This is even worse!' + term.normal - - You can also call them as wrappers, which sets everything back to normal at the - end:: - - print term.red_on_green('Red on green? Ick!') - print term.yellow('I can barely see it.') - - The available colors are... - - * ``black`` - * ``red`` - * ``green`` - * ``yellow`` - * ``blue`` - * ``magenta`` - * ``cyan`` - * ``white`` - - You can set the background color instead of the foreground by prepending - ``on_``, as in ``on_blue``. There is also a ``bright`` version of each color: - for example, ``on_bright_blue``. - - There is also a numerical interface to colors, which takes an integer from - 0-15:: - - term.color(5) + 'Hello' + term.normal - term.on_color(3) + 'Hello' + term.normal - - term.color(5)('Hello') - term.on_color(3)('Hello') - - If some color is unsupported (for instance, if only the normal colors are - available, not the bright ones), trying to use it will, on most terminals, have - no effect: the foreground and background colors will stay as they were. You can - get fancy and do different things depending on the supported colors by checking - `number_of_colors`_. - - .. _`number_of_colors`: http://packages.python.org/blessings/#blessings.Terminal.number_of_colors - - Compound Formatting - ------------------- - - If you want to do lots of crazy formatting all at once, you can just mash it - all together:: - - from blessings import Terminal - - term = Terminal() - print term.bold_underline_green_on_yellow + 'Woo' + term.normal - - Or you can use your newly coined attribute as a wrapper, which implicitly sets - everything back to normal afterward:: - - print term.bold_underline_green_on_yellow('Woo') - - This compound notation comes in handy if you want to allow users to customize - the formatting of your app: just have them pass in a format specifier like - "bold_green" on the command line, and do a quick ``getattr(term, - that_option)('Your text')`` when you do your formatting. - - I'd be remiss if I didn't credit couleur_, where I probably got the idea for - all this mashing. - - .. _couleur: http://pypi.python.org/pypi/couleur - - Parametrized Capabilities - ------------------------- - - Some capabilities take parameters. Rather than making you dig up ``tparm()`` - all the time, we simply make such capabilities into callable strings. You can - pass the parameters right in:: - - from blessings import Terminal - - term = Terminal() - print term.move(10, 1) - - Here are some of interest: - - ``move`` - Position the cursor elsewhere. Parameters are y coordinate, then x - coordinate. - ``move_x`` - Move the cursor to the given column. - ``move_y`` - Move the cursor to the given row. - - You can also reference any other string-returning capability listed on the - `terminfo man page`_ by its name under the "Cap-name" column. - - .. _`terminfo man page`: http://www.manpagez.com/man/5/terminfo/ - - Height and Width - ---------------- - - It's simple to get the height and width of the terminal, in characters:: - - from blessings import Terminal - - term = Terminal() - height = term.height - width = term.width - - These are newly updated each time you ask for them, so they're safe to use from - SIGWINCH handlers. - - Temporary Repositioning - ----------------------- - - Sometimes you need to flit to a certain location, print something, and then - return: for example, when updating a progress bar at the bottom of the screen. - ``Terminal`` provides a context manager for doing this concisely:: - - from blessings import Terminal - - term = Terminal() - with term.location(0, term.height - 1): - print 'Here is the bottom.' - print 'This is back where I came from.' - - Parameters to ``location()`` are ``x`` and then ``y``, but you can also pass - just one of them, leaving the other alone. For example... :: - - with term.location(y=10): - print 'We changed just the row.' - - If you want to reposition permanently, see ``move``, in an example above. - - Pipe Savvy - ---------- - - If your program isn't attached to a terminal, like if it's being piped to - another command or redirected to a file, all the capability attributes on - ``Terminal`` will return empty strings. You'll get a nice-looking file without - any formatting codes gumming up the works. - - If you want to override this--like if you anticipate your program being piped - through ``less -r``, which handles terminal escapes just fine--pass - ``force_styling=True`` to the ``Terminal`` constructor. - - In any case, there is an ``is_a_tty`` attribute on ``Terminal`` that lets you - see whether the attached stream seems to be a terminal. If it's false, you - might refrain from drawing progress bars and other frippery, since you're - apparently headed into a pipe:: - - from blessings import Terminal - - term = Terminal() - if term.is_a_tty: - with term.location(0, term.height - 1): - print 'Progress: [=======> ]' - print term.bold('Important stuff') - - Shopping List - ============= - - There are decades of legacy tied up in terminal interaction, so attention to - detail and behavior in edge cases make a difference. Here are some ways - Blessings has your back: - - * Uses the terminfo database so it works with any terminal type - * Provides up-to-the-moment terminal height and width, so you can respond to - terminal size changes (SIGWINCH signals). (Most other libraries query the - ``COLUMNS`` and ``LINES`` environment variables or the ``cols`` or ``lines`` - terminal capabilities, which don't update promptly, if at all.) - * Avoids making a mess if the output gets piped to a non-terminal - * Works great with standard Python string templating - * Provides convenient access to all terminal capabilities, not just a sugared - few - * Outputs to any file-like object, not just stdout - * Keeps a minimum of internal state, so you can feel free to mix and match with - calls to curses or whatever other terminal libraries you like - - Blessings does not provide... - - * Native color support on the Windows command prompt. However, it should work - when used in concert with colorama_. - - .. _colorama: http://pypi.python.org/pypi/colorama/0.2.4 - - Bugs - ==== - - Bugs or suggestions? Visit the `issue tracker`_. - - .. _`issue tracker`: https://github.com/erikrose/blessings/issues/new - - License - ======= - - Blessings is under the MIT License. See the LICENSE file. - - Version History - =============== - - 1.3 - * Add ``number_of_colors``, which tells you how many colors the terminal - supports. - * Made ``color(n)`` and ``on_color(n)`` callable to wrap a string, like the - named colors can. Also, make them both fall back to the ``setf`` and - ``setb`` capabilities (like the named colors do) if the ANSI ``setaf`` and - ``setab`` aren't available. - * Allow ``color`` attr to act as an unparametrized string, not just a - callable. - * Make ``height`` and ``width`` examine any passed-in stream before falling - back to stdout. (This rarely if ever affects actual behavior; it's mostly - philosophical.) - * Make caching simpler and slightly more efficient. - * Get rid of a reference cycle between Terminals and FormattingStrings. - * Update docs to reflect that terminal addressing (as in ``location()``) is - 0-based. - - 1.2 - * Added support for Python 3! We need 3.2.3 or greater, because the curses - library couldn't decide whether to accept strs or bytes before that - (http://bugs.python.org/issue10570). - * Everything that comes out of the library is now unicode. This lets us - support Python 3 without making a mess of the code, and Python 2 should - continue to work unless you were testing types (and badly). Please file a - bug if this causes trouble for you. - * Changed to the MIT License for better world domination. - * Added Sphinx docs. - - 1.1 - * Added nicely named attributes for colors. - * Introduced compound formatting. - * Added wrapper behavior for styling and colors. - * Let you force capabilities to be non-empty, even if the output stream is - not a terminal. - * Added the ``is_a_tty`` attribute for telling whether the output stream is a - terminal. - * Sugared the remaining interesting string capabilities. - * Let ``location()`` operate on just an x *or* y coordinate. - - 1.0 - * Extracted Blessings from nose-progressive, my `progress-bar-having, - traceback-shortcutting, rootin', tootin' testrunner`_. It provided the - tootin' functionality. - - .. _`progress-bar-having, traceback-shortcutting, rootin', tootin' testrunner`: http://pypi.python.org/pypi/nose-progressive/ - -Keywords: terminal,tty,curses,ncurses,formatting,style,color,console -Platform: UNKNOWN -Classifier: Intended Audience :: Developers -Classifier: Natural Language :: English -Classifier: Development Status :: 5 - Production/Stable -Classifier: Environment :: Console -Classifier: Environment :: Console :: Curses -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: POSIX -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.2 -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development :: User Interfaces -Classifier: Topic :: Terminals diff --git a/build/pylib/blessings/blessings.egg-info/SOURCES.txt b/build/pylib/blessings/blessings.egg-info/SOURCES.txt deleted file mode 100644 index ec94260cf453..000000000000 --- a/build/pylib/blessings/blessings.egg-info/SOURCES.txt +++ /dev/null @@ -1,12 +0,0 @@ -LICENSE -MANIFEST.in -README.rst -setup.cfg -setup.py -tox.ini -blessings/__init__.py -blessings/tests.py -blessings.egg-info/PKG-INFO -blessings.egg-info/SOURCES.txt -blessings.egg-info/dependency_links.txt -blessings.egg-info/top_level.txt \ No newline at end of file diff --git a/build/pylib/blessings/blessings.egg-info/dependency_links.txt b/build/pylib/blessings/blessings.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791f..000000000000 --- a/build/pylib/blessings/blessings.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build/pylib/blessings/blessings.egg-info/top_level.txt b/build/pylib/blessings/blessings.egg-info/top_level.txt deleted file mode 100644 index 57087d01d6d7..000000000000 --- a/build/pylib/blessings/blessings.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -blessings diff --git a/dom/file/FileHandle.cpp b/dom/file/FileHandle.cpp index d074258fbfc8..345932051662 100644 --- a/dom/file/FileHandle.cpp +++ b/dom/file/FileHandle.cpp @@ -72,8 +72,8 @@ NS_INTERFACE_MAP_END_INHERITING(nsDOMEventTargetHelper) NS_IMPL_ADDREF_INHERITED(FileHandle, nsDOMEventTargetHelper) NS_IMPL_RELEASE_INHERITED(FileHandle, nsDOMEventTargetHelper) -NS_IMPL_EVENT_HANDLER(FileHandle, abort); -NS_IMPL_EVENT_HANDLER(FileHandle, error); +NS_IMPL_EVENT_HANDLER(FileHandle, abort) +NS_IMPL_EVENT_HANDLER(FileHandle, error) NS_IMETHODIMP FileHandle::GetName(nsAString& aName) diff --git a/dom/file/FileRequest.cpp b/dom/file/FileRequest.cpp index 4d38e5f500e6..327a56daf3c4 100644 --- a/dom/file/FileRequest.cpp +++ b/dom/file/FileRequest.cpp @@ -135,7 +135,7 @@ NS_IMPL_RELEASE_INHERITED(FileRequest, DOMRequest) DOMCI_DATA(FileRequest, FileRequest) -NS_IMPL_EVENT_HANDLER(FileRequest, progress); +NS_IMPL_EVENT_HANDLER(FileRequest, progress) void FileRequest::FireProgressEvent(PRUint64 aLoaded, PRUint64 aTotal) diff --git a/dom/file/LockedFile.cpp b/dom/file/LockedFile.cpp index 3f41ef1db551..2367434d084d 100644 --- a/dom/file/LockedFile.cpp +++ b/dom/file/LockedFile.cpp @@ -349,9 +349,9 @@ NS_IMPL_RELEASE_INHERITED(LockedFile, nsDOMEventTargetHelper) DOMCI_DATA(LockedFile, LockedFile) -NS_IMPL_EVENT_HANDLER(LockedFile, complete); -NS_IMPL_EVENT_HANDLER(LockedFile, abort); -NS_IMPL_EVENT_HANDLER(LockedFile, error); +NS_IMPL_EVENT_HANDLER(LockedFile, complete) +NS_IMPL_EVENT_HANDLER(LockedFile, abort) +NS_IMPL_EVENT_HANDLER(LockedFile, error) nsresult LockedFile::PreHandleEvent(nsEventChainPreVisitor& aVisitor) @@ -662,7 +662,9 @@ LockedFile::Append(const jsval& aValue, } NS_IMETHODIMP -LockedFile::Truncate(nsIDOMFileRequest** _retval) +LockedFile::Truncate(PRUint64 aLocation, + PRUint8 aOptionalArgCount, + nsIDOMFileRequest** _retval) { NS_ASSERTION(NS_IsMainThread(), "Wrong thread!"); @@ -682,12 +684,18 @@ LockedFile::Truncate(nsIDOMFileRequest** _retval) nsRefPtr fileRequest = GenerateFileRequest(); NS_ENSURE_TRUE(fileRequest, NS_ERROR_DOM_FILEHANDLE_UNKNOWN_ERR); + PRUint64 location = aOptionalArgCount ? aLocation : mLocation; + nsRefPtr helper = - new TruncateHelper(this, fileRequest, mLocation); + new TruncateHelper(this, fileRequest, location); nsresult rv = helper->Enqueue(); NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_FILEHANDLE_UNKNOWN_ERR); + if (aOptionalArgCount) { + mLocation = aLocation; + } + fileRequest.forget(_retval); return NS_OK; } diff --git a/dom/file/nsIDOMLockedFile.idl b/dom/file/nsIDOMLockedFile.idl index 2e91d1631495..22564ca01777 100644 --- a/dom/file/nsIDOMLockedFile.idl +++ b/dom/file/nsIDOMLockedFile.idl @@ -16,7 +16,7 @@ dictionary DOMFileMetadataParameters boolean lastModified; }; -[scriptable, builtinclass, uuid(63055eeb-cc19-468b-bafa-7b7961796340)] +[scriptable, builtinclass, uuid(e1f69cc5-c6ce-4850-bc09-c4211b1d4290)] interface nsIDOMLockedFile : nsISupports { readonly attribute nsIDOMFileHandle fileHandle; @@ -49,8 +49,9 @@ interface nsIDOMLockedFile : nsISupports nsIDOMFileRequest append(in jsval value); + [optional_argc] nsIDOMFileRequest - truncate(); + truncate([optional] in unsigned long long location); nsIDOMFileRequest flush(); diff --git a/dom/file/test/test_truncate.html b/dom/file/test/test_truncate.html index a358df28b229..7b73e08dca24 100644 --- a/dom/file/test/test_truncate.html +++ b/dom/file/test/test_truncate.html @@ -31,7 +31,7 @@ is(lockedFile.location, 100000, "Correct location"); for (let i = 0; i < 10; i++) { - let location = lockedFile.location - 10000 + let location = lockedFile.location - 10000; lockedFile.location = location; request = lockedFile.truncate(); @@ -46,6 +46,27 @@ is(event.target.result.size, location, "Correct size"); } + + request = lockedFile.write(testBuffer); + request.onsuccess = grabEventAndContinueHandler; + event = yield; + + let location = lockedFile.location; + for (let i = 0; i < 10; i++) { + location -= 10000; + + request = lockedFile.truncate(location); + request.onsuccess = grabEventAndContinueHandler; + event = yield; + + is(lockedFile.location, location, "Correct location"); + + request = lockedFile.getMetadata({ size: true }); + request.onsuccess = grabEventAndContinueHandler; + event = yield; + + is(event.target.result.size, location, "Correct size"); + } } finishTest(); diff --git a/dom/indexedDB/ipc/IndexedDBParent.cpp b/dom/indexedDB/ipc/IndexedDBParent.cpp index 27cfee855882..01265780dce4 100644 --- a/dom/indexedDB/ipc/IndexedDBParent.cpp +++ b/dom/indexedDB/ipc/IndexedDBParent.cpp @@ -239,7 +239,7 @@ IndexedDBDatabaseParent::HandleRequestEvent(nsIDOMEvent* aEvent, nsCOMPtr changeEvent = do_QueryInterface(aEvent); NS_ENSURE_TRUE(changeEvent, NS_ERROR_FAILURE); - uint64_t oldVersion; + PRUint64 oldVersion; rv = changeEvent->GetOldVersion(&oldVersion); NS_ENSURE_SUCCESS(rv, rv); @@ -333,7 +333,7 @@ IndexedDBDatabaseParent::HandleRequestEvent(nsIDOMEvent* aEvent, nsCOMPtr changeEvent = do_QueryInterface(aEvent); NS_ENSURE_TRUE(changeEvent, NS_ERROR_FAILURE); - uint64_t oldVersion; + PRUint64 oldVersion; rv = changeEvent->GetOldVersion(&oldVersion); NS_ENSURE_SUCCESS(rv, rv); @@ -383,7 +383,7 @@ IndexedDBDatabaseParent::HandleDatabaseEvent(nsIDOMEvent* aEvent, nsCOMPtr changeEvent = do_QueryInterface(aEvent); NS_ENSURE_TRUE(changeEvent, NS_ERROR_FAILURE); - uint64_t oldVersion; + PRUint64 oldVersion; rv = changeEvent->GetOldVersion(&oldVersion); NS_ENSURE_SUCCESS(rv, rv); @@ -1757,7 +1757,7 @@ IndexedDBDeleteDatabaseRequestParent::HandleEvent(nsIDOMEvent* aEvent) nsCOMPtr event = do_QueryInterface(aEvent); MOZ_ASSERT(event); - uint64_t currentVersion; + PRUint64 currentVersion; rv = event->GetOldVersion(¤tVersion); NS_ENSURE_SUCCESS(rv, rv); diff --git a/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest b/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest new file mode 100644 index 000000000000..a8a7a3bf6f0b --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest @@ -0,0 +1,37 @@ +CACHE MANIFEST + +http://mochi.test:8888/tests/SimpleTest/SimpleTest.js +http://mochi.test:8888/tests/dom/tests/mochitest/ajax/offline/offlineTests.js + +# more than 15 what is a number of parallel loads +subresource744719.html?001 +subresource744719.html?002 +subresource744719.html?003 +subresource744719.html?004 +subresource744719.html?005 +subresource744719.html?006 +subresource744719.html?007 +subresource744719.html?008 +subresource744719.html?009 +# this one is non existing and should cancel the load +nonexistent744719.html?010 +subresource744719.html?011 +subresource744719.html?012 +subresource744719.html?013 +subresource744719.html?014 +subresource744719.html?015 +subresource744719.html?016 +subresource744719.html?017 +subresource744719.html?018 +subresource744719.html?019 +subresource744719.html?020 +subresource744719.html?021 +subresource744719.html?022 +subresource744719.html?023 +subresource744719.html?024 +subresource744719.html?025 +subresource744719.html?026 +subresource744719.html?027 +subresource744719.html?028 +subresource744719.html?029 +subresource744719.html?030 diff --git a/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest^headers^ b/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest^headers^ new file mode 100644 index 000000000000..5efde3c5b01e --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/744719-cancel.cacheManifest^headers^ @@ -0,0 +1,2 @@ +Content-Type: text/cache-manifest + diff --git a/dom/tests/mochitest/ajax/offline/744719.cacheManifest b/dom/tests/mochitest/ajax/offline/744719.cacheManifest new file mode 100644 index 000000000000..c8246cb44d61 --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/744719.cacheManifest @@ -0,0 +1,36 @@ +CACHE MANIFEST + +http://mochi.test:8888/tests/SimpleTest/SimpleTest.js +http://mochi.test:8888/tests/dom/tests/mochitest/ajax/offline/offlineTests.js + +# more than 15 what is a number of parallel loads +subresource744719.html?001 +subresource744719.html?002 +subresource744719.html?003 +subresource744719.html?004 +subresource744719.html?005 +subresource744719.html?006 +subresource744719.html?007 +subresource744719.html?008 +subresource744719.html?009 +subresource744719.html?010 +subresource744719.html?011 +subresource744719.html?012 +subresource744719.html?013 +subresource744719.html?014 +subresource744719.html?015 +subresource744719.html?016 +subresource744719.html?017 +subresource744719.html?018 +subresource744719.html?019 +subresource744719.html?020 +subresource744719.html?021 +subresource744719.html?022 +subresource744719.html?023 +subresource744719.html?024 +subresource744719.html?025 +subresource744719.html?026 +subresource744719.html?027 +subresource744719.html?028 +subresource744719.html?029 +subresource744719.html?030 diff --git a/dom/tests/mochitest/ajax/offline/744719.cacheManifest^headers^ b/dom/tests/mochitest/ajax/offline/744719.cacheManifest^headers^ new file mode 100644 index 000000000000..5efde3c5b01e --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/744719.cacheManifest^headers^ @@ -0,0 +1,2 @@ +Content-Type: text/cache-manifest + diff --git a/dom/tests/mochitest/ajax/offline/Makefile.in b/dom/tests/mochitest/ajax/offline/Makefile.in index 80a2b495be68..8186ea4026cd 100644 --- a/dom/tests/mochitest/ajax/offline/Makefile.in +++ b/dom/tests/mochitest/ajax/offline/Makefile.in @@ -31,6 +31,13 @@ _TEST_FILES = \ test_bug460353.html \ test_bug474696.html \ test_bug544462.html \ + test_bug744719.html \ + 744719.cacheManifest \ + 744719.cacheManifest^headers^ \ + test_bug744719-cancel.html \ + 744719-cancel.cacheManifest \ + 744719-cancel.cacheManifest^headers^ \ + subresource744719.html \ test_foreign.html \ test_fallback.html \ test_overlap.html \ diff --git a/dom/tests/mochitest/ajax/offline/subresource744719.html b/dom/tests/mochitest/ajax/offline/subresource744719.html new file mode 100644 index 000000000000..86ba068ec2cf --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/subresource744719.html @@ -0,0 +1 @@ +Dummy subresource \ No newline at end of file diff --git a/dom/tests/mochitest/ajax/offline/test_bug744719-cancel.html b/dom/tests/mochitest/ajax/offline/test_bug744719-cancel.html new file mode 100644 index 000000000000..8487c0ee1ff1 --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/test_bug744719-cancel.html @@ -0,0 +1,83 @@ + + +parallel load canceled + + + + + + + + + + + + + diff --git a/dom/tests/mochitest/ajax/offline/test_bug744719.html b/dom/tests/mochitest/ajax/offline/test_bug744719.html new file mode 100644 index 000000000000..17f0300ee112 --- /dev/null +++ b/dom/tests/mochitest/ajax/offline/test_bug744719.html @@ -0,0 +1,76 @@ + + +parallel load + + + + + + + + + + + + + diff --git a/netwerk/base/public/nsIApplicationCache.idl b/netwerk/base/public/nsIApplicationCache.idl index 0910a3483d76..4a206f3725de 100644 --- a/netwerk/base/public/nsIApplicationCache.idl +++ b/netwerk/base/public/nsIApplicationCache.idl @@ -7,6 +7,7 @@ #include "nsISupports.idl" interface nsIArray; +interface nsILocalFile; /** * Application caches can store a set of namespace entries that affect @@ -78,7 +79,7 @@ interface nsIApplicationCacheNamespace : nsISupports * loads. Inactive caches will be removed from the cache when they are * no longer referenced. */ -[scriptable, uuid(32f83e3f-470c-4423-a86a-d35d1c215ccb)] +[scriptable, uuid(231e1e53-05c1-41b6-b9de-dbbcce9385c9)] interface nsIApplicationCache : nsISupports { /** @@ -189,4 +190,10 @@ interface nsIApplicationCache : nsISupports * Get the most specific namespace matching a given key. */ nsIApplicationCacheNamespace getMatchingNamespace(in ACString key); + + /** + * If set, this offline cache is placed in a different directory + * than the current application profile. + */ + readonly attribute nsILocalFile cacheDirectory; }; diff --git a/netwerk/base/public/nsIApplicationCacheService.idl b/netwerk/base/public/nsIApplicationCacheService.idl index b1cae077aaa5..6e80001b1bb7 100644 --- a/netwerk/base/public/nsIApplicationCacheService.idl +++ b/netwerk/base/public/nsIApplicationCacheService.idl @@ -7,12 +7,13 @@ #include "nsISupports.idl" interface nsIApplicationCache; +interface nsILocalFile; /** * The application cache service manages the set of application cache * groups. */ -[scriptable, uuid(10fdea21-1224-4c29-8507-8f3205a121d5)] +[scriptable, uuid(28adfdc7-6718-4b3e-bdb2-ecfefa3c8910)] interface nsIApplicationCacheService : nsISupports { /** @@ -21,6 +22,22 @@ interface nsIApplicationCacheService : nsISupports */ nsIApplicationCache createApplicationCache(in ACString group); + /** + * Create a new, empty application cache for the given cache + * group residing in a custom directory with a custom quota. + * + * @param group + * URL of the manifest + * @param directory + * Actually a reference to a profile directory where to + * create the OfflineCache sub-dir. + * @param quota + * Optional override of the default quota. + */ + nsIApplicationCache createCustomApplicationCache(in ACString group, + in nsILocalFile profileDir, + in PRInt32 quota); + /** * Get an application cache object for the given client ID. */ diff --git a/netwerk/base/public/nsICachingChannel.idl b/netwerk/base/public/nsICachingChannel.idl index c943f71cbb64..96a8aef8d97f 100644 --- a/netwerk/base/public/nsICachingChannel.idl +++ b/netwerk/base/public/nsICachingChannel.idl @@ -6,6 +6,7 @@ #include "nsICacheInfoChannel.idl" interface nsIFile; +interface nsILocalFile; /** * A channel may optionally implement this interface to allow clients @@ -17,7 +18,7 @@ interface nsIFile; * 3) Support for uniquely identifying cached data in cases when the URL * is insufficient (e.g., HTTP form submission). */ -[scriptable, uuid(830D4BCB-3E46-4011-9BDA-51A5D1AF891F)] +[scriptable, uuid(E2143B61-62FE-4da5-BE2E-E31981095889)] interface nsICachingChannel : nsICacheInfoChannel { /** @@ -87,6 +88,12 @@ interface nsICachingChannel : nsICacheInfoChannel */ attribute ACString offlineCacheClientID; + /** + * Override base (profile) directory to work with when accessing the cache. + * When not specified, the current process' profile directory will be used. + */ + attribute nsILocalFile profileDirectory; + /** * Get the "file" where the cached data can be found. This is valid for * as long as a reference to the cache token is held. This may return diff --git a/netwerk/cache/nsApplicationCacheService.cpp b/netwerk/cache/nsApplicationCacheService.cpp index ae9887e2096e..4a016aec0f4d 100644 --- a/netwerk/cache/nsApplicationCacheService.cpp +++ b/netwerk/cache/nsApplicationCacheService.cpp @@ -34,6 +34,23 @@ nsApplicationCacheService::CreateApplicationCache(const nsACString &group, return device->CreateApplicationCache(group, out); } +NS_IMETHODIMP +nsApplicationCacheService::CreateCustomApplicationCache(const nsACString & group, + nsILocalFile *profileDir, + PRInt32 quota, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + nsRefPtr device; + nsresult rv = mCacheService->GetCustomOfflineDevice(profileDir, + quota, + getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->CreateApplicationCache(group, out); +} + NS_IMETHODIMP nsApplicationCacheService::GetApplicationCache(const nsACString &clientID, nsIApplicationCache **out) diff --git a/netwerk/cache/nsCacheEntry.cpp b/netwerk/cache/nsCacheEntry.cpp index 1e6226197dc2..65fc061f825a 100644 --- a/netwerk/cache/nsCacheEntry.cpp +++ b/netwerk/cache/nsCacheEntry.cpp @@ -32,6 +32,7 @@ nsCacheEntry::nsCacheEntry(nsCString * key, mPredictedDataSize(-1), mDataSize(0), mCacheDevice(nsnull), + mCustomDevice(nsnull), mData(nsnull) { MOZ_COUNT_CTOR(nsCacheEntry); diff --git a/netwerk/cache/nsCacheEntry.h b/netwerk/cache/nsCacheEntry.h index 1622d894bde3..047d7f36913c 100644 --- a/netwerk/cache/nsCacheEntry.h +++ b/netwerk/cache/nsCacheEntry.h @@ -63,6 +63,9 @@ public: nsCacheDevice * CacheDevice() { return mCacheDevice; } void SetCacheDevice( nsCacheDevice * device) { mCacheDevice = device; } + void SetCustomCacheDevice( nsCacheDevice * device ) + { mCustomDevice = device; } + nsCacheDevice * CustomCacheDevice() { return mCustomDevice; } const char * GetDeviceID(); /** @@ -216,6 +219,7 @@ private: PRInt64 mPredictedDataSize; // Size given by ContentLength. PRUint32 mDataSize; // 4 nsCacheDevice * mCacheDevice; // 4 + nsCacheDevice * mCustomDevice; // 4 nsCOMPtr mSecurityInfo; // nsISupports * mData; // strong ref nsCOMPtr mThread; diff --git a/netwerk/cache/nsCacheRequest.h b/netwerk/cache/nsCacheRequest.h index 753eb2510945..0cdc14786ee0 100644 --- a/netwerk/cache/nsCacheRequest.h +++ b/netwerk/cache/nsCacheRequest.h @@ -37,7 +37,8 @@ private: mInfo(0), mListener(listener), mLock("nsCacheRequest.mLock"), - mCondVar(mLock, "nsCacheRequest.mCondVar") + mCondVar(mLock, "nsCacheRequest.mCondVar"), + mProfileDir(session->ProfileDir()) { MOZ_COUNT_CTOR(nsCacheRequest); PR_INIT_CLIST(this); @@ -152,6 +153,7 @@ private: nsCOMPtr mThread; Mutex mLock; CondVar mCondVar; + nsCOMPtr mProfileDir; }; #endif // _nsCacheRequest_h_ diff --git a/netwerk/cache/nsCacheService.cpp b/netwerk/cache/nsCacheService.cpp index a0fc17455c15..639983495691 100644 --- a/netwerk/cache/nsCacheService.cpp +++ b/netwerk/cache/nsCacheService.cpp @@ -1089,6 +1089,7 @@ nsCacheService::nsCacheService() // create list of cache devices PR_INIT_CLIST(&mDoomedEntries); + mCustomOfflineDevices.Init(); } nsCacheService::~nsCacheService() @@ -1143,6 +1144,15 @@ nsCacheService::Init() return NS_OK; } +// static +PLDHashOperator +nsCacheService::ShutdownCustomCacheDeviceEnum(const nsAString& aProfileDir, + nsRefPtr& aDevice, + void* aUserArg) +{ + aDevice->Shutdown(); + return PL_DHASH_REMOVE; +} void nsCacheService::Shutdown() @@ -1198,6 +1208,8 @@ nsCacheService::Shutdown() NS_IF_RELEASE(mOfflineDevice); + mCustomOfflineDevices.Enumerate(&nsCacheService::ShutdownCustomCacheDeviceEnum, nsnull); + #ifdef PR_LOGGING LogCacheStatistics(); #endif @@ -1532,32 +1544,75 @@ nsCacheService::GetOfflineDevice(nsOfflineCacheDevice **aDevice) return NS_OK; } +nsresult +nsCacheService::GetCustomOfflineDevice(nsILocalFile *aProfileDir, + PRInt32 aQuota, + nsOfflineCacheDevice **aDevice) +{ + nsresult rv; + + nsAutoString profilePath; + rv = aProfileDir->GetPath(profilePath); + NS_ENSURE_SUCCESS(rv, rv); + + if (!mCustomOfflineDevices.Get(profilePath, aDevice)) { + rv = CreateCustomOfflineDevice(aProfileDir, aQuota, aDevice); + NS_ENSURE_SUCCESS(rv, rv); + + mCustomOfflineDevices.Put(profilePath, *aDevice); + } + + return NS_OK; +} + nsresult nsCacheService::CreateOfflineDevice() { - CACHE_LOG_ALWAYS(("Creating offline device")); + CACHE_LOG_ALWAYS(("Creating default offline device")); + + if (mOfflineDevice) return NS_OK; + if (!mObserver) return NS_ERROR_NOT_AVAILABLE; + + nsresult rv = CreateCustomOfflineDevice( + mObserver->OfflineCacheParentDirectory(), + mObserver->OfflineCacheCapacity(), + &mOfflineDevice); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsCacheService::CreateCustomOfflineDevice(nsILocalFile *aProfileDir, + PRInt32 aQuota, + nsOfflineCacheDevice **aDevice) +{ + NS_ENSURE_ARG(aProfileDir); + +#if defined(PR_LOGGING) + nsCAutoString profilePath; + aProfileDir->GetNativePath(profilePath); + CACHE_LOG_ALWAYS(("Creating custom offline device, %s, %d", + profilePath.BeginReading(), aQuota)); +#endif if (!mInitialized) return NS_ERROR_NOT_AVAILABLE; if (!mEnableOfflineDevice) return NS_ERROR_NOT_AVAILABLE; - if (mOfflineDevice) return NS_OK; - mOfflineDevice = new nsOfflineCacheDevice; - if (!mOfflineDevice) return NS_ERROR_OUT_OF_MEMORY; + *aDevice = new nsOfflineCacheDevice; - NS_ADDREF(mOfflineDevice); + NS_ADDREF(*aDevice); // set the preferences - mOfflineDevice->SetCacheParentDirectory( - mObserver->OfflineCacheParentDirectory()); - mOfflineDevice->SetCapacity(mObserver->OfflineCacheCapacity()); + (*aDevice)->SetCacheParentDirectory(aProfileDir); + (*aDevice)->SetCapacity(aQuota); - nsresult rv = mOfflineDevice->Init(); + nsresult rv = (*aDevice)->Init(); if (NS_FAILED(rv)) { - CACHE_LOG_DEBUG(("mOfflineDevice->Init() failed (0x%.8x)\n", rv)); + CACHE_LOG_DEBUG(("OfflineDevice->Init() failed (0x%.8x)\n", rv)); CACHE_LOG_DEBUG((" - disabling offline cache for this session.\n")); - mEnableOfflineDevice = false; - NS_RELEASE(mOfflineDevice); + NS_RELEASE(*aDevice); } return rv; } @@ -1734,6 +1789,20 @@ nsCacheService::ProcessRequest(nsCacheRequest * request, // loop back around to look for another entry } + if (NS_SUCCEEDED(rv) && request->mProfileDir) { + // Custom cache directory has been demanded. Preset the cache device. + if (entry->StoragePolicy() != nsICache::STORE_OFFLINE) { + // Failsafe check: this is implemented only for offline cache atm. + rv = NS_ERROR_FAILURE; + } else { + nsRefPtr customCacheDevice; + rv = GetCustomOfflineDevice(request->mProfileDir, -1, + getter_AddRefs(customCacheDevice)); + if (NS_SUCCEEDED(rv)) + entry->SetCustomCacheDevice(customCacheDevice); + } + } + nsICacheEntryDescriptor *descriptor = nsnull; if (NS_SUCCEEDED(rv)) @@ -2046,12 +2115,16 @@ nsCacheService::EnsureEntryHasDevice(nsCacheEntry * entry) (void)CreateOfflineDevice(); // ignore the error (check for mOfflineDevice instead) } - if (mOfflineDevice) { + device = entry->CustomCacheDevice() + ? entry->CustomCacheDevice() + : mOfflineDevice; + + if (device) { entry->MarkBinding(); - nsresult rv = mOfflineDevice->BindEntry(entry); + nsresult rv = device->BindEntry(entry); entry->ClearBinding(); - if (NS_SUCCEEDED(rv)) - device = mOfflineDevice; + if (NS_FAILED(rv)) + device = nsnull; } } @@ -2146,6 +2219,9 @@ nsCacheService::OnProfileShutdown(bool cleanse) gService->mOfflineDevice->Shutdown(); } + gService->mCustomOfflineDevices.Enumerate( + &nsCacheService::ShutdownCustomCacheDeviceEnum, nsnull); + gService->mEnableOfflineDevice = false; if (gService->mMemoryDevice) { diff --git a/netwerk/cache/nsCacheService.h b/netwerk/cache/nsCacheService.h index 30c857e5ab6a..238a66e82c00 100644 --- a/netwerk/cache/nsCacheService.h +++ b/netwerk/cache/nsCacheService.h @@ -17,6 +17,7 @@ #include "nsIObserver.h" #include "nsString.h" #include "nsTArray.h" +#include "nsRefPtrHashtable.h" #include "mozilla/CondVar.h" #include "mozilla/Mutex.h" @@ -112,6 +113,15 @@ public: nsresult GetOfflineDevice(nsOfflineCacheDevice ** aDevice); + /** + * Creates an offline cache device that works over a specific profile directory. + * A tool to preload offline cache for profiles different from the current + * application's profile directory. + */ + nsresult GetCustomOfflineDevice(nsILocalFile *aProfileDir, + PRInt32 aQuota, + nsOfflineCacheDevice **aDevice); + // This method may be called to release an object while the cache service // lock is being held. If a non-null target is specified and the target // does not correspond to the current thread, then the release will be @@ -183,6 +193,9 @@ private: nsresult CreateDiskDevice(); nsresult CreateOfflineDevice(); + nsresult CreateCustomOfflineDevice(nsILocalFile *aProfileDir, + PRInt32 aQuota, + nsOfflineCacheDevice **aDevice); nsresult CreateMemoryDevice(); nsresult CreateRequest(nsCacheSession * session, @@ -237,6 +250,11 @@ private: PLDHashEntryHdr * hdr, PRUint32 number, void * arg); + + static + PLDHashOperator ShutdownCustomCacheDeviceEnum(const nsAString& aProfileDir, + nsRefPtr& aDevice, + void* aUserArg); #if defined(PR_LOGGING) void LogCacheStatistics(); #endif @@ -270,6 +288,8 @@ private: nsDiskCacheDevice * mDiskDevice; nsOfflineCacheDevice * mOfflineDevice; + nsRefPtrHashtable mCustomOfflineDevices; + nsCacheEntryHashTable mActiveEntries; PRCList mDoomedEntries; diff --git a/netwerk/cache/nsCacheSession.cpp b/netwerk/cache/nsCacheSession.cpp index 0781be9faba1..b5905cedfe76 100644 --- a/netwerk/cache/nsCacheSession.cpp +++ b/netwerk/cache/nsCacheSession.cpp @@ -41,6 +41,31 @@ NS_IMETHODIMP nsCacheSession::GetDoomEntriesIfExpired(bool *result) } +NS_IMETHODIMP nsCacheSession::SetProfileDirectory(nsILocalFile *profileDir) +{ + if (StoragePolicy() != nsICache::STORE_OFFLINE && profileDir) { + // Profile directory override is currently implemented only for + // offline cache. This is an early failure to prevent the request + // being processed before it would fail later because of inability + // to assign a cache base dir. + return NS_ERROR_UNEXPECTED; + } + + mProfileDir = profileDir; + return NS_OK; +} + +NS_IMETHODIMP nsCacheSession::GetProfileDirectory(nsILocalFile **profileDir) +{ + if (mProfileDir) + NS_ADDREF(*profileDir = mProfileDir); + else + *profileDir = nsnull; + + return NS_OK; +} + + NS_IMETHODIMP nsCacheSession::SetDoomEntriesIfExpired(bool doomEntriesIfExpired) { if (doomEntriesIfExpired) MarkDoomEntriesIfExpired(); diff --git a/netwerk/cache/nsCacheSession.h b/netwerk/cache/nsCacheSession.h index 4990cea88847..25ed4cedddb9 100644 --- a/netwerk/cache/nsCacheSession.h +++ b/netwerk/cache/nsCacheSession.h @@ -9,7 +9,9 @@ #include "nspr.h" #include "nsError.h" +#include "nsCOMPtr.h" #include "nsICacheSession.h" +#include "nsILocalFile.h" #include "nsString.h" class nsCacheSession : public nsICacheSession @@ -53,9 +55,12 @@ public: mInfo |= policy; } + nsILocalFile* ProfileDir() { return mProfileDir; } + private: nsCString mClientID; PRUint32 mInfo; + nsCOMPtr mProfileDir; }; #endif // _nsCacheSession_h_ diff --git a/netwerk/cache/nsDiskCacheDeviceSQL.cpp b/netwerk/cache/nsDiskCacheDeviceSQL.cpp index f6052ff53329..5e899a777f7c 100644 --- a/netwerk/cache/nsDiskCacheDeviceSQL.cpp +++ b/netwerk/cache/nsDiskCacheDeviceSQL.cpp @@ -629,6 +629,17 @@ nsApplicationCache::GetClientID(nsACString &out) return NS_OK; } +NS_IMETHODIMP +nsApplicationCache::GetCacheDirectory(nsILocalFile **out) +{ + if (mDevice->BaseDirectory()) + NS_ADDREF(*out = mDevice->BaseDirectory()); + else + *out = nsnull; + + return NS_OK; +} + NS_IMETHODIMP nsApplicationCache::GetActive(bool *out) { @@ -2361,6 +2372,8 @@ nsOfflineCacheDevice::SetCacheParentDirectory(nsILocalFile *parentDir) return; } + mBaseDirectory = parentDir; + // cache dir may not exist, but that's ok nsCOMPtr dir; rv = parentDir->Clone(getter_AddRefs(dir)); diff --git a/netwerk/cache/nsDiskCacheDeviceSQL.h b/netwerk/cache/nsDiskCacheDeviceSQL.h index db29087a0339..85c392eb1e51 100644 --- a/netwerk/cache/nsDiskCacheDeviceSQL.h +++ b/netwerk/cache/nsDiskCacheDeviceSQL.h @@ -168,6 +168,7 @@ public: void SetCacheParentDirectory(nsILocalFile * parentDir); void SetCapacity(PRUint32 capacity); + nsILocalFile * BaseDirectory() { return mBaseDirectory; } nsILocalFile * CacheDirectory() { return mCacheDirectory; } PRUint32 CacheCapacity() { return mCacheCapacity; } PRUint32 CacheSize(); @@ -252,6 +253,7 @@ private: nsCOMPtr mStatement_EnumerateGroups; nsCOMPtr mStatement_EnumerateGroupsTimeOrder; + nsCOMPtr mBaseDirectory; nsCOMPtr mCacheDirectory; PRUint32 mCacheCapacity; // in bytes PRInt32 mDeltaCounter; diff --git a/netwerk/cache/nsICacheSession.idl b/netwerk/cache/nsICacheSession.idl index 47425f7da51b..f97994e6fb9e 100644 --- a/netwerk/cache/nsICacheSession.idl +++ b/netwerk/cache/nsICacheSession.idl @@ -9,6 +9,7 @@ interface nsICacheEntryDescriptor; interface nsICacheListener; +interface nsILocalFile; [scriptable, uuid(1dd7708c-de48-4ffe-b5aa-cd218c762887)] interface nsICacheSession : nsISupports @@ -21,6 +22,14 @@ interface nsICacheSession : nsISupports */ attribute boolean doomEntriesIfExpired; + /** + * When set, entries created with this session will be placed to a cache + * based at this directory. Use when storing entries to a different + * profile than the active profile of the the current running application + * process. + */ + attribute nsILocalFile profileDirectory; + /** * A cache session can only give out one descriptor with WRITE access * to a given cache entry at a time. Until the client calls MarkValid on diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp index d5c306c66d29..64989ae7b6ff 100644 --- a/netwerk/protocol/http/nsHttpChannel.cpp +++ b/netwerk/protocol/http/nsHttpChannel.cpp @@ -2617,6 +2617,11 @@ nsHttpChannel::OpenOfflineCacheEntryForWriting() getter_AddRefs(session)); if (NS_FAILED(rv)) return rv; + if (mProfileDirectory) { + rv = session->SetProfileDirectory(mProfileDirectory); + if (NS_FAILED(rv)) return rv; + } + mOnCacheEntryAvailableCallback = &nsHttpChannel::OnOfflineCacheEntryForWritingAvailable; rv = session->AsyncOpenCacheEntry(cacheKey, nsICache::ACCESS_READ_WRITE, @@ -3934,6 +3939,7 @@ nsHttpChannel::SetupReplacementChannel(nsIURI *newURI, // cacheClientID, cacheForOfflineUse cachingChannel->SetOfflineCacheClientID(mOfflineCacheClientID); cachingChannel->SetCacheForOfflineUse(mCacheForOfflineUse); + cachingChannel->SetProfileDirectory(mProfileDirectory); } } @@ -5361,6 +5367,22 @@ nsHttpChannel::SetOfflineCacheClientID(const nsACString &value) return NS_OK; } +NS_IMETHODIMP +nsHttpChannel::GetProfileDirectory(nsILocalFile **_result) +{ + NS_ENSURE_ARG(_result); + + NS_ADDREF(*_result = mProfileDirectory); + return NS_OK; +} + +NS_IMETHODIMP +nsHttpChannel::SetProfileDirectory(nsILocalFile *value) +{ + mProfileDirectory = value; + return NS_OK; +} + NS_IMETHODIMP nsHttpChannel::GetCacheFile(nsIFile **cacheFile) { diff --git a/netwerk/protocol/http/nsHttpChannel.h b/netwerk/protocol/http/nsHttpChannel.h index 55f8e31bc38f..e79279650f68 100644 --- a/netwerk/protocol/http/nsHttpChannel.h +++ b/netwerk/protocol/http/nsHttpChannel.h @@ -27,6 +27,7 @@ #include "nsIHttpChannelAuthProvider.h" #include "nsIAsyncVerifyRedirectCallback.h" #include "nsITimedChannel.h" +#include "nsILocalFile.h" #include "nsDNSPrefetch.h" #include "TimingStruct.h" #include "AutoClose.h" @@ -304,6 +305,8 @@ private: nsCacheAccessMode mOfflineCacheAccess; nsCString mOfflineCacheClientID; + nsCOMPtr mProfileDirectory; + // auth specific data nsCOMPtr mAuthProvider; diff --git a/netwerk/test/unit/test_offlinecache_custom-directory.js b/netwerk/test/unit/test_offlinecache_custom-directory.js new file mode 100644 index 000000000000..43becb2a2e8e --- /dev/null +++ b/netwerk/test/unit/test_offlinecache_custom-directory.js @@ -0,0 +1,125 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This test executes nsIOfflineCacheUpdateService.scheduleCustomProfileUpdate API + * 1. preloads an app with a manifest to a custom sudir in the profile (for simplicity) + * 2. observes progress and completion of the update + * 3. checks presence of index.sql and files in the expected location + */ + +do_load_httpd_js(); + +var httpServer = null; +var cacheUpdateObserver = null; + +function make_channel(url, callback, ctx) { + var ios = Cc["@mozilla.org/network/io-service;1"]. + getService(Ci.nsIIOService); + return ios.newChannel(url, "", null); +} + +function make_uri(url) { + var ios = Cc["@mozilla.org/network/io-service;1"]. + getService(Ci.nsIIOService); + return ios.newURI(url, null, null); +} + +// start the test with loading this master entry referencing the manifest +function masterEntryHandler(metadata, response) +{ + var masterEntryContent = ""; + response.setHeader("Content-Type", "text/html"); + response.bodyOutputStream.write(masterEntryContent, masterEntryContent.length); +} + +// manifest defines fallback namespace from any /redirect path to /content +function manifestHandler(metadata, response) +{ + var manifestContent = "CACHE MANIFEST\n"; + response.setHeader("Content-Type", "text/cache-manifest"); + response.bodyOutputStream.write(manifestContent, manifestContent.length); +} + +// finally check we got fallback content +function finish_test(customDir) +{ + var offlineCacheDir = customDir.clone(); + offlineCacheDir.append("OfflineCache"); + + var indexSqlFile = offlineCacheDir.clone(); + indexSqlFile.append('index.sqlite'); + do_check_eq(indexSqlFile.exists(), true); + + var file1 = offlineCacheDir.clone(); + file1.append("2"); + file1.append("E"); + file1.append("2C99DE6E7289A5-0"); + do_check_eq(file1.exists(), true); + + var file2 = offlineCacheDir.clone(); + file2.append("8"); + file2.append("6"); + file2.append("0B457F75198B29-0"); + do_check_eq(file2.exists(), true); + + httpServer.stop(do_test_finished); +} + +function run_test() +{ + httpServer = new nsHttpServer(); + httpServer.registerPathHandler("/masterEntry", masterEntryHandler); + httpServer.registerPathHandler("/manifest", manifestHandler); + httpServer.start(4444); + + var profileDir = do_get_profile(); + var customDir = profileDir.clone(); + customDir.append("customOfflineCacheDir" + Math.random()); + + var pm = Cc["@mozilla.org/permissionmanager;1"] + .getService(Ci.nsIPermissionManager); + var uri = make_uri("http://localhost:4444"); + if (pm.testPermission(uri, "offline-app") != 0) { + dump("Previous test failed to clear offline-app permission! Expect failures.\n"); + } + pm.add(uri, "offline-app", Ci.nsIPermissionManager.ALLOW_ACTION); + + var ps = Cc["@mozilla.org/preferences-service;1"] + .getService(Ci.nsIPrefBranch); + ps.setBoolPref("browser.cache.offline.enable", true); + // Set this pref to mimic the default browser behavior. + ps.setComplexValue("browser.cache.offline.parent_directory", Ci.nsILocalFile, profileDir); + + var us = Cc["@mozilla.org/offlinecacheupdate-service;1"]. + getService(Ci.nsIOfflineCacheUpdateService); + var update = us.scheduleCustomProfileUpdate( + make_uri("http://localhost:4444/manifest"), + make_uri("http://localhost:4444/masterEntry"), + customDir); + + var expectedStates = [ + Ci.nsIOfflineCacheUpdateObserver.STATE_DOWNLOADING, + Ci.nsIOfflineCacheUpdateObserver.STATE_ITEMSTARTED, + Ci.nsIOfflineCacheUpdateObserver.STATE_ITEMPROGRESS, + Ci.nsIOfflineCacheUpdateObserver.STATE_ITEMCOMPLETED, + Ci.nsIOfflineCacheUpdateObserver.STATE_FINISHED + ]; + + update.addObserver({ + updateStateChanged: function(update, state) + { + do_check_eq(state, expectedStates.shift()); + + if (state == Ci.nsIOfflineCacheUpdateObserver.STATE_FINISHED) + finish_test(customDir); + }, + + applicationCacheAvailable: function(appCache) + { + } + }, false); + + do_test_pending(); +} diff --git a/netwerk/test/unit/xpcshell.ini b/netwerk/test/unit/xpcshell.ini index fea8eadae10d..08bab3a9a984 100644 --- a/netwerk/test/unit/xpcshell.ini +++ b/netwerk/test/unit/xpcshell.ini @@ -187,3 +187,4 @@ run-if = hasNode [test_xmlhttprequest.js] [test_XHR_redirects.js] [test_pinned_app_cache.js] +[test_offlinecache_custom-directory.js] diff --git a/other-licenses/virtualenv/virtualenv.egg-info/PKG-INFO b/other-licenses/virtualenv/virtualenv.egg-info/PKG-INFO deleted file mode 100644 index 4be2b3229f24..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/PKG-INFO +++ /dev/null @@ -1,1022 +0,0 @@ -Metadata-Version: 1.0 -Name: virtualenv -Version: 1.7.1.2 -Summary: Virtual Python Environment builder -Home-page: http://www.virtualenv.org -Author: Jannis Leidel, Carl Meyer and Brian Rosner -Author-email: python-virtualenv@groups.google.com -License: MIT -Description: - - Installation - ------------ - - You can install virtualenv with ``pip install virtualenv``, or the `latest - development version `_ - with ``pip install virtualenv==dev``. - - You can also use ``easy_install``, or if you have no Python package manager - available at all, you can just grab the single file `virtualenv.py`_ and run - it with ``python virtualenv.py``. - - .. _virtualenv.py: https://raw.github.com/pypa/virtualenv/master/virtualenv.py - - What It Does - ------------ - - ``virtualenv`` is a tool to create isolated Python environments. - - The basic problem being addressed is one of dependencies and versions, - and indirectly permissions. Imagine you have an application that - needs version 1 of LibFoo, but another application requires version - 2. How can you use both these applications? If you install - everything into ``/usr/lib/python2.7/site-packages`` (or whatever your - platform's standard location is), it's easy to end up in a situation - where you unintentionally upgrade an application that shouldn't be - upgraded. - - Or more generally, what if you want to install an application *and - leave it be*? If an application works, any change in its libraries or - the versions of those libraries can break the application. - - Also, what if you can't install packages into the global - ``site-packages`` directory? For instance, on a shared host. - - In all these cases, ``virtualenv`` can help you. It creates an - environment that has its own installation directories, that doesn't - share libraries with other virtualenv environments (and optionally - doesn't access the globally installed libraries either). - - The basic usage is:: - - $ python virtualenv.py ENV - - If you install it you can also just do ``virtualenv ENV``. - - This creates ``ENV/lib/pythonX.X/site-packages``, where any libraries you - install will go. It also creates ``ENV/bin/python``, which is a Python - interpreter that uses this environment. Anytime you use that interpreter - (including when a script has ``#!/path/to/ENV/bin/python`` in it) the libraries - in that environment will be used. - - It also installs either `Setuptools - `_ or `distribute - `_ into the environment. To use - Distribute instead of setuptools, just call virtualenv like this:: - - $ python virtualenv.py --distribute ENV - - You can also set the environment variable VIRTUALENV_USE_DISTRIBUTE. - - A new virtualenv also includes the `pip `_ - installer, so you can use ``ENV/bin/pip`` to install additional packages into - the environment. - - Environment variables and configuration files - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - virtualenv can not only be configured by passing command line options such as - ``--distribute`` but also by two other means: - - - Environment variables - - Each command line option is automatically used to look for environment - variables with the name format ``VIRTUALENV_``. That means - the name of the command line options are capitalized and have dashes - (``'-'``) replaced with underscores (``'_'``). - - For example, to automatically install Distribute instead of setuptools - you can also set an environment variable:: - - $ export VIRTUALENV_USE_DISTRIBUTE=true - $ python virtualenv.py ENV - - It's the same as passing the option to virtualenv directly:: - - $ python virtualenv.py --distribute ENV - - This also works for appending command line options, like ``--find-links``. - Just leave an empty space between the passsed values, e.g.:: - - $ export VIRTUALENV_EXTRA_SEARCH_DIR="/path/to/dists /path/to/other/dists" - $ virtualenv ENV - - is the same as calling:: - - $ python virtualenv.py --extra-search-dir=/path/to/dists --extra-search-dir=/path/to/other/dists ENV - - - Config files - - virtualenv also looks for a standard ini config file. On Unix and Mac OS X - that's ``$HOME/.virtualenv/virtualenv.ini`` and on Windows, it's - ``%HOME%\\virtualenv\\virtualenv.ini``. - - The names of the settings are derived from the long command line option, - e.g. the option ``--distribute`` would look like this:: - - [virtualenv] - distribute = true - - Appending options like ``--extra-search-dir`` can be written on multiple - lines:: - - [virtualenv] - extra-search-dir = - /path/to/dists - /path/to/other/dists - - Please have a look at the output of ``virtualenv --help`` for a full list - of supported options. - - Windows Notes - ~~~~~~~~~~~~~ - - Some paths within the virtualenv are slightly different on Windows: scripts and - executables on Windows go in ``ENV\Scripts\`` instead of ``ENV/bin/`` and - libraries go in ``ENV\Lib\`` rather than ``ENV/lib/``. - - To create a virtualenv under a path with spaces in it on Windows, you'll need - the `win32api `_ library installed. - - PyPy Support - ~~~~~~~~~~~~ - - Beginning with virtualenv version 1.5 `PyPy `_ is - supported. To use PyPy 1.4 or 1.4.1, you need a version of virtualenv >= 1.5. - To use PyPy 1.5, you need a version of virtualenv >= 1.6.1. - - Creating Your Own Bootstrap Scripts - ----------------------------------- - - While this creates an environment, it doesn't put anything into the - environment. Developers may find it useful to distribute a script - that sets up a particular environment, for example a script that - installs a particular web application. - - To create a script like this, call - ``virtualenv.create_bootstrap_script(extra_text)``, and write the - result to your new bootstrapping script. Here's the documentation - from the docstring: - - Creates a bootstrap script, which is like this script but with - extend_parser, adjust_options, and after_install hooks. - - This returns a string that (written to disk of course) can be used - as a bootstrap script with your own customizations. The script - will be the standard virtualenv.py script, with your extra text - added (your extra text should be Python code). - - If you include these functions, they will be called: - - ``extend_parser(optparse_parser)``: - You can add or remove options from the parser here. - - ``adjust_options(options, args)``: - You can change options here, or change the args (if you accept - different kinds of arguments, be sure you modify ``args`` so it is - only ``[DEST_DIR]``). - - ``after_install(options, home_dir)``: - - After everything is installed, this function is called. This - is probably the function you are most likely to use. An - example would be:: - - def after_install(options, home_dir): - if sys.platform == 'win32': - bin = 'Scripts' - else: - bin = 'bin' - subprocess.call([join(home_dir, bin, 'easy_install'), - 'MyPackage']) - subprocess.call([join(home_dir, bin, 'my-package-script'), - 'setup', home_dir]) - - This example immediately installs a package, and runs a setup - script from that package. - - Bootstrap Example - ~~~~~~~~~~~~~~~~~ - - Here's a more concrete example of how you could use this:: - - import virtualenv, textwrap - output = virtualenv.create_bootstrap_script(textwrap.dedent(""" - import os, subprocess - def after_install(options, home_dir): - etc = join(home_dir, 'etc') - if not os.path.exists(etc): - os.makedirs(etc) - subprocess.call([join(home_dir, 'bin', 'easy_install'), - 'BlogApplication']) - subprocess.call([join(home_dir, 'bin', 'paster'), - 'make-config', 'BlogApplication', - join(etc, 'blog.ini')]) - subprocess.call([join(home_dir, 'bin', 'paster'), - 'setup-app', join(etc, 'blog.ini')]) - """)) - f = open('blog-bootstrap.py', 'w').write(output) - - Another example is available `here - `_. - - activate script - ~~~~~~~~~~~~~~~ - - In a newly created virtualenv there will be a ``bin/activate`` shell - script. For Windows systems, activation scripts are provided for CMD.exe - and Powershell. - - On Posix systems you can do:: - - $ source bin/activate - - This will change your ``$PATH`` to point to the virtualenv's ``bin/`` - directory. (You have to use ``source`` because it changes your shell - environment in-place.) This is all it does; it's purely a convenience. If - you directly run a script or the python interpreter from the virtualenv's - ``bin/`` directory (e.g. ``path/to/env/bin/pip`` or - ``/path/to/env/bin/python script.py``) there's no need for activation. - - After activating an environment you can use the function ``deactivate`` to - undo the changes to your ``$PATH``. - - The ``activate`` script will also modify your shell prompt to indicate - which environment is currently active. You can disable this behavior, - which can be useful if you have your own custom prompt that already - displays the active environment name. To do so, set the - ``VIRTUAL_ENV_DISABLE_PROMPT`` environment variable to any non-empty - value before running the ``activate`` script. - - On Windows you just do:: - - > \path\to\env\Scripts\activate - - And type `deactivate` to undo the changes. - - Based on your active shell (CMD.exe or Powershell.exe), Windows will use - either activate.bat or activate.ps1 (as appropriate) to activate the - virtual environment. If using Powershell, see the notes about code signing - below. - - .. note:: - - If using Powershell, the ``activate`` script is subject to the - `execution policies`_ on the system. By default on Windows 7, the system's - excution policy is set to ``Restricted``, meaning no scripts like the - ``activate`` script are allowed to be executed. But that can't stop us - from changing that slightly to allow it to be executed. - - In order to use the script, you have to relax your system's execution - policy to ``AllSigned``, meaning all scripts on the system must be - digitally signed to be executed. Since the virtualenv activation - script is signed by one of the authors (Jannis Leidel) this level of - the execution policy suffices. As an adminstrator run:: - - PS C:\> Set-ExecutionPolicy AllSigned - - Then you'll be asked to trust the signer, when executing the script. - You will be prompted with the following:: - - PS C:\> virtualenv .\foo - New python executable in C:\foo\Scripts\python.exe - Installing setuptools................done. - Installing pip...................done. - PS C:\> .\foo\scripts\activate - - Do you want to run software from this untrusted publisher? - File C:\foo\scripts\activate.ps1 is published by E=jannis@leidel.info, - CN=Jannis Leidel, L=Berlin, S=Berlin, C=DE, Description=581796-Gh7xfJxkxQSIO4E0 - and is not trusted on your system. Only run scripts from trusted publishers. - [V] Never run [D] Do not run [R] Run once [A] Always run [?] Help - (default is "D"):A - (foo) PS C:\> - - If you select ``[A] Always Run``, the certificate will be added to the - Trusted Publishers of your user account, and will be trusted in this - user's context henceforth. If you select ``[R] Run Once``, the script will - be run, but you will be prometed on a subsequent invocation. Advanced users - can add the signer's certificate to the Trusted Publishers of the Computer - account to apply to all users (though this technique is out of scope of this - document). - - Alternatively, you may relax the system execution policy to allow running - of local scripts without verifying the code signature using the following:: - - PS C:\> Set-ExecutionPolicy RemoteSigned - - Since the ``activate.ps1`` script is generated locally for each virtualenv, - it is not considered a remote script and can then be executed. - - .. _`execution policies`: http://technet.microsoft.com/en-us/library/dd347641.aspx - - The ``--system-site-packages`` Option - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - If you build with ``virtualenv --system-site-packages ENV``, your virtual - environment will inherit packages from ``/usr/lib/python2.7/site-packages`` - (or wherever your global site-packages directory is). - - This can be used if you have control over the global site-packages directory, - and you want to depend on the packages there. If you want isolation from the - global system, do not use this flag. - - Using Virtualenv without ``bin/python`` - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sometimes you can't or don't want to use the Python interpreter - created by the virtualenv. For instance, in a `mod_python - `_ or `mod_wsgi `_ - environment, there is only one interpreter. - - Luckily, it's easy. You must use the custom Python interpreter to - *install* libraries. But to *use* libraries, you just have to be sure - the path is correct. A script is available to correct the path. You - can setup the environment like:: - - activate_this = '/path/to/env/bin/activate_this.py' - execfile(activate_this, dict(__file__=activate_this)) - - This will change ``sys.path`` and even change ``sys.prefix``, but also allow - you to use an existing interpreter. Items in your environment will show up - first on ``sys.path``, before global items. However, global items will - always be accessible (as if the ``--system-site-packages`` flag had been used - in creating the environment, whether it was or not). Also, this cannot undo - the activation of other environments, or modules that have been imported. - You shouldn't try to, for instance, activate an environment before a web - request; you should activate *one* environment as early as possible, and not - do it again in that process. - - Making Environments Relocatable - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Note: this option is somewhat experimental, and there are probably - caveats that have not yet been identified. Also this does not - currently work on Windows. - - Normally environments are tied to a specific path. That means that - you cannot move an environment around or copy it to another computer. - You can fix up an environment to make it relocatable with the - command:: - - $ virtualenv --relocatable ENV - - This will make some of the files created by setuptools or distribute - use relative paths, and will change all the scripts to use ``activate_this.py`` - instead of using the location of the Python interpreter to select the - environment. - - **Note:** you must run this after you've installed *any* packages into - the environment. If you make an environment relocatable, then - install a new package, you must run ``virtualenv --relocatable`` - again. - - Also, this **does not make your packages cross-platform**. You can - move the directory around, but it can only be used on other similar - computers. Some known environmental differences that can cause - incompatibilities: a different version of Python, when one platform - uses UCS2 for its internal unicode representation and another uses - UCS4 (a compile-time option), obvious platform changes like Windows - vs. Linux, or Intel vs. ARM, and if you have libraries that bind to C - libraries on the system, if those C libraries are located somewhere - different (either different versions, or a different filesystem - layout). - - If you use this flag to create an environment, currently, the - ``--system-site-packages`` option will be implied. - - The ``--extra-search-dir`` Option - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - When it creates a new environment, virtualenv installs either - setuptools or distribute, and pip. In normal operation, the latest - releases of these packages are fetched from the `Python Package Index - `_ (PyPI). In some circumstances, this - behavior may not be wanted, for example if you are using virtualenv - during a deployment and do not want to depend on Internet access and - PyPI availability. - - As an alternative, you can provide your own versions of setuptools, - distribute and/or pip on the filesystem, and tell virtualenv to use - those distributions instead of downloading them from the Internet. To - use this feature, pass one or more ``--extra-search-dir`` options to - virtualenv like this:: - - $ virtualenv --extra-search-dir=/path/to/distributions ENV - - The ``/path/to/distributions`` path should point to a directory that - contains setuptools, distribute and/or pip distributions. Setuptools - distributions must be ``.egg`` files; distribute and pip distributions - should be `.tar.gz` source distributions. - - Virtualenv will still download these packages if no satisfactory local - distributions are found. - - If you are really concerned about virtualenv fetching these packages - from the Internet and want to ensure that it never will, you can also - provide an option ``--never-download`` like so:: - - $ virtualenv --extra-search-dir=/path/to/distributions --never-download ENV - - If this option is provided, virtualenv will never try to download - setuptools/distribute or pip. Instead, it will exit with status code 1 - if it fails to find local distributions for any of these required - packages. The local distribution lookup is done in this order and the - following locations: - - #. The current directory. - #. The directory where virtualenv.py is located. - #. A ``virtualenv_support`` directory relative to the directory where - virtualenv.py is located. - #. If the file being executed is not named virtualenv.py (i.e. is a boot - script), a ``virtualenv_support`` directory relative to wherever - virtualenv.py is actually installed. - - Compare & Contrast with Alternatives - ------------------------------------ - - There are several alternatives that create isolated environments: - - * ``workingenv`` (which I do not suggest you use anymore) is the - predecessor to this library. It used the main Python interpreter, - but relied on setting ``$PYTHONPATH`` to activate the environment. - This causes problems when running Python scripts that aren't part of - the environment (e.g., a globally installed ``hg`` or ``bzr``). It - also conflicted a lot with Setuptools. - - * `virtual-python - `_ - is also a predecessor to this library. It uses only symlinks, so it - couldn't work on Windows. It also symlinks over the *entire* - standard library and global ``site-packages``. As a result, it - won't see new additions to the global ``site-packages``. - - This script only symlinks a small portion of the standard library - into the environment, and so on Windows it is feasible to simply - copy these files over. Also, it creates a new/empty - ``site-packages`` and also adds the global ``site-packages`` to the - path, so updates are tracked separately. This script also installs - Setuptools automatically, saving a step and avoiding the need for - network access. - - * `zc.buildout `_ doesn't - create an isolated Python environment in the same style, but - achieves similar results through a declarative config file that sets - up scripts with very particular packages. As a declarative system, - it is somewhat easier to repeat and manage, but more difficult to - experiment with. ``zc.buildout`` includes the ability to setup - non-Python systems (e.g., a database server or an Apache instance). - - I *strongly* recommend anyone doing application development or - deployment use one of these tools. - - Contributing - ------------ - - Refer to the `contributing to pip`_ documentation - it applies equally to - virtualenv. - - Virtualenv's release schedule is tied to pip's -- each time there's a new pip - release, there will be a new virtualenv release that bundles the new version of - pip. - - .. _contributing to pip: http://www.pip-installer.org/en/latest/contributing.html - - Running the tests - ~~~~~~~~~~~~~~~~~ - - Virtualenv's test suite is small and not yet at all comprehensive, but we aim - to grow it. - - The easy way to run tests (handles test dependencies automatically):: - - $ python setup.py test - - If you want to run only a selection of the tests, you'll need to run them - directly with nose instead. Create a virtualenv, and install required - packages:: - - $ pip install nose mock - - Run nosetests:: - - $ nosetests - - Or select just a single test file to run:: - - $ nosetests tests.test_virtualenv - - - Other Documentation and Links - ----------------------------- - - * James Gardner has written a tutorial on using `virtualenv with - Pylons - `_. - - * `Blog announcement - `_. - - * Doug Hellmann wrote a description of his `command-line work flow - using virtualenv (virtualenvwrapper) - `_ - including some handy scripts to make working with multiple - environments easier. He also wrote `an example of using virtualenv - to try IPython - `_. - - * Chris Perkins created a `showmedo video including virtualenv - `_. - - * `Using virtualenv with mod_wsgi - `_. - - * `virtualenv commands - `_ for some more - workflow-related tools around virtualenv. - - Status and License - ------------------ - - ``virtualenv`` is a successor to `workingenv - `_, and an extension - of `virtual-python - `_. - - It was written by Ian Bicking, sponsored by the `Open Planning - Project `_ and is now maintained by a - `group of developers `_. - It is licensed under an - `MIT-style permissive license `_. - - Changes & News - -------------- - - 1.7.1.2 (2012-02-17) - ~~~~~~~~~~~~~~~~~~~~ - - * Fixed minor issue in `--relocatable`. Thanks, Cap Petschulat. - - 1.7.1.1 (2012-02-16) - ~~~~~~~~~~~~~~~~~~~~ - - * Bumped the version string in ``virtualenv.py`` up, too. - - * Fixed rST rendering bug of long description. - - 1.7.1 (2012-02-16) - ~~~~~~~~~~~~~~~~~~ - - * Update embedded pip to version 1.1. - - * Fix `--relocatable` under Python 3. Thanks Doug Hellmann. - - * Added environ PATH modification to activate_this.py. Thanks Doug - Napoleone. Fixes #14. - - * Support creating virtualenvs directly from a Python build directory on - Windows. Thanks CBWhiz. Fixes #139. - - * Use non-recursive symlinks to fix things up for posix_local install - scheme. Thanks michr. - - * Made activate script available for use with msys and cygwin on Windows. - Thanks Greg Haskins, Cliff Xuan, Jonathan Griffin and Doug Napoleone. - Fixes #176. - - * Fixed creation of virtualenvs on Windows when Python is not installed for - all users. Thanks Anatoly Techtonik for report and patch and Doug - Napoleone for testing and confirmation. Fixes #87. - - * Fixed creation of virtualenvs using -p in installs where some modules - that ought to be in the standard library (e.g. `readline`) are actually - installed in `site-packages` next to `virtualenv.py`. Thanks Greg Haskins - for report and fix. Fixes #167. - - * Added activation script for Powershell (signed by Jannis Leidel). Many - thanks to Jason R. Coombs. - - 1.7 (2011-11-30) - ~~~~~~~~~~~~~~~~ - - * Gave user-provided ``--extra-search-dir`` priority over default dirs for - finding setuptools/distribute (it already had priority for finding pip). - Thanks Ethan Jucovy. - - * Updated embedded Distribute release to 0.6.24. Thanks Alex Gronholm. - - * Made ``--no-site-packages`` behavior the default behavior. The - ``--no-site-packages`` flag is still permitted, but displays a warning when - used. Thanks Chris McDonough. - - * New flag: ``--system-site-packages``; this flag should be passed to get the - previous default global-site-package-including behavior back. - - * Added ability to set command options as environment variables and options - in a ``virtualenv.ini`` file. - - * Fixed various encoding related issues with paths. Thanks Gunnlaugur Thor Briem. - - * Made ``virtualenv.py`` script executable. - - 1.6.4 (2011-07-21) - ~~~~~~~~~~~~~~~~~~ - - * Restored ability to run on Python 2.4, too. - - 1.6.3 (2011-07-16) - ~~~~~~~~~~~~~~~~~~ - - * Restored ability to run on Python < 2.7. - - 1.6.2 (2011-07-16) - ~~~~~~~~~~~~~~~~~~ - - * Updated embedded distribute release to 0.6.19. - - * Updated embedded pip release to 1.0.2. - - * Fixed #141 - Be smarter about finding pkg_resources when using the - non-default Python intepreter (by using the ``-p`` option). - - * Fixed #112 - Fixed path in docs. - - * Fixed #109 - Corrected doctests of a Logger method. - - * Fixed #118 - Fixed creating virtualenvs on platforms that use the - "posix_local" install scheme, such as Ubuntu with Python 2.7. - - * Add missing library to Python 3 virtualenvs (``_dummy_thread``). - - - 1.6.1 (2011-04-30) - ~~~~~~~~~~~~~~~~~~ - - * Start to use git-flow. - - * Added support for PyPy 1.5 - - * Fixed #121 -- added sanity-checking of the -p argument. Thanks Paul Nasrat. - - * Added progress meter for pip installation as well as setuptools. Thanks Ethan - Jucovy. - - * Added --never-download and --search-dir options. Thanks Ethan Jucovy. - - 1.6 - ~~~ - - * Added Python 3 support! Huge thanks to Vinay Sajip and Vitaly Babiy. - - * Fixed creation of virtualenvs on Mac OS X when standard library modules - (readline) are installed outside the standard library. - - * Updated bundled pip to 1.0. - - 1.5.2 - ~~~~~ - - * Moved main repository to Github: https://github.com/pypa/virtualenv - - * Transferred primary maintenance from Ian to Jannis Leidel, Carl Meyer and Brian Rosner - - * Fixed a few more pypy related bugs. - - * Updated bundled pip to 0.8.2. - - * Handed project over to new team of maintainers. - - * Moved virtualenv to Github at https://github.com/pypa/virtualenv - - 1.5.1 - ~~~~~ - - * Added ``_weakrefset`` requirement for Python 2.7.1. - - * Fixed Windows regression in 1.5 - - 1.5 - ~~~ - - * Include pip 0.8.1. - - * Add support for PyPy. - - * Uses a proper temporary dir when installing environment requirements. - - * Add ``--prompt`` option to be able to override the default prompt prefix. - - * Fix an issue with ``--relocatable`` on Windows. - - * Fix issue with installing the wrong version of distribute. - - * Add fish and csh activate scripts. - - 1.4.9 - ~~~~~ - - * Include pip 0.7.2 - - 1.4.8 - ~~~~~ - - * Fix for Mac OS X Framework builds that use - ``--universal-archs=intel`` - - * Fix ``activate_this.py`` on Windows. - - * Allow ``$PYTHONHOME`` to be set, so long as you use ``source - bin/activate`` it will get unset; if you leave it set and do not - activate the environment it will still break the environment. - - * Include pip 0.7.1 - - 1.4.7 - ~~~~~ - - * Include pip 0.7 - - 1.4.6 - ~~~~~ - - * Allow ``activate.sh`` to skip updating the prompt (by setting - ``$VIRTUAL_ENV_DISABLE_PROMPT``). - - 1.4.5 - ~~~~~ - - * Include pip 0.6.3 - - * Fix ``activate.bat`` and ``deactivate.bat`` under Windows when - ``PATH`` contained a parenthesis - - 1.4.4 - ~~~~~ - - * Include pip 0.6.2 and Distribute 0.6.10 - - * Create the ``virtualenv`` script even when Setuptools isn't - installed - - * Fix problem with ``virtualenv --relocate`` when ``bin/`` has - subdirectories (e.g., ``bin/.svn/``); from Alan Franzoni. - - * If you set ``$VIRTUALENV_USE_DISTRIBUTE`` then virtualenv will use - Distribute by default (so you don't have to remember to use - ``--distribute``). - - 1.4.3 - ~~~~~ - - * Include pip 0.6.1 - - 1.4.2 - ~~~~~ - - * Fix pip installation on Windows - - * Fix use of stand-alone ``virtualenv.py`` (and boot scripts) - - * Exclude ~/.local (user site-packages) from environments when using - ``--no-site-packages`` - - 1.4.1 - ~~~~~ - - * Include pip 0.6 - - 1.4 - ~~~ - - * Updated setuptools to 0.6c11 - - * Added the --distribute option - - * Fixed packaging problem of support-files - - 1.3.4 - ~~~~~ - - * Virtualenv now copies the actual embedded Python binary on - Mac OS X to fix a hang on Snow Leopard (10.6). - - * Fail more gracefully on Windows when ``win32api`` is not installed. - - * Fix site-packages taking precedent over Jython's ``__classpath__`` - and also specially handle the new ``__pyclasspath__`` entry in - ``sys.path``. - - * Now copies Jython's ``registry`` file to the virtualenv if it exists. - - * Better find libraries when compiling extensions on Windows. - - * Create ``Scripts\pythonw.exe`` on Windows. - - * Added support for the Debian/Ubuntu - ``/usr/lib/pythonX.Y/dist-packages`` directory. - - * Set ``distutils.sysconfig.get_config_vars()['LIBDIR']`` (based on - ``sys.real_prefix``) which is reported to help building on Windows. - - * Make ``deactivate`` work on ksh - - * Fixes for ``--python``: make it work with ``--relocatable`` and the - symlink created to the exact Python version. - - 1.3.3 - ~~~~~ - - * Use Windows newlines in ``activate.bat``, which has been reported to help - when using non-ASCII directory names. - - * Fixed compatibility with Jython 2.5b1. - - * Added a function ``virtualenv.install_python`` for more fine-grained - access to what ``virtualenv.create_environment`` does. - - * Fix `a problem `_ - with Windows and paths that contain spaces. - - * If ``/path/to/env/.pydistutils.cfg`` exists (or - ``/path/to/env/pydistutils.cfg`` on Windows systems) then ignore - ``~/.pydistutils.cfg`` and use that other file instead. - - * Fix ` a problem - `_ picking up - some ``.so`` libraries in ``/usr/local``. - - 1.3.2 - ~~~~~ - - * Remove the ``[install] prefix = ...`` setting from the virtualenv - ``distutils.cfg`` -- this has been causing problems for a lot of - people, in rather obscure ways. - - * If you use a `boot script <./index.html#boot-script>`_ it will attempt to import ``virtualenv`` - and find a pre-downloaded Setuptools egg using that. - - * Added platform-specific paths, like ``/usr/lib/pythonX.Y/plat-linux2`` - - 1.3.1 - ~~~~~ - - * Real Python 2.6 compatibility. Backported the Python 2.6 updates to - ``site.py``, including `user directories - `_ - (this means older versions of Python will support user directories, - whether intended or not). - - * Always set ``[install] prefix`` in ``distutils.cfg`` -- previously - on some platforms where a system-wide ``distutils.cfg`` was present - with a ``prefix`` setting, packages would be installed globally - (usually in ``/usr/local/lib/pythonX.Y/site-packages``). - - * Sometimes Cygwin seems to leave ``.exe`` off ``sys.executable``; a - workaround is added. - - * Fix ``--python`` option. - - * Fixed handling of Jython environments that use a - jython-complete.jar. - - 1.3 - ~~~ - - * Update to Setuptools 0.6c9 - * Added an option ``virtualenv --relocatable EXISTING_ENV``, which - will make an existing environment "relocatable" -- the paths will - not be absolute in scripts, ``.egg-info`` and ``.pth`` files. This - may assist in building environments that can be moved and copied. - You have to run this *after* any new packages installed. - * Added ``bin/activate_this.py``, a file you can use like - ``execfile("path_to/activate_this.py", - dict(__file__="path_to/activate_this.py"))`` -- this will activate - the environment in place, similar to what `the mod_wsgi example - does `_. - * For Mac framework builds of Python, the site-packages directory - ``/Library/Python/X.Y/site-packages`` is added to ``sys.path``, from - Andrea Rech. - * Some platform-specific modules in Macs are added to the path now - (``plat-darwin/``, ``plat-mac/``, ``plat-mac/lib-scriptpackages``), - from Andrea Rech. - * Fixed a small Bashism in the ``bin/activate`` shell script. - * Added ``__future__`` to the list of required modules, for Python - 2.3. You'll still need to backport your own ``subprocess`` module. - * Fixed the ``__classpath__`` entry in Jython's ``sys.path`` taking - precedent over virtualenv's libs. - - 1.2 - ~~~ - - * Added a ``--python`` option to select the Python interpreter. - * Add ``warnings`` to the modules copied over, for Python 2.6 support. - * Add ``sets`` to the module copied over for Python 2.3 (though Python - 2.3 still probably doesn't work). - - 1.1.1 - ~~~~~ - - * Added support for Jython 2.5. - - 1.1 - ~~~ - - * Added support for Python 2.6. - * Fix a problem with missing ``DLLs/zlib.pyd`` on Windows. Create - * ``bin/python`` (or ``bin/python.exe``) even when you run virtualenv - with an interpreter named, e.g., ``python2.4`` - * Fix MacPorts Python - * Added --unzip-setuptools option - * Update to Setuptools 0.6c8 - * If the current directory is not writable, run ez_setup.py in ``/tmp`` - * Copy or symlink over the ``include`` directory so that packages will - more consistently compile. - - 1.0 - ~~~ - - * Fix build on systems that use ``/usr/lib64``, distinct from - ``/usr/lib`` (specifically CentOS x64). - * Fixed bug in ``--clear``. - * Fixed typos in ``deactivate.bat``. - * Preserve ``$PYTHONPATH`` when calling subprocesses. - - 0.9.2 - ~~~~~ - - * Fix include dir copying on Windows (makes compiling possible). - * Include the main ``lib-tk`` in the path. - * Patch ``distutils.sysconfig``: ``get_python_inc`` and - ``get_python_lib`` to point to the global locations. - * Install ``distutils.cfg`` before Setuptools, so that system - customizations of ``distutils.cfg`` won't effect the installation. - * Add ``bin/pythonX.Y`` to the virtualenv (in addition to - ``bin/python``). - * Fixed an issue with Mac Framework Python builds, and absolute paths - (from Ronald Oussoren). - - 0.9.1 - ~~~~~ - - * Improve ability to create a virtualenv from inside a virtualenv. - * Fix a little bug in ``bin/activate``. - * Actually get ``distutils.cfg`` to work reliably. - - 0.9 - ~~~ - - * Added ``lib-dynload`` and ``config`` to things that need to be - copied over in an environment. - * Copy over or symlink the ``include`` directory, so that you can - build packages that need the C headers. - * Include a ``distutils`` package, so you can locally update - ``distutils.cfg`` (in ``lib/pythonX.Y/distutils/distutils.cfg``). - * Better avoid downloading Setuptools, and hitting PyPI on environment - creation. - * Fix a problem creating a ``lib64/`` directory. - * Should work on MacOSX Framework builds (the default Python - installations on Mac). Thanks to Ronald Oussoren. - - 0.8.4 - ~~~~~ - - * Windows installs would sometimes give errors about ``sys.prefix`` that - were inaccurate. - * Slightly prettier output. - - 0.8.3 - ~~~~~ - - * Added support for Windows. - - 0.8.2 - ~~~~~ - - * Give a better warning if you are on an unsupported platform (Mac - Framework Pythons, and Windows). - * Give error about running while inside a workingenv. - * Give better error message about Python 2.3. - - 0.8.1 - ~~~~~ - - Fixed packaging of the library. - - 0.8 - ~~~ - - Initial release. Everything is changed and new! - -Keywords: setuptools deployment installation distutils -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.4 -Classifier: Programming Language :: Python :: 2.5 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.1 -Classifier: Programming Language :: Python :: 3.2 diff --git a/other-licenses/virtualenv/virtualenv.egg-info/SOURCES.txt b/other-licenses/virtualenv/virtualenv.egg-info/SOURCES.txt deleted file mode 100644 index 0959ae6f99a7..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/SOURCES.txt +++ /dev/null @@ -1,33 +0,0 @@ -AUTHORS.txt -LICENSE.txt -MANIFEST.in -setup.py -virtualenv.py -docs/index.txt -docs/news.txt -scripts/virtualenv -virtualenv.egg-info/PKG-INFO -virtualenv.egg-info/SOURCES.txt -virtualenv.egg-info/dependency_links.txt -virtualenv.egg-info/entry_points.txt -virtualenv.egg-info/not-zip-safe -virtualenv.egg-info/top_level.txt -virtualenv_embedded/activate.bat -virtualenv_embedded/activate.csh -virtualenv_embedded/activate.fish -virtualenv_embedded/activate.ps1 -virtualenv_embedded/activate.sh -virtualenv_embedded/activate_this.py -virtualenv_embedded/deactivate.bat -virtualenv_embedded/distribute_setup.py -virtualenv_embedded/distutils-init.py -virtualenv_embedded/distutils.cfg -virtualenv_embedded/ez_setup.py -virtualenv_embedded/site.py -virtualenv_support/__init__.py -virtualenv_support/distribute-0.6.24.tar.gz -virtualenv_support/pip-1.1.tar.gz -virtualenv_support/setuptools-0.6c11-py2.4.egg -virtualenv_support/setuptools-0.6c11-py2.5.egg -virtualenv_support/setuptools-0.6c11-py2.6.egg -virtualenv_support/setuptools-0.6c11-py2.7.egg \ No newline at end of file diff --git a/other-licenses/virtualenv/virtualenv.egg-info/dependency_links.txt b/other-licenses/virtualenv/virtualenv.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791f..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/other-licenses/virtualenv/virtualenv.egg-info/entry_points.txt b/other-licenses/virtualenv/virtualenv.egg-info/entry_points.txt deleted file mode 100644 index 46e4bb50eb5a..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -virtualenv = virtualenv:main diff --git a/other-licenses/virtualenv/virtualenv.egg-info/not-zip-safe b/other-licenses/virtualenv/virtualenv.egg-info/not-zip-safe deleted file mode 100644 index 8b137891791f..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/other-licenses/virtualenv/virtualenv.egg-info/top_level.txt b/other-licenses/virtualenv/virtualenv.egg-info/top_level.txt deleted file mode 100644 index 2fe6b5d992bd..000000000000 --- a/other-licenses/virtualenv/virtualenv.egg-info/top_level.txt +++ /dev/null @@ -1,2 +0,0 @@ -virtualenv_support -virtualenv diff --git a/services/Makefile.in b/services/Makefile.in index 8646dc91eaea..e2a0f94ebe08 100644 --- a/services/Makefile.in +++ b/services/Makefile.in @@ -11,7 +11,7 @@ VPATH = @srcdir@ include $(DEPTH)/config/autoconf.mk ifdef MOZ_SERVICES_SYNC -PARALLEL_DIRS += common crypto sync +PARALLEL_DIRS += aitc common crypto sync endif include $(topsrcdir)/config/rules.mk diff --git a/services/aitc/Aitc.js b/services/aitc/Aitc.js new file mode 100644 index 000000000000..ac9a6a42d995 --- /dev/null +++ b/services/aitc/Aitc.js @@ -0,0 +1,121 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/PlacesUtils.jsm"); + +Cu.import("resource://services-common/utils.js"); + +function AitcService() { + this.aitc = null; + this.wrappedJSObject = this; +} +AitcService.prototype = { + classID: Components.ID("{a3d387ca-fd26-44ca-93be-adb5fda5a78d}"), + + QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, + Ci.nsINavHistoryObserver, + Ci.nsISupportsWeakReference]), + + observe: function observe(subject, topic, data) { + switch (topic) { + case "app-startup": + // We listen for this event beacause Aitc won't work until there is + // atleast 1 visible top-level XUL window. + Services.obs.addObserver(this, "sessionstore-windows-restored", true); + break; + case "sessionstore-windows-restored": + Services.obs.removeObserver(this, "sessionstore-windows-restored"); + + // Don't start AITC if classic sync is on. + Cu.import("resource://services-common/preferences.js"); + if (Preferences.get("services.sync.engine.apps", false)) { + return; + } + // Start AITC only if it is enabled. + if (!Preferences.get("services.aitc.enabled", false)) { + return; + } + + // Start AITC service if apps.enabled is true. If false, we look + // in the browser history to determine if they're an "apps user". If + // an entry wasn't found, we'll watch for navigation to either the + // marketplace or dashboard and switch ourselves on then. + + if (Preferences.get("apps.enabled", false)) { + this.start(); + return; + } + + // Set commonly used URLs. + this.DASHBOARD_URL = CommonUtils.makeURI( + Preferences.get("services.aitc.dashboard.url") + ); + this.MARKETPLACE_URL = CommonUtils.makeURI( + Preferences.get("services.aitc.marketplace.url") + ); + + if (this.hasUsedApps()) { + Preferences.set("apps.enabled", true); + this.start(); + return; + } + + // Wait and see if the user wants anything apps related. + PlacesUtils.history.addObserver(this, true); + break; + } + }, + + start: function start() { + Cu.import("resource://services-aitc/main.js"); + if (!this.aitc) { + this.aitc = new Aitc(); + } + }, + + hasUsedApps: function hasUsedApps() { + // There is no easy way to determine whether a user is "using apps". + // The best we can do right now is to see if they have visited either + // the Mozilla dashboard or Marketplace. See bug 760898. + let gh = PlacesUtils.ghistory2; + if (gh.isVisited(this.DASHBOARD_URL)) { + return true; + } + if (gh.isVisited(this.MARKETPLACE_URL)) { + return true; + } + return false; + }, + + // nsINavHistoryObserver. We are only interested in onVisit(). + onBeforeDeleteURI: function() {}, + onBeginUpdateBatch: function() {}, + onClearHistory: function() {}, + onDeleteURI: function() {}, + onDeleteVisits: function() {}, + onEndUpdateBatch: function() {}, + onPageChanged: function() {}, + onPageExpired: function() {}, + onTitleChanged: function() {}, + + onVisit: function onVisit(uri) { + if (!uri.equals(this.MARKETPLACE_URL) && !uri.equals(this.DASHBOARD_URL)) { + return; + } + + PlacesUtils.history.removeObserver(this); + Preferences.set("apps.enabled", true); + this.start(); + return; + }, +}; + +const components = [AitcService]; +const NSGetFactory = XPCOMUtils.generateNSGetFactory(components); diff --git a/services/aitc/AitcComponents.manifest b/services/aitc/AitcComponents.manifest new file mode 100644 index 000000000000..3f435ba7f1ac --- /dev/null +++ b/services/aitc/AitcComponents.manifest @@ -0,0 +1,6 @@ +# Aitc.js +component {a3d387ca-fd26-44ca-93be-adb5fda5a78d} Aitc.js +contract @mozilla.org/services/aitc;1 {a3d387ca-fd26-44ca-93be-adb5fda5a78d} +category app-startup AitcService service,@mozilla.org/services/aitc;1 +# Register resource aliases +resource services-aitc resource:///modules/services-aitc/ diff --git a/services/aitc/Makefile.in b/services/aitc/Makefile.in new file mode 100644 index 000000000000..e336b2fc0478 --- /dev/null +++ b/services/aitc/Makefile.in @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +DEPTH = ../.. +topsrcdir = @top_srcdir@ +srcdir = @srcdir@ +VPATH = @srcdir@ + +include $(DEPTH)/config/autoconf.mk + +EXTRA_COMPONENTS = \ + AitcComponents.manifest \ + Aitc.js \ + $(NULL) + +PREF_JS_EXPORTS = $(srcdir)/services-aitc.js + +libs:: + $(NSINSTALL) $(srcdir)/modules/* $(FINAL_TARGET)/modules/services-aitc + +TEST_DIRS += tests + +include $(topsrcdir)/config/rules.mk diff --git a/services/aitc/modules/browserid.js b/services/aitc/modules/browserid.js new file mode 100644 index 000000000000..0a961d79df80 --- /dev/null +++ b/services/aitc/modules/browserid.js @@ -0,0 +1,458 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["BrowserID"]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +Cu.import("resource://services-common/utils.js"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/preferences.js"); + +const PREFS = new Preferences("services.aitc.browserid."); + +/** + * This implementation will be replaced with native crypto and assertion + * generation goodness. See bug 753238. + */ +function BrowserIDService() { + this._log = Log4Moz.repository.getLogger("Services.BrowserID"); + this._log.level = Log4Moz.Level[PREFS.get("log")]; +} +BrowserIDService.prototype = { + /** + * Getter that returns the freshest value for ID_URI. + */ + get ID_URI() { + return PREFS.get("url"); + }, + + /** + * Obtain a BrowserID assertion with the specified characteristics. + * + * @param cb + * (Function) Callback to be called with (err, assertion) where 'err' + * can be an Error or NULL, and 'assertion' can be NULL or a valid + * BrowserID assertion. If no callback is provided, an exception is + * thrown. + * + * @param options + * (Object) An object that may contain the following properties: + * + * "requiredEmail" : An email for which the assertion is to be + * issued. If one could not be obtained, the call + * will fail. If this property is not specified, + * the default email as set by the user will be + * chosen. If both this property and "sameEmailAs" + * are set, an exception will be thrown. + * + * "sameEmailAs" : If set, instructs the function to issue an + * assertion for the same email that was provided + * to the domain specified by this value. If this + * information could not be obtained, the call + * will fail. If both this property and + * "requiredEmail" are set, an exception will be + * thrown. + * + * "audience" : The audience for which the assertion is to be + * issued. If this property is not set an exception + * will be thrown. + * + * Any properties not listed above will be ignored. + * + * (This function could use some love in terms of what arguments it accepts. + * See bug 746401.) + */ + getAssertion: function getAssertion(cb, options) { + if (!cb) { + throw new Error("getAssertion called without a callback"); + } + if (!options) { + throw new Error("getAssertion called without any options"); + } + if (!options.audience) { + throw new Error("getAssertion called without an audience"); + } + if (options.sameEmailAs && options.requiredEmail) { + throw new Error( + "getAssertion sameEmailAs and requiredEmail are mutually exclusive" + ); + } + + new Sandbox(this._getEmails.bind(this, cb, options), this.ID_URI); + }, + + /** + * Obtain a BrowserID assertion by asking the user to login and select an + * email address. + * + * @param cb + * (Function) Callback to be called with (err, assertion) where 'err' + * can be an Error or NULL, and 'assertion' can be NULL or a valid + * BrowserID assertion. If no callback is provided, an exception is + * thrown. + * + * @param win + * (Window) A contentWindow that has a valid document loaded. If this + * argument is provided the user will be asked to login in the context + * of the document currently loaded in this window. + * + * The audience of the assertion will be set to the domain of the + * loaded document, and the "audience" property in the "options" + * argument (if provided), will be ignored. The email to which this + * assertion issued will be selected by the user when they login (and + * "requiredEmail" or "sameEmailAs", if provided, will be ignored). If + * the user chooses to not login, this call will fail. + * + * Be aware! The provided contentWindow must also have loaded the + * BrowserID include.js shim for this to work! This behavior is + * temporary until we implement native support for navigator.id. + * + * @param options + * (Object) Currently an empty object. Present for future compatiblity + * when options for a login case may be added. Any properties, if + * present, are ignored. + */ + getAssertionWithLogin: function getAssertionWithLogin(cb, win, options) { + if (!cb) { + throw new Error("getAssertionWithLogin called without a callback"); + } + if (!win) { + throw new Error("getAssertionWithLogin called without a window"); + } + this._getAssertionWithLogin(cb, win); + }, + + + /** + * Internal implementation methods begin here + */ + + // Try to get the user's email(s). If user isn't logged in, this will be empty + _getEmails: function _getEmails(cb, options, sandbox) { + let self = this; + + function callback(res) { + let emails = {}; + try { + emails = JSON.parse(res); + } catch (e) { + self._log.error("Exception in JSON.parse for _getAssertion: " + e); + } + self._gotEmails(emails, sandbox, cb, options); + } + sandbox.box.importFunction(callback); + + let scriptText = + "var list = window.BrowserID.User.getStoredEmailKeypairs();" + + "callback(JSON.stringify(list));"; + Cu.evalInSandbox(scriptText, sandbox.box, "1.8", this.ID_URI, 1); + }, + + // Received a list of emails from BrowserID for current user + _gotEmails: function _gotEmails(emails, sandbox, cb, options) { + let keys = Object.keys(emails); + + // If list is empty, user is not logged in, or doesn't have a default email. + if (!keys.length) { + sandbox.free(); + + let err = "User is not logged in, or no emails were found"; + this._log.error(err); + try { + cb(new Error(err), null); + } catch(e) { + this._log.warn("Callback threw in _gotEmails " + + CommonUtils.exceptionStr(e)); + } + return; + } + + // User is logged in. For which email shall we get an assertion? + + // Case 1: Explicitely provided + if (options.requiredEmail) { + this._getAssertionWithEmail( + sandbox, cb, options.requiredEmail, options.audience + ); + return; + } + + // Case 2: Derive from a given domain + if (options.sameEmailAs) { + this._getAssertionWithDomain( + sandbox, cb, options.sameEmailAs, options.audience + ); + return; + } + + // Case 3: Default email + this._getAssertionWithEmail( + sandbox, cb, keys[0], options.audience + ); + return; + }, + + /** + * Open a login window and ask the user to login, returning the assertion + * generated as a result to the caller. + */ + _getAssertionWithLogin: function _getAssertionWithLogin(cb, win) { + // We're executing navigator.id.get as a content script in win. + // This results in a popup that we will temporarily unblock. + let pm = Services.perms; + let origin = Services.io.newURI( + win.wrappedJSObject.location.toString(), null, null + ); + let oldPerm = pm.testExactPermission(origin, "popup"); + try { + pm.add(origin, "popup", pm.ALLOW_ACTION); + } catch(e) { + this._log.warn("Setting popup blocking to false failed " + e); + } + + // Open sandbox and execute script. This sandbox will be GC'ed. + let sandbox = new Cu.Sandbox(win, { + wantXrays: false, + sandboxPrototype: win + }); + + let self = this; + function callback(val) { + // Set popup blocker permission to original value. + try { + pm.add(origin, "popup", oldPerm); + } catch(e) { + this._log.warn("Setting popup blocking to original value failed " + e); + } + + if (val) { + self._log.info("_getAssertionWithLogin succeeded"); + try { + cb(null, val); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithLogin " + + CommonUtils.exceptionStr(e)); + } + } else { + let msg = "Could not obtain assertion in _getAssertionWithLogin"; + self._log.error(msg); + try { + cb(new Error(msg), null); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithLogin " + + CommonUtils.exceptionStr(e)); + } + } + } + sandbox.importFunction(callback); + + function doGetAssertion() { + self._log.info("_getAssertionWithLogin Started"); + let scriptText = "window.navigator.id.get(" + + " callback, {allowPersistent: true}" + + ");"; + Cu.evalInSandbox(scriptText, sandbox, "1.8", self.ID_URI, 1); + } + + // Sometimes the provided win hasn't fully loaded yet + if (!win.document || (win.document.readyState != "complete")) { + win.addEventListener("DOMContentLoaded", function _contentLoaded() { + win.removeEventListener("DOMContentLoaded", _contentLoaded, false); + doGetAssertion(); + }, false); + } else { + doGetAssertion(); + } + }, + + /** + * Gets an assertion for the specified 'email' and 'audience' + */ + _getAssertionWithEmail: function _getAssertionWithEmail(sandbox, cb, email, + audience) { + let self = this; + + function onSuccess(res) { + // Cleanup first. + sandbox.free(); + + // The internal API sometimes calls onSuccess even though no assertion + // could be obtained! Double check: + if (!res) { + let msg = "BrowserID.User.getAssertion empty assertion for " + email; + self._log.error(msg); + try { + cb(new Error(msg), null); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithEmail " + + CommonUtils.exceptionStr(e)); + } + return; + } + + // Success + self._log.info("BrowserID.User.getAssertion succeeded"); + try { + cb(null, res); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithEmail " + + CommonUtils.exceptionStr(e)); + } + } + + function onError(err) { + sandbox.free(); + + self._log.info("BrowserID.User.getAssertion failed"); + try { + cb(err, null); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithEmail " + + CommonUtils.exceptionStr(e)); + } + } + sandbox.box.importFunction(onSuccess); + sandbox.box.importFunction(onError); + + self._log.info("_getAssertionWithEmail Started"); + let scriptText = + "window.BrowserID.User.getAssertion(" + + "'" + email + "', " + + "'" + audience + "', " + + "onSuccess, " + + "onError" + + ");"; + Cu.evalInSandbox(scriptText, sandbox.box, "1.8", this.ID_URI, 1); + }, + + /** + * Gets the email which was used to login to 'domain'. If one was found, + * _getAssertionWithEmail is called to obtain the assertion. + */ + _getAssertionWithDomain: function _getAssertionWithDomain(sandbox, cb, domain, + audience) { + let self = this; + + function onDomainSuccess(email) { + if (email) { + self._getAssertionWithEmail(sandbox, cb, email, audience); + } else { + sandbox.free(); + try { + cb(new Error("No email found for _getAssertionWithDomain"), null); + } catch(e) { + self._log.warn("Callback threw in _getAssertionWithDomain " + + CommonUtils.exceptionStr(e)); + } + } + } + sandbox.box.importFunction(onDomainSuccess); + + // This wil tell us which email was used to login to "domain", if any. + self._log.info("_getAssertionWithDomain Started"); + let scriptText = + "onDomainSuccess(window.BrowserID.Storage.site.get(" + + "'" + domain + "', " + + "'email'" + + "));"; + Cu.evalInSandbox(scriptText, sandbox.box, "1.8", this.ID_URI, 1); + }, +}; + +/** + * An object that represents a sandbox in an iframe loaded with uri. The + * callback provided to the constructor will be invoked when the sandbox is + * ready to be used. The callback will receive this object as its only argument + * and the prepared sandbox may be accessed via the "box" property. + * + * Please call free() when you are finished with the sandbox to explicitely free + * up all associated resources. + * + * @param cb + * (function) Callback to be invoked with a Sandbox, when ready. + * @param uri + * (String) URI to be loaded in the Sandbox. + */ +function Sandbox(cb, uri) { + this._uri = uri; + this._createFrame(); + this._createSandbox(cb, uri); +} +Sandbox.prototype = { + /** + * Frees the sandbox and releases the iframe created to host it. + */ + free: function free() { + delete this.box; + this._container.removeChild(this._frame); + this._frame = null; + this._container = null; + }, + + /** + * Creates an empty, hidden iframe and sets it to the _iframe + * property of this object. + * + * @return frame + * (iframe) An empty, hidden iframe + */ + _createFrame: function _createFrame() { + let doc = Services.wm.getMostRecentWindow("navigator:browser").document; + + // Insert iframe in to create docshell. + let frame = doc.createElement("iframe"); + frame.setAttribute("type", "content"); + frame.setAttribute("collapsed", "true"); + doc.documentElement.appendChild(frame); + + // Stop about:blank from being loaded. + let webNav = frame.docShell.QueryInterface(Ci.nsIWebNavigation); + webNav.stop(Ci.nsIWebNavigation.STOP_NETWORK); + + // Set instance properties. + this._frame = frame; + this._container = doc.documentElement; + }, + + _createSandbox: function _createSandbox(cb, uri) { + let self = this; + this._frame.addEventListener( + "DOMContentLoaded", + function _makeSandboxContentLoaded(event) { + if (event.target.location.toString() != uri) { + return; + } + event.target.removeEventListener( + "DOMContentLoaded", _makeSandboxContentLoaded, false + ); + let workerWindow = self._frame.contentWindow; + self.box = new Cu.Sandbox(workerWindow, { + wantXrays: false, + sandboxPrototype: workerWindow + }); + cb(self); + }, + true + ); + + // Load the iframe. + this._frame.docShell.loadURI( + uri, + this._frame.docShell.LOAD_FLAGS_NONE, + null, // referrer + null, // postData + null // headers + ); + }, +}; + +XPCOMUtils.defineLazyGetter(this, "BrowserID", function() { + return new BrowserIDService(); +}); diff --git a/services/aitc/modules/client.js b/services/aitc/modules/client.js new file mode 100644 index 000000000000..bc116369840f --- /dev/null +++ b/services/aitc/modules/client.js @@ -0,0 +1,387 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["AitcClient"]; + +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +Cu.import("resource://gre/modules/Webapps.jsm"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/preferences.js"); +Cu.import("resource://services-common/rest.js"); +Cu.import("resource://services-common/utils.js"); +Cu.import("resource://services-crypto/utils.js"); + +/** + * First argument is a token as returned by CommonUtils.TokenServerClient. + * This is a convenience, so you don't have to call updateToken immediately + * after making a new client (though you must when the token expires and you + * intend to reuse the same client instance). + * + * Second argument is a key-value store object that exposes two methods: + * set(key, value); Sets the value of a given key + * get(key, default); Returns the value of key, if it doesn't exist, + * return default + * The values should be stored persistently. The Preferences object from + * services-common/preferences.js is an example of such an object. + */ +function AitcClient(token, state) { + this.updateToken(token); + + this._log = Log4Moz.repository.getLogger("Service.AITC.Client"); + this._log.level = Log4Moz.Level[ + Preferences.get("services.aitc.client.log.level") + ]; + + this._state = state; + this._backoff = !!state.get("backoff", false); + + this._timeout = state.get("timeout", 120); + this._appsLastModified = parseInt(state.get("lastModified", "0"), 10); + this._log.info("Client initialized with token endpoint: " + this.uri); +} +AitcClient.prototype = { + _requiredLocalKeys: [ + "origin", "receipts", "manifestURL", "installOrigin" + ], + _requiredRemoteKeys: [ + "origin", "name", "receipts", "manifestPath", "installOrigin", + "installedAt", "modifiedAt" + ], + + /** + * Updates the token the client must use to authenticate outgoing requests. + * + * @param token + * (Object) A token as returned by CommonUtils.TokenServerClient. + */ + updateToken: function updateToken(token) { + this.uri = token.endpoint.replace(/\/+$/, ""); + this.token = {id: token.id, key: token.key}; + }, + + /** + * Initiates an update of a newly installed app to the AITC server. Call this + * when an application is installed locally. + * + * @param app + * (Object) The app record of the application that was just installed. + */ + remoteInstall: function remoteInstall(app, cb) { + if (!cb) { + throw new Error("remoteInstall called without callback"); + } + + // Fetch the name of the app because it's not in the event app object. + let self = this; + DOMApplicationRegistry.getManifestFor(app.origin, function gotManifest(m) { + app.name = m.name; + self._putApp(self._makeRemoteApp(app), cb); + }); + }, + + /** + * Initiates an update of an uinstalled app to the AITC server. Call this + * when an application is uninstalled locally. + * + * @param app + * (Object) The app record of the application that was uninstalled. + */ + remoteUninstall: function remoteUninstall(app, cb) { + if (!cb) { + throw new Error("remoteUninstall called without callback"); + } + + app.name = "Uninstalled"; // Bug 760262 + let record = this._makeRemoteApp(app); + record.deleted = true; + this._putApp(record, cb); + }, + + /** + * Fetch remote apps from server with GET. The provided callback will receive + * an array of app objects in the format expected by DOMApplicationRegistry, + * if successful, or an Error; in the usual (err, result) way. + */ + getApps: function getApps(cb) { + if (!cb) { + throw new Error("getApps called but no callback provided"); + } + + if (!this._isRequestAllowed()) { + cb(null, null); + return; + } + + let uri = this.uri + "/apps/?full=1"; + let req = new TokenAuthenticatedRESTRequest(uri, this.token); + req.timeout = this._timeout; + req.setHeader("Content-Type", "application/json"); + + if (this._appsLastModified) { + req.setHeader("X-If-Modified-Since", this._appsLastModified); + } + + let self = this; + req.get(function _getAppsCb(err) { + self._processGetApps(err, cb, req); + }); + }, + + /** + * GET request returned from getApps, process. + */ + _processGetApps: function _processGetApps(err, cb, req) { + // Set X-Backoff or Retry-After, if needed. + this._setBackoff(req); + + if (err) { + this._log.error("getApps request error " + err); + cb(err, null); + return; + } + + // Bubble auth failure back up so new token can be acquired. + if (req.response.status == 401) { + let msg = new Error("getApps failed due to 401 authentication failure"); + this._log.info(msg); + msg.authfailure = true; + cb(msg, null); + return; + } + // Process response. + if (req.response.status == 304) { + this._log.info("getApps returned 304"); + cb(null, null); + return; + } + if (req.response.status != 200) { + this._log.error(req); + cb(new Error("Unexpected error with getApps"), null); + return; + } + + let apps; + try { + let tmp = JSON.parse(req.response.body); + tmp = tmp["apps"]; + // Convert apps from remote to local format. + apps = tmp.map(this._makeLocalApp, this); + this._log.info("getApps succeeded and got " + apps.length); + } catch (e) { + this._log.error(CommonUtils.exceptionStr(e)); + cb(new Error("Exception in getApps " + e), null); + return; + } + + // Return success. + try { + cb(null, apps); + // Don't update lastModified until we know cb succeeded. + this._appsLastModified = parseInt(req.response.headers["X-Timestamp"], 10); + this._state.set("lastModified", "" + this._appsLastModified); + } catch (e) { + this._log.error("Exception in getApps callback " + e); + } + }, + + /** + * Change a given app record to match what the server expects. + * Change manifestURL to manifestPath, and trim out manifests since we + * don't store them on the server. + */ + _makeRemoteApp: function _makeRemoteApp(app) { + for each (let key in this.requiredLocalKeys) { + if (!(key in app)) { + throw new Error("Local app missing key " + key); + } + } + + let record = { + name: app.name, + origin: app.origin, + receipts: app.receipts, + manifestPath: app.manifestURL, + installOrigin: app.installOrigin + }; + if ("modifiedAt" in app) { + record.modifiedAt = app.modifiedAt; + } + if ("installedAt" in app) { + record.installedAt = app.installedAt; + } + return record; + }, + + /** + * Change a given app record received from the server to match what the local + * registry expects. (Inverse of _makeRemoteApp) + */ + _makeLocalApp: function _makeLocalApp(app) { + for each (let key in this._requiredRemoteKeys) { + if (!(key in app)) { + throw new Error("Remote app missing key " + key); + } + } + + let record = { + origin: app.origin, + installOrigin: app.installOrigin, + installedAt: app.installedAt, + modifiedAt: app.modifiedAt, + manifestURL: app.manifestPath, + receipts: app.receipts + }; + if ("deleted" in app) { + record.deleted = app.deleted; + } + return record; + }, + + /** + * Try PUT for an app on the server and determine if we should retry + * if it fails. + */ + _putApp: function _putApp(app, cb) { + if (!this._isRequestAllowed()) { + // PUT requests may qualify as the "minimum number of additional requests + // required to maintain consistency of their stored data". However, it's + // better to keep server load low, even if it means user's apps won't + // reach their other devices during the early days of AITC. We should + // revisit this when we have a better of idea of server load curves. + err = new Error("Backoff in effect, aborting PUT"); + err.processed = false; + cb(err, null); + return; + } + + let uri = this._makeAppURI(app.origin); + let req = new TokenAuthenticatedRESTRequest(uri, this.token); + req.timeout = this._timeout; + req.setHeader("Content-Type", "application/json"); + + if (app.modifiedAt) { + req.setHeader("X-If-Unmodified-Since", "" + app.modified); + } + + let self = this; + this._log.info("Trying to _putApp to " + uri); + req.put(JSON.stringify(app), function _putAppCb(err) { + self._processPutApp(err, cb, req); + }); + }, + + /** + * PUT from _putApp finished, process. + */ + _processPutApp: function _processPutApp(error, cb, req) { + this._setBackoff(req); + + if (error) { + this._log.error("_putApp request error " + error); + cb(error, null); + return; + } + + let err = null; + switch (req.response.status) { + case 201: + case 204: + this._log.info("_putApp succeeded"); + cb(null, true); + break; + + case 401: + // Bubble auth failure back up so new token can be acquired + err = new Error("_putApp failed due to 401 authentication failure"); + this._log.warn(err); + err.authfailure = true; + cb(err, null); + break; + + case 409: + // Retry on server conflict + err = new Error("_putApp failed due to 409 conflict"); + this._log.warn(err); + cb(err,null); + break; + + case 400: + case 412: + case 413: + let msg = "_putApp returned: " + req.response.status; + this._log.warn(msg); + err = new Error(msg); + err.processed = true; + cb(err, null); + break; + + default: + this._error(req); + err = new Error("Unexpected error with _putApp"); + err.processed = false; + cb(err, null); + break; + } + }, + + /** + * Utility methods. + */ + _error: function _error(req) { + this._log.error("Catch-all error for request: " + + req.uri.asciiSpec + req.response.status + " with: " + req.response.body); + }, + + _makeAppURI: function _makeAppURI(origin) { + let part = CommonUtils.encodeBase64URL( + CryptoUtils.UTF8AndSHA1(origin) + ).replace("=", ""); + return this.uri + "/apps/" + part; + }, + + // Before making a request, check if we are allowed to. + _isRequestAllowed: function _isRequestAllowed() { + if (!this._backoff) { + return true; + } + + let time = Date.now(); + let backoff = parseInt(this._state.get("backoff", 0), 10); + + if (time < backoff) { + this._log.warn(backoff - time + "ms left for backoff, aborting request"); + return false; + } + + this._backoff = false; + this._state.set("backoff", "0"); + return true; + }, + + // Set values from X-Backoff and Retry-After headers, if present + _setBackoff: function _setBackoff(req) { + let backoff = 0; + + let val; + if (req.response.headers["Retry-After"]) { + val = req.response.headers["Retry-After"]; + backoff = parseInt(val, 10); + this._log.warn("Retry-Header header was seen: " + val); + } else if (req.response.headers["X-Backoff"]) { + val = req.response.headers["X-Backoff"]; + backoff = parseInt(val, 10); + this._log.warn("X-Backoff header was seen: " + val); + } + if (backoff) { + this._backoff = true; + let time = Date.now(); + // Fuzz backoff time so all client don't retry at the same time + backoff = Math.floor((Math.random() * backoff + backoff) * 1000); + this._state.set("backoff", "" + (time + backoff)); + } + }, +}; \ No newline at end of file diff --git a/services/aitc/modules/main.js b/services/aitc/modules/main.js new file mode 100644 index 000000000000..81acff2e8ff4 --- /dev/null +++ b/services/aitc/modules/main.js @@ -0,0 +1,161 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["Aitc"]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/Webapps.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +Cu.import("resource://services-aitc/manager.js"); +Cu.import("resource://services-common/utils.js"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/preferences.js"); + +function Aitc() { + this._log = Log4Moz.repository.getLogger("Service.AITC"); + this._log.level = Log4Moz.Level[Preferences.get( + "services.aitc.service.log.level" + )]; + this._log.info("Loading AitC"); + + this.DASHBOARD_ORIGIN = CommonUtils.makeURI( + Preferences.get("services.aitc.dashboard.url") + ).prePath; + + this._manager = new AitcManager(this._init.bind(this)); +} +Aitc.prototype = { + // The goal of the init function is to be ready to activate the AITC + // client whenever the user is looking at the dashboard. + _init: function init() { + let self = this; + + // This is called iff the user is currently looking the dashboard. + function dashboardLoaded(browser) { + let win = browser.contentWindow; + self._log.info("Dashboard was accessed " + win); + + // If page is ready to go, fire immediately. + if (win.document && win.document.readyState == "complete") { + self._manager.userActive(win); + return; + } + + // Only fire event after the page fully loads. + browser.contentWindow.addEventListener( + "DOMContentLoaded", + function _contentLoaded(event) { + self._manager.userActive(win); + }, + false + ); + } + + // This is called when the user's attention is elsewhere. + function dashboardUnloaded() { + self._log.info("Dashboard closed or in background"); + self._manager.userIdle(); + } + + // Called when a URI is loaded in any tab. We have to listen for this + // because tabSelected is not called if I open a new tab which loads + // about:home and then navigate to the dashboard, or navigation via + // links on the currently open tab. + let listener = { + onLocationChange: function onLocationChange(browser, pr, req, loc, flag) { + let win = Services.wm.getMostRecentWindow("navigator:browser"); + if (win.gBrowser.selectedBrowser == browser) { + if (loc.prePath == self.DASHBOARD_ORIGIN) { + dashboardLoaded(browser); + } + } + } + }; + // Called when the current tab selection changes. + function tabSelected(event) { + let browser = event.target.linkedBrowser; + if (browser.currentURI.prePath == self.DASHBOARD_ORIGIN) { + dashboardLoaded(browser); + } else { + dashboardUnloaded(); + } + } + + // Add listeners for all windows opened in the future. + function winWatcher(subject, topic) { + if (topic != "domwindowopened") { + return; + } + subject.addEventListener("load", function winWatcherLoad() { + subject.removeEventListener("load", winWatcherLoad, false); + let doc = subject.document.documentElement; + if (doc.getAttribute("windowtype") == "navigator:browser") { + let browser = subject.gBrowser; + browser.addTabsProgressListener(listener); + browser.tabContainer.addEventListener("TabSelect", tabSelected); + } + }, false); + } + Services.ww.registerNotification(winWatcher); + + // Add listeners for all current open windows. + let enumerator = Services.wm.getEnumerator("navigator:browser"); + while (enumerator.hasMoreElements()) { + let browser = enumerator.getNext().gBrowser; + browser.addTabsProgressListener(listener); + browser.tabContainer.addEventListener("TabSelect", tabSelected); + + // Also check the currently open URI. + if (browser.currentURI.prePath == this.DASHBOARD_ORIGIN) { + dashboardLoaded(browser); + } + } + + // Add listeners for app installs/uninstall. + Services.obs.addObserver(this, "webapps-sync-install", false); + Services.obs.addObserver(this, "webapps-sync-uninstall", false); + + // Add listener for idle service. + let idleSvc = Cc["@mozilla.org/widget/idleservice;1"]. + getService(Ci.nsIIdleService); + idleSvc.addIdleObserver(this, + Preferences.get("services.aitc.main.idleTime")); + }, + + observe: function(subject, topic, data) { + let app; + switch (topic) { + case "webapps-sync-install": + app = JSON.parse(data); + this._log.info(app.origin + " was installed, initiating PUT"); + this._manager.appEvent("install", app); + break; + case "webapps-sync-uninstall": + app = JSON.parse(data); + this._log.info(app.origin + " was uninstalled, initiating PUT"); + this._manager.appEvent("uninstall", app); + break; + case "idle": + this._log.info("User went idle"); + if (this._manager) { + this._manager.userIdle(); + } + break; + case "back": + this._log.info("User is no longer idle"); + let win = Services.wm.getMostRecentWindow("navigator:browser"); + if (win && win.gBrowser.currentURI.prePath == this.DASHBOARD_ORIGIN && + this._manager) { + this._manager.userActive(); + } + break; + } + }, + +}; diff --git a/services/aitc/modules/manager.js b/services/aitc/modules/manager.js new file mode 100644 index 000000000000..3f9d58b6cd46 --- /dev/null +++ b/services/aitc/modules/manager.js @@ -0,0 +1,573 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["AitcManager"]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/Webapps.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/FileUtils.jsm"); + +Cu.import("resource://services-aitc/client.js"); +Cu.import("resource://services-aitc/browserid.js"); +Cu.import("resource://services-aitc/storage.js"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/preferences.js"); +Cu.import("resource://services-common/tokenserverclient.js"); +Cu.import("resource://services-common/utils.js"); + +const PREFS = new Preferences("services.aitc."); +const TOKEN_TIMEOUT = 240000; // 4 minutes +const DASHBOARD_URL = PREFS.get("dashboard.url"); +const MARKETPLACE_URL = PREFS.get("marketplace.url"); + +/** + * The constructor for the manager takes a callback, which will be invoked when + * the manager is ready (construction is asynchronous). *DO NOT* call any + * methods on this object until the callback has been invoked, doing so will + * lead to undefined behaviour. + */ +function AitcManager(cb) { + this._client = null; + this._getTimer = null; + this._putTimer = null; + + this._lastToken = 0; + this._lastEmail = null; + this._dashboardWindow = null; + + this._log = Log4Moz.repository.getLogger("Service.AITC.Manager"); + this._log.level = Log4Moz.Level[Preferences.get("manager.log.level")]; + this._log.info("Loading AitC manager module"); + + // Check if we have pending PUTs from last time. + let self = this; + this._pending = new AitcQueue("webapps-pending.json", function _queueDone() { + // Inform the AitC service that we're good to go! + self._log.info("AitC manager has finished loading"); + try { + cb(true); + } catch (e) { + self._log.error(new Error("AitC manager callback threw " + e)); + } + + // Schedule them, but only if we can get a silent assertion. + self._makeClient(function(err, client) { + if (!err && client) { + self._client = client; + self._processQueue(); + } + }, false); + }); +} +AitcManager.prototype = { + /** + * State of the user. ACTIVE implies user is looking at the dashboard, + * PASSIVE means either not at the dashboard or the idle timer started. + */ + _ACTIVE: 1, + _PASSIVE: 2, + + /** + * Smart setter that will only call _setPoll is the value changes. + */ + _clientState: null, + get _state() { + return this._clientState; + }, + set _state(value) { + if (this._clientState == value) { + return; + } + this._clientState = value; + this._setPoll(); + }, + + /** + * Local app was just installed or uninstalled, ask client to PUT if user + * is logged in. + */ + appEvent: function appEvent(type, app) { + // Add this to the equeue. + let self = this; + let obj = {type: type, app: app, retries: 0, lastTime: 0}; + this._pending.enqueue(obj, function _enqueued(err, rec) { + if (err) { + self._log.error("Could not add " + type + " " + app + " to queue"); + return; + } + + // If we already have a client (i.e. user is logged in), attempt to PUT. + if (self._client) { + self._processQueue(); + return; + } + + // If not, try a silent client creation. + self._makeClient(function(err, client) { + if (!err && client) { + self._client = client; + self._processQueue(); + } + // If user is not logged in, we'll just have to try later. + }); + }); + }, + + /** + * User is looking at dashboard. Start polling actively, but if user isn't + * logged in, prompt for them to login via a dialog. + */ + userActive: function userActive(win) { + // Stash a reference to the dashboard window in case we need to prompt + this._dashboardWindow = win; + + if (this._client) { + this._state = this._ACTIVE; + return; + } + + // Make client will first try silent login, if it doesn't work, a popup + // will be shown in the context of the dashboard. We shouldn't be + // trying to make a client every time this function is called, there is + // room for optimization (Bug 750607). + let self = this; + this._makeClient(function(err, client) { + if (err) { + // Notify user of error (Bug 750610). + self._log.error("Client not created at Dashboard"); + return; + } + self._client = client; + self._state = self._ACTIVE; + }, true, win); + }, + + /** + * User is idle, (either by idle observer, or by not being on the dashboard). + * When the user is no longer idle and the dashboard is the current active + * page, a call to userActive MUST be made. + */ + userIdle: function userIdle() { + this._state = this._PASSIVE; + this._dashboardWindow = null; + }, + + /** + * Poll the AITC server for any changes and process them. It is safe to call + * this function multiple times. Last caller wins. The function will + * grab the current user state from _state and act accordingly. + * + * Invalid states will cause this function to throw. + */ + _setPoll: function _setPoll() { + if (this._state == this._ACTIVE && !this._client) { + throw new Error("_setPoll(ACTIVE) called without client"); + } + if (this._state != this._ACTIVE && this._state != this._PASSIVE) { + throw new Error("_state is invalid " + this._state); + } + + if (!this._client) { + // User is not logged in, we cannot do anything. + self._log.warn("_setPoll called but user not logged in, ignoring"); + return; + } + + // Check if there are any PUTs pending first. + if (this._pending.length && !(this._putTimer)) { + // There are pending PUTs and no timer, so let's process them. GETs will + // resume after the PUTs finish (see processQueue) + this._processQueue(); + return; + } + + // Do one GET soon, but only if user is active. + let getFreq; + if (this._state == this._ACTIVE) { + CommonUtils.nextTick(this._checkServer, this); + getFreq = PREFS.get("manager.getActiveFreq"); + } else { + getFreq = PREFS.get("manager.getPassiveFreq"); + } + + // Cancel existing timer, if any. + if (this._getTimer) { + this._getTimer.cancel(); + this._getTimer = null; + } + + // Start the timer for GETs. + let self = this; + this._log.info("Starting GET timer"); + this._getTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer); + this._getTimer.initWithCallback({notify: this._checkServer.bind(this)}, + getFreq, Ci.nsITimer.TYPE_REPEATING_SLACK); + + this._log.info("GET timer set, next attempt in " + getFreq + "ms"); + }, + + /** + * Checks if the current token we hold is valid. If not, we obtain a new one + * and execute the provided func. If a token could not be obtained, func will + * not be called and an error will be logged. + */ + _validateToken: function _validateToken(func) { + if (Date.now() - this._lastToken < TOKEN_TIMEOUT) { + func(); + return; + } + + let win; + if (this._state == this.ACTIVE) { + win = this._dashboardWindow; + } + + let self = this; + this._refreshToken(function(err, done) { + if (!done) { + this._log.warn("_checkServer could not refresh token, aborting"); + return; + } + func(); + }, win); + }, + + /** + * Do a GET check on the server to see if we have any new apps. Abort if + * there are pending PUTs. If we GET some apps, send to storage for + * further processing. + */ + _checkServer: function _checkServer() { + if (!this._client) { + throw new Error("_checkServer called without a client"); + } + + if (this._pending.length) { + this._log.warn("_checkServer aborted because of pending PUTs"); + return; + } + + this._validateToken(this._getApps.bind(this)); + }, + + _getApps: function _getApps() { + // Do a GET + this._log.info("Attempting to getApps"); + + let self = this; + this._client.getApps(function gotApps(err, apps) { + if (err) { + // Error was logged in client. + return; + } + if (!apps) { + // No changes, got 304. + return; + } + if (!apps.length) { + // Empty array, nothing to process + self._log.info("No apps found on remote server"); + return; + } + + // Send list of remote apps to storage to apply locally + AitcStorage.processApps(apps, function processedApps() { + self._log.info("processApps completed successfully, changes applied"); + }); + }); + }, + + /** + * Go through list of apps to PUT and attempt each one. If we fail, try + * again in PUT_FREQ. Will throw if called with an empty, _reschedule() + * makes sure that we don't. + */ + _processQueue: function _processQueue() { + if (!this._client) { + throw new Error("_processQueue called without a client"); + } + if (!this._pending.length) { + throw new Error("_processQueue called with an empty queue"); + } + + if (this._putInProgress) { + // The network request sent out as a result to the last call to + // _processQueue still isn't done. A timer is created they all + // finish to make sure this function is called again if neccessary. + return; + } + + this._validateToken(this._putApps.bind(this)); + }, + + _putApps: function _putApps() { + this._putInProgress = true; + let record = this._pending.peek(); + + this._log.info("Processing record type " + record.type); + + let self = this; + function _clientCallback(err, done) { + // Send to end of queue if unsuccessful or err.removeFromQueue is false. + if (err && !err.removeFromQueue) { + self._log.info("PUT failed, re-adding to queue"); + + // Update retries and time + record.retries += 1; + record.lastTime = new Date().getTime(); + + // Add updated record to the end of the queue. + self._pending.enqueue(record, function(err, done) { + if (err) { + self._log.error("Enqueue failed " + err); + _reschedule(); + return; + } + // If record was successfully added, remove old record. + self._pending.dequeue(function(err, done) { + if (err) { + self._log.error("Dequeue failed " + err); + } + _reschedule(); + return; + }); + }); + } + + // If succeeded or client told us to remove from queue + self._log.info("_putApp asked us to remove it from queue"); + self._pending.dequeue(function(err, done) { + if (err) { + self._log.error("Dequeue failed " + e); + } + _reschedule(); + }); + } + + function _reschedule() { + // Release PUT lock + self._putInProgress = false; + + // We just finished PUTting an object, try the next one immediately, + // but only if haven't tried it already in the last putFreq (ms). + if (!self._pending.length) { + // Start GET timer now that we're done with PUTs. + self._setPoll(); + return; + } + + let obj = self._pending.peek(); + let cTime = new Date().getTime(); + let freq = PREFS.get("manager.putFreq"); + + // We tried this object recently, we'll come back to it later. + if (obj.lastTime && ((cTime - obj.lastTime) < freq)) { + self._log.info("Scheduling next processQueue in " + freq); + CommonUtils.namedTimer(self._processQueue, freq, self, "_putTimer"); + return; + } + + // Haven't tried this PUT yet, do it immediately. + self._log.info("Queue non-empty, processing next PUT"); + self._processQueue(); + } + + switch (record.type) { + case "install": + this._client.remoteInstall(record.app, _clientCallback); + break; + case "uninstall": + record.app.deleted = true; + this._client.remoteUninstall(record.app, _clientCallback); + break; + default: + this._log.warn( + "Unrecognized type " + record.type + " in queue, removing" + ); + let self = this; + this._pending.dequeue(function _dequeued(err) { + if (err) { + self._log.error("Dequeue of unrecognized app type failed"); + } + _reschedule(); + }); + } + }, + + /* Obtain a (new) token from the Sagrada token server. If win is is specified, + * the user will be asked to login via UI, if required. The callback's + * signature is cb(err, done). If a token is obtained successfully, done will + * be true and err will be null. + */ + _refreshToken: function _refreshToken(cb, win) { + if (!this._client) { + throw new Error("_refreshToken called without an active client"); + } + + this._log.info("Token refresh requested"); + + let self = this; + function refreshedAssertion(err, assertion) { + if (!err) { + self._getToken(assertion, function(err, token) { + if (err) { + cb(err, null); + return; + } + self._lastToken = Date.now(); + self._client.updateToken(token); + cb(null, true); + }); + return; + } + + // Silent refresh was asked for. + if (!win) { + cb(err, null); + return; + } + + // Prompt user to login. + self._makeClient(function(err, client) { + if (err) { + cb(err, null); + return; + } + + // makeClient sets an updated token. + self._client = client; + cb(null, true); + }, win); + } + + let options = { audience: DASHBOARD_URL }; + if (this._lastEmail) { + options.requiredEmail = this._lastEmail; + } else { + options.sameEmailAs = MARKETPLACE_URL; + } + BrowserID.getAssertion(refreshedAssertion, options); + }, + + /* Obtain a token from Sagrada token server, given a BrowserID assertion + * cb(err, token) will be invoked on success or failure. + */ + _getToken: function _getToken(assertion, cb) { + let url = PREFS.get("tokenServer.url") + "/1.0/aitc/1.0"; + let client = new TokenServerClient(); + + this._log.info("Obtaining token from " + url); + + let self = this; + try { + client.getTokenFromBrowserIDAssertion(url, assertion, function(err, tok) { + self._gotToken(err, tok, cb); + }); + } catch (e) { + cb(new Error(e), null); + } + }, + + // Token recieved from _getToken. + _gotToken: function _gotToken(err, tok, cb) { + if (!err) { + this._log.info("Got token from server: " + JSON.stringify(tok)); + cb(null, tok); + return; + } + + let msg = err.name + " in _getToken: " + err.error; + this._log.error(msg); + cb(msg, null); + }, + + // Extract the email address from a BrowserID assertion. + _extractEmail: function _extractEmail(assertion) { + // Please look the other way while I do this. Thanks. + let chain = assertion.split("~"); + let len = chain.length; + if (len < 2) { + return null; + } + + try { + // We need CommonUtils.decodeBase64URL. + let cert = JSON.parse(atob( + chain[0].split(".")[1].replace("-", "+", "g").replace("_", "/", "g") + )); + return cert.principal.email; + } catch (e) { + return null; + } + }, + + /* To start the AitcClient we need a token, for which we need a BrowserID + * assertion. If login is true, makeClient will ask the user to login in + * the context of win. cb is called with (err, client). + */ + _makeClient: function makeClient(cb, login, win) { + if (!cb) { + throw new Error("makeClient called without callback"); + } + if (login && !win) { + throw new Error("makeClient called with login as true but no win"); + } + + let self = this; + let ctxWin = win; + function processAssertion(val) { + // Store the email we got the token for so we can refresh. + self._lastEmail = self._extractEmail(val); + self._log.info("Got assertion from BrowserID, creating token"); + self._getToken(val, function(err, token) { + if (err) { + cb(err, null); + return; + } + + // Store when we got the token so we can refresh it as needed. + self._lastToken = Date.now(); + + // We only create one client instance, store values in a pref tree + cb(null, new AitcClient( + token, new Preferences("services.aitc.client.") + )); + }); + } + function gotSilentAssertion(err, val) { + self._log.info("gotSilentAssertion called"); + if (err) { + // If we were asked to let the user login, do the popup method. + if (login) { + self._log.info("Could not obtain silent assertion, retrying login"); + BrowserID.getAssertionWithLogin(function gotAssertion(err, val) { + if (err) { + self._log.error(err); + cb(err, false); + return; + } + processAssertion(val); + }, ctxWin); + return; + } + self._log.warn("Could not obtain assertion in _makeClient"); + cb(err, false); + } else { + processAssertion(val); + } + } + + // Check if we can get assertion silently first + self._log.info("Attempting to obtain assertion silently") + BrowserID.getAssertion(gotSilentAssertion, { + audience: DASHBOARD_URL, + sameEmailAs: MARKETPLACE_URL + }); + }, + +}; diff --git a/services/aitc/modules/storage.js b/services/aitc/modules/storage.js new file mode 100644 index 000000000000..193e47e3ee38 --- /dev/null +++ b/services/aitc/modules/storage.js @@ -0,0 +1,452 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const EXPORTED_SYMBOLS = ["AitcStorage", "AitcQueue"]; + +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +Cu.import("resource://gre/modules/NetUtil.jsm"); +Cu.import("resource://gre/modules/Webapps.jsm"); +Cu.import("resource://gre/modules/FileUtils.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/preferences.js"); +Cu.import("resource://services-common/rest.js"); + +/** + * Provides a file-backed queue. Currently used by manager.js as persistent + * storage to manage pending installs and uninstalls. + * + * @param filename + * (String) The file backing this queue will be named as this string. + * + * @param cb + * (Function) This function will be called when the queue is ready to + * use. *DO NOT* call any methods on this object until the + * callback is invoked, if you do so, none of your operations + * will be persisted on disk. + * + */ +function AitcQueue(filename, cb) { + if (!cb) { + throw new Error("AitcQueue constructor called without callback"); + } + + this._log = Log4Moz.repository.getLogger("Service.AITC.Storage.Queue"); + this._log.level = Log4Moz.Level[Preferences.get( + "services.aitc.storage.log.level" + )]; + + this._queue = []; + this._writeLock = false; + this._file = FileUtils.getFile("ProfD", ["webapps", filename], true); + + this._log.info("AitcQueue instance loading"); + + let self = this; + if (this._file.exists()) { + this._getFile(function gotFile(data) { + if (data && Array.isArray(data)) { + self._queue = data; + } + self._log.info("AitcQueue instance created"); + cb(true); + }); + } else { + self._log.info("AitcQueue instance created"); + cb(true); + } +} +AitcQueue.prototype = { + /** + * Add an object to the queue, and data is saved to disk. + */ + enqueue: function enqueue(obj, cb) { + this._log.info("Adding to queue " + obj); + + if (!cb) { + throw new Error("enqueue called without callback"); + } + + let self = this; + this._queue.push(obj); + + try { + this._putFile(this._queue, function _enqueuePutFile(err) { + if (err) { + // Write unsuccessful, don't add to queue. + self._queue.pop(); + cb(err, false); + return; + } + // Successful write. + cb(null, true); + return; + }); + } catch (e) { + self._queue.pop(); + cb(e, false); + } + }, + + /** + * Remove the object at the head of the queue, and data is saved to disk. + */ + dequeue: function dequeue(cb) { + this._log.info("Removing head of queue"); + + if (!cb) { + throw new Error("dequeue called without callback"); + } + if (!this._queue.length) { + throw new Error("Queue is empty"); + } + + let self = this; + let obj = this._queue.shift(); + + try { + this._putFile(this._queue, function _dequeuePutFile(err) { + if (!err) { + // Successful write. + cb(null, true); + return; + } + // Unsuccessful write, put back in queue. + self._queue.unshift(obj); + cb(err, false); + }); + } catch (e) { + self._queue.unshift(obj); + cb(e, false); + } + }, + + /** + * Return the object at the front of the queue without removing it. + */ + peek: function peek() { + this._log.info("Peek called when length " + this._queue.length); + if (!this._queue.length) { + throw new Error("Queue is empty"); + } + return this._queue[0]; + }, + + /** + * Find out the length of the queue. + */ + get length() { + return this._queue.length; + }, + + /** + * Get contents of cache file and parse it into an array. Will throw an + * exception if there is an error while reading the file. + */ + _getFile: function _getFile(cb) { + let channel = NetUtil.newChannel(this._file); + channel.contentType = "application/json"; + + let self = this; + NetUtil.asyncFetch(channel, function _asyncFetched(stream, res) { + if (!Components.isSuccessCode(res)) { + self._log.error("Could not read from json file " + this._file.path); + cb(null); + return; + } + + let data = []; + try { + data = JSON.parse( + NetUtil.readInputStreamToString(stream, stream.available()) + ); + stream.close(); + cb(data); + } catch (e) { + self._log.error("Could not parse JSON " + e); + cb(null); + } + }); + }, + + /** + * Put an array into the cache file. Will throw an exception if there is + * an error while trying to write to the file. + */ + _putFile: function _putFile(value, cb) { + if (this._writeLock) { + throw new Error("_putFile already in progress"); + } + + this._writeLock = true; + try { + let ostream = FileUtils.openSafeFileOutputStream(this._file); + + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]. + createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + let istream = converter.convertToInputStream(JSON.stringify(value)); + + // Asynchronously copy the data to the file. + let self = this; + this._log.info("Writing queue to disk"); + NetUtil.asyncCopy(istream, ostream, function _asyncCopied(result) { + self._writeLock = false; + if (Components.isSuccessCode(result)) { + self._log.info("asyncCopy succeeded"); + cb(null); + } else { + let msg = new Error("asyncCopy failed with " + result); + self._log.info(msg); + cb(msg); + } + }); + } catch (e) { + this._writeLock = false; + cb(msg); + } + }, +}; + +/** + * An interface to DOMApplicationRegistry, used by manager.js to process + * remote changes received and apply them to the local registry. + */ +function AitcStorageImpl() { + this._log = Log4Moz.repository.getLogger("Service.AITC.Storage"); + this._log.level = Log4Moz.Level[Preferences.get( + "services.aitc.storage.log.level" + )]; + this._log.info("Loading AitC storage module"); +} +AitcStorageImpl.prototype = { + /** + * Determines what changes are to be made locally, given a list of + * remote apps. + * + * @param remoteApps + * (Array) An array of app records fetched from the AITC server. + * + * @param callback + * (function) A callback to be invoked when processing is finished. + */ + processApps: function processApps(remoteApps, callback) { + let self = this; + this._log.info("Server check got " + remoteApps.length + " apps"); + + // Get the set of local apps, and then pass to _processApps. + // _processApps will check for the validity of remoteApps. + DOMApplicationRegistry.getAllWithoutManifests( + function _processAppsGotLocalApps(localApps) { + self._processApps(remoteApps, localApps, callback); + } + ); + }, + + /** + * Take a list of remote and local apps and figured out what changes (if any) + * are to be made to the local DOMApplicationRegistry. + * + * General algorithm: + * 1. Put all remote apps in a dictionary of origin->app. + * 2. Put all local apps in a dictionary of origin->app. + * 3. Mark all local apps as "to be deleted". + * 4. Go through each remote app: + * 4a. If remote app is not marked as deleted, remove from the "to be + * deleted" set. + * 4b. If remote app is marked as deleted, but isn't present locally, + * process the next remote app. + * 4c. If remote app is not marked as deleted and isn't present locally, + * add to the "to be installed" set. + * 5. For each app either in the "to be installed" or "to be deleted" set, + * apply the changes locally. For apps to be installed, we must also + * fetch the manifest. + * + */ + _processApps: function _processApps(remoteApps, lApps, callback) { + let toDelete = {}; + let localApps = {}; + + // If remoteApps is empty, do nothing. The correct thing to do is to + // delete all local apps, but we'll play it safe for now since we are + // marking apps as deleted anyway. In a subsequent version (when the + // deleted flag is no longer in use), this check can be removed. + if (!Object.keys(remoteApps).length) { + this._log.warn("Empty set of remote apps to _processApps, returning"); + callback(); + return; + } + + // Convert lApps to a dictionary of origin -> app (instead of id -> app). + for (let [id, app] in Iterator(lApps)) { + app.id = id; + toDelete[app.origin] = app; + localApps[app.origin] = app; + } + + // Iterate over remote apps, and find out what changes we must apply. + let toInstall = []; + for each (let app in remoteApps) { + // Don't delete apps that are both local & remote. + let origin = app.origin; + if (!app.deleted) { + delete toDelete[origin]; + } + + // A remote app that was deleted, but also isn't present locally is NOP. + if (app.deleted && !localApps[origin]) { + continue; + } + + // If there is a remote app that isn't local or if the remote app was + // installed or updated later. + let id; + if (!localApps[origin]) { + id = DOMApplicationRegistry.makeAppId(); + } + if (localApps[origin] && + (localApps[origin].installTime < app.installTime)) { + id = localApps[origin].id; + } + + // We should (re)install this app locally + if (id) { + toInstall.push({id: id, value: app}); + } + } + + // Uninstalls only need the ID & deleted flag. + let toUninstall = []; + for (let origin in toDelete) { + toUninstall.push({id: toDelete[origin].id, deleted: true}); + } + + // Apply uninstalls first, we do not need to fetch manifests. + if (toUninstall.length) { + this._log.info("Applying uninstalls to registry"); + + let self = this; + DOMApplicationRegistry.updateApps(toUninstall, function() { + // If there are installs, proceed to apply each on in parallel. + if (toInstall.length) { + self._applyInstalls(toInstall, callback); + return; + } + callback(); + }); + + return; + } + + // If there were no uninstalls, only apply installs + if (toInstall.length) { + this._applyInstalls(toInstall, callback); + return; + } + + this._log.info("There were no changes to be applied, returning"); + callback(); + return; + }, + + /** + * Apply a set of installs to the local registry. Fetch each app's manifest + * in parallel (don't retry on failure) and insert into registry. + */ + _applyInstalls: function _applyInstalls(toInstall, callback) { + let done = 0; + let total = toInstall.length; + this._log.info("Applying " + total + " installs to registry"); + + /** + * The purpose of _checkIfDone is to invoke the callback after all the + * installs have been applied. They all fire in parallel, and each will + * check-in when it is done. + */ + let self = this; + function _checkIfDone() { + done += 1; + self._log.debug(done + "/" + total + " apps processed"); + if (done == total) { + callback(); + } + } + + function _makeManifestCallback(appObj) { + return function(err, manifest) { + if (err) { + self._log.warn("Could not fetch manifest for " + appObj.name); + _checkIfDone(); + return; + } + appObj.value.manifest = manifest; + DOMApplicationRegistry.updateApps([appObj], _checkIfDone); + } + } + + /** + * Now we get a manifest for each record, and add it to the local registry + * when we receive it. If a manifest GET times out, we will not add + * the app to the registry but count as "success" anyway. The app will + * be added on the next GET poll, hopefully the manifest will be + * available then. + */ + for each (let app in toInstall) { + let url = app.value.manifestURL; + if (url[0] == "/") { + url = app.value.origin + app.value.manifestURL; + } + this._getManifest(url, _makeManifestCallback(app)); + } + }, + + /** + * Fetch a manifest from given URL. No retries are made on failure. We'll + * timeout after 20 seconds. + */ + _getManifest: function _getManifest(url, callback) { + let req = new RESTRequest(url); + req.timeout = 20; + + let self = this; + req.get(function(error) { + if (error) { + callback(error, null); + return; + } + if (!req.response.success) { + callback(new Error("Non-200 while fetching manifest"), null); + return; + } + + let err = null; + let manifest = null; + try { + manifest = JSON.parse(req.response.body); + if (!manifest.name) { + self._log.warn( + "_getManifest got invalid manifest: " + req.response.body + ); + err = new Error("Invalid manifest fetched"); + } + } catch (e) { + self._log.warn( + "_getManifest got invalid JSON response: " + req.response.body + ); + err = new Error("Invalid manifest fetched"); + } + + callback(err, manifest); + }); + }, + +}; + +XPCOMUtils.defineLazyGetter(this, "AitcStorage", function() { + return new AitcStorageImpl(); +}); \ No newline at end of file diff --git a/services/aitc/services-aitc.js b/services/aitc/services-aitc.js new file mode 100644 index 000000000000..6a0931e73342 --- /dev/null +++ b/services/aitc/services-aitc.js @@ -0,0 +1,23 @@ +pref("services.aitc.browserid.url", "https://browserid.org/sign_in"); +pref("services.aitc.browserid.log.level", "Debug"); + +pref("services.aitc.client.log.level", "Debug"); +pref("services.aitc.client.timeout", 120); // 120 seconds + +pref("services.aitc.dashboard.url", "https://myapps.mozillalabs.com"); + +pref("services.aitc.main.idleTime", 120000); // 2 minutes + +pref("services.aitc.manager.putFreq", 10000); // 10 seconds +pref("services.aitc.manager.getActiveFreq", 120000); // 2 minutes +pref("services.aitc.manager.getPassiveFreq", 7200000); // 2 hours +pref("services.aitc.manager.log.level", "Debug"); + +pref("services.aitc.marketplace.url", "https://marketplace.mozilla.org"); + +pref("services.aitc.service.log.level", "Debug"); + +// Temporary value. Change to the production server when we get the OK from server ops +pref("services.aitc.tokenServer.url", "https://stage-token.services.mozilla.com"); + +pref("services.aitc.storage.log.level", "Debug"); diff --git a/services/aitc/tests/Makefile.in b/services/aitc/tests/Makefile.in new file mode 100644 index 000000000000..5cd02f741be5 --- /dev/null +++ b/services/aitc/tests/Makefile.in @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +DEPTH = ../../.. +topsrcdir = @top_srcdir@ +srcdir = @srcdir@ +VPATH = @srcdir@ +relativesrcdir = services/aitc/tests + +include $(DEPTH)/config/autoconf.mk + +MODULE = test_services_aitc +XPCSHELL_TESTS = unit + +include $(topsrcdir)/config/rules.mk + +_browser_files = \ + mochitest/head.js \ + mochitest/browser_id_simple.js \ + mochitest/file_browser_id_mock.html \ + $(NULL) + +libs:: $(_browser_files) + $(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/browser/$(relativesrcdir) diff --git a/services/aitc/tests/mochitest/browser_id_simple.js b/services/aitc/tests/mochitest/browser_id_simple.js new file mode 100644 index 000000000000..0b325dd6b0b3 --- /dev/null +++ b/services/aitc/tests/mochitest/browser_id_simple.js @@ -0,0 +1,44 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const AUD = "http://foo.net"; + +function test() { + waitForExplicitFinish(); + setEndpoint("browser_id_mock"); + + // Get an assertion for default email. + BrowserID.getAssertion(gotDefaultAssertion, {audience: AUD}); +} + +function gotDefaultAssertion(err, ast) { + is(err, null, "gotDefaultAssertion failed with " + err); + is(ast, "default@example.org_assertion_" + AUD, + "gotDefaultAssertion returned wrong assertion"); + + // Get an assertion for a specific email. + BrowserID.getAssertion(gotSpecificAssertion, { + requiredEmail: "specific@example.org", + audience: AUD + }); +} + +function gotSpecificAssertion(err, ast) { + is(err, null, "gotSpecificAssertion failed with " + err); + is(ast, "specific@example.org_assertion_" + AUD, + "gotSpecificAssertion returned wrong assertion"); + + // Get an assertion using sameEmailAs for another domain. + BrowserID.getAssertion(gotSameEmailAssertion, { + sameEmailAs: "http://zombo.com", + audience: AUD + }); +} + +function gotSameEmailAssertion(err, ast) { + is(err, null, "gotSameEmailAssertion failed with " + err); + is(ast, "assertion_for_sameEmailAs", + "gotSameEmailAssertion returned wrong assertion"); + + finish(); +} diff --git a/services/aitc/tests/mochitest/file_browser_id_mock.html b/services/aitc/tests/mochitest/file_browser_id_mock.html new file mode 100644 index 000000000000..a86adf1f116e --- /dev/null +++ b/services/aitc/tests/mochitest/file_browser_id_mock.html @@ -0,0 +1,52 @@ + + + + + +

Mock BrowserID endpoint for a logged-in user

+ + + \ No newline at end of file diff --git a/services/aitc/tests/mochitest/head.js b/services/aitc/tests/mochitest/head.js new file mode 100644 index 000000000000..e4cdb1efe3e3 --- /dev/null +++ b/services/aitc/tests/mochitest/head.js @@ -0,0 +1,23 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +let tmp = {}; +Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://services-aitc/browserid.js", tmp); + +const BrowserID = tmp.BrowserID; +const testPath = "http://mochi.test:8888/browser/services/aitc/tests/"; + +function loadURL(aURL, aCB) { + gBrowser.selectedBrowser.addEventListener("load", function () { + gBrowser.selectedBrowser.removeEventListener("load", arguments.callee, true); + is(gBrowser.currentURI.spec, aURL, "loaded expected URL"); + aCB(); + }, true); + gBrowser.loadURI(aURL); +} + +function setEndpoint(name) { + let fullPath = testPath + "file_" + name + ".html"; + Services.prefs.setCharPref("services.aitc.browserid.url", fullPath); +} \ No newline at end of file diff --git a/services/aitc/tests/unit/test_load_modules.js b/services/aitc/tests/unit/test_load_modules.js new file mode 100644 index 000000000000..266e8bbb422d --- /dev/null +++ b/services/aitc/tests/unit/test_load_modules.js @@ -0,0 +1,13 @@ +const modules = [ + "client.js", + "browserid.js", + "main.js", + "manager.js", + "storage.js" +]; + +function run_test() { + for each (let m in modules) { + Cu.import("resource://services-aitc/" + m, {}); + } +} diff --git a/services/aitc/tests/unit/test_storage_queue.js b/services/aitc/tests/unit/test_storage_queue.js new file mode 100644 index 000000000000..21ddf41708dc --- /dev/null +++ b/services/aitc/tests/unit/test_storage_queue.js @@ -0,0 +1,117 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Cu.import("resource://services-aitc/storage.js"); +Cu.import("resource://services-common/async.js"); + +let queue = null; + +function run_test() { + queue = new AitcQueue("test", run_next_test); +} + +add_test(function test_queue_create() { + do_check_eq(queue._queue.length, 0); + do_check_eq(queue._writeLock, false); + run_next_test(); +}); + +add_test(function test_queue_enqueue() { + // Add to queue. + let testObj = {foo: "bar"}; + queue.enqueue(testObj, function(err, done) { + do_check_eq(err, null); + do_check_true(done); + + // Check if peek value is correct. + do_check_eq(queue.peek(), testObj); + // Peek should be idempotent. + do_check_eq(queue.peek(), testObj); + + run_next_test(); + }); +}); + +add_test(function test_queue_dequeue() { + // Remove an item and see if queue is empty. + queue.dequeue(function(err, done) { + do_check_eq(err, null); + do_check_true(done); + do_check_eq(queue.length, 0); + try { + queue.peek(); + } catch (e) { + do_check_eq(e.toString(), "Error: Queue is empty"); + run_next_test(); + } + }); +}); + +add_test(function test_queue_multiaddremove() { + // Queues should handle objects, strings and numbers. + let items = [{test:"object"}, "teststring", 42]; + + // Two random numbers: how many items to queue and how many to remove. + let num = Math.floor(Math.random() * 100 + 1); + let rem = Math.floor(Math.random() * num + 1); + + // First insert all the items we will remove later. + for (let i = 0; i < rem; i++) { + let ins = items[Math.round(Math.random() * 2)]; + let cb = Async.makeSpinningCallback(); + queue.enqueue(ins, cb); + do_check_true(cb.wait()); + } + + do_check_eq(queue.length, rem); + + // Now insert the items we won't remove. + let check = []; + for (let i = 0; i < (num - rem); i++) { + check.push(items[Math.round(Math.random() * 2)]); + let cb = Async.makeSpinningCallback(); + queue.enqueue(check[check.length - 1], cb); + do_check_true(cb.wait()); + } + + do_check_eq(queue.length, num); + + // Now dequeue rem items. + for (let i = 0; i < rem; i++) { + let cb = Async.makeSpinningCallback(); + queue.dequeue(cb); + do_check_true(cb.wait()); + } + + do_check_eq(queue.length, num - rem); + + // Check that the items left are the right ones. + do_check_eq(JSON.stringify(queue._queue), JSON.stringify(check)); + + // Another instance of the same queue should return correct data. + let queue2 = new AitcQueue("test", function(done) { + do_check_true(done); + do_check_eq(queue2.length, queue.length); + do_check_eq(JSON.stringify(queue._queue), JSON.stringify(queue2._queue)); + run_next_test(); + }); +}); + +/* TODO Bug 760905 - Temporarily disabled for orange. +add_test(function test_queue_writelock() { + // Queue should not enqueue or dequeue if lock is enabled. + queue._writeLock = true; + let len = queue.length; + + queue.enqueue("writeLock test", function(err, done) { + do_check_eq(err.toString(), "Error: _putFile already in progress"); + do_check_eq(queue.length, len); + + queue.dequeue(function(err, done) { + do_check_eq(err.toString(), "Error: _putFile already in progress"); + do_check_eq(queue.length, len); + run_next_test(); + }); + }); +}); +*/ diff --git a/services/aitc/tests/unit/test_storage_registry.js b/services/aitc/tests/unit/test_storage_registry.js new file mode 100644 index 000000000000..6ef5ed2dd4f8 --- /dev/null +++ b/services/aitc/tests/unit/test_storage_registry.js @@ -0,0 +1,124 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Cu.import("resource://gre/modules/Webapps.jsm"); +Cu.import("resource://services-aitc/storage.js"); + +const START_PORT = 8080; +const SERVER = "http://localhost"; + +let fakeApp1 = { + origin: SERVER + ":" + START_PORT, + receipts: [], + manifestURL: "/manifest.webapp", + installOrigin: "http://localhost", + installedAt: Date.now(), + modifiedAt: Date.now() +}; + +// Valid manifest for app1 +let manifest1 = { + name: "Appasaurus", + description: "Best fake app ever", + launch_path: "/", + fullscreen: true, + required_features: ["webgl"] +}; + +let fakeApp2 = { + origin: SERVER + ":" + (START_PORT + 1), + receipts: ["fake.jwt.token"], + manifestURL: "/manifest.webapp", + installOrigin: "http://localhost", + installedAt: Date.now(), + modifiedAt: Date.now() +}; + +// Invalid manifest for app2 +let manifest2_bad = { + not: "a manifest", + fullscreen: true +}; + +// Valid manifest for app2 +let manifest2_good = { + name: "Supercalifragilisticexpialidocious", + description: "Did we blow your mind yet?", + launch_path: "/" +}; + +let fakeApp3 = { + origin: SERVER + ":" + (START_PORT + 3), // 8082 is for the good manifest2 + receipts: [], + manifestURL: "/manifest.webapp", + installOrigin: "http://localhost", + installedAt: Date.now(), + modifiedAt: Date.now() +}; + +let manifest3 = { + name: "Taumatawhakatangihangakoauauotamateapokaiwhenuakitanatahu", + description: "Find your way around this beautiful hill", + launch_path: "/" +}; + +function create_servers() { + // Setup servers to server manifests at each port + let manifests = [manifest1, manifest2_bad, manifest2_good, manifest3]; + for (let i = 0; i < manifests.length; i++) { + let response = JSON.stringify(manifests[i]); + httpd_setup({"/manifest.webapp": function(req, res) { + res.setStatusLine(req.httpVersion, 200, "OK"); + res.setHeader("Content-Type", "application/x-web-app-manifest+json"); + res.bodyOutputStream.write(response, response.length); + }}, START_PORT + i); + } +} + +function run_test() { + create_servers(); + run_next_test(); +} + +add_test(function test_storage_install() { + let apps = [fakeApp1, fakeApp2]; + AitcStorage.processApps(apps, function() { + // Verify that app1 got added to registry + let id = DOMApplicationRegistry._appId(fakeApp1.origin); + do_check_eq(DOMApplicationRegistry.itemExists(id), true); + + // app2 should be missing because of bad manifest + do_check_eq(DOMApplicationRegistry._appId(fakeApp2.origin), null); + + // Now associate fakeApp2 with a good manifest and process again + fakeApp2.origin = SERVER + ":8082"; + AitcStorage.processApps([fakeApp1, fakeApp2], function() { + // Both apps must be installed + let id1 = DOMApplicationRegistry._appId(fakeApp1.origin); + let id2 = DOMApplicationRegistry._appId(fakeApp2.origin); + do_check_eq(DOMApplicationRegistry.itemExists(id1), true); + do_check_eq(DOMApplicationRegistry.itemExists(id2), true); + run_next_test(); + }); + }); +}); + +add_test(function test_storage_uninstall() { + // Set app1 as deleted. + fakeApp1.deleted = true; + AitcStorage.processApps([fakeApp2], function() { + // It should be missing. + do_check_eq(DOMApplicationRegistry._appId(fakeApp1.origin), null); + run_next_test(); + }); +}); + +add_test(function test_storage_uninstall_empty() { + // Now remove app2 by virtue of it missing in the remote list. + AitcStorage.processApps([fakeApp3], function() { + let id3 = DOMApplicationRegistry._appId(fakeApp3.origin); + do_check_eq(DOMApplicationRegistry.itemExists(id3), true); + do_check_eq(DOMApplicationRegistry._appId(fakeApp2.origin), null); + run_next_test(); + }); +}); \ No newline at end of file diff --git a/services/aitc/tests/unit/xpcshell.ini b/services/aitc/tests/unit/xpcshell.ini new file mode 100644 index 000000000000..8f744aeba0bb --- /dev/null +++ b/services/aitc/tests/unit/xpcshell.ini @@ -0,0 +1,7 @@ +[DEFAULT] +head = ../../../common/tests/unit/head_global.js ../../../common/tests/unit/head_helpers.js +tail = + +[test_load_modules.js] +[test_storage_queue.js] +[test_storage_registry.js] diff --git a/services/common/Makefile.in b/services/common/Makefile.in index 45948d7fa6bf..d30f7499960a 100644 --- a/services/common/Makefile.in +++ b/services/common/Makefile.in @@ -31,4 +31,30 @@ libs:: TEST_DIRS += tests +# TODO enable once build infra supports testing modules. +#TESTING_JS_MODULES := aitcserver.js storageserver.js +#TESTING_JS_MODULE_DIR := services-common + +# What follows is a helper to launch a standalone storage server instance. +# Most of the code lives in a Python script in the tests directory. If we +# ever consolidate our Python code, and/or have a supplemental driver for the +# build system, this can go away. + +storage_server_hostname := localhost +storage_server_port := 8080 + +head_path = $(srcdir)/tests/unit + +storage-server: + $(PYTHON) $(srcdir)/tests/run_server.py $(topsrcdir) \ + $(MOZ_BUILD_ROOT) run_storage_server.js --port $(storage_server_port) + +# And the same thing for an AITC server. +aitc_server_hostname := localhost +aitc_server_port := 8080 + +aitc-server: + $(PYTHON) $(srcdir)/tests/run_server.py $(topsrcdir) \ + $(MOZ_BUILD_ROOT) run_aitc_server.js --port $(aitc_server_port) + include $(topsrcdir)/config/rules.mk diff --git a/services/common/tests/run_aitc_server.js b/services/common/tests/run_aitc_server.js new file mode 100644 index 000000000000..c63c5e50b6ad --- /dev/null +++ b/services/common/tests/run_aitc_server.js @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file runs a standalone AITC server. + * + * It is meant to be executed with an xpcshell. + * + * The Makefile in this directory contains a target to run it: + * + * $ make aitc-server + */ + +Cu.import("resource://testing-common/services-common/aitcserver.js"); + +initTestLogging(); + +let server = new AITCServer10Server(); +server.autoCreateUsers = true; +server.start(SERVER_PORT); + +_("AITC server started on port " + SERVER_PORT); + +// Launch the thread manager. +_do_main(); diff --git a/services/common/tests/run_server.py b/services/common/tests/run_server.py new file mode 100755 index 000000000000..3eb386e0966e --- /dev/null +++ b/services/common/tests/run_server.py @@ -0,0 +1,79 @@ +#!/usr/bin/python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from argparse import ArgumentParser +from shutil import rmtree +from subprocess import Popen +from sys import argv +from sys import exit +from tempfile import mkdtemp + +DEFAULT_PORT = 8080 +DEFAULT_HOSTNAME = 'localhost' + +def run_server(srcdir, objdir, js_file, hostname=DEFAULT_HOSTNAME, + port=DEFAULT_PORT): + + dist_dir = '%s/dist' % objdir + head_dir = '%s/services/common/tests/unit' % srcdir + + head_paths = [ + 'head_global.js', + 'head_helpers.js', + 'head_http.js', + ] + + head_paths = ['"%s/%s"' % (head_dir, path) for path in head_paths] + + args = [ + '%s/bin/xpcshell' % dist_dir, + '-g', '%s/bin' % dist_dir, + '-a', '%s/bin' % dist_dir, + '-r', '%s/bin/components/httpd.manifest' % dist_dir, + '-m', + '-n', + '-s', + '-f', '%s/testing/xpcshell/head.js' % srcdir, + '-e', 'const _SERVER_ADDR = "%s";' % hostname, + '-e', 'const _TESTING_MODULES_DIR = "%s/_tests/modules";' % objdir, + '-e', 'const SERVER_PORT = "%s";' % port, + '-e', 'const INCLUDE_FILES = [%s];' % ', '.join(head_paths), + '-e', '_register_protocol_handlers();', + '-e', 'for each (let name in INCLUDE_FILES) load(name);', + '-e', '_fakeIdleService.activate();', + '-f', '%s/services/common/tests/%s' % (srcdir, js_file) + ] + + profile_dir = mkdtemp() + print 'Created profile directory: %s' % profile_dir + + try: + env = {'XPCSHELL_TEST_PROFILE_DIR': profile_dir} + proc = Popen(args, env=env) + + return proc.wait() + + finally: + print 'Removing profile directory %s' % profile_dir + rmtree(profile_dir) + +if __name__ == '__main__': + parser = ArgumentParser(description="Run a standalone JS server.") + parser.add_argument('srcdir', + help="Root directory of Firefox source code.") + parser.add_argument('objdir', + help="Root directory object directory created during build.") + parser.add_argument('js_file', + help="JS file (in this directory) to execute.") + parser.add_argument('--port', default=DEFAULT_PORT, type=int, + help="Port to run server on.") + parser.add_argument('--address', default=DEFAULT_HOSTNAME, + help="Hostname to bind server to.") + + args = parser.parse_args() + + exit(run_server(args.srcdir, args.objdir, args.js_file, args.address, + args.port)) diff --git a/services/common/tests/run_storage_server.js b/services/common/tests/run_storage_server.js new file mode 100644 index 000000000000..2cfda3e3215d --- /dev/null +++ b/services/common/tests/run_storage_server.js @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file runs a Storage Service server. + * + * It is meant to be executed with an xpcshell. + * + * The Makefile in this directory contains a target to run it: + * + * $ make storage-server + */ + +Cu.import("resource://testing-common/services-common/storageserver.js"); + +initTestLogging(); + +let server = new StorageServer(); +server.allowAllUsers = true; +server.startSynchronous(SERVER_PORT); +_("Storage server started on port " + SERVER_PORT); + +// Launch the thread manager. +_do_main(); diff --git a/services/common/tests/unit/aitcserver.js b/services/common/tests/unit/aitcserver.js new file mode 100644 index 000000000000..f88bafe9b8b0 --- /dev/null +++ b/services/common/tests/unit/aitcserver.js @@ -0,0 +1,528 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +// TODO enable once build infra supports test modules. +/* +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +const EXPORTED_SYMBOLS = [ + "AITCServer10User", + "AITCServer10Server", +]; + +Cu.import("resource://testing-common/httpd.js"); +*/ +Cu.import("resource://services-crypto/utils.js"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/utils.js"); + +/** + * Represents an individual user on an AITC 1.0 server. + * + * This type provides convenience APIs for interacting with an individual + * user's data. + */ +function AITCServer10User() { + this._log = Log4Moz.repository.getLogger("Services.Common.AITCServer"); + this.apps = {}; +} +AITCServer10User.prototype = { + appRecordProperties: { + origin: true, + manifestPath: true, + installOrigin: true, + installedAt: true, + modifiedAt: true, + receipts: true, + name: true, + deleted: true, + }, + + requiredAppProperties: [ + "origin", + "manifestPath", + "installOrigin", + "installedAt", + "modifiedAt", + "name", + "receipts", + ], + + /** + * Obtain the apps for this user. + * + * This is a generator of objects representing the apps. Returns the original + * apps object normally or an abbreviated version if `minimal` is truthy. + */ + getApps: function getApps(minimal) { + let result; + + for (let id in this.apps) { + let app = this.apps[id]; + + if (!minimal) { + yield app; + continue; + } + + yield {origin: app.origin, modifiedAt: app.modifiedAt}; + } + }, + + getAppByID: function getAppByID(id) { + return this.apps[id]; + }, + + /** + * Adds an app to this user. + * + * The app record should be an object (likely from decoded JSON). + */ + addApp: function addApp(app) { + for (let k in app) { + if (!(k in this.appRecordProperties)) { + throw new Error("Unexpected property in app record: " + k); + } + } + + for each (let k in this.requiredAppProperties) { + if (!(k in app)) { + throw new Error("Required property not in app record: " + k); + } + } + + this.apps[this.originToID(app.origin)] = app; + }, + + /** + * Returns whether a user has an app with the specified ID. + */ + hasAppID: function hasAppID(id) { + return id in this.apps; + }, + + /** + * Delete an app having the specified ID. + */ + deleteAppWithID: function deleteAppWithID(id) { + delete this.apps[id]; + }, + + /** + * Convert an origin string to an ID. + */ + originToID: function originToID(origin) { + let hash = CryptoUtils.UTF8AndSHA1(origin); + return CommonUtils.encodeBase64URL(hash, false); + }, +}; + +/** + * A fully-functional AITC 1.0 server implementation. + * + * Each server instance is capable of serving requests for multiple users. + * By default, users do not exist and requests to URIs for a specific user + * will result in 404s. To register a new user with an empty account, call + * createUser(). If you wish for HTTP requests for non-existing users to + * work, set autoCreateUsers to true and am empty user will be + * provisioned at request time. + */ +function AITCServer10Server() { + this._log = Log4Moz.repository.getLogger("Services.Common.AITCServer"); + + this.server = new nsHttpServer(); + this.port = null; + this.users = {}; + this.autoCreateUsers = false; + + this._appsAppHandlers = { + GET: this._appsAppGetHandler, + PUT: this._appsAppPutHandler, + DELETE: this._appsAppDeleteHandler, + }; +} +AITCServer10Server.prototype = { + ID_REGEX: /^[a-zA-Z0-9_-]{27}$/, + VERSION_PATH: "/1.0/", + + /** + * Obtain the base URL the server can be accessed at as a string. + */ + get url() { + // Is this available on the nsHttpServer instance? + return "http://localhost:" + this.port + this.VERSION_PATH; + }, + + /** + * Start the server on a specified port. + */ + start: function start(port) { + if (!port) { + throw new Error("port argument must be specified."); + } + + this.port = port; + + this.server.registerPrefixHandler(this.VERSION_PATH, + this._generalHandler.bind(this)); + this.server.start(port); + }, + + /** + * Stop the server. + * + * Calls the specified callback when the server is stopped. + */ + stop: function stop(cb) { + let handler = {onStopped: cb}; + + this.server.stop(handler); + }, + + createUser: function createUser(username) { + if (username in this.users) { + throw new Error("User already exists: " + username); + } + + this._log.info("Registering user: " + username); + + this.users[username] = new AITCServer10User(); + this.server.registerPrefixHandler(this.VERSION_PATH + username + "/", + this._userHandler.bind(this, username)); + + return this.users[username]; + }, + + /** + * Returns information for an individual user. + * + * The returned object contains functions to access and manipulate an + * individual user. + */ + getUser: function getUser(username) { + if (!(username in this.users)) { + throw new Error("user is not present in server: " + username); + } + + return this.users[username]; + }, + + /** + * HTTP handler for requests to /1.0/ which don't have a specific user + * registered. + */ + _generalHandler: function _generalHandler(request, response) { + let path = request.path; + this._log.info("Request: " + request.method + " " + path); + + if (path.indexOf(this.VERSION_PATH) != 0) { + throw new Error("generalHandler invoked improperly."); + } + + let rest = request.path.substr(this.VERSION_PATH.length); + if (!rest.length) { + throw HTTP_404; + } + + if (!this.autoCreateUsers) { + throw HTTP_404; + } + + let username; + let index = rest.indexOf("/"); + if (index == -1) { + username = rest; + } else { + username = rest.substr(0, index); + } + + this.createUser(username); + this._userHandler(username, request, response); + }, + + /** + * HTTP handler for requests for a specific user. + * + * This handles request routing to the appropriate handler. + */ + _userHandler: function _userHandler(username, request, response) { + this._log.info("Request: " + request.method + " " + request.path); + let path = request.path; + let prefix = this.VERSION_PATH + username + "/"; + + if (path.indexOf(prefix) != 0) { + throw new Error("userHandler invoked improperly."); + } + + let user = this.users[username]; + if (!user) { + throw new Error("User handler should not have been invoked for an " + + "unknown user!"); + } + + let requestTime = Date.now(); + response.dispatchTime = requestTime; + response.setHeader("X-Timestamp", "" + requestTime); + + let handler; + let remaining = path.substr(prefix.length); + + if (remaining == "apps" || remaining == "apps/") { + this._log.info("Dispatching to apps index handler."); + handler = this._appsIndexHandler.bind(this, user, request, response); + } else if (!remaining.indexOf("apps/")) { + let id = remaining.substr("apps/".length); + + this._log.info("Dispatching to app handler."); + handler = this._appsAppHandler.bind(this, user, id, request, response); + } else if (remaining == "devices" || !remaining.indexOf("devices/")) { + this._log.info("Dispatching to devices handler."); + handler = this._devicesHandler.bind(this, user, + remaining.substr("devices".length), + request, response); + } else { + throw HTTP_404; + } + + try { + handler(); + } catch (ex) { + if (ex instanceof HttpError) { + response.setStatusLine(request.httpVersion, ex.code, ex.description); + return; + } + + this._log.warn("Exception when processing request: " + + CommonUtils.exceptionStr(ex)); + throw ex; + } + }, + + _appsIndexHandler: function _appsIndexHandler(user, request, response) { + if (request.method != "GET") { + response.setStatusLine(request.httpVersion, 405, "Method Not Allowed"); + response.setHeader("Accept", "GET"); + + return; + } + + let options = this._getQueryStringParams(request); + for (let key in options) { + let value = options[key]; + + switch (key) { + case "after": + let time = parseInt(value, 10); + if (isNaN(time)) { + throw HTTP_400; + } + + options.after = time; + break; + + case "full": + // Value is irrelevant. + break; + + default: + this._log.info("Unknown query string parameter: " + key); + throw HTTP_400; + } + } + + let apps = []; + let newest = 0; + for each (let app in user.getApps(!("full" in options))) { + if (app.modifiedAt > newest) { + newest = app.modifiedAt; + } + + if ("after" in options && app.modifiedAt <= options.after) { + continue; + } + + apps.push(app); + } + + if (request.hasHeader("X-If-Modified-Since")) { + let modified = parseInt(request.getHeader("X-If-Modified-Since"), 10); + if (modified >= newest) { + response.setStatusLine(request.httpVersion, 304, "Not Modified"); + return; + } + } + + let body = JSON.stringify({apps: apps}); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.setHeader("X-Last-Modified", "" + newest); + response.setHeader("Content-Type", "application/json"); + response.bodyOutputStream.write(body, body.length); + }, + + _appsAppHandler: function _appAppHandler(user, id, request, response) { + if (!(request.method in this._appsAppHandlers)) { + response.setStatusLine(request.httpVersion, 405, "Method Not Allowed"); + response.setHeader("Accept", Object.keys(this._appsAppHandlers).join(",")); + + return; + } + + let handler = this._appsAppHandlers[request.method]; + return handler.call(this, user, id, request, response); + }, + + _appsAppGetHandler: function _appsAppGetHandler(user, id, request, response) { + if (!user.hasAppID(id)) { + throw HTTP_404; + } + + let app = user.getAppByID(id); + + if (request.hasHeader("X-If-Modified-Since")) { + let modified = parseInt(request.getHeader("X-If-Modified-Since"), 10); + + this._log.debug("Client time: " + modified + "; Server time: " + + app.modifiedAt); + + if (modified >= app.modifiedAt) { + response.setStatusLine(request.httpVersion, 304, "Not Modified"); + return; + } + } + + let body = JSON.stringify(app); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.setHeader("X-Last-Modified", "" + response.dispatchTime); + response.setHeader("Content-Type", "application/json"); + response.bodyOutputStream.write(body, body.length); + }, + + _appsAppPutHandler: function _appsAppPutHandler(user, id, request, response) { + if (!request.hasHeader("Content-Type")) { + this._log.info("Request does not have Content-Type header."); + throw HTTP_400; + } + + let ct = request.getHeader("Content-Type"); + if (ct != "application/json" && ct.indexOf("application/json;") !== 0) { + this._log.info("Unknown media type: " + ct); + // TODO proper response headers. + throw HTTP_415; + } + + let requestBody = CommonUtils.readBytesFromInputStream(request.bodyInputStream); + this._log.debug("Request body: " + requestBody); + if (requestBody.length > 8192) { + this._log.info("Request body too long: " + requestBody.length); + throw HTTP_413; + } + + let hadApp = user.hasAppID(id); + + let app; + try { + app = JSON.parse(requestBody); + } catch (e) { + this._log.info("JSON parse error."); + throw HTTP_400; + } + + // URL and record mismatch. + if (user.originToID(app.origin) != id) { + this._log.warn("URL ID and origin mismatch. URL: " + id + "; Record: " + + user.originToID(app.origin)); + throw HTTP_403; + } + + if (request.hasHeader("X-If-Unmodified-Since") && hadApp) { + let modified = parseInt(request.getHeader("X-If-Unmodified-Since"), 10); + let existing = user.getAppByID(id); + + if (existing.modifiedAt > modified) { + this._log.info("Server modified after client."); + throw HTTP_412; + } + } + + try { + app.modifiedAt = response.dispatchTime; + + if (hadApp) { + app.installedAt = user.getAppByID(id).installedAt; + } else { + app.installedAt = response.dispatchTime; + } + + user.addApp(app); + } catch (e) { + this._log.info("Error adding app: " + CommonUtils.exceptionStr(e)); + throw HTTP_400; + } + + let code = 201; + let status = "Created"; + + if (hadApp) { + code = 204; + status = "No Content"; + } + + response.setHeader("X-Last-Modified", "" + response.dispatchTime); + response.setStatusLine(request.httpVersion, code, status); + }, + + _appsAppDeleteHandler: function _appsAppDeleteHandler(user, id, request, + response) { + if (!user.hasAppID(id)) { + throw HTTP_404; + } + + let existing = user.getAppByID(id); + if (request.hasHeader("X-If-Unmodified-Since")) { + let modified = parseInt(request.getHeader("X-If-Unmodified-Since"), 10); + + if (existing.modifiedAt > modified) { + throw HTTP_412; + } + } + + user.deleteAppWithID(id); + + response.setHeader("X-Last-Modified", "" + response.dispatchTime); + response.setStatusLine(request.httpVersion, 204, "No Content"); + }, + + _devicesHandler: function _devicesHandler(user, path, request, response) { + // TODO need to support full API. + // For now, we just assume it is a request for /. + response.setHeader("Content-Type", "application/json"); + let body = JSON.stringify({devices: []}); + + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + }, + + // Surely this exists elsewhere in the Mozilla source tree... + _getQueryStringParams: function _getQueryStringParams(request) { + let params = {}; + for each (let chunk in request.queryString.split("&")) { + if (!chunk) { + continue; + } + + let parts = chunk.split("="); + // TODO URL decode key and value. + if (parts.length == 1) { + params[parts[0]] = ""; + } else { + params[parts[0]] = parts[1]; + } + } + + return params; + }, +}; + diff --git a/services/common/tests/unit/head_global.js b/services/common/tests/unit/head_global.js index deb593fd6283..114557352582 100644 --- a/services/common/tests/unit/head_global.js +++ b/services/common/tests/unit/head_global.js @@ -47,7 +47,7 @@ function addResourceAlias() { const handler = Services.io.getProtocolHandler("resource") .QueryInterface(Ci.nsIResProtocolHandler); - let modules = ["common", "crypto"]; + let modules = ["aitc", "common", "crypto"]; for each (let module in modules) { let uri = Services.io.newURI("resource:///modules/services-" + module + "/", null, null); diff --git a/services/common/tests/unit/head_helpers.js b/services/common/tests/unit/head_helpers.js index 1873e74e858d..c884c32a05fd 100644 --- a/services/common/tests/unit/head_helpers.js +++ b/services/common/tests/unit/head_helpers.js @@ -3,6 +3,7 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/utils.js"); let btoa = Cu.import("resource://services-common/log4moz.js").btoa; let atob = Cu.import("resource://services-common/log4moz.js").atob; @@ -57,13 +58,15 @@ function initTestLogging(level) { }; LogStats.prototype.__proto__ = new Log4Moz.Formatter(); - var log = Log4Moz.repository.rootLogger; - var logStats = new LogStats(); - var appender = new Log4Moz.DumpAppender(logStats); + let log = Log4Moz.repository.rootLogger; + let logStats = new LogStats(); + let appender = new Log4Moz.DumpAppender(logStats); - if (typeof(level) == "undefined") + if (typeof(level) == "undefined") { level = "Debug"; + } getTestLogger().level = Log4Moz.Level[level]; + Log4Moz.repository.getLogger("Services").level = Log4Moz.Level[level]; log.level = Log4Moz.Level.Trace; appender.level = Log4Moz.Level.Trace; @@ -78,6 +81,16 @@ function getTestLogger(component) { return Log4Moz.repository.getLogger("Testing"); } +/** + * Obtain a port number to run a server on. + * + * In the ideal world, this would be dynamic so multiple servers could be run + * in parallel. + */ +function get_server_port() { + return 8080; +} + function httpd_setup (handlers, port) { let port = port || 8080; let server = new nsHttpServer(); @@ -89,7 +102,7 @@ function httpd_setup (handlers, port) { } catch (ex) { _("=========================================="); _("Got exception starting HTTP server on port " + port); - _("Error: " + Utils.exceptionStr(ex)); + _("Error: " + CommonUtils.exceptionStr(ex)); _("Is there a process already listening on port " + port + "?"); _("=========================================="); do_throw(ex); @@ -117,14 +130,7 @@ function httpd_handler(statusCode, status, body) { * all available input is read. */ function readBytesFromInputStream(inputStream, count) { - var BinaryInputStream = Components.Constructor( - "@mozilla.org/binaryinputstream;1", - "nsIBinaryInputStream", - "setInputStream"); - if (!count) { - count = inputStream.available(); - } - return new BinaryInputStream(inputStream).readBytes(count); + return CommonUtils.readBytesFromInputStream(inputStream, count); } /* diff --git a/services/common/tests/unit/head_http.js b/services/common/tests/unit/head_http.js new file mode 100644 index 000000000000..f590e86cb64e --- /dev/null +++ b/services/common/tests/unit/head_http.js @@ -0,0 +1,29 @@ + /* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Cu.import("resource://services-common/utils.js"); + +function basic_auth_header(user, password) { + return "Basic " + btoa(user + ":" + CommonUtils.encodeUTF8(password)); +} + +function basic_auth_matches(req, user, password) { + if (!req.hasHeader("Authorization")) { + return false; + } + + let expected = basic_auth_header(user, CommonUtils.encodeUTF8(password)); + return req.getHeader("Authorization") == expected; +} + +function httpd_basic_auth_handler(body, metadata, response) { + if (basic_auth_matches(metadata, "guest", "guest")) { + response.setStatusLine(metadata.httpVersion, 200, "OK, authorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } else { + body = "This path exists and is protected - failed"; + response.setStatusLine(metadata.httpVersion, 401, "Unauthorized"); + response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false); + } + response.bodyOutputStream.write(body, body.length); +} diff --git a/services/common/tests/unit/storageserver.js b/services/common/tests/unit/storageserver.js new file mode 100644 index 000000000000..791c4d7bed97 --- /dev/null +++ b/services/common/tests/unit/storageserver.js @@ -0,0 +1,1638 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file contains an implementation of the Storage Server in JavaScript. + * + * The server should not be used for any production purposes. + */ + +// TODO enable once build infra supports testing modules. +/* +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +const EXPORTED_SYMBOLS = [ + "ServerBSO", + "StorageServerCallback", + "StorageServerCollection", + "StorageServer", + +]; + +Cu.import("resource://testing-common/httpd.js"); +*/ +Cu.import("resource://services-common/async.js"); +Cu.import("resource://services-common/log4moz.js"); +Cu.import("resource://services-common/utils.js"); + +const STORAGE_HTTP_LOGGER = "Services.Common.Test.Server"; +const STORAGE_API_VERSION = "2.0"; + +// Use the same method that record.js does, which mirrors the server. +function new_timestamp() { + return Math.round(Date.now()); +} + +function isInteger(s) { + let re = /^[0-9]+$/; + return re.test(s); +} + +function writeHttpBody(response, body) { + if (!body) { + return; + } + + response.bodyOutputStream.write(body, body.length); +} + +function sendMozSvcError(request, response, code) { + response.setStatusLine(request.httpVersion, 400, "Bad Request"); + response.setHeader("Content-Type", "text/plain", false); + response.bodyOutputStream.write(code, code.length); +} + +/** + * Represent a BSO on the server. + * + * A BSO is constructed from an ID, content, and a modified time. + * + * @param id + * (string) ID of the BSO being created. + * @param payload + * (strong|object) Payload for the BSO. Should ideally be a string. If + * an object is passed, it will be fed into JSON.stringify and that + * output will be set as the payload. + * @param modified + * (number) Milliseconds since UNIX epoch that the BSO was last + * modified. If not defined or null, the current time will be used. + */ +function ServerBSO(id, payload, modified) { + if (!id) { + throw new Error("No ID for ServerBSO!"); + } + + if (!id.match(/^[a-zA-Z0-9_-]{1,64}$/)) { + throw new Error("BSO ID is invalid: " + id); + } + + this._log = Log4Moz.repository.getLogger(STORAGE_HTTP_LOGGER); + + this.id = id; + if (!payload) { + return; + } + + CommonUtils.ensureMillisecondsTimestamp(modified); + + if (typeof payload == "object") { + payload = JSON.stringify(payload); + } + + this.payload = payload; + this.modified = modified || new_timestamp(); +} +ServerBSO.prototype = { + FIELDS: [ + "id", + "modified", + "payload", + "ttl", + "sortindex", + ], + + toJSON: function toJSON() { + let obj = {}; + + for each (let key in this.FIELDS) { + if (this[key] !== undefined) { + obj[key] = this[key]; + } + } + + return obj; + }, + + delete: function delete() { + this.deleted = true; + + delete this.payload; + delete this.modified; + }, + + /** + * Handler for GET requests for this BSO. + */ + getHandler: function getHandler(request, response) { + let code = 200; + let status = "OK"; + let body; + + function sendResponse() { + response.setStatusLine(request.httpVersion, code, status); + writeHttpBody(response, body); + } + + if (request.hasHeader("x-if-modified-since")) { + let headerModified = parseInt(request.getHeader("x-if-modified-since")); + CommonUtils.ensureMillisecondsTimestamp(headerModified); + + if (headerModified >= this.modified) { + code = 304; + status = "Not Modified"; + + sendResponse(); + return; + } + } + + if (!this.deleted) { + body = JSON.stringify(this.toJSON()); + response.setHeader("Content-Type", "application/json", false); + response.setHeader("X-Last-Modified", "" + this.modified, false); + } else { + code = 404; + status = "Not Found"; + } + + sendResponse(); + }, + + /** + * Handler for PUT requests for this BSO. + */ + putHandler: function putHandler(request, response) { + if (request.hasHeader("Content-Type")) { + let ct = request.getHeader("Content-Type"); + if (ct != "application/json") { + throw HTTP_415; + } + } + + let input = CommonUtils.readBytesFromInputStream(request.bodyInputStream); + let parsed; + try { + parsed = JSON.parse(input); + } catch (ex) { + return sendMozSvcError(request, response, "8"); + } + + if (typeof(parsed) != "object") { + return sendMozSvcError(request, response, "8"); + } + + // Don't update if a conditional request fails preconditions. + if (request.hasHeader("x-if-unmodified-since")) { + let reqModified = parseInt(request.getHeader("x-if-unmodified-since")); + + if (reqModified < this.modified) { + response.setStatusLine(request.httpVersion, 412, "Precondition Failed"); + return; + } + } + + let code, status; + if (this.payload) { + code = 204; + status = "No Content"; + } else { + code = 201; + status = "Created"; + } + + // Alert when we see unrecognized fields. + for (let [key, value] in Iterator(parsed)) { + switch (key) { + case "payload": + if (typeof(value) != "string") { + sendMozSvcError(request, response, "8"); + return true; + } + + this.payload = value; + break; + + case "ttl": + if (!isInteger(value)) { + sendMozSvcError(request, response, "8"); + return true; + } + this.ttl = parseInt(value, 10); + break; + + case "sortindex": + if (!isInteger(value) || value.length > 9) { + sendMozSvcError(request, response, "8"); + return true; + } + this.sortindex = parseInt(value, 10); + break; + + case "id": + break; + + default: + this._log.warn("Unexpected field in BSO record: " + key); + sendMozSvcError(request, response, "8"); + return true; + } + } + + this.modified = request.timestamp; + response.newModified = request.timestamp; + response.setHeader("X-Last-Modified", "" + this.modified, false); + + response.setStatusLine(request.httpVersion, code, status); + }, +}; + +/** + * Represent a collection on the server. + * + * The '_bsos' attribute is a mapping of id -> ServerBSO objects. + * + * Note that if you want these records to be accessible individually, + * you need to register their handlers with the server separately, or use a + * containing HTTP server that will do so on your behalf. + * + * @param bsos + * An object mapping BSO IDs to ServerBSOs. + * @param acceptNew + * If true, POSTs to this collection URI will result in new BSOs being + * created and wired in on the fly. + * @param timestamp + * An optional timestamp value to initialize the modified time of the + * collection. This should be in the format returned by new_timestamp(). + */ +function StorageServerCollection(bsos, acceptNew, timestamp) { + this._bsos = bsos || {}; + this.acceptNew = acceptNew || false; + + /* + * Track modified timestamp. + * We can't just use the timestamps of contained BSOs: an empty collection + * has a modified time. + */ + CommonUtils.ensureMillisecondsTimestamp(timestamp); + + this.timestamp = timestamp || new_timestamp(); + this._log = Log4Moz.repository.getLogger(STORAGE_HTTP_LOGGER); +} +StorageServerCollection.prototype = { + BATCH_MAX_COUNT: 100, // # of records. + BATCH_MAX_SIZE: 1024 * 1024, // # bytes. + + _timestamp: null, + + get timestamp() { + return this._timestamp; + }, + + set timestamp(timestamp) { + CommonUtils.ensureMillisecondsTimestamp(timestamp); + this._timestamp = timestamp; + }, + + get totalPayloadSize() { + let size = 0; + for each (let bso in this.bsos()) { + size += bso.payload.length; + } + + return size; + }, + + /** + * Convenience accessor for our BSO keys. + * Excludes deleted items, of course. + * + * @param filter + * A predicate function (applied to the ID and BSO) which dictates + * whether to include the BSO's ID in the output. + * + * @return an array of IDs. + */ + keys: function keys(filter) { + return [id for ([id, bso] in Iterator(this._bsos)) + if (!bso.deleted && (!filter || filter(id, bso)))]; + }, + + /** + * Convenience method to get an array of BSOs. + * Optionally provide a filter function. + * + * @param filter + * A predicate function, applied to the BSO, which dictates whether to + * include the BSO in the output. + * + * @return an array of ServerBSOs. + */ + bsos: function bsos(filter) { + let os = [bso for ([id, bso] in Iterator(this._bsos)) + if (!bso.deleted)]; + + if (!filter) { + return os; + } + + return os.filter(filter); + }, + + /** + * Obtain a BSO by ID. + */ + bso: function bso(id) { + return this._bsos[id]; + }, + + /** + * Obtain the payload of a specific BSO. + * + * Raises if the specified BSO does not exist. + */ + payload: function payload(id) { + return this.bso(id).payload; + }, + + /** + * Insert the provided BSO under its ID. + * + * @return the provided BSO. + */ + insertBSO: function insertBSO(bso) { + return this._bsos[bso.id] = bso; + }, + + /** + * Insert the provided payload as part of a new ServerBSO with the provided + * ID. + * + * @param id + * The GUID for the BSO. + * @param payload + * The payload, as provided to the ServerBSO constructor. + * @param modified + * An optional modified time for the ServerBSO. If not specified, the + * current time will be used. + * + * @return the inserted BSO. + */ + insert: function insert(id, payload, modified) { + return this.insertBSO(new ServerBSO(id, payload, modified)); + }, + + /** + * Removes an object entirely from the collection. + * + * @param id + * (string) ID to remove. + */ + remove: function remove(id) { + delete this._bsos[id]; + }, + + _inResultSet: function _inResultSet(bso, options) { + if (!bso.payload) { + return false; + } + + if (options.ids) { + if (options.ids.indexOf(bso.id) == -1) { + return false; + } + } + + if (options.newer) { + if (bso.modified < options.newer) { + return false; + } + } + + if (options.older) { + if (bso.modified > options.older) { + return false; + } + } + + if (options.index_above) { + if (bso.sortindex === undefined) { + return false; + } + + if (bso.sortindex <= options.index_above) { + return false; + } + } + + if (options.index_below) { + if (bso.sortindex === undefined) { + return false; + } + + if (bso.sortindex >= options.index_below) { + return false; + } + } + + return true; + }, + + count: function count(options) { + options = options || {}; + let c = 0; + for (let [id, bso] in Iterator(this._bsos)) { + if (bso.modified && this._inResultSet(bso, options)) { + c++; + } + } + return c; + }, + + get: function get(options) { + let data = []; + for each (let bso in this._bsos) { + if (!bso.modified) { + continue; + } + + if (!this._inResultSet(bso, options)) { + continue; + } + + data.push(bso); + } + + if (options.sort) { + if (options.sort == "oldest") { + data.sort(function sortOldest(a, b) { + if (a.modified == b.modified) { + return 0; + } + + return a.modified < b.modified ? -1 : 1; + }); + } else if (options.sort == "newest") { + data.sort(function sortNewest(a, b) { + if (a.modified == b.modified) { + return 0; + } + + return a.modified > b.modified ? -1 : 1; + }); + } else if (options.sort == "index") { + data.sort(function sortIndex(a, b) { + if (a.sortindex == b.sortindex) { + return 0; + } + + if (a.sortindex !== undefined && b.sortindex == undefined) { + return 1; + } + + if (a.sortindex === undefined && b.sortindex !== undefined) { + return -1; + } + + return a.sortindex > b.sortindex ? -1 : 1; + }); + } + } + + if (options.limit) { + data = data.slice(0, options.limit); + } + + return data; + }, + + post: function post(input, timestamp) { + let success = []; + let failed = {}; + let count = 0; + let size = 0; + + // This will count records where we have an existing ServerBSO + // registered with us as successful and all other records as failed. + for each (let record in input) { + count += 1; + if (count > this.BATCH_MAX_COUNT) { + failed[record.id] = "Max record count exceeded."; + continue; + } + + if (typeof(record.payload) != "string") { + failed[record.id] = "Payload is not a string!"; + continue; + } + + size += record.payload.length; + if (size > this.BATCH_MAX_SIZE) { + failed[record.id] = "Payload max size exceeded!"; + continue; + } + + if (record.sortindex) { + if (!isInteger(record.sortindex)) { + failed[record.id] = "sortindex is not an integer."; + continue; + } + + if (record.sortindex.length > 9) { + failed[record.id] = "sortindex is too long."; + continue; + } + } + + if ("ttl" in record) { + if (!isInteger(record.ttl)) { + failed[record.id] = "ttl is not an integer."; + continue; + } + } + + try { + let bso = this.bso(record.id); + if (!bso && this.acceptNew) { + this._log.debug("Creating BSO " + JSON.stringify(record.id) + + " on the fly."); + bso = new ServerBSO(record.id); + this.insertBSO(bso); + } + if (bso) { + bso.payload = record.payload; + bso.modified = timestamp; + success.push(record.id); + + if (record.sortindex) { + bso.sortindex = parseInt(record.sortindex, 10); + } + + } else { + failed[record.id] = "no bso configured"; + } + } catch (ex) { + this._log.info("Exception when processing BSO: " + + CommonUtils.exceptionStr(ex)); + failed[record.id] = "Exception when processing."; + } + } + return {success: success, failed: failed}; + }, + + delete: function delete(options) { + options = options || {}; + + // Protocol 2.0 only allows the "ids" query string argument. + let keys = Object.keys(options).filter(function(k) { + return k != "ids"; + }); + if (keys.length) { + this._log.warn("Invalid query string parameter to collection delete: " + + keys.join(", ")); + throw new Error("Malformed client request."); + } + + if (options.ids && options.ids.length > this.BATCH_MAX_COUNT) { + throw HTTP_400; + } + + let deleted = []; + for (let [id, bso] in Iterator(this._bsos)) { + if (this._inResultSet(bso, options)) { + this._log.debug("Deleting " + JSON.stringify(bso)); + deleted.push(bso.id); + bso.delete(); + } + } + return deleted; + }, + + parseOptions: function parseOptions(request) { + let options = {}; + + for each (let chunk in request.queryString.split("&")) { + if (!chunk) { + continue; + } + chunk = chunk.split("="); + if (chunk.length == 1) { + options[chunk[0]] = ""; + } else { + options[chunk[0]] = chunk[1]; + } + } + + if (options.ids) { + options.ids = options.ids.split(","); + } + + if (options.newer) { + if (!isInteger(options.newer)) { + throw HTTP_400; + } + + CommonUtils.ensureMillisecondsTimestamp(options.newer); + options.newer = parseInt(options.newer, 10); + } + + if (options.older) { + if (!isInteger(options.older)) { + throw HTTP_400; + } + + CommonUtils.ensureMillisecondsTimestamp(options.older); + options.older = parseInt(options.older, 10); + } + + if (options.index_above) { + if (!isInteger(options.index_above)) { + throw HTTP_400; + } + } + + if (options.index_below) { + if (!isInteger(options.index_below)) { + throw HTTP_400; + } + } + + if (options.limit) { + if (!isInteger(options.limit)) { + throw HTTP_400; + } + + options.limit = parseInt(options.limit, 10); + } + + return options; + }, + + getHandler: function getHandler(request, response) { + let options = this.parseOptions(request); + let data = this.get(options); + + if (request.hasHeader("x-if-modified-since")) { + let requestModified = parseInt(request.getHeader("x-if-modified-since"), + 10); + let newestBSO = 0; + for each (let bso in data) { + if (bso.modified > newestBSO) { + newestBSO = bso.modified; + } + } + + if (requestModified >= newestBSO) { + response.setHeader("X-Last-Modified", "" + newestBSO); + response.setStatusLine(request.httpVersion, 304, "Not Modified"); + return; + } + } + + if (options.full) { + data = data.map(function map(bso) { + return bso.toJSON(); + }); + } else { + data = data.map(function map(bso) { + return bso.id; + }); + } + + // application/json is default media type. + let newlines = false; + if (request.hasHeader("accept")) { + let accept = request.getHeader("accept"); + if (accept == "application/newlines") { + newlines = true; + } else if (accept != "application/json") { + throw HTTP_406; + } + } + + let body; + if (newlines) { + response.setHeader("Content-Type", "application/newlines", false); + let normalized = data.map(function map(d) { + let result = JSON.stringify(d); + + return result.replace("\n", "\\u000a"); + }); + + body = normalized.join("\n") + "\n"; + _(body); + } else { + response.setHeader("Content-Type", "application/json", false); + body = JSON.stringify({items: data}); + } + + this._log.info("Records: " + data.length); + response.setHeader("X-Num-Records", "" + data.length, false); + response.setHeader("X-Last-Modified", "" + this.timestamp, false); + response.setStatusLine(request.httpVersion, 200, "OK"); + response.bodyOutputStream.write(body, body.length); + }, + + postHandler: function postHandler(request, response) { + let options = this.parseOptions(request); + + if (!request.hasHeader("content-type")) { + this._log.info("No Content-Type request header!"); + throw HTTP_400; + } + + let inputStream = request.bodyInputStream; + let inputBody = CommonUtils.readBytesFromInputStream(inputStream); + let input = []; + + let inputMediaType = request.getHeader("content-type"); + if (inputMediaType == "application/json") { + try { + input = JSON.parse(inputBody); + } catch (ex) { + this._log.info("JSON parse error on input body!"); + throw HTTP_400; + } + + if (!Array.isArray(input)) { + this._log.info("Input JSON type not an array!"); + return sendMozSvcError(request, response, "8"); + } + } else if (inputMediaType == "application/newlines") { + for each (let line in inputBody.split("\n")) { + let json = line.replace("\\u000a", "\n"); + let record; + try { + record = JSON.parse(json); + } catch (ex) { + this._log.info("JSON parse error on line!"); + return sendMozSvcError(request, response, "8"); + } + + input.push(record); + } + } else { + this._log.info("Unknown media type: " + inputMediaType); + throw HTTP_415; + } + + let res = this.post(input, request.timestamp); + let body = JSON.stringify(res); + response.setHeader("Content-Type", "application/json", false); + this.timestamp = request.timestamp; + response.setHeader("X-Last-Modified", "" + this.timestamp, false); + + response.setStatusLine(request.httpVersion, "200", "OK"); + response.bodyOutputStream.write(body, body.length); + }, + + deleteHandler: function deleteHandler(request, response) { + this._log.debug("Invoking StorageServerCollection.DELETE."); + + let options = this.parseOptions(request); + + let deleted = this.delete(options); + response.deleted = deleted; + this.timestamp = request.timestamp; + + response.setStatusLine(request.httpVersion, 204, "No Content"); + }, + + handler: function handler() { + let self = this; + + return function(request, response) { + switch(request.method) { + case "GET": + return self.getHandler(request, response); + + case "POST": + return self.postHandler(request, response); + + case "DELETE": + return self.deleteHandler(request, response); + + } + + request.setHeader("Allow", "GET,POST,DELETE"); + response.setStatusLine(request.httpVersion, 405, "Method Not Allowed"); + }; + } +}; + + +//===========================================================================// +// httpd.js-based Storage server. // +//===========================================================================// + +/** + * In general, the preferred way of using StorageServer is to directly + * introspect it. Callbacks are available for operations which are hard to + * verify through introspection, such as deletions. + * + * One of the goals of this server is to provide enough hooks for test code to + * find out what it needs without monkeypatching. Use this object as your + * prototype, and override as appropriate. + */ +let StorageServerCallback = { + onCollectionDeleted: function onCollectionDeleted(user, collection) {}, + onItemDeleted: function onItemDeleted(user, collection, bsoID) {}, + + /** + * Called at the top of every request. + * + * Allows the test to inspect the request. Hooks should be careful not to + * modify or change state of the request or they may impact future processing. + */ + onRequest: function onRequest(request) {}, +}; + +/** + * Construct a new test Storage server. Takes a callback object (e.g., + * StorageServerCallback) as input. + */ +function StorageServer(callback) { + this.callback = callback || {__proto__: StorageServerCallback}; + this.server = new nsHttpServer(); + this.started = false; + this.users = {}; + this._log = Log4Moz.repository.getLogger(STORAGE_HTTP_LOGGER); + + // Install our own default handler. This allows us to mess around with the + // whole URL space. + let handler = this.server._handler; + handler._handleDefault = this.handleDefault.bind(this, handler); +} +StorageServer.prototype = { + DEFAULT_QUOTA: 1024 * 1024, // # bytes. + + port: 8080, + server: null, // HttpServer. + users: null, // Map of username => {collections, password}. + + /** + * If true, the server will allow any arbitrary user to be used. + * + * No authentication will be performed. Whatever user is detected from the + * URL or auth headers will be created (if needed) and used. + */ + allowAllUsers: false, + + /** + * Start the StorageServer's underlying HTTP server. + * + * @param port + * The numeric port on which to start. A falsy value implies the + * default (8080). + * @param cb + * A callback function (of no arguments) which is invoked after + * startup. + */ + start: function start(port, cb) { + if (this.started) { + this._log.warn("Warning: server already started on " + this.port); + return; + } + if (port) { + this.port = port; + } + try { + this.server.start(this.port); + this.started = true; + if (cb) { + cb(); + } + } catch (ex) { + _("=========================================="); + _("Got exception starting Storage HTTP server on port " + this.port); + _("Error: " + CommonUtils.exceptionStr(ex)); + _("Is there a process already listening on port " + this.port + "?"); + _("=========================================="); + do_throw(ex); + } + }, + + /** + * Start the server synchronously. + * + * @param port + * The numeric port on which to start. A falsy value implies the + * default (8080). + */ + startSynchronous: function startSynchronous(port) { + let cb = Async.makeSpinningCallback(); + this.start(port, cb); + cb.wait(); + }, + + /** + * Stop the StorageServer's HTTP server. + * + * @param cb + * A callback function. Invoked after the server has been stopped. + * + */ + stop: function stop(cb) { + if (!this.started) { + this._log.warn("StorageServer: Warning: server not running. Can't stop " + + "me now!"); + return; + } + + this.server.stop(cb); + this.started = false; + }, + + serverTime: function serverTime() { + return new_timestamp(); + }, + + /** + * Create a new user, complete with an empty set of collections. + * + * @param username + * The username to use. An Error will be thrown if a user by that name + * already exists. + * @param password + * A password string. + * + * @return a user object, as would be returned by server.user(username). + */ + registerUser: function registerUser(username, password) { + if (username in this.users) { + throw new Error("User already exists."); + } + + if (!isFinite(parseInt(username))) { + throw new Error("Usernames must be numeric: " + username); + } + + this._log.info("Registering new user with server: " + username); + this.users[username] = { + password: password, + collections: {}, + quota: this.DEFAULT_QUOTA, + }; + return this.user(username); + }, + + userExists: function userExists(username) { + return username in this.users; + }, + + getCollection: function getCollection(username, collection) { + return this.users[username].collections[collection]; + }, + + _insertCollection: function _insertCollection(collections, collection, bsos) { + let coll = new StorageServerCollection(bsos, true); + coll.collectionHandler = coll.handler(); + collections[collection] = coll; + return coll; + }, + + createCollection: function createCollection(username, collection, bsos) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let collections = this.users[username].collections; + if (collection in collections) { + throw new Error("Collection already exists."); + } + return this._insertCollection(collections, collection, bsos); + }, + + deleteCollection: function deleteCollection(username, collection) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + delete this.users[username].collections[collection]; + }, + + /** + * Accept a map like the following: + * { + * meta: {global: {version: 1, ...}}, + * crypto: {"keys": {}, foo: {bar: 2}}, + * bookmarks: {} + * } + * to cause collections and BSOs to be created. + * If a collection already exists, no error is raised. + * If a BSO already exists, it will be updated to the new contents. + */ + createContents: function createContents(username, collections) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for (let [id, contents] in Iterator(collections)) { + let coll = userCollections[id] || + this._insertCollection(userCollections, id); + for (let [bsoID, payload] in Iterator(contents)) { + coll.insert(bsoID, payload); + } + } + }, + + /** + * Insert a BSO in an existing collection. + */ + insertBSO: function insertBSO(username, collection, bso) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + if (!(collection in userCollections)) { + throw new Error("Unknown collection."); + } + userCollections[collection].insertBSO(bso); + return bso; + }, + + /** + * Delete all of the collections for the named user. + * + * @param username + * The name of the affected user. + */ + deleteCollections: function deleteCollections(username) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + let userCollections = this.users[username].collections; + for each (let [name, coll] in Iterator(userCollections)) { + this._log.trace("Bulk deleting " + name + " for " + username + "..."); + coll.delete({}); + } + this.users[username].collections = {}; + }, + + getQuota: function getQuota(username) { + if (!(username in this.users)) { + throw new Error("Unknown user."); + } + + return this.users[username].quota; + }, + + /** + * Obtain the newest timestamp of all collections for a user. + */ + newestCollectionTimestamp: function newestCollectionTimestamp(username) { + let collections = this.users[username].collections; + let newest = 0; + for each (let collection in collections) { + if (collection.timestamp > newest) { + newest = collection.timestamp; + } + } + + return newest; + }, + + /** + * Compute the object that is returned for an info/collections request. + */ + infoCollections: function infoCollections(username) { + let responseObject = {}; + let colls = this.users[username].collections; + for (let coll in colls) { + responseObject[coll] = colls[coll].timestamp; + } + this._log.trace("StorageServer: info/collections returning " + + JSON.stringify(responseObject)); + return responseObject; + }, + + infoCounts: function infoCounts(username) { + let data = {}; + let collections = this.users[username].collections; + for (let [k, v] in Iterator(collections)) { + let count = v.count(); + if (!count) { + continue; + } + + data[k] = count; + } + + return data; + }, + + infoUsage: function infoUsage(username) { + let data = {}; + let collections = this.users[username].collections; + for (let [k, v] in Iterator(collections)) { + data[k] = v.totalPayloadSize; + } + + return data; + }, + + infoQuota: function infoQuota(username) { + let total = 0; + for each (let value in this.infoUsage(username)) { + total += value; + } + + return { + quota: this.getQuota(username), + usage: total + }; + }, + + /** + * Simple accessor to allow collective binding and abbreviation of a bunch of + * methods. Yay! + * Use like this: + * + * let u = server.user("john"); + * u.collection("bookmarks").bso("abcdefg").payload; // Etc. + * + * @return a proxy for the user data stored in this server. + */ + user: function user(username) { + let collection = this.getCollection.bind(this, username); + let createCollection = this.createCollection.bind(this, username); + let createContents = this.createContents.bind(this, username); + let modified = function (collectionName) { + return collection(collectionName).timestamp; + } + let deleteCollections = this.deleteCollections.bind(this, username); + let quota = this.getQuota.bind(this, username); + return { + collection: collection, + createCollection: createCollection, + createContents: createContents, + deleteCollections: deleteCollections, + modified: modified, + quota: quota, + }; + }, + + _pruneExpired: function _pruneExpired() { + let now = Date.now(); + + for each (let user in this.users) { + for each (let collection in user.collections) { + for each (let bso in collection.bsos()) { + // ttl === 0 is a special case, so we can't simply !ttl. + if (typeof(bso.ttl) != "number") { + continue; + } + + let ttlDate = bso.modified + (bso.ttl * 1000); + if (ttlDate < now) { + this._log.info("Deleting BSO because TTL expired: " + bso.id); + bso.delete(); + } + } + } + } + }, + + /* + * Regular expressions for splitting up Storage request paths. + * Storage URLs are of the form: + * /$apipath/$version/$userid/$further + * where $further is usually: + * storage/$collection/$bso + * or + * storage/$collection + * or + * info/$op + * + * We assume for the sake of simplicity that $apipath is empty. + * + * N.B., we don't follow any kind of username spec here, because as far as I + * can tell there isn't one. See Bug 689671. Instead we follow the Python + * server code. + * + * Path: [all, version, first, rest] + * Storage: [all, collection?, id?] + */ + pathRE: /^\/([0-9]+(?:\.[0-9]+)?)(?:\/([0-9]+)\/([^\/]+)(?:\/(.+))?)?$/, + storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/, + + defaultHeaders: {}, + + /** + * HTTP response utility. + */ + respond: function respond(req, resp, code, status, body, headers, timestamp) { + resp.setStatusLine(req.httpVersion, code, status); + for each (let [header, value] in Iterator(headers || this.defaultHeaders)) { + resp.setHeader(header, value, false); + } + + if (timestamp) { + resp.setHeader("X-Timestamp", "" + timestamp, false); + } + + if (body) { + resp.bodyOutputStream.write(body, body.length); + } + }, + + /** + * This is invoked by the HttpServer. `this` is bound to the StorageServer; + * `handler` is the HttpServer's handler. + * + * TODO: need to use the correct Storage API response codes and errors here. + */ + handleDefault: function handleDefault(handler, req, resp) { + let timestamp = new_timestamp(); + try { + this._handleDefault(handler, req, resp, timestamp); + } catch (e) { + if (e instanceof HttpError) { + this.respond(req, resp, e.code, e.description, "", {}, timestamp); + } else { + this._log.warn(CommonUtils.exceptionStr(e)); + throw e; + } + } + }, + + _handleDefault: function _handleDefault(handler, req, resp, timestamp) { + let path = req.path; + if (req.queryString.length) { + path += "?" + req.queryString; + } + + this._log.debug("StorageServer: Handling request: " + req.method + " " + + path); + + if (this.callback.onRequest) { + this.callback.onRequest(req); + } + + // Prune expired records for all users at top of request. This is the + // easiest way to process TTLs since all requests go through here. + this._pruneExpired(); + + req.timestamp = timestamp; + resp.setHeader("X-Timestamp", "" + timestamp, false); + + let parts = this.pathRE.exec(req.path); + if (!parts) { + this._log.debug("StorageServer: Unexpected request: bad URL " + req.path); + throw HTTP_404; + } + + let [all, version, userPath, first, rest] = parts; + if (version != STORAGE_API_VERSION) { + this._log.debug("StorageServer: Unknown version."); + throw HTTP_404; + } + + let username; + + // By default, the server requires users to be authenticated. When a + // request arrives, the user must have been previously configured and + // the request must have authentication. In "allow all users" mode, we + // take the username from the URL, create the user on the fly, and don't + // perform any authentication. + if (!this.allowAllUsers) { + // Enforce authentication. + if (!req.hasHeader("authorization")) { + this.respond(req, resp, 401, "Authorization Required", "{}", { + "WWW-Authenticate": 'Basic realm="secret"' + }); + return; + } + + let ensureUserExists = function ensureUserExists(username) { + if (this.userExists(username)) { + return; + } + + this._log.info("StorageServer: Unknown user: " + username); + throw HTTP_401; + }.bind(this); + + let auth = req.getHeader("authorization"); + this._log.debug("Authorization: " + auth); + + if (auth.indexOf("Basic ") == 0) { + let decoded = CommonUtils.safeAtoB(auth.substr(6)); + this._log.debug("Decoded Basic Auth: " + decoded); + let [user, password] = decoded.split(":", 2); + + if (!password) { + this._log.debug("Malformed HTTP Basic Authorization header: " + auth); + throw HTTP_400; + } + + this._log.debug("Got HTTP Basic auth for user: " + user); + ensureUserExists(user); + username = user; + + if (this.users[user].password != password) { + this._log.debug("StorageServer: Provided password is not correct."); + throw HTTP_401; + } + // TODO support token auth. + } else { + this._log.debug("Unsupported HTTP authorization type: " + auth); + throw HTTP_500; + } + // All users mode. + } else { + // Auto create user with dummy password. + if (!this.userExists(userPath)) { + this.registerUser(userPath, "DUMMY-PASSWORD-*&%#"); + } + + username = userPath; + } + + // Hand off to the appropriate handler for this path component. + if (first in this.toplevelHandlers) { + let handler = this.toplevelHandlers[first]; + try { + return handler.call(this, handler, req, resp, version, username, rest); + } catch (ex) { + this._log.warn("Got exception during request: " + + CommonUtils.exceptionStr(ex)); + throw ex; + } + } + this._log.debug("StorageServer: Unknown top-level " + first); + throw HTTP_404; + }, + + /** + * Collection of the handler methods we use for top-level path components. + */ + toplevelHandlers: { + "storage": function handleStorage(handler, req, resp, version, username, + rest) { + let respond = this.respond.bind(this, req, resp); + if (!rest || !rest.length) { + this._log.debug("StorageServer: top-level storage " + + req.method + " request."); + + if (req.method != "DELETE") { + respond(405, "Method Not Allowed", null, {"Allow": "DELETE"}); + return; + } + + this.user(username).deleteCollections(); + + respond(204, "No Content"); + return; + } + + let match = this.storageRE.exec(rest); + if (!match) { + this._log.warn("StorageServer: Unknown storage operation " + rest); + throw HTTP_404; + } + let [all, collection, bsoID] = match; + let coll = this.getCollection(username, collection); + let collectionExisted = !!coll; + + switch (req.method) { + case "GET": + // Tried to GET on a collection that doesn't exist. + if (!coll) { + respond(404, "Not Found"); + return; + } + + // No BSO URL parameter goes to collection handler. + if (!bsoID) { + return coll.collectionHandler(req, resp); + } + + // Handle non-existent BSO. + let bso = coll.bso(bsoID); + if (!bso) { + respond(404, "Not Found"); + return; + } + + // Proxy to BSO handler. + return bso.getHandler(req, resp); + + case "DELETE": + // Collection doesn't exist. + if (!coll) { + respond(404, "Not Found"); + return; + } + + // Deleting a specific BSO. + if (bsoID) { + let bso = coll.bso(bsoID); + + // BSO does not exist on the server. Nothing to do. + if (!bso) { + respond(404, "Not Found"); + return; + } + + if (req.hasHeader("x-if-unmodified-since")) { + let modified = parseInt(req.getHeader("x-if-unmodified-since")); + CommonUtils.ensureMillisecondsTimestamp(modified); + + if (bso.modified > modified) { + respond(412, "Precondition Failed"); + return; + } + } + + bso.delete(); + coll.timestamp = req.timestamp; + this.callback.onItemDeleted(username, collection, bsoID); + respond(204, "No Content"); + return; + } + + // Proxy to collection handler. + coll.collectionHandler(req, resp); + + // Spot if this is a DELETE for some IDs, and don't blow away the + // whole collection! + // + // We already handled deleting the BSOs by invoking the deleted + // collection's handler. However, in the case of + // + // DELETE storage/foobar + // + // we also need to remove foobar from the collections map. This + // clause tries to differentiate the above request from + // + // DELETE storage/foobar?ids=foo,baz + // + // and do the right thing. + // TODO: less hacky method. + if (-1 == req.queryString.indexOf("ids=")) { + // When you delete the entire collection, we drop it. + this._log.debug("Deleting entire collection."); + delete this.users[username].collections[collection]; + this.callback.onCollectionDeleted(username, collection); + } + + // Notify of item deletion. + let deleted = resp.deleted || []; + for (let i = 0; i < deleted.length; ++i) { + this.callback.onItemDeleted(username, collection, deleted[i]); + } + return; + + case "POST": + case "PUT": + // Auto-create collection if it doesn't exist. + if (!coll) { + coll = this.createCollection(username, collection); + } + + try { + if (bsoID) { + let bso = coll.bso(bsoID); + if (!bso) { + this._log.trace("StorageServer: creating BSO " + collection + + "/" + bsoID); + try { + bso = coll.insert(bsoID); + } catch (ex) { + return sendMozSvcError(req, resp, "8"); + } + } + + bso.putHandler(req, resp); + + coll.timestamp = resp.newModified; + return resp; + } + + return coll.collectionHandler(req, resp); + } catch (ex) { + if (ex instanceof HttpError) { + if (!collectionExisted) { + this.deleteCollection(username, collection); + } + } + + throw ex; + } + + default: + throw new Error("Request method " + req.method + " not implemented."); + } + }, + + "info": function handleInfo(handler, req, resp, version, username, rest) { + switch (rest) { + case "collections": + return this.handleInfoCollections(req, resp, username); + + case "collection_counts": + return this.handleInfoCounts(req, resp, username); + + case "collection_usage": + return this.handleInfoUsage(req, resp, username); + + case "quota": + return this.handleInfoQuota(req, resp, username); + + default: + this._log.warn("StorageServer: Unknown info operation " + rest); + throw HTTP_404; + } + } + }, + + handleInfoConditional: function handleInfoConditional(request, response, + user) { + if (!request.hasHeader("x-if-modified-since")) { + return false; + } + + let requestModified = request.getHeader("x-if-modified-since"); + requestModified = parseInt(requestModified, 10); + + let serverModified = this.newestCollectionTimestamp(user); + + this._log.info("Server mtime: " + serverModified + "; Client modified: " + + requestModified); + if (serverModified > requestModified) { + return false; + } + + this.respond(request, response, 304, "Not Modified", null, { + "X-Last-Modified": "" + serverModified + }); + + return true; + }, + + handleInfoCollections: function handleInfoCollections(request, response, + user) { + if (this.handleInfoConditional(request, response, user)) { + return; + } + + let info = this.infoCollections(user); + let body = JSON.stringify(info); + this.respond(request, response, 200, "OK", body, { + "Content-Type": "application/json", + "X-Last-Modified": "" + this.newestCollectionTimestamp(user), + }); + }, + + handleInfoCounts: function handleInfoCounts(request, response, user) { + if (this.handleInfoConditional(request, response, user)) { + return; + } + + let counts = this.infoCounts(user); + let body = JSON.stringify(counts); + + this.respond(request, response, 200, "OK", body, { + "Content-Type": "application/json", + "X-Last-Modified": "" + this.newestCollectionTimestamp(user), + }); + }, + + handleInfoUsage: function handleInfoUsage(request, response, user) { + if (this.handleInfoConditional(request, response, user)) { + return; + } + + let body = JSON.stringify(this.infoUsage(user)); + this.respond(request, response, 200, "OK", body, { + "Content-Type": "application/json", + "X-Last-Modified": "" + this.newestCollectionTimestamp(user), + }); + }, + + handleInfoQuota: function handleInfoQuota(request, response, user) { + if (this.handleInfoConditional(request, response, user)) { + return; + } + + let body = JSON.stringify(this.infoQuota(user)); + this.respond(request, response, 200, "OK", body, { + "Content-Type": "application/json", + "X-Last-Modified": "" + this.newestCollectionTimestamp(user), + }); + }, +}; + +/** + * Helper to create a storage server for a set of users. + * + * Each user is specified by a map of username to password. + */ +function storageServerForUsers(users, contents, callback) { + let server = new StorageServer(callback); + for (let [user, pass] in Iterator(users)) { + server.registerUser(user, pass); + server.createContents(user, contents); + } + server.start(); + return server; +} diff --git a/services/common/tests/unit/test_aitc_server.js b/services/common/tests/unit/test_aitc_server.js new file mode 100644 index 000000000000..d5fd6d57fcea --- /dev/null +++ b/services/common/tests/unit/test_aitc_server.js @@ -0,0 +1,169 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://services-common/rest.js"); +Cu.import("resource://services-common/utils.js"); + +// TODO enable once build infra supports testing modules. +//Cu.import("resource://testing-common/services-common/aitcserver.js"); + +function run_test() { + initTestLogging("Trace"); + run_next_test(); +} + +function get_aitc_server() { + let server = new AITCServer10Server(); + server.start(get_server_port()); + + return server; +} + +function get_server_with_user(username) { + let server = get_aitc_server(); + server.createUser(username); + + return server; +} + +add_test(function test_origin_conversion() { + let mapping = { + "www.mozilla.org": "xSMmiFEpg4b4TRtzJZd6Mvy4hGc", + "foo": "C-7Hteo_D9vJXQ3UfzxbwnXaijM", + }; + + for (let k in mapping) { + do_check_eq(AITCServer10User.prototype.originToID(k), mapping[k]); + } + + run_next_test(); +}); + +add_test(function test_empty_user() { + _("Ensure user instances can be created."); + + let user = new AITCServer10User(); + + let apps = user.getApps(); + do_check_eq([app for (app in apps)].length, 0); + do_check_false(user.hasAppID("foobar")); + + run_next_test(); +}); + +add_test(function test_user_add_app() { + _("Ensure apps can be added to users."); + + let user = new AITCServer10User(); + let threw = false; + try { + user.addApp({}); + } catch (ex) { + threw = true; + } finally { + do_check_true(threw); + threw = false; + } + + run_next_test(); +}); + +add_test(function test_server_run() { + _("Ensure server can be started properly."); + + let server = new AITCServer10Server(); + server.start(get_server_port()); + + server.stop(run_next_test); +}); + +add_test(function test_create_user() { + _("Ensure users can be created properly."); + + let server = get_aitc_server(); + + let u1 = server.createUser("123"); + do_check_true(u1 instanceof AITCServer10User); + + let u2 = server.getUser("123"); + do_check_eq(u1, u2); + + server.stop(run_next_test); +}); + +add_test(function test_empty_server_404() { + _("Ensure empty server returns 404."); + + let server = get_aitc_server(); + let request = new RESTRequest(server.url + "123/"); + request.get(function onComplete(error) { + do_check_eq(this.response.status, 404); + + let request = new RESTRequest(server.url + "123/apps/"); + request.get(function onComplete(error) { + do_check_eq(this.response.status, 404); + + server.stop(run_next_test); + }); + }); +}); + +add_test(function test_empty_user_apps() { + _("Ensure apps request for empty user has appropriate content."); + + const username = "123"; + + let server = get_server_with_user(username); + let request = new RESTRequest(server.url + username + "/apps/"); + _("Performing request..."); + request.get(function onComplete(error) { + _("Got response"); + do_check_eq(error, null); + + do_check_eq(200, this.response.status); + let headers = this.response.headers; + do_check_true("content-type" in headers); + do_check_eq(headers["content-type"], "application/json"); + do_check_true("x-timestamp" in headers); + + let body = this.response.body; + let parsed = JSON.parse(body); + do_check_attribute_count(parsed, 1); + do_check_true("apps" in parsed); + do_check_true(Array.isArray(parsed.apps)); + do_check_eq(parsed.apps.length, 0); + + server.stop(run_next_test); + }); +}); + +add_test(function test_invalid_request_method() { + _("Ensure HTTP 405 works as expected."); + + const username = "12345"; + + let server = get_server_with_user(username); + let request = new RESTRequest(server.url + username + "/apps/foobar"); + request.dispatch("SILLY", null, function onComplete(error) { + do_check_eq(error, null); + do_check_eq(this.response.status, 405); + + let headers = this.response.headers; + do_check_true("accept" in headers); + + let allowed = new Set(); + + for (let method of headers["accept"].split(",")) { + allowed.add(method); + } + + do_check_eq(allowed.size(), 3); + for (let method of ["GET", "PUT", "DELETE"]) { + do_check_true(allowed.has(method)); + } + + run_next_test(); + }); +}); diff --git a/services/common/tests/unit/test_load_modules.js b/services/common/tests/unit/test_load_modules.js index 7685126efaf1..5885d7f1d280 100644 --- a/services/common/tests/unit/test_load_modules.js +++ b/services/common/tests/unit/test_load_modules.js @@ -11,9 +11,22 @@ const modules = [ "utils.js", ]; +const test_modules = [ + "aitcserver.js", + "storageserver.js", +]; + function run_test() { for each (let m in modules) { let resource = "resource://services-common/" + m; Components.utils.import(resource, {}); } + + // TODO enable once build infra supports testing modules. + /* + for each (let m in test_modules) { + let resource = "resource://testing-common/services-common/" + m; + Components.utils.import(resource, {}); + } + */ } diff --git a/services/common/tests/unit/test_storage_server.js b/services/common/tests/unit/test_storage_server.js new file mode 100644 index 000000000000..021f3cb89d66 --- /dev/null +++ b/services/common/tests/unit/test_storage_server.js @@ -0,0 +1,517 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Cu.import("resource://services-common/async.js"); +Cu.import("resource://services-common/rest.js"); +Cu.import("resource://services-common/utils.js"); +// TODO enable once build infra supports testing modules. +//Cu.import("resource://testing-common/services-common/storageserver.js"); + +const PORT = 8080; +const DEFAULT_USER = "123"; +const DEFAULT_PASSWORD = "password"; + +/** + * Helper function to prepare a RESTRequest against the server. + */ +function localRequest(path, user=DEFAULT_USER, password=DEFAULT_PASSWORD) { + _("localRequest: " + path); + let url = "http://127.0.0.1:" + PORT + path; + _("url: " + url); + let req = new RESTRequest(url); + + let header = basic_auth_header(user, password); + req.setHeader("Authorization", header); + req.setHeader("Accept", "application/json"); + + return req; +} + +/** + * Helper function to validate an HTTP response from the server. + */ +function validateResponse(response) { + do_check_true("x-timestamp" in response.headers); + + if ("content-length" in response.headers) { + let cl = parseInt(response.headers["content-length"]); + + if (cl != 0) { + do_check_true("content-type" in response.headers); + do_check_eq("application/json", response.headers["content-type"]); + } + } + + if (response.status == 204 || response.status == 304) { + do_check_false("content-type" in response.headers); + + if ("content-length" in response.headers) { + do_check_eq(response.headers["content-length"], "0"); + } + } + + if (response.status == 405) { + do_check_true("allow" in response.headers); + } +} + +/** + * Helper function to synchronously wait for a response and validate it. + */ +function waitAndValidateResponse(cb, request) { + let error = cb.wait(); + + if (!error) { + validateResponse(request.response); + } + + return error; +} + +/** + * Helper function to synchronously perform a GET request. + * + * @return Error instance or null if no error. + */ +function doGetRequest(request) { + let cb = Async.makeSpinningCallback(); + request.get(cb); + + return waitAndValidateResponse(cb, request); +} + +/** + * Helper function to synchronously perform a PUT request. + * + * @return Error instance or null if no error. + */ +function doPutRequest(request, data) { + let cb = Async.makeSpinningCallback(); + request.put(data, cb); + + return waitAndValidateResponse(cb, request); +} + +/** + * Helper function to synchronously perform a DELETE request. + * + * @return Error or null if no error was encountered. + */ +function doDeleteRequest(request) { + let cb = Async.makeSpinningCallback(); + request.delete(cb); + + return waitAndValidateResponse(cb, request); +} + +function run_test() { + Log4Moz.repository.getLogger("Services.Common.Test.StorageServer").level = + Log4Moz.Level.Trace; + initTestLogging(); + + run_next_test(); +} + +add_test(function test_creation() { + _("Ensure a simple server can be created."); + + // Explicit callback for this one. + let server = new StorageServer({ + __proto__: StorageServerCallback, + }); + do_check_true(!!server); + + server.start(PORT, function () { + _("Started on " + server.port); + do_check_eq(server.port, PORT); + server.stop(run_next_test); + }); +}); + +add_test(function test_synchronous_start() { + _("Ensure starting using startSynchronous works."); + + let server = new StorageServer(); + server.startSynchronous(PORT); + server.stop(run_next_test); +}); + +add_test(function test_url_parsing() { + _("Ensure server parses URLs properly."); + + let server = new StorageServer(); + + // Check that we can parse a BSO URI. + let parts = server.pathRE.exec("/2.0/12345/storage/crypto/keys"); + let [all, version, user, first, rest] = parts; + do_check_eq(all, "/2.0/12345/storage/crypto/keys"); + do_check_eq(version, "2.0"); + do_check_eq(user, "12345"); + do_check_eq(first, "storage"); + do_check_eq(rest, "crypto/keys"); + do_check_eq(null, server.pathRE.exec("/nothing/else")); + + // Check that we can parse a collection URI. + parts = server.pathRE.exec("/2.0/123/storage/crypto"); + let [all, version, user, first, rest] = parts; + do_check_eq(all, "/2.0/123/storage/crypto"); + do_check_eq(version, "2.0"); + do_check_eq(user, "123"); + do_check_eq(first, "storage"); + do_check_eq(rest, "crypto"); + + // We don't allow trailing slash on storage URI. + parts = server.pathRE.exec("/2.0/1234/storage/"); + do_check_eq(parts, undefined); + + // storage alone is a valid request. + parts = server.pathRE.exec("/2.0/123456/storage"); + let [all, version, user, first, rest] = parts; + do_check_eq(all, "/2.0/123456/storage"); + do_check_eq(version, "2.0"); + do_check_eq(user, "123456"); + do_check_eq(first, "storage"); + do_check_eq(rest, undefined); + + parts = server.storageRE.exec("storage"); + let [all, storage, collection, id] = parts; + do_check_eq(all, "storage"); + do_check_eq(collection, undefined); + + run_next_test(); +}); + +add_test(function test_basic_http() { + let server = new StorageServer(); + server.registerUser("345", "password"); + do_check_true(server.userExists("345")); + server.startSynchronous(PORT); + + _("Started on " + server.port); + let req = localRequest("/2.0/storage/crypto/keys"); + _("req is " + req); + req.get(function (err) { + do_check_eq(null, err); + server.stop(run_next_test); + }); +}); + +add_test(function test_info_collections() { + let server = new StorageServer(); + server.registerUser("123", "password"); + server.startSynchronous(PORT); + + let path = "/2.0/123/info/collections"; + + _("info/collections on empty server should be empty object."); + let request = localRequest(path, "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 200); + do_check_eq(request.response.body, "{}"); + + _("Creating an empty collection should result in collection appearing."); + let coll = server.createCollection("123", "col1"); + let request = localRequest(path, "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 200); + let info = JSON.parse(request.response.body); + do_check_attribute_count(info, 1); + do_check_true("col1" in info); + do_check_eq(info.col1, coll.timestamp); + + server.stop(run_next_test); +}); + +add_test(function test_bso_get_existing() { + _("Ensure that BSO retrieval works."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.createContents("123", { + test: {"bso": {"foo": "bar"}} + }); + server.startSynchronous(PORT); + + let coll = server.user("123").collection("test"); + + let request = localRequest("/2.0/123/storage/test/bso", "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 200); + do_check_eq(request.response.headers["content-type"], "application/json"); + let bso = JSON.parse(request.response.body); + do_check_attribute_count(bso, 3); + do_check_eq(bso.id, "bso"); + do_check_eq(bso.modified, coll.bso("bso").modified); + let payload = JSON.parse(bso.payload); + do_check_attribute_count(payload, 1); + do_check_eq(payload.foo, "bar"); + + server.stop(run_next_test); +}); + +add_test(function test_bso_404() { + _("Ensure the server responds with a 404 if a BSO does not exist."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.createContents("123", { + test: {} + }); + server.startSynchronous(PORT); + + let request = localRequest("/2.0/123/storage/test/foo"); + let error = doGetRequest(request); + do_check_eq(error, null); + + do_check_eq(request.response.status, 404); + do_check_false("content-type" in request.response.headers); + + server.stop(run_next_test); +}); + +add_test(function test_bso_if_modified_since_304() { + _("Ensure the server responds properly to X-If-Modified-Since for BSOs."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.createContents("123", { + test: {bso: {foo: "bar"}} + }); + server.startSynchronous(PORT); + + let coll = server.user("123").collection("test"); + do_check_neq(coll, null); + + // Rewind clock just in case. + coll.timestamp -= 10000; + coll.bso("bso").modified -= 10000; + + let request = localRequest("/2.0/123/storage/test/bso", "123", "password"); + request.setHeader("X-If-Modified-Since", "" + server.serverTime()); + let error = doGetRequest(request); + do_check_eq(null, error); + + do_check_eq(request.response.status, 304); + do_check_false("content-type" in request.response.headers); + + let request = localRequest("/2.0/123/storage/test/bso", "123", "password"); + request.setHeader("X-If-Modified-Since", "" + (server.serverTime() - 20000)); + let error = doGetRequest(request); + do_check_eq(null, error); + do_check_eq(request.response.status, 200); + do_check_eq(request.response.headers["content-type"], "application/json"); + + server.stop(run_next_test); +}); + +add_test(function test_bso_if_unmodified_since() { + _("Ensure X-If-Unmodified-Since works properly on BSOs."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.createContents("123", { + test: {bso: {foo: "bar"}} + }); + server.startSynchronous(PORT); + + let coll = server.user("123").collection("test"); + do_check_neq(coll, null); + + let time = coll.timestamp; + + _("Ensure we get a 412 for specified times older than server time."); + let request = localRequest("/2.0/123/storage/test/bso", "123", "password"); + request.setHeader("X-If-Unmodified-Since", time - 5000); + request.setHeader("Content-Type", "application/json"); + let payload = JSON.stringify({"payload": "foobar"}); + let error = doPutRequest(request, payload); + do_check_eq(null, error); + do_check_eq(request.response.status, 412); + + _("Ensure we get a 204 if update goes through."); + let request = localRequest("/2.0/123/storage/test/bso", "123", "password"); + request.setHeader("Content-Type", "application/json"); + request.setHeader("X-If-Unmodified-Since", time + 1); + let error = doPutRequest(request, payload); + do_check_eq(null, error); + do_check_eq(request.response.status, 204); + do_check_true(coll.timestamp > time); + + // Not sure why a client would send X-If-Unmodified-Since if a BSO doesn't + // exist. But, why not test it? + _("Ensure we get a 201 if creation goes through."); + let request = localRequest("/2.0/123/storage/test/none", "123", "password"); + request.setHeader("Content-Type", "application/json"); + request.setHeader("X-If-Unmodified-Since", time); + let error = doPutRequest(request, payload); + do_check_eq(null, error); + do_check_eq(request.response.status, 201); + + server.stop(run_next_test); +}); + +add_test(function test_bso_delete_not_exist() { + _("Ensure server behaves properly when deleting a BSO that does not exist."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.user("123").createCollection("empty"); + server.startSynchronous(PORT); + + server.callback.onItemDeleted = function onItemDeleted(username, collection, + id) { + do_throw("onItemDeleted should not have been called."); + }; + + let request = localRequest("/2.0/123/storage/empty/nada", "123", "password"); + let error = doDeleteRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 404); + do_check_false("content-type" in request.response.headers); + + server.stop(run_next_test); +}); + +add_test(function test_bso_delete_exists() { + _("Ensure proper semantics when deleting a BSO that exists."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.startSynchronous(PORT); + + let coll = server.user("123").createCollection("test"); + let bso = coll.insert("myid", {foo: "bar"}); + let timestamp = coll.timestamp; + + server.callback.onItemDeleted = function onDeleted(username, collection, id) { + delete server.callback.onItemDeleted; + do_check_eq(username, "123"); + do_check_eq(collection, "test"); + do_check_eq(id, "myid"); + }; + + let request = localRequest("/2.0/123/storage/test/myid", "123", "password"); + let error = doDeleteRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 204); + do_check_eq(coll.bsos().length, 0); + do_check_true(coll.timestamp > timestamp); + + _("On next request the BSO should not exist."); + let request = localRequest("/2.0/123/storage/test/myid", "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 404); + + server.stop(run_next_test); +}); + +add_test(function test_bso_delete_unmodified() { + _("Ensure X-If-Unmodified-Since works when deleting BSOs."); + + let server = new StorageServer(); + server.startSynchronous(PORT); + server.registerUser("123", "password"); + let coll = server.user("123").createCollection("test"); + let bso = coll.insert("myid", {foo: "bar"}); + + let modified = bso.modified; + + _("Issuing a DELETE with an older time should fail."); + let path = "/2.0/123/storage/test/myid"; + let request = localRequest(path, "123", "password"); + request.setHeader("X-If-Unmodified-Since", modified - 1000); + let error = doDeleteRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 412); + do_check_false("content-type" in request.response.headers); + do_check_neq(coll.bso("myid"), null); + + _("Issuing a DELETE with a newer time should work."); + let request = localRequest(path, "123", "password"); + request.setHeader("X-If-Unmodified-Since", modified + 1000); + let error = doDeleteRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 204); + do_check_true(coll.bso("myid").deleted); + + server.stop(run_next_test); +}); + +add_test(function test_missing_collection_404() { + _("Ensure a missing collection returns a 404."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.startSynchronous(PORT); + + let request = localRequest("/2.0/123/storage/none", "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 404); + do_check_false("content-type" in request.response.headers); + + server.stop(run_next_test); +}); + +add_test(function test_get_storage_405() { + _("Ensure that a GET on /storage results in a 405."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.startSynchronous(PORT); + + let request = localRequest("/2.0/123/storage", "123", "password"); + let error = doGetRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 405); + do_check_eq(request.response.headers["allow"], "DELETE"); + + server.stop(run_next_test); +}); + +add_test(function test_delete_storage() { + _("Ensure that deleting all of storage works."); + + let server = new StorageServer(); + server.registerUser("123", "password"); + server.createContents("123", { + foo: {a: {foo: "bar"}, b: {bar: "foo"}}, + baz: {c: {bob: "law"}, blah: {law: "blog"}} + }); + + server.startSynchronous(PORT); + + let request = localRequest("/2.0/123/storage", "123", "password"); + let error = doDeleteRequest(request); + do_check_eq(error, null); + do_check_eq(request.response.status, 204); + do_check_attribute_count(server.users["123"].collections, 0); + + server.stop(run_next_test); +}); + +add_test(function test_x_num_records() { + let server = new StorageServer(); + server.registerUser("123", "password"); + + server.createContents("123", { + crypto: {foos: {foo: "bar"}, + bars: {foo: "baz"}} + }); + server.startSynchronous(PORT); + let bso = localRequest("/2.0/123/storage/crypto/foos"); + bso.get(function (err) { + // BSO fetches don't have one. + do_check_false("x-num-records" in this.response.headers); + let col = localRequest("/2.0/123/storage/crypto"); + col.get(function (err) { + // Collection fetches do. + do_check_eq(this.response.headers["x-num-records"], "2"); + server.stop(run_next_test); + }); + }); +}); diff --git a/services/common/tests/unit/test_utils_encodeBase64URL.js b/services/common/tests/unit/test_utils_encodeBase64URL.js new file mode 100644 index 000000000000..fd0e559f384e --- /dev/null +++ b/services/common/tests/unit/test_utils_encodeBase64URL.js @@ -0,0 +1,27 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +Cu.import("resource://services-common/utils.js"); + +function run_test() { + run_next_test(); +} + +add_test(function test_simple() { + let expected = { + hello: "aGVsbG8=", + "<>?": "PD4_", + }; + + for (let [k, v] in Iterator(expected)) { + do_check_eq(CommonUtils.encodeBase64URL(k), v); + } + + run_next_test(); +}); + +add_test(function test_no_padding() { + do_check_eq(CommonUtils.encodeBase64URL("hello", false), "aGVsbG8"); + + run_next_test(); +}); diff --git a/services/common/tests/unit/test_utils_json.js b/services/common/tests/unit/test_utils_json.js index d255fda3e420..6ba0f403a218 100644 --- a/services/common/tests/unit/test_utils_json.js +++ b/services/common/tests/unit/test_utils_json.js @@ -11,7 +11,9 @@ function run_test() { add_test(function test_roundtrip() { _("Do a simple write of an array to json and read"); - CommonUtils.jsonSave("foo", {}, ["v1", "v2"], ensureThrows(function() { + CommonUtils.jsonSave("foo", {}, ["v1", "v2"], ensureThrows(function(error) { + do_check_eq(error, null); + CommonUtils.jsonLoad("foo", {}, ensureThrows(function(val) { let foo = val; do_check_eq(typeof foo, "object"); @@ -25,7 +27,9 @@ add_test(function test_roundtrip() { add_test(function test_string() { _("Try saving simple strings"); - CommonUtils.jsonSave("str", {}, "hi", ensureThrows(function() { + CommonUtils.jsonSave("str", {}, "hi", ensureThrows(function(error) { + do_check_eq(error, null); + CommonUtils.jsonLoad("str", {}, ensureThrows(function(val) { let str = val; do_check_eq(typeof str, "string"); @@ -39,7 +43,9 @@ add_test(function test_string() { add_test(function test_number() { _("Try saving a number"); - CommonUtils.jsonSave("num", {}, 42, ensureThrows(function() { + CommonUtils.jsonSave("num", {}, 42, ensureThrows(function(error) { + do_check_eq(error, null); + CommonUtils.jsonLoad("num", {}, ensureThrows(function(val) { let num = val; do_check_eq(typeof num, "number"); diff --git a/services/common/tests/unit/xpcshell.ini b/services/common/tests/unit/xpcshell.ini index 52bcb7a723bd..ea625be45da1 100644 --- a/services/common/tests/unit/xpcshell.ini +++ b/services/common/tests/unit/xpcshell.ini @@ -1,5 +1,5 @@ [DEFAULT] -head = head_global.js head_helpers.js +head = head_global.js head_helpers.js head_http.js aitcserver.js storageserver.js tail = # Test load modules first so syntax failures are caught early. @@ -7,12 +7,14 @@ tail = [test_utils_atob.js] [test_utils_encodeBase32.js] +[test_utils_encodeBase64URL.js] [test_utils_json.js] [test_utils_makeURI.js] [test_utils_namedTimer.js] [test_utils_stackTrace.js] [test_utils_utf8.js] +[test_aitc_server.js] [test_async_chain.js] [test_async_querySpinningly.js] [test_log4moz.js] @@ -21,3 +23,5 @@ tail = [test_restrequest.js] [test_tokenauthenticatedrequest.js] [test_tokenserverclient.js] + +[test_storage_server.js] diff --git a/services/common/utils.js b/services/common/utils.js index 26d88b4c4d65..dee816cad3f8 100644 --- a/services/common/utils.js +++ b/services/common/utils.js @@ -59,11 +59,23 @@ let CommonUtils = { /** * Encode byte string as base64URL (RFC 4648). + * + * @param bytes + * (string) Raw byte string to encode. + * @param pad + * (bool) Whether to include padding characters (=). Defaults + * to true for historical reasons. */ - encodeBase64URL: function encodeBase64URL(bytes) { - return btoa(bytes).replace("+", "-", "g").replace("/", "_", "g"); + encodeBase64URL: function encodeBase64URL(bytes, pad=true) { + let s = btoa(bytes).replace("+", "-", "g").replace("/", "_", "g"); + + if (!pad) { + s = s.replace("=", ""); + } + + return s; }, - + /** * Create a nsIURI instance from a string. */ @@ -94,6 +106,20 @@ let CommonUtils = { Services.tm.currentThread.dispatch(callback, Ci.nsIThread.DISPATCH_NORMAL); }, + /** + * Spin the event loop and return once the next tick is executed. + * + * This is an evil function and should not be used in production code. It + * exists in this module for ease-of-use. + */ + waitForNextTick: function waitForNextTick() { + let cb = Async.makeSyncCallback(); + this.nextTick(cb); + Async.waitForSyncCallback(cb); + + return; + }, + /** * Return a timer that is scheduled to call the callback after waiting the * provided time or as soon as possible. The timer will be set as a property @@ -364,6 +390,9 @@ let CommonUtils = { * function, it'll be used as the object to make a json string. * @param callback * Function called when the write has been performed. Optional. + * The first argument will be a Components.results error + * constant on error or null if no error was encountered (and + * the file saved successfully). */ jsonSave: function jsonSave(filePath, that, obj, callback) { let path = filePath + ".json"; @@ -379,10 +408,63 @@ let CommonUtils = { let is = this._utf8Converter.convertToInputStream(out); NetUtil.asyncCopy(is, fos, function (result) { if (typeof callback == "function") { - callback.call(that); + let error = (result == Cr.NS_OK) ? null : result; + callback.call(that, error); } }); }, + + /** + * Ensure that the specified value is defined in integer milliseconds since + * UNIX epoch. + * + * This throws an error if the value is not an integer, is negative, or looks + * like seconds, not milliseconds. + * + * If the value is null or 0, no exception is raised. + * + * @param value + * Value to validate. + */ + ensureMillisecondsTimestamp: function ensureMillisecondsTimestamp(value) { + if (!value) { + return; + } + + if (value < 0) { + throw new Error("Timestamp value is negative: " + value); + } + + // Catch what looks like seconds, not milliseconds. + if (value < 10000000000) { + throw new Error("Timestamp appears to be in seconds: " + value); + } + + if (Math.floor(value) != Math.ceil(value)) { + throw new Error("Timestamp value is not an integer: " + value); + } + }, + + /** + * Read bytes from an nsIInputStream into a string. + * + * @param stream + * (nsIInputStream) Stream to read from. + * @param count + * (number) Integer number of bytes to read. If not defined, or + * 0, all available input is read. + */ + readBytesFromInputStream: function readBytesFromInputStream(stream, count) { + let BinaryInputStream = Components.Constructor( + "@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream"); + if (!count) { + count = stream.available(); + } + + return new BinaryInputStream(stream).readBytes(count); + }, }; XPCOMUtils.defineLazyGetter(CommonUtils, "_utf8Converter", function() { diff --git a/testing/xpcshell/xpcshell.ini b/testing/xpcshell/xpcshell.ini index a2ded891bda0..927f02a9497f 100644 --- a/testing/xpcshell/xpcshell.ini +++ b/testing/xpcshell/xpcshell.ini @@ -73,11 +73,12 @@ skip-if = os == "android" [include:content/base/test/unit/xpcshell.ini] [include:content/test/unit/xpcshell.ini] [include:toolkit/components/url-classifier/tests/unit/xpcshell.ini] +[include:services/aitc/tests/unit/xpcshell.ini] [include:services/common/tests/unit/xpcshell.ini] [include:services/crypto/tests/unit/xpcshell.ini] [include:services/crypto/components/tests/unit/xpcshell.ini] [include:services/sync/tests/unit/xpcshell.ini] -# Bug 676978: tests hang on Android +# Bug 676978: tests hang on Android skip-if = os == "android" [include:browser/components/dirprovider/tests/unit/xpcshell.ini] [include:browser/components/downloads/test/unit/xpcshell.ini] diff --git a/uriloader/prefetch/OfflineCacheUpdateChild.cpp b/uriloader/prefetch/OfflineCacheUpdateChild.cpp index 5cf2f7e3748f..e3e670a152df 100644 --- a/uriloader/prefetch/OfflineCacheUpdateChild.cpp +++ b/uriloader/prefetch/OfflineCacheUpdateChild.cpp @@ -177,8 +177,9 @@ OfflineCacheUpdateChild::AssociateDocument(nsIDOMDocument *aDocument, NS_IMETHODIMP OfflineCacheUpdateChild::Init(nsIURI *aManifestURI, - nsIURI *aDocumentURI, - nsIDOMDocument *aDocument) + nsIURI *aDocumentURI, + nsIDOMDocument *aDocument, + nsILocalFile *aCustomProfileDir) { nsresult rv; @@ -188,6 +189,11 @@ OfflineCacheUpdateChild::Init(nsIURI *aManifestURI, if (!service) return NS_ERROR_FAILURE; + if (aCustomProfileDir) { + NS_ERROR("Custom Offline Cache Update not supported on child process"); + return NS_ERROR_NOT_IMPLEMENTED; + } + LOG(("OfflineCacheUpdateChild::Init [%p]", this)); // Only http and https applications are supported. diff --git a/uriloader/prefetch/OfflineCacheUpdateGlue.cpp b/uriloader/prefetch/OfflineCacheUpdateGlue.cpp index f089b40c9426..49becdf23512 100644 --- a/uriloader/prefetch/OfflineCacheUpdateGlue.cpp +++ b/uriloader/prefetch/OfflineCacheUpdateGlue.cpp @@ -91,7 +91,8 @@ OfflineCacheUpdateGlue::Schedule() NS_IMETHODIMP OfflineCacheUpdateGlue::Init(nsIURI *aManifestURI, nsIURI *aDocumentURI, - nsIDOMDocument *aDocument) + nsIDOMDocument *aDocument, + nsILocalFile *aCustomProfileDir) { if (!EnsureUpdate()) return NS_ERROR_NULL_POINTER; @@ -101,7 +102,7 @@ OfflineCacheUpdateGlue::Init(nsIURI *aManifestURI, if (aDocument) SetDocument(aDocument); - return mUpdate->Init(aManifestURI, aDocumentURI, nsnull); + return mUpdate->Init(aManifestURI, aDocumentURI, nsnull, aCustomProfileDir); } void diff --git a/uriloader/prefetch/OfflineCacheUpdateGlue.h b/uriloader/prefetch/OfflineCacheUpdateGlue.h index cbf55246392e..5a5a045563eb 100644 --- a/uriloader/prefetch/OfflineCacheUpdateGlue.h +++ b/uriloader/prefetch/OfflineCacheUpdateGlue.h @@ -35,8 +35,8 @@ namespace docshell { NS_SCRIPTABLE NS_IMETHOD GetByteProgress(PRUint64 * _result) { return !_to ? NS_ERROR_NULL_POINTER : _to->GetByteProgress(_result); } class OfflineCacheUpdateGlue : public nsSupportsWeakReference - , public nsIOfflineCacheUpdate - , public nsIOfflineCacheUpdateObserver + , public nsIOfflineCacheUpdate + , public nsIOfflineCacheUpdateObserver { public: NS_DECL_ISUPPORTS @@ -49,7 +49,8 @@ public: NS_SCRIPTABLE NS_IMETHOD Schedule(void); NS_SCRIPTABLE NS_IMETHOD Init(nsIURI *aManifestURI, nsIURI *aDocumentURI, - nsIDOMDocument *aDocument); + nsIDOMDocument *aDocument, + nsILocalFile *aCustomProfileDir); NS_DECL_NSIOFFLINECACHEUPDATEOBSERVER diff --git a/uriloader/prefetch/OfflineCacheUpdateParent.cpp b/uriloader/prefetch/OfflineCacheUpdateParent.cpp index 516611d8c07e..49881cfb79f0 100644 --- a/uriloader/prefetch/OfflineCacheUpdateParent.cpp +++ b/uriloader/prefetch/OfflineCacheUpdateParent.cpp @@ -83,7 +83,7 @@ OfflineCacheUpdateParent::Schedule(const URI& aManifestURI, nsresult rv; // Leave aDocument argument null. Only glues and children keep // document instances. - rv = update->Init(manifestURI, documentURI, nsnull); + rv = update->Init(manifestURI, documentURI, nsnull, nsnull); NS_ENSURE_SUCCESS(rv, rv); rv = update->Schedule(); diff --git a/uriloader/prefetch/nsIOfflineCacheUpdate.idl b/uriloader/prefetch/nsIOfflineCacheUpdate.idl index 8745bc63d425..efa43b4b10ba 100644 --- a/uriloader/prefetch/nsIOfflineCacheUpdate.idl +++ b/uriloader/prefetch/nsIOfflineCacheUpdate.idl @@ -14,6 +14,7 @@ interface nsIOfflineCacheUpdate; interface nsIPrincipal; interface nsIPrefBranch; interface nsIApplicationCache; +interface nsILocalFile; [scriptable, uuid(47360d57-8ef4-4a5d-8865-1a27a739ad1a)] interface nsIOfflineCacheUpdateObserver : nsISupports { @@ -105,7 +106,8 @@ interface nsIOfflineCacheUpdate : nsISupports { * @param aDocumentURI * The page that is requesting the update. */ - void init(in nsIURI aManifestURI, in nsIURI aDocumentURI, in nsIDOMDocument aDocument); + void init(in nsIURI aManifestURI, in nsIURI aDocumentURI, in nsIDOMDocument aDocument, + [optional] in nsILocalFile aCustomProfileDir); /** * Initialize the update for partial processing. @@ -162,7 +164,7 @@ interface nsIOfflineCacheUpdate : nsISupports { readonly attribute PRUint64 byteProgress; }; -[scriptable, uuid(6fd2030f-7b00-4102-a0e3-d73078821eb1)] +[scriptable, uuid(dc5de18c-197c-41d2-9584-dd7ac7494611)] interface nsIOfflineCacheUpdateService : nsISupports { /** * Constants for the offline-app permission. @@ -192,6 +194,15 @@ interface nsIOfflineCacheUpdateService : nsISupports { in nsIURI aDocumentURI, in nsIDOMWindow aWindow); + /** + * Schedule a cache update for a given offline manifest and let the data + * be stored to a custom profile directory. There is no coalescing of + * manifests by manifest URL. + */ + nsIOfflineCacheUpdate scheduleCustomProfileUpdate(in nsIURI aManifestURI, + in nsIURI aDocumentURI, + in nsILocalFile aProfileDir); + /** * Schedule a cache update for a manifest when the document finishes * loading. diff --git a/uriloader/prefetch/nsOfflineCacheUpdate.cpp b/uriloader/prefetch/nsOfflineCacheUpdate.cpp index 25fd3128551f..006ee402b7e3 100644 --- a/uriloader/prefetch/nsOfflineCacheUpdate.cpp +++ b/uriloader/prefetch/nsOfflineCacheUpdate.cpp @@ -45,6 +45,11 @@ using namespace mozilla; static const PRUint32 kRescheduleLimit = 3; // Max number of retries for every entry of pinned app. static const PRUint32 kPinnedEntryRetriesLimit = 3; +// Maximum number of parallel items loads +static const PRUint32 kParallelLoadLimit = 15; + +// Quota for offline apps when preloading +static const PRInt32 kCustomProfileQuota = 512000; #if defined(PR_LOGGING) // @@ -280,18 +285,18 @@ NS_IMPL_ISUPPORTS6(nsOfflineCacheUpdateItem, // nsOfflineCacheUpdateItem //----------------------------------------------------------------------------- -nsOfflineCacheUpdateItem::nsOfflineCacheUpdateItem(nsOfflineCacheUpdate *aUpdate, - nsIURI *aURI, +nsOfflineCacheUpdateItem::nsOfflineCacheUpdateItem(nsIURI *aURI, nsIURI *aReferrerURI, + nsIApplicationCache *aApplicationCache, nsIApplicationCache *aPreviousApplicationCache, const nsACString &aClientID, PRUint32 type) : mURI(aURI) , mReferrerURI(aReferrerURI) + , mApplicationCache(aApplicationCache) , mPreviousApplicationCache(aPreviousApplicationCache) , mClientID(aClientID) , mItemType(type) - , mUpdate(aUpdate) , mChannel(nsnull) , mState(nsIDOMLoadStatus::UNINITIALIZED) , mBytesRead(0) @@ -303,7 +308,7 @@ nsOfflineCacheUpdateItem::~nsOfflineCacheUpdateItem() } nsresult -nsOfflineCacheUpdateItem::OpenChannel() +nsOfflineCacheUpdateItem::OpenChannel(nsOfflineCacheUpdate *aUpdate) { #if defined(PR_LOGGING) if (LOG_ENABLED()) { @@ -350,6 +355,13 @@ nsOfflineCacheUpdateItem::OpenChannel() rv = cachingChannel->SetCacheForOfflineUse(true); NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr cacheDirectory; + rv = mApplicationCache->GetCacheDirectory(getter_AddRefs(cacheDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = cachingChannel->SetProfileDirectory(cacheDirectory); + NS_ENSURE_SUCCESS(rv, rv); + if (!mClientID.IsEmpty()) { rv = cachingChannel->SetOfflineCacheClientID(mClientID); NS_ENSURE_SUCCESS(rv, rv); @@ -359,6 +371,8 @@ nsOfflineCacheUpdateItem::OpenChannel() rv = mChannel->AsyncOpen(this, nsnull); NS_ENSURE_SUCCESS(rv, rv); + mUpdate = aUpdate; + mState = nsIDOMLoadStatus::REQUESTED; return NS_OK; @@ -415,8 +429,6 @@ nsOfflineCacheUpdateItem::OnStopRequest(nsIRequest *aRequest, { LOG(("done fetching offline item [status=%x]\n", aStatus)); - mState = nsIDOMLoadStatus::LOADED; - if (mBytesRead == 0 && aStatus == NS_OK) { // we didn't need to read (because LOAD_ONLY_IF_MODIFIED was // specified), but the object should report loadedSize as if it @@ -439,7 +451,17 @@ nsOfflineCacheUpdateItem::OnStopRequest(nsIRequest *aRequest, NS_IMETHODIMP nsOfflineCacheUpdateItem::Run() { - mUpdate->LoadCompleted(); + // Set mState to LOADED here rather than in OnStopRequest to prevent + // race condition when checking state of all mItems in ProcessNextURI(). + // If state would have been set in OnStopRequest we could mistakenly + // take this item as already finished and finish the update process too + // early when ProcessNextURI() would get called between OnStopRequest() + // and Run() of this item. Finish() would then have been called twice. + mState = nsIDOMLoadStatus::LOADED; + + nsRefPtr update; + update.swap(mUpdate); + update->LoadCompleted(this); return NS_OK; } @@ -487,10 +509,18 @@ nsOfflineCacheUpdateItem::AsyncOnChannelRedirect(nsIChannel *aOldChannel, if (newCachingChannel) { rv = newCachingChannel->SetCacheForOfflineUse(true); NS_ENSURE_SUCCESS(rv, rv); + if (!mClientID.IsEmpty()) { rv = newCachingChannel->SetOfflineCacheClientID(mClientID); NS_ENSURE_SUCCESS(rv, rv); } + + nsCOMPtr cacheDirectory; + rv = mApplicationCache->GetCacheDirectory(getter_AddRefs(cacheDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = newCachingChannel->SetProfileDirectory(cacheDirectory); + NS_ENSURE_SUCCESS(rv, rv); } nsCAutoString oldScheme; @@ -599,6 +629,25 @@ nsOfflineCacheUpdateItem::GetRequestSucceeded(bool * succeeded) return NS_OK; } +bool +nsOfflineCacheUpdateItem::IsScheduled() +{ + return mState == nsIDOMLoadStatus::UNINITIALIZED; +} + +bool +nsOfflineCacheUpdateItem::IsInProgress() +{ + return mState == nsIDOMLoadStatus::REQUESTED || + mState == nsIDOMLoadStatus::RECEIVING; +} + +bool +nsOfflineCacheUpdateItem::IsCompleted() +{ + return mState == nsIDOMLoadStatus::LOADED; +} + NS_IMETHODIMP nsOfflineCacheUpdateItem::GetStatus(PRUint16 *aStatus) { @@ -631,13 +680,13 @@ nsOfflineCacheUpdateItem::GetStatus(PRUint16 *aStatus) // nsOfflineManifestItem //----------------------------------------------------------------------------- -nsOfflineManifestItem::nsOfflineManifestItem(nsOfflineCacheUpdate *aUpdate, - nsIURI *aURI, +nsOfflineManifestItem::nsOfflineManifestItem(nsIURI *aURI, nsIURI *aReferrerURI, + nsIApplicationCache *aApplicationCache, nsIApplicationCache *aPreviousApplicationCache, const nsACString &aClientID) - : nsOfflineCacheUpdateItem(aUpdate, aURI, aReferrerURI, - aPreviousApplicationCache, aClientID, + : nsOfflineCacheUpdateItem(aURI, aReferrerURI, + aApplicationCache, aPreviousApplicationCache, aClientID, nsIApplicationCache::ITEM_MANIFEST) , mParserState(PARSE_INIT) , mNeedsUpdate(true) @@ -709,6 +758,7 @@ nsOfflineManifestItem::ReadManifest(nsIInputStream *aInputStream, if (NS_FAILED(rv)) { LOG(("HandleManifestLine failed with 0x%08x", rv)); + *aBytesConsumed = 0; // Avoid assertion failure in stream tee return NS_ERROR_ABORT; } @@ -1096,9 +1146,10 @@ nsOfflineManifestItem::OnStopRequest(nsIRequest *aRequest, // nsOfflineCacheUpdate::nsISupports //----------------------------------------------------------------------------- -NS_IMPL_ISUPPORTS2(nsOfflineCacheUpdate, +NS_IMPL_ISUPPORTS3(nsOfflineCacheUpdate, nsIOfflineCacheUpdateObserver, - nsIOfflineCacheUpdate) + nsIOfflineCacheUpdate, + nsIRunnable) //----------------------------------------------------------------------------- // nsOfflineCacheUpdate @@ -1111,7 +1162,7 @@ nsOfflineCacheUpdate::nsOfflineCacheUpdate() , mPartialUpdate(false) , mSucceeded(true) , mObsolete(false) - , mCurrentItem(-1) + , mItemsInProgress(0) , mRescheduleCount(0) , mPinnedEntryRetriesCount(0) , mPinned(false) @@ -1142,7 +1193,8 @@ nsOfflineCacheUpdate::GetCacheKey(nsIURI *aURI, nsACString &aKey) nsresult nsOfflineCacheUpdate::Init(nsIURI *aManifestURI, nsIURI *aDocumentURI, - nsIDOMDocument *aDocument) + nsIDOMDocument *aDocument, + nsILocalFile *aCustomProfileDir) { nsresult rv; @@ -1184,13 +1236,32 @@ nsOfflineCacheUpdate::Init(nsIURI *aManifestURI, do_GetService(NS_APPLICATIONCACHESERVICE_CONTRACTID, &rv); NS_ENSURE_SUCCESS(rv, rv); - rv = cacheService->GetActiveCache(manifestSpec, - getter_AddRefs(mPreviousApplicationCache)); - NS_ENSURE_SUCCESS(rv, rv); + if (aCustomProfileDir) { + // Create only a new offline application cache in the custom profile + // This is a preload of a new cache. - rv = cacheService->CreateApplicationCache(manifestSpec, - getter_AddRefs(mApplicationCache)); - NS_ENSURE_SUCCESS(rv, rv); + // XXX Custom updates don't support "updating" of an existing cache + // in the custom profile at the moment. This support can be, though, + // simply added as well when needed. + mPreviousApplicationCache = nsnull; + + rv = cacheService->CreateCustomApplicationCache(manifestSpec, + aCustomProfileDir, + kCustomProfileQuota, + getter_AddRefs(mApplicationCache)); + NS_ENSURE_SUCCESS(rv, rv); + + mCustomProfileDir = aCustomProfileDir; + } + else { + rv = cacheService->GetActiveCache(manifestSpec, + getter_AddRefs(mPreviousApplicationCache)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = cacheService->CreateApplicationCache(manifestSpec, + getter_AddRefs(mApplicationCache)); + NS_ENSURE_SUCCESS(rv, rv); + } rv = mApplicationCache->GetClientID(mClientID); NS_ENSURE_SUCCESS(rv, rv); @@ -1317,15 +1388,20 @@ nsOfflineCacheUpdate::HandleManifest(bool *aDoUpdate) } void -nsOfflineCacheUpdate::LoadCompleted() +nsOfflineCacheUpdate::LoadCompleted(nsOfflineCacheUpdateItem *aItem) { nsresult rv; + LOG(("nsOfflineCacheUpdate::LoadCompleted [%p]", this)); + + if (mState == STATE_FINISHED) { + LOG((" after completion, ignoring")); + return; + } + // Keep the object alive through a Finish() call. nsCOMPtr kungFuDeathGrip(this); - LOG(("nsOfflineCacheUpdate::LoadCompleted [%p]", this)); - if (mState == STATE_CANCELLED) { Finish(); return; @@ -1336,6 +1412,8 @@ nsOfflineCacheUpdate::LoadCompleted() NS_ASSERTION(mManifestItem, "Must have a manifest item in STATE_CHECKING."); + NS_ASSERTION(mManifestItem == aItem, + "Unexpected aItem in nsOfflineCacheUpdate::LoadCompleted"); // A 404 or 410 is interpreted as an intentional removal of // the manifest file, rather than a transient server error. @@ -1403,21 +1481,21 @@ nsOfflineCacheUpdate::LoadCompleted() } // Normal load finished. - - nsRefPtr item = mItems[mCurrentItem]; + if (mItemsInProgress) // Just to be safe here! + --mItemsInProgress; bool succeeded; - rv = item->GetRequestSucceeded(&succeeded); + rv = aItem->GetRequestSucceeded(&succeeded); - if (mPinned) { + if (mPinned && NS_SUCCEEDED(rv) && succeeded) { PRUint32 dummy_cache_type; - rv = mApplicationCache->GetTypes(item->mCacheKey, &dummy_cache_type); + rv = mApplicationCache->GetTypes(aItem->mCacheKey, &dummy_cache_type); bool item_doomed = NS_FAILED(rv); // can not find it? -> doomed if (item_doomed && mPinnedEntryRetriesCount < kPinnedEntryRetriesLimit && - (item->mItemType & (nsIApplicationCache::ITEM_EXPLICIT | - nsIApplicationCache::ITEM_FALLBACK))) { + (aItem->mItemType & (nsIApplicationCache::ITEM_EXPLICIT | + nsIApplicationCache::ITEM_FALLBACK))) { rv = EvictOneNonPinned(); if (NS_FAILED(rv)) { mSucceeded = false; @@ -1426,7 +1504,9 @@ nsOfflineCacheUpdate::LoadCompleted() return; } - rv = item->Cancel(); + // This reverts the item state to UNINITIALIZED that makes it to + // be scheduled for download again. + rv = aItem->Cancel(); if (NS_FAILED(rv)) { mSucceeded = false; NotifyState(nsIOfflineCacheUpdateObserver::STATE_ERROR); @@ -1435,26 +1515,29 @@ nsOfflineCacheUpdate::LoadCompleted() } mPinnedEntryRetriesCount++; - // Retry current item, so mCurrentItem is not advanced. + + // Retry this item. ProcessNextURI(); return; } } - // Advance to next item. - mCurrentItem++; + // According to parallelism this may imply more pinned retries count, + // but that is not critical, since at one moment the algoritm will + // stop anyway. Also, this code may soon be completely removed + // after we have a separate storage for pinned apps. mPinnedEntryRetriesCount = 0; // Check for failures. 3XX, 4XX and 5XX errors on items explicitly // listed in the manifest will cause the update to fail. if (NS_FAILED(rv) || !succeeded) { - if (item->mItemType & + if (aItem->mItemType & (nsIApplicationCache::ITEM_EXPLICIT | nsIApplicationCache::ITEM_FALLBACK)) { mSucceeded = false; } } else { - rv = mApplicationCache->MarkEntry(item->mCacheKey, item->mItemType); + rv = mApplicationCache->MarkEntry(aItem->mCacheKey, aItem->mItemType); if (NS_FAILED(rv)) { mSucceeded = false; } @@ -1505,7 +1588,7 @@ nsOfflineCacheUpdate::ManifestCheckCompleted(nsresult aStatus, new nsOfflineCacheUpdate(); // Leave aDocument argument null. Only glues and children keep // document instances. - newUpdate->Init(mManifestURI, mDocumentURI, nsnull); + newUpdate->Init(mManifestURI, mDocumentURI, nsnull, mCustomProfileDir); // In a rare case the manifest will not be modified on the next refetch // transfer all master document URIs to the new update to ensure that @@ -1531,7 +1614,7 @@ nsOfflineCacheUpdate::Begin() // Keep the object alive through a ProcessNextURI()/Finish() call. nsCOMPtr kungFuDeathGrip(this); - mCurrentItem = 0; + mItemsInProgress = 0; if (mPartialUpdate) { mState = STATE_DOWNLOADING; @@ -1543,8 +1626,9 @@ nsOfflineCacheUpdate::Begin() // Start checking the manifest. nsCOMPtr uri; - mManifestItem = new nsOfflineManifestItem(this, mManifestURI, + mManifestItem = new nsOfflineManifestItem(mManifestURI, mDocumentURI, + mApplicationCache, mPreviousApplicationCache, mClientID); if (!mManifestItem) { @@ -1555,9 +1639,9 @@ nsOfflineCacheUpdate::Begin() mByteProgress = 0; NotifyState(nsIOfflineCacheUpdateObserver::STATE_CHECKING); - nsresult rv = mManifestItem->OpenChannel(); + nsresult rv = mManifestItem->OpenChannel(this); if (NS_FAILED(rv)) { - LoadCompleted(); + LoadCompleted(mManifestItem); } return NS_OK; @@ -1571,10 +1655,12 @@ nsOfflineCacheUpdate::Cancel() mState = STATE_CANCELLED; mSucceeded = false; - if (mCurrentItem >= 0 && - mCurrentItem < static_cast(mItems.Length())) { - // Load might be running - mItems[mCurrentItem]->Cancel(); + // Cancel all running downloads + for (PRUint32 i = 0; i < mItems.Length(); ++i) { + nsOfflineCacheUpdateItem * item = mItems[i]; + + if (item->IsInProgress()) + item->Cancel(); } return NS_OK; @@ -1634,13 +1720,29 @@ nsOfflineCacheUpdate::ProcessNextURI() // Keep the object alive through a Finish() call. nsCOMPtr kungFuDeathGrip(this); - LOG(("nsOfflineCacheUpdate::ProcessNextURI [%p, current=%d, numItems=%d]", - this, mCurrentItem, mItems.Length())); + LOG(("nsOfflineCacheUpdate::ProcessNextURI [%p, inprogress=%d, numItems=%d]", + this, mItemsInProgress, mItems.Length())); NS_ASSERTION(mState == STATE_DOWNLOADING, "ProcessNextURI should only be called from the DOWNLOADING state"); - if (mCurrentItem >= static_cast(mItems.Length())) { + nsOfflineCacheUpdateItem * runItem = nsnull; + PRUint32 completedItems = 0; + for (PRUint32 i = 0; i < mItems.Length(); ++i) { + nsOfflineCacheUpdateItem * item = mItems[i]; + + if (item->IsScheduled()) { + runItem = item; + break; + } + + if (item->IsCompleted()) + ++completedItems; + } + + if (completedItems == mItems.Length()) { + LOG(("nsOfflineCacheUpdate::ProcessNextURI [%p]: all items loaded", this)); + if (mPartialUpdate) { return Finish(); } else { @@ -1660,23 +1762,38 @@ nsOfflineCacheUpdate::ProcessNextURI() } } + if (!runItem) { + LOG(("nsOfflineCacheUpdate::ProcessNextURI [%p]:" + " No more items to include in parallel load", this)); + return NS_OK; + } + #if defined(PR_LOGGING) if (LOG_ENABLED()) { nsCAutoString spec; - mItems[mCurrentItem]->mURI->GetSpec(spec); + runItem->mURI->GetSpec(spec); LOG(("%p: Opening channel for %s", this, spec.get())); } #endif + ++mItemsInProgress; NotifyState(nsIOfflineCacheUpdateObserver::STATE_ITEMSTARTED); - nsresult rv = mItems[mCurrentItem]->OpenChannel(); + nsresult rv = runItem->OpenChannel(this); if (NS_FAILED(rv)) { - LoadCompleted(); + LoadCompleted(runItem); return rv; } - return NS_OK; + if (mItemsInProgress >= kParallelLoadLimit) { + LOG(("nsOfflineCacheUpdate::ProcessNextURI [%p]:" + " At parallel load limit", this)); + return NS_OK; + } + + // This calls this method again via a post triggering + // a parallel item load + return NS_DispatchToCurrentThread(this); } nsresult @@ -1836,7 +1953,7 @@ nsOfflineCacheUpdate::FinishNoNotify() mApplicationCache->GetGroupID(groupID); appCacheService->DeactivateGroup(groupID); } - } + } if (!mSucceeded) { // Update was not merged, mark all the loads as failures @@ -2016,8 +2133,11 @@ nsOfflineCacheUpdate::AddURI(nsIURI *aURI, PRUint32 aType) } nsRefPtr item = - new nsOfflineCacheUpdateItem(this, aURI, mDocumentURI, - mPreviousApplicationCache, mClientID, + new nsOfflineCacheUpdateItem(aURI, + mDocumentURI, + mApplicationCache, + mPreviousApplicationCache, + mClientID, aType); if (!item) return NS_ERROR_OUT_OF_MEMORY; @@ -2146,3 +2266,14 @@ nsOfflineCacheUpdate::ApplicationCacheAvailable(nsIApplicationCache *application { return AssociateDocuments(applicationCache); } + +//----------------------------------------------------------------------------- +// nsOfflineCacheUpdate::nsIRunable +//----------------------------------------------------------------------------- + +NS_IMETHODIMP +nsOfflineCacheUpdate::Run() +{ + ProcessNextURI(); + return NS_OK; +} diff --git a/uriloader/prefetch/nsOfflineCacheUpdate.h b/uriloader/prefetch/nsOfflineCacheUpdate.h index 1c4fe8c49bea..3efe9690436c 100644 --- a/uriloader/prefetch/nsOfflineCacheUpdate.h +++ b/uriloader/prefetch/nsOfflineCacheUpdate.h @@ -52,9 +52,9 @@ public: NS_DECL_NSIINTERFACEREQUESTOR NS_DECL_NSICHANNELEVENTSINK - nsOfflineCacheUpdateItem(nsOfflineCacheUpdate *aUpdate, - nsIURI *aURI, + nsOfflineCacheUpdateItem(nsIURI *aURI, nsIURI *aReferrerURI, + nsIApplicationCache *aApplicationCache, nsIApplicationCache *aPreviousApplicationCache, const nsACString &aClientID, PRUint32 aType); @@ -62,17 +62,22 @@ public: nsCOMPtr mURI; nsCOMPtr mReferrerURI; + nsCOMPtr mApplicationCache; nsCOMPtr mPreviousApplicationCache; nsCString mClientID; nsCString mCacheKey; PRUint32 mItemType; - nsresult OpenChannel(); + nsresult OpenChannel(nsOfflineCacheUpdate *aUpdate); nsresult Cancel(); nsresult GetRequestSucceeded(bool * succeeded); + bool IsInProgress(); + bool IsScheduled(); + bool IsCompleted(); + private: - nsOfflineCacheUpdate* mUpdate; + nsRefPtr mUpdate; nsCOMPtr mChannel; PRUint16 mState; @@ -87,9 +92,9 @@ public: NS_DECL_NSISTREAMLISTENER NS_DECL_NSIREQUESTOBSERVER - nsOfflineManifestItem(nsOfflineCacheUpdate *aUpdate, - nsIURI *aURI, + nsOfflineManifestItem(nsIURI *aURI, nsIURI *aReferrerURI, + nsIApplicationCache *aApplicationCache, nsIApplicationCache *aPreviousApplicationCache, const nsACString &aClientID); virtual ~nsOfflineManifestItem(); @@ -179,12 +184,14 @@ public: class nsOfflineCacheUpdate : public nsIOfflineCacheUpdate , public nsIOfflineCacheUpdateObserver + , public nsIRunnable , public nsOfflineCacheUpdateOwner { public: NS_DECL_ISUPPORTS NS_DECL_NSIOFFLINECACHEUPDATE NS_DECL_NSIOFFLINECACHEUPDATEOBSERVER + NS_DECL_NSIRUNNABLE nsOfflineCacheUpdate(); ~nsOfflineCacheUpdate(); @@ -196,7 +203,7 @@ public: nsresult Begin(); nsresult Cancel(); - void LoadCompleted(); + void LoadCompleted(nsOfflineCacheUpdateItem *aItem); void ManifestCheckCompleted(nsresult aStatus, const nsCString &aManifestHash); void StickDocument(nsIURI *aDocumentURI); @@ -250,6 +257,7 @@ private: nsCString mUpdateDomain; nsCOMPtr mManifestURI; nsCOMPtr mDocumentURI; + nsCOMPtr mCustomProfileDir; nsCString mClientID; nsCOMPtr mApplicationCache; @@ -260,7 +268,7 @@ private: nsRefPtr mManifestItem; /* Items being updated */ - PRInt32 mCurrentItem; + PRUint32 mItemsInProgress; nsTArray > mItems; /* Clients watching this update for changes */ @@ -309,6 +317,7 @@ public: nsIURI *aDocumentURI, nsIDOMDocument *aDocument, nsIDOMWindow* aWindow, + nsILocalFile* aCustomProfileDir, nsIOfflineCacheUpdate **aUpdate); virtual nsresult UpdateFinished(nsOfflineCacheUpdate *aUpdate); diff --git a/uriloader/prefetch/nsOfflineCacheUpdateService.cpp b/uriloader/prefetch/nsOfflineCacheUpdateService.cpp index c2d1fcbe57e7..06cdb372741d 100644 --- a/uriloader/prefetch/nsOfflineCacheUpdateService.cpp +++ b/uriloader/prefetch/nsOfflineCacheUpdateService.cpp @@ -161,7 +161,7 @@ nsOfflineCachePendingUpdate::OnStateChange(nsIWebProgress* aWebProgress, if (NS_SUCCEEDED(aStatus)) { nsCOMPtr update; mService->Schedule(mManifestURI, mDocumentURI, - updateDoc, window, getter_AddRefs(update)); + updateDoc, window, nsnull, getter_AddRefs(update)); } aWebProgress->RemoveProgressListener(this); @@ -436,6 +436,7 @@ nsOfflineCacheUpdateService::Schedule(nsIURI *aManifestURI, nsIURI *aDocumentURI, nsIDOMDocument *aDocument, nsIDOMWindow* aWindow, + nsILocalFile* aCustomProfileDir, nsIOfflineCacheUpdate **aUpdate) { nsCOMPtr update; @@ -448,7 +449,7 @@ nsOfflineCacheUpdateService::Schedule(nsIURI *aManifestURI, nsresult rv; - rv = update->Init(aManifestURI, aDocumentURI, aDocument); + rv = update->Init(aManifestURI, aDocumentURI, aDocument, aCustomProfileDir); NS_ENSURE_SUCCESS(rv, rv); rv = update->Schedule(); @@ -465,7 +466,19 @@ nsOfflineCacheUpdateService::ScheduleUpdate(nsIURI *aManifestURI, nsIDOMWindow *aWindow, nsIOfflineCacheUpdate **aUpdate) { - return Schedule(aManifestURI, aDocumentURI, nsnull, aWindow, aUpdate); + return Schedule(aManifestURI, aDocumentURI, nsnull, aWindow, nsnull, aUpdate); +} + +NS_IMETHODIMP +nsOfflineCacheUpdateService::ScheduleCustomProfileUpdate(nsIURI *aManifestURI, + nsIURI *aDocumentURI, + nsILocalFile *aProfileDir, + nsIOfflineCacheUpdate **aUpdate) +{ + // The profile directory is mandatory + NS_ENSURE_ARG(aProfileDir); + + return Schedule(aManifestURI, aDocumentURI, nsnull, nsnull, aProfileDir, aUpdate); } //-----------------------------------------------------------------------------