Backed out changeset 1686b901daa4 (bug 1237983) for bustage

This commit is contained in:
Carsten "Tomcat" Book 2016-01-28 09:36:00 +01:00
parent 9e296bba83
commit 0ce9718a21
9 changed files with 825 additions and 0 deletions

View File

@ -0,0 +1,278 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* This file contains a client API for the Bagheera data storage service.
*
* Information about Bagheera is available at
* https://github.com/mozilla-metrics/bagheera
*/
#ifndef MERGED_COMPARTMENT
"use strict";
this.EXPORTED_SYMBOLS = [
"BagheeraClient",
"BagheeraClientRequestResult",
];
var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
#endif
Cu.import("resource://gre/modules/Promise.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
Cu.import("resource://services-common/utils.js");
/**
* Represents the result of a Bagheera request.
*/
this.BagheeraClientRequestResult = function BagheeraClientRequestResult() {
this.transportSuccess = false;
this.serverSuccess = false;
this.request = null;
};
Object.freeze(BagheeraClientRequestResult.prototype);
/**
* Wrapper around RESTRequest so logging is sane.
*/
function BagheeraRequest(uri) {
RESTRequest.call(this, uri);
this._log = Log.repository.getLogger("Services.BagheeraClient");
this._log.level = Log.Level.Debug;
}
BagheeraRequest.prototype = Object.freeze({
__proto__: RESTRequest.prototype,
});
/**
* Create a new Bagheera client instance.
*
* Each client is associated with a specific Bagheera HTTP URI endpoint.
*
* @param baseURI
* (string) The base URI of the Bagheera HTTP endpoint.
*/
this.BagheeraClient = function BagheeraClient(baseURI) {
if (!baseURI) {
throw new Error("baseURI argument must be defined.");
}
this._log = Log.repository.getLogger("Services.BagheeraClient");
this._log.level = Log.Level.Debug;
this.baseURI = baseURI;
if (!baseURI.endsWith("/")) {
this.baseURI += "/";
}
};
BagheeraClient.prototype = Object.freeze({
/**
* Channel load flags for all requests.
*
* Caching is not applicable, so we bypass and disable it. We also
* ignore any cookies that may be present for the domain because
* Bagheera does not utilize cookies and the release of cookies may
* inadvertantly constitute unncessary information disclosure.
*/
_loadFlags: Ci.nsIRequest.LOAD_BYPASS_CACHE |
Ci.nsIRequest.INHIBIT_CACHING |
Ci.nsIRequest.LOAD_ANONYMOUS,
DEFAULT_TIMEOUT_MSEC: 5 * 60 * 1000, // 5 minutes.
_RE_URI_IDENTIFIER: /^[a-zA-Z0-9_-]+$/,
/**
* Upload a JSON payload to the server.
*
* The return value is a Promise which will be resolved with a
* BagheeraClientRequestResult when the request has finished.
*
* @param namespace
* (string) The namespace to post this data to.
* @param id
* (string) The ID of the document being uploaded. This is typically
* a UUID in hex form.
* @param payload
* (string|object) Data to upload. Can be specified as a string (which
* is assumed to be JSON) or an object. If an object, it will be fed into
* JSON.stringify() for serialization.
* @param options
* (object) Extra options to control behavior. Recognized properties:
*
* deleteIDs -- (array) Old document IDs to delete as part of
* upload. If not specified, no old documents will be deleted as
* part of upload. The array values are typically UUIDs in hex
* form.
*
* telemetryCompressed -- (string) Telemetry histogram to record
* compressed size of payload under. If not defined, no telemetry
* data for the compressed size will be recorded.
*
* @return Promise<BagheeraClientRequestResult>
*/
uploadJSON: function uploadJSON(namespace, id, payload, options={}) {
if (!namespace) {
throw new Error("namespace argument must be defined.");
}
if (!id) {
throw new Error("id argument must be defined.");
}
if (!payload) {
throw new Error("payload argument must be defined.");
}
if (options && typeof(options) != "object") {
throw new Error("Unexpected type for options argument. Expected object. " +
"Got: " + typeof(options));
}
let uri = this._submitURI(namespace, id);
let data = payload;
if (typeof(payload) == "object") {
data = JSON.stringify(payload);
}
if (typeof(data) != "string") {
throw new Error("Unknown type for payload: " + typeof(data));
}
this._log.info("Uploading data to " + uri);
let request = new BagheeraRequest(uri);
request.loadFlags = this._loadFlags;
request.timeout = this.DEFAULT_TIMEOUT_MSEC;
// Since API changed, throw on old API usage.
if ("deleteID" in options) {
throw new Error("API has changed, use (array) deleteIDs instead");
}
let deleteIDs;
if (options.deleteIDs && options.deleteIDs.length > 0) {
deleteIDs = options.deleteIDs;
this._log.debug("Will delete " + deleteIDs.join(", "));
request.setHeader("X-Obsolete-Document", deleteIDs.join(","));
}
let deferred = Promise.defer();
// The string converter service used by CommonUtils.convertString()
// silently throws away high bytes. We need to convert the string to
// consist of only low bytes first.
data = CommonUtils.encodeUTF8(data);
data = CommonUtils.convertString(data, "uncompressed", "deflate");
if (options.telemetryCompressed) {
try {
let h = Services.telemetry.getHistogramById(options.telemetryCompressed);
h.add(data.length);
} catch (ex) {
this._log.warn("Unable to record telemetry for compressed payload size", ex);
}
}
// TODO proper header per bug 807134.
request.setHeader("Content-Type", "application/json+zlib; charset=utf-8");
this._log.info("Request body length: " + data.length);
let result = new BagheeraClientRequestResult();
result.namespace = namespace;
result.id = id;
result.deleteIDs = deleteIDs ? deleteIDs.slice(0) : null;
request.onComplete = this._onComplete.bind(this, request, deferred, result);
request.post(data);
return deferred.promise;
},
/**
* Delete the specified document.
*
* @param namespace
* (string) Namespace from which to delete the document.
* @param id
* (string) ID of document to delete.
*
* @return Promise<BagheeraClientRequestResult>
*/
deleteDocument: function deleteDocument(namespace, id) {
let uri = this._submitURI(namespace, id);
let request = new BagheeraRequest(uri);
request.loadFlags = this._loadFlags;
request.timeout = this.DEFAULT_TIMEOUT_MSEC;
let result = new BagheeraClientRequestResult();
result.namespace = namespace;
result.id = id;
let deferred = Promise.defer();
request.onComplete = this._onComplete.bind(this, request, deferred, result);
request.delete();
return deferred.promise;
},
_submitURI: function _submitURI(namespace, id) {
if (!this._RE_URI_IDENTIFIER.test(namespace)) {
throw new Error("Illegal namespace name. Must be alphanumeric + [_-]: " +
namespace);
}
if (!this._RE_URI_IDENTIFIER.test(id)) {
throw new Error("Illegal id value. Must be alphanumeric + [_-]: " + id);
}
return this.baseURI + "1.0/submit/" + namespace + "/" + id;
},
_onComplete: function _onComplete(request, deferred, result, error) {
result.request = request;
if (error) {
this._log.info("Transport failure on request", error);
result.transportSuccess = false;
deferred.resolve(result);
return;
}
result.transportSuccess = true;
let response = request.response;
switch (response.status) {
case 200:
case 201:
result.serverSuccess = true;
break;
default:
result.serverSuccess = false;
this._log.info("Received unexpected status code: " + response.status);
this._log.debug("Response body: " + response.body);
}
deferred.resolve(result);
},
});

View File

@ -0,0 +1,296 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var {utils: Cu} = Components;
this.EXPORTED_SYMBOLS = ["BagheeraServer"];
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/utils.js");
Cu.import("resource://testing-common/httpd.js");
/**
* This is an implementation of the Bagheera server.
*
* The purpose of the server is to facilitate testing of the Bagheera
* client and the Firefox Health report. It is *not* meant to be a
* production grade server.
*
* The Bagheera server is essentially a glorified document store.
*/
this.BagheeraServer = function BagheeraServer() {
this._log = Log.repository.getLogger("metrics.BagheeraServer");
this.server = new HttpServer();
this.namespaces = {};
this.allowAllNamespaces = false;
}
BagheeraServer.prototype = {
/**
* Whether this server has a namespace defined.
*
* @param ns
* (string) Namepsace whose existence to query for.
* @return bool
*/
hasNamespace: function hasNamespace(ns) {
return ns in this.namespaces;
},
/**
* Whether this server has an ID in a particular namespace.
*
* @param ns
* (string) Namespace to look for item in.
* @param id
* (string) ID of object to look for.
* @return bool
*/
hasDocument: function hasDocument(ns, id) {
let namespace = this.namespaces[ns];
if (!namespace) {
return false;
}
return id in namespace;
},
/**
* Obtain a document from the server.
*
* @param ns
* (string) Namespace to retrieve document from.
* @param id
* (string) ID of document to retrieve.
*
* @return string The content of the document or null if the document
* does not exist.
*/
getDocument: function getDocument(ns, id) {
let namespace = this.namespaces[ns];
if (!namespace) {
return null;
}
return namespace[id];
},
/**
* Set the contents of a document in the server.
*
* @param ns
* (string) Namespace to add document to.
* @param id
* (string) ID of document being added.
* @param payload
* (string) The content of the document.
*/
setDocument: function setDocument(ns, id, payload) {
let namespace = this.namespaces[ns];
if (!namespace) {
if (!this.allowAllNamespaces) {
throw new Error("Namespace does not exist: " + ns);
}
this.createNamespace(ns);
namespace = this.namespaces[ns];
}
namespace[id] = payload;
},
/**
* Create a namespace in the server.
*
* The namespace will initially be empty.
*
* @param ns
* (string) The name of the namespace to create.
*/
createNamespace: function createNamespace(ns) {
if (ns in this.namespaces) {
throw new Error("Namespace already exists: " + ns);
}
this.namespaces[ns] = {};
},
start: function start(port=-1) {
this.server.registerPrefixHandler("/", this._handleRequest.bind(this));
this.server.start(port);
let i = this.server.identity;
this.serverURI = i.primaryScheme + "://" + i.primaryHost + ":" +
i.primaryPort + "/";
this.port = i.primaryPort;
},
stop: function stop(cb) {
let handler = {onStopped: cb};
this.server.stop(handler);
},
/**
* Our root path handler.
*/
_handleRequest: function _handleRequest(request, response) {
let path = request.path;
this._log.info("Received request: " + request.method + " " + path + " " +
"HTTP/" + request.httpVersion);
try {
if (path.startsWith("/1.0/submit/")) {
return this._handleV1Submit(request, response,
path.substr("/1.0/submit/".length));
} else {
throw HTTP_404;
}
} catch (ex) {
if (ex instanceof HttpError) {
this._log.info("HttpError thrown: " + ex.code + " " + ex.description);
} else {
this._log.warn("Exception processing request", ex);
}
throw ex;
}
},
/**
* Handles requests to /submit/*.
*/
_handleV1Submit: function _handleV1Submit(request, response, rest) {
if (!rest.length) {
throw HTTP_404;
}
let namespace;
let index = rest.indexOf("/");
if (index == -1) {
namespace = rest;
rest = "";
} else {
namespace = rest.substr(0, index);
rest = rest.substr(index + 1);
}
this._handleNamespaceSubmit(namespace, rest, request, response);
},
_handleNamespaceSubmit: function _handleNamespaceSubmit(namespace, rest,
request, response) {
if (!this.hasNamespace(namespace)) {
if (!this.allowAllNamespaces) {
this._log.info("Request to unknown namespace: " + namespace);
throw HTTP_404;
}
this.createNamespace(namespace);
}
if (!rest) {
this._log.info("No ID defined.");
throw HTTP_404;
}
let id = rest;
if (id.includes("/")) {
this._log.info("URI has too many components.");
throw HTTP_404;
}
if (request.method == "POST") {
return this._handleNamespaceSubmitPost(namespace, id, request, response);
}
if (request.method == "DELETE") {
return this._handleNamespaceSubmitDelete(namespace, id, request, response);
}
this._log.info("Unsupported HTTP method on namespace handler: " +
request.method);
response.setHeader("Allow", "POST,DELETE");
throw HTTP_405;
},
_handleNamespaceSubmitPost:
function _handleNamespaceSubmitPost(namespace, id, request, response) {
this._log.info("Handling data upload for " + namespace + ":" + id);
let requestBody = CommonUtils.readBytesFromInputStream(request.bodyInputStream);
this._log.info("Raw body length: " + requestBody.length);
if (!request.hasHeader("Content-Type")) {
this._log.info("Request does not have Content-Type header.");
throw HTTP_400;
}
const ALLOWED_TYPES = [
// TODO proper content types from bug 807134.
"application/json; charset=utf-8",
"application/json+zlib; charset=utf-8",
];
let ct = request.getHeader("Content-Type");
if (ALLOWED_TYPES.indexOf(ct) == -1) {
this._log.info("Unknown media type: " + ct);
// Should generate proper HTTP response headers for this error.
throw HTTP_415;
}
if (ct.startsWith("application/json+zlib")) {
this._log.debug("Uncompressing entity body with deflate.");
requestBody = CommonUtils.convertString(requestBody, "deflate",
"uncompressed");
}
requestBody = CommonUtils.decodeUTF8(requestBody);
this._log.debug("HTTP request body: " + requestBody);
let doc;
try {
doc = JSON.parse(requestBody);
} catch(ex) {
this._log.info("JSON parse error.");
throw HTTP_400;
}
this.namespaces[namespace][id] = doc;
if (request.hasHeader("X-Obsolete-Document")) {
let obsolete = request.getHeader("X-Obsolete-Document");
this._log.info("Deleting from X-Obsolete-Document header: " + obsolete);
for (let obsolete_id of obsolete.split(",")) {
delete this.namespaces[namespace][obsolete_id];
}
}
response.setStatusLine(request.httpVersion, 201, "Created");
response.setHeader("Content-Type", "text/plain");
let body = id;
response.bodyOutputStream.write(body, body.length);
},
_handleNamespaceSubmitDelete:
function _handleNamespaceSubmitDelete(namespace, id, request, response) {
delete this.namespaces[namespace][id];
let body = id;
response.bodyOutputStream.write(body, body.length);
},
};
Object.freeze(BagheeraServer.prototype);

View File

@ -31,6 +31,14 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android' or CONFIG['MOZ_B2GDROID']:
'modules-testing/storageserver.js',
]
if CONFIG['MOZ_SERVICES_HEALTHREPORT']:
EXTRA_PP_JS_MODULES['services-common'] += [
'bagheeraclient.js',
]
TESTING_JS_MODULES.services.common += [
'modules-testing/bagheeraserver.js',
]
EXTRA_PP_JS_MODULES['services-common'] += [
'async.js',
'observers.js',

View File

@ -34,6 +34,7 @@ DEFAULT_HOSTNAME = 'localhost'
SRCDIR = mozpath.abspath(mozpath.dirname(__file__))
STORAGE_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_storage_server.js')
BAGHEERA_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_bagheera_server.js')
def SyncStorageCommand(func):
"""Decorator that adds shared command arguments to services commands."""
@ -109,3 +110,8 @@ class SyncTestCommands(MachCommandBase):
@SyncStorageCommand
def run_storage_server(self, port=DEFAULT_PORT, address=DEFAULT_HOSTNAME):
exit(self.run_server(STORAGE_SERVER_SCRIPT, address, port))
@Command('bagheera-server', category='services',
description='Run a bagheera server.')
def run_bagheera_server(self, port=DEFAULT_PORT, address=DEFAULT_HOSTNAME):
exit(self.run_server(BAGHEERA_SERVER_SCRIPT, address, port))

View File

@ -0,0 +1,26 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* This file runs a stub Bagheera server.
*
* It is meant to be executed with an xpcshell.
*
* The Makefile in this directory contains a target to run it:
*
* $ make bagheera-server
*/
Cu.import("resource://testing-common/services/common/bagheeraserver.js");
initTestLogging();
var server = new BagheeraServer();
server.allowAllNamespaces = true;
server.start(SERVER_PORT);
_("Bagheera server started on port " + SERVER_PORT);
// Launch the thread manager.
_do_main();

View File

@ -0,0 +1,161 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
Cu.import("resource://services-common/bagheeraclient.js");
Cu.import("resource://services-common/rest.js");
Cu.import("resource://testing-common/services/common/bagheeraserver.js");
Cu.import("resource://gre/modules/Promise.jsm");
Cu.import("resource://gre/modules/Task.jsm");
function getClientAndServer() {
let server = new BagheeraServer();
server.start();
let client = new BagheeraClient(server.serverURI);
return [client, server];
}
function run_test() {
initTestLogging("Trace");
run_next_test();
}
add_test(function test_constructor() {
let client = new BagheeraClient("http://localhost:1234/");
run_next_test();
});
add_test(function test_post_json_transport_failure() {
let client = new BagheeraClient("http://localhost:1234/");
client.uploadJSON("foo", "bar", {}).then(function onResult(result) {
do_check_false(result.transportSuccess);
run_next_test();
});
});
add_test(function test_post_json_simple() {
let [client, server] = getClientAndServer();
server.createNamespace("foo");
let promise = client.uploadJSON("foo", "bar", {foo: "bar", biz: "baz"});
promise.then(function onSuccess(result) {
do_check_true(result instanceof BagheeraClientRequestResult);
do_check_true(result.request instanceof RESTRequest);
do_check_true(result.transportSuccess);
do_check_true(result.serverSuccess);
server.stop(run_next_test);
}, do_check_null);
});
add_test(function test_post_json_bad_data() {
let [client, server] = getClientAndServer();
server.createNamespace("foo");
client.uploadJSON("foo", "bar", "{this is invalid json}").then(
function onResult(result) {
do_check_true(result.transportSuccess);
do_check_false(result.serverSuccess);
server.stop(run_next_test);
});
});
add_task(function* test_unicode_payload() {
let [client, server] = getClientAndServer();
server.createNamespace("foo");
const EXPECTED = "πόλλ' οἶδ' ἀλώπηξ, ἀλλ' ἐχῖνος ἓν μέγα";
let result = yield client.uploadJSON("foo", "bar", {test: EXPECTED});
Assert.ok(result.transportSuccess);
Assert.ok(result.serverSuccess);
let p = server.getDocument("foo", "bar");
Assert.equal(p.test, EXPECTED);
result = yield client.uploadJSON("foo", "baz", JSON.stringify({test: EXPECTED}));
Assert.ok(result.transportSuccess);
Assert.ok(result.serverSuccess);
p = server.getDocument("foo", "baz");
Assert.equal(p.test, EXPECTED);
let deferred = Promise.defer();
server.stop(() => deferred.resolve());
yield deferred.promise;
});
add_task(function test_post_delete_multiple_obsolete_documents () {
let [client, server] = getClientAndServer();
let namespace = "foo";
let documents = [
[namespace, "one", "{v:1}"],
[namespace, "two", "{v:2}"],
[namespace, "three", "{v:3}"],
[namespace, "four", "{v:4}"],
];
try {
// create initial documents
server.createNamespace(namespace);
for (let [ns, id, payload] of documents) {
server.setDocument(ns, id, payload);
do_check_true(server.hasDocument(ns, id));
}
// Test uploading with deleting some documents.
let deleteIDs = [0, 1].map((no) => { return documents[no][1]; });
let result = yield client.uploadJSON(namespace, "new-1", {foo: "bar"}, {deleteIDs: deleteIDs});
do_check_true(result.transportSuccess);
do_check_true(result.serverSuccess);
do_check_true(server.hasDocument(namespace, "new-1"));
for (let id of deleteIDs) {
do_check_false(server.hasDocument(namespace, id));
}
// Check if the documents that were not staged for deletion are still there.
for (let [,id,] of documents) {
if (deleteIDs.indexOf(id) == -1) {
do_check_true(server.hasDocument(namespace, id));
}
}
// Test upload without deleting documents.
let ids = Object.keys(server.namespaces[namespace]);
result = yield client.uploadJSON(namespace, "new-2", {foo: "bar"});
do_check_true(result.transportSuccess);
do_check_true(result.serverSuccess);
do_check_true(server.hasDocument(namespace, "new-2"));
// Check to see if all the original documents are still there.
for (let id of ids) {
do_check_true(deleteIDs.indexOf(id) !== -1 || server.hasDocument(namespace, id));
}
} finally {
let deferred = Promise.defer();
server.stop(deferred.resolve.bind(deferred));
yield deferred.promise;
}
});
add_test(function test_delete_document() {
let [client, server] = getClientAndServer();
server.createNamespace("foo");
server.setDocument("foo", "bar", "{}");
client.deleteDocument("foo", "bar").then(function onResult(result) {
do_check_true(result.transportSuccess);
do_check_true(result.serverSuccess);
do_check_null(server.getDocument("foo", "bar"));
server.stop(run_next_test);
});
});

View File

@ -0,0 +1,30 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
Cu.import("resource://testing-common/services/common/bagheeraserver.js");
function run_test() {
run_next_test();
}
add_test(function test_server_empty() {
let server = new BagheeraServer();
do_check_false(server.hasNamespace("foo"));
do_check_false(server.hasDocument("foo", "bar"));
do_check_null(server.getDocument("foo", "bar"));
server.createNamespace("foo");
do_check_true(server.hasNamespace("foo"));
run_next_test();
});
add_test(function test_server_start() {
let server = new BagheeraServer();
server.start();
server.stop(run_next_test);
});

View File

@ -16,6 +16,10 @@ const non_android_modules = [
"tokenserverclient.js",
];
const non_android_healthreport_modules = [
"bagheeraclient.js",
];
const TEST_BASE = "resource://testing-common/services/common/";
const shared_test_modules = [
"logging.js",
@ -25,6 +29,10 @@ const non_android_test_modules = [
"storageserver.js",
];
const non_android_healthreport_test_modules = [
"bagheeraserver.js",
];
function expectImportsToSucceed(mm, base=MODULE_BASE) {
for (let m of mm) {
let resource = base + m;
@ -62,8 +70,14 @@ function run_test() {
if (AppConstants.platform != "android") {
expectImportsToSucceed(non_android_modules);
expectImportsToSucceed(non_android_test_modules, TEST_BASE);
if (AppConstants.MOZ_SERVICES_HEALTHREPORT) {
expectImportsToSucceed(non_android_healthreport_modules);
expectImportsToSucceed(non_android_healthreport_test_modules, TEST_BASE);
}
} else {
expectImportsToFail(non_android_modules);
expectImportsToFail(non_android_test_modules, TEST_BASE);
expectImportsToFail(non_android_healthreport_modules);
expectImportsToFail(non_android_healthreport_test_modules, TEST_BASE);
}
}

View File

@ -29,6 +29,12 @@ support-files =
[test_async_chain.js]
[test_async_querySpinningly.js]
[test_bagheera_server.js]
skip-if = (os == "android" || !healthreport)
[test_bagheera_client.js]
skip-if = (os == "android" || !healthreport)
[test_hawkclient.js]
skip-if = os == "android"
[test_hawkrequest.js]