Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 13:51:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1333255: Task for indexing other tasks. r=jonasfj

Browse Source 
This introduces a new docker image: `index-task`,
which given a taskId and a set of namespaces will
index the given taskId under said namespaces.

Modified to include a script with a descriptive name that curious users can
find in the source code.

MozReview-Commit-ID: KPHVT0XPfsb

--HG--
extra : rebase_source : ed3abde6082f98b83bc5b13f65172562b8875263
extra : source : f8ca244c170f587ead852bb9fbdc99c9dec52f31
This commit is contained in:
Jonas Finnemann Jensen 2017-03-14 15:25:57 +00:00
parent 378a0e0712
commit 654a2a08db
6 changed files with 411 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
taskcluster/ci/docker-image/kind.yml
View File

@ -26,3 +26,5 @@ jobs:
symbol: I(lnt)
android-gradle-build:
symbol: I(agb)
index-task:
symbol: I(idx)

11
taskcluster/docker/index-task/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM node:6-alpine
ENV NODE_ENV production
RUN mkdir /app
ADD insert-indexes.js /app/
ADD package.json /app/
ADD npm-shrinkwrap.json /app/
WORKDIR /app
RUN npm install && npm cache clean
ENTRYPOINT ["node"]

36
taskcluster/docker/index-task/README Normal file
View File

@ -0,0 +1,36 @@
Index-Image
===========
This image is designed to be used for indexing other tasks. It takes a task
definition as follows:
```js
{
...,
scopes: [
'index:insert-task:my-index.namespace',
'index:insert-task:...',
],
payload: {
image: '...',
env: {
TARGET_TASKID: '<taskId-to-be-indexed>',
},
command: [
'insert-indexes.js',
'my-index.namespace.one',
'my-index.namespace.two',
'....',
],
features: {
taskclusterProxy: true,
},
maxRunTime: 600,
},
}
```
As can be seen the `taskId` to be indexed is given by the environment variable
`TARGET_TASKID` and the `command` arguments specifies namespaces that it must
be index under. It is **important** to also include scopes on the form
`index:insert-task:<...>` for all namespaces `<...>` given as `command`
arguments.

44
taskcluster/docker/index-task/insert-indexes.js Normal file
View File

@ -0,0 +1,44 @@
let taskcluster = require('taskcluster-client');
// Create instance of index client
let index = new taskcluster.Index({
delayFactor: 750, // Good solid delay for background process
retries: 8, // A few extra retries for robustness
baseUrl: 'taskcluster/index/v1',
});
// Create queue instance for fetching taskId
let queue = new taskcluster.Queue();
// Load input
let taskId = process.env.TARGET_TASKID;
let namespaces = process.argv.slice(2);
// Validate input
if (!taskId) {
console.log('Expected target task as environment variable: TARGET_TASKID');
process.exit(1);
}
// Fetch task definition to get expiration and then insert into index
queue.task(taskId).then(task => task.expires).then(expires => {
return Promise.all(namespaces.map(namespace => {
console.log('Inserting %s into index under: %s', taskId, namespace);
return index.insertTask(namespace, {
taskId,
rank: 0,
data: {},
expires,
});
}));
}).then(() => {
console.log('indexing successfully completed.');
process.exit(0);
}).catch(err => {
console.log('Error:\n%s', err);
if (err.stack) {
console.log('Stack:\n%s', err.stack());
}
console.log('Properties:\n%j', err);
throw err;
}).catch(() => process.exit(1));

309
taskcluster/docker/index-task/npm-shrinkwrap.json generated Normal file
View File

@ -0,0 +1,309 @@
{
"dependencies": {
"amqplib": {
"version": "0.5.1",
"from": "amqplib@>=0.5.1 <0.6.0",
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.1.tgz"
},
"asap": {
"version": "1.0.0",
"from": "asap@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz"
},
"async": {
"version": "0.9.2",
"from": "async@>=0.9.0 <0.10.0",
"resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
},
"bitsyntax": {
"version": "0.0.4",
"from": "bitsyntax@>=0.0.4 <0.1.0",
"resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.0.4.tgz"
},
"bluebird": {
"version": "3.4.7",
"from": "bluebird@>=3.4.6 <4.0.0",
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz"
},
"boom": {
"version": "2.10.1",
"from": "boom@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
},
"buffer-more-ints": {
"version": "0.0.2",
"from": "buffer-more-ints@0.0.2",
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz"
},
"combined-stream": {
"version": "0.0.7",
"from": "combined-stream@>=0.0.4 <0.1.0",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz"
},
"component-emitter": {
"version": "1.2.1",
"from": "component-emitter@>=1.2.0 <1.3.0",
"resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz"
},
"cookiejar": {
"version": "2.0.6",
"from": "cookiejar@2.0.6",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.6.tgz"
},
"core-util-is": {
"version": "1.0.2",
"from": "core-util-is@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
},
"cryptiles": {
"version": "2.0.5",
"from": "cryptiles@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
},
"debug": {
"version": "2.6.0",
"from": "debug@>=2.1.3 <3.0.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.0.tgz"
},
"delayed-stream": {
"version": "0.0.5",
"from": "delayed-stream@0.0.5",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
},
"eventsource": {
"version": "0.1.6",
"from": "eventsource@0.1.6",
"resolved": "https://registry.npmjs.org/eventsource/-/eventsource-0.1.6.tgz",
"optional": true
},
"extend": {
"version": "3.0.0",
"from": "extend@3.0.0",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
},
"faye-websocket": {
"version": "0.11.1",
"from": "faye-websocket@>=0.11.0 <0.12.0",
"resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.1.tgz",
"optional": true
},
"form-data": {
"version": "0.2.0",
"from": "form-data@0.2.0",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-0.2.0.tgz"
},
"formidable": {
"version": "1.0.17",
"from": "formidable@>=1.0.14 <1.1.0",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz"
},
"hawk": {
"version": "2.3.1",
"from": "hawk@>=2.3.1 <3.0.0",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz"
},
"hoek": {
"version": "2.16.3",
"from": "hoek@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
},
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
},
"isarray": {
"version": "0.0.1",
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
},
"json3": {
"version": "3.3.2",
"from": "json3@>=3.3.2 <4.0.0",
"resolved": "https://registry.npmjs.org/json3/-/json3-3.3.2.tgz",
"optional": true
},
"lodash": {
"version": "3.10.1",
"from": "lodash@>=3.6.0 <4.0.0",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz"
},
"methods": {
"version": "1.1.2",
"from": "methods@>=1.1.1 <1.2.0",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
},
"mime": {
"version": "1.3.4",
"from": "mime@1.3.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
},
"mime-db": {
"version": "1.12.0",
"from": "mime-db@>=1.12.0 <1.13.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.12.0.tgz"
},
"mime-types": {
"version": "2.0.14",
"from": "mime-types@>=2.0.3 <2.1.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz"
},
"ms": {
"version": "0.7.2",
"from": "ms@0.7.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz"
},
"original": {
"version": "1.0.0",
"from": "original@>=0.0.5",
"resolved": "https://registry.npmjs.org/original/-/original-1.0.0.tgz",
"optional": true,
"dependencies": {
"url-parse": {
"version": "1.0.5",
"from": "url-parse@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.0.5.tgz",
"optional": true
}
}
},
"promise": {
"version": "6.1.0",
"from": "promise@>=6.1.0 <7.0.0",
"resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz"
},
"qs": {
"version": "2.3.3",
"from": "qs@2.3.3",
"resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
},
"querystringify": {
"version": "0.0.4",
"from": "querystringify@>=0.0.0 <0.1.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-0.0.4.tgz"
},
"readable-stream": {
"version": "1.1.14",
"from": "readable-stream@>=1.0.0 <2.0.0 >=1.1.9",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz"
},
"reduce-component": {
"version": "1.0.1",
"from": "reduce-component@1.0.1",
"resolved": "https://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
},
"requires-port": {
"version": "1.0.0",
"from": "requires-port@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz"
},
"slugid": {
"version": "1.1.0",
"from": "slugid@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/slugid/-/slugid-1.1.0.tgz"
},
"sntp": {
"version": "1.0.9",
"from": "sntp@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
},
"sockjs-client": {
"version": "1.1.2",
"from": "sockjs-client@>=1.0.3 <2.0.0",
"resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.1.2.tgz",
"optional": true
},
"string_decoder": {
"version": "0.10.31",
"from": "string_decoder@>=0.10.0 <0.11.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
},
"superagent": {
"version": "1.7.2",
"from": "superagent@>=1.7.0 <1.8.0",
"resolved": "https://registry.npmjs.org/superagent/-/superagent-1.7.2.tgz",
"dependencies": {
"readable-stream": {
"version": "1.0.27-1",
"from": "readable-stream@1.0.27-1",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz"
}
}
},
"superagent-hawk": {
"version": "0.0.6",
"from": "superagent-hawk@>=0.0.6 <0.0.7",
"resolved": "https://registry.npmjs.org/superagent-hawk/-/superagent-hawk-0.0.6.tgz",
"dependencies": {
"boom": {
"version": "0.4.2",
"from": "boom@>=0.4.0 <0.5.0",
"resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz"
},
"cryptiles": {
"version": "0.2.2",
"from": "cryptiles@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz"
},
"hawk": {
"version": "1.0.0",
"from": "hawk@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-1.0.0.tgz"
},
"hoek": {
"version": "0.9.1",
"from": "hoek@>=0.9.0 <0.10.0",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz"
},
"qs": {
"version": "0.6.6",
"from": "qs@>=0.6.6 <0.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-0.6.6.tgz"
},
"sntp": {
"version": "0.2.4",
"from": "sntp@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz"
}
}
},
"superagent-promise": {
"version": "0.2.0",
"from": "superagent-promise@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/superagent-promise/-/superagent-promise-0.2.0.tgz"
},
"taskcluster-client": {
"version": "1.6.3",
"from": "taskcluster-client@>=1.6.2 <2.0.0",
"resolved": "https://registry.npmjs.org/taskcluster-client/-/taskcluster-client-1.6.3.tgz"
},
"url-join": {
"version": "0.0.1",
"from": "url-join@>=0.0.1 <0.0.2",
"resolved": "https://registry.npmjs.org/url-join/-/url-join-0.0.1.tgz"
},
"url-parse": {
"version": "1.1.7",
"from": "url-parse@>=1.1.1 <2.0.0",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.1.7.tgz",
"optional": true
},
"uuid": {
"version": "2.0.3",
"from": "uuid@>=2.0.1 <3.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz"
},
"websocket-driver": {
"version": "0.6.5",
"from": "websocket-driver@>=0.5.1",
"resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.6.5.tgz",
"optional": true
},
"websocket-extensions": {
"version": "0.1.1",
"from": "websocket-extensions@>=0.1.1",
"resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.1.tgz",
"optional": true
}
}
}

9
taskcluster/docker/index-task/package.json Normal file
View File

@ -0,0 +1,9 @@
{
"private": true,
"scripts":{
"start": "node index.js"
},
"dependencies": {
"taskcluster-client": "^1.6.2"
}
}