mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-25 05:41:12 +00:00
Bug 1566070 Remove references to datadog from partials r=mtabara
Our datadog contract is coming to an end. Soon we'll add influxdb data collection Differential Revision: https://phabricator.services.mozilla.com/D38086 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Simon Fraser
parent
0233c4cddc
commit
b4e6cbf3bc
@ -3,7 +3,6 @@ python_version = '3.6'
|
||||
|
||||
[packages]
|
||||
aiohttp = "*"
|
||||
datadog = "*"
|
||||
mar = "*"
|
||||
redo = "*"
|
||||
requests = "*"
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "8ae855f2ef3d4d9dcaa6dd796ee176980724e8215eea64494710ef603cadd3c5"
|
||||
"sha256": "99aadbfad074e19038b66e1ac7d268cdf10db18212b3e02fe497b8cc60d8bfec"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@ -181,21 +181,6 @@
|
||||
],
|
||||
"version": "==2.7"
|
||||
},
|
||||
"datadog": {
|
||||
"hashes": [
|
||||
"sha256:0e33727bd9ef0b8201359accdf12f61fb58ab541bf4062693aca16653f8e7a0a",
|
||||
"sha256:995e51d142ae3624c86c78369b268f23386bb207df5345d718c241718387875c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.29.3"
|
||||
},
|
||||
"decorator": {
|
||||
"hashes": [
|
||||
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de",
|
||||
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6"
|
||||
],
|
||||
"version": "==4.4.0"
|
||||
},
|
||||
"dictdiffer": {
|
||||
"hashes": [
|
||||
"sha256:97cf4ef98ebc1acf737074aed41e379cf48ab5ff528c92109dfb8e2e619e6809",
|
||||
@ -316,6 +301,17 @@
|
||||
},
|
||||
"pyasn1": {
|
||||
"hashes": [
|
||||
"sha256:061442c60842f6d11051d4fdae9bc197b64bd41573a12234a753a0cb80b4f30b",
|
||||
"sha256:0ee2449bf4c4e535823acc25624c45a8b454f328d59d3f3eeb82d3567100b9bd",
|
||||
"sha256:5f9fb05c33e53b9a6ee3b1ed1d292043f83df465852bec876e93b47fd2df7eed",
|
||||
"sha256:65201d28e081f690a32401e6253cca4449ccacc8f3988e811fae66bd822910ee",
|
||||
"sha256:79b336b073a52fa3c3d8728e78fa56b7d03138ef59f44084de5f39650265b5ff",
|
||||
"sha256:8ec20f61483764de281e0b4aba7d12716189700debcfa9e7935780850bf527f3",
|
||||
"sha256:9458d0273f95d035de4c0d5e0643f25daba330582cc71bb554fe6969c015042a",
|
||||
"sha256:98d97a1833a29ca61cd04a60414def8f02f406d732f9f0bcb49f769faff1b699",
|
||||
"sha256:b00d7bfb6603517e189d1ad76967c7e805139f63e43096e5f871d1277f50aea5",
|
||||
"sha256:b06c0cfd708b806ea025426aace45551f91ea7f557e0c2d4fbd9a4b346873ce0",
|
||||
"sha256:d14d05984581770333731690f5453efd4b82e1e5d824a1d7976b868a2e5c38e8",
|
||||
"sha256:da2420fe13a9452d8ae97a0e478adde1dee153b11ba832a95b223a2ba01c10f7",
|
||||
"sha256:da6b43a8c9ae93bc80e2739efb38cc776ba74a886e3e9318d65fe81a8b8a2c6e"
|
||||
],
|
||||
@ -338,7 +334,6 @@
|
||||
"sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb",
|
||||
"sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e"
|
||||
],
|
||||
"markers": "python_version >= '2.7'",
|
||||
"version": "==2.8.0"
|
||||
},
|
||||
"pyyaml": {
|
||||
@ -355,7 +350,6 @@
|
||||
"sha256:c68ea4d3ba1705da1e0d85da6684ac657912679a649e8868bd850d2c299cce13",
|
||||
"sha256:e23d0cc5299223dcc37885dae624f382297717e459ea24053709675a976a3e19"
|
||||
],
|
||||
"markers": "python_version != '2.6'",
|
||||
"version": "==5.1"
|
||||
},
|
||||
"redo": {
|
||||
@ -456,7 +450,6 @@
|
||||
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
|
||||
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
|
||||
],
|
||||
"markers": "python_version >= '3.4'",
|
||||
"version": "==1.25.3"
|
||||
},
|
||||
"yarl": {
|
||||
|
||||
@ -25,15 +25,9 @@ from scriptworker.utils import retry_async
|
||||
from mardor.reader import MarReader
|
||||
from mardor.signing import get_keysize
|
||||
|
||||
from datadog import initialize, ThreadStats
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Create this even when not sending metrics, so the context manager
|
||||
# statements work.
|
||||
ddstats = ThreadStats(namespace='releng.releases.partials')
|
||||
|
||||
|
||||
ROOT_URL = os.environ['TASKCLUSTER_ROOT_URL']
|
||||
QUEUE_PREFIX = ("https://queue.taskcluster.net/"
|
||||
@ -64,18 +58,6 @@ BCJ_OPTIONS = {
|
||||
}
|
||||
|
||||
|
||||
def write_dogrc(api_key):
|
||||
"""Datadog .dogrc file for command line interface."""
|
||||
dogrc_path = os.path.join(os.path.expanduser('~'), '.dogrc')
|
||||
config = configparser.ConfigParser()
|
||||
config['Connection'] = {
|
||||
'apikey': api_key,
|
||||
'appkey': '',
|
||||
}
|
||||
with open(dogrc_path, 'w') as f:
|
||||
config.write(f)
|
||||
|
||||
|
||||
def verify_signature(mar, certs):
|
||||
log.info("Checking %s signature", mar)
|
||||
with open(mar, 'rb') as mar_fh:
|
||||
@ -334,11 +316,7 @@ async def manage_partial(partial_def, filename_template, artifacts_dir,
|
||||
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
|
||||
unpack_dir = os.path.join(work_env.workdir, mar_type)
|
||||
|
||||
metric_tags = [
|
||||
"platform:{}".format(partial_def['platform']),
|
||||
]
|
||||
with ddstats.timer('mar.download.time', tags=metric_tags):
|
||||
await retry_download(f, dest)
|
||||
await retry_download(f, dest)
|
||||
|
||||
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
|
||||
verify_signature(dest, signing_certs)
|
||||
@ -346,8 +324,7 @@ async def manage_partial(partial_def, filename_template, artifacts_dir,
|
||||
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
|
||||
complete_mars["%s_hash" % mar_type] = get_hash(dest)
|
||||
|
||||
with ddstats.timer('mar.unpack.time'):
|
||||
await unpack(work_env, dest, unpack_dir)
|
||||
await unpack(work_env, dest, unpack_dir)
|
||||
|
||||
if mar_type == 'to':
|
||||
check_channels_in_files.append(dest)
|
||||
@ -413,25 +390,11 @@ async def manage_partial(partial_def, filename_template, artifacts_dir,
|
||||
await work_env.download_buildsystem_bits(repo=mar_data["repo"],
|
||||
revision=mar_data["revision"])
|
||||
|
||||
metric_tags = [
|
||||
"branch:{}".format(mar_data['branch']),
|
||||
"platform:{}".format(mar_data['platform']),
|
||||
# If required. Shouldn't add much useful info, but increases
|
||||
# cardinality of metrics substantially, so avoided.
|
||||
# "locale:{}".format(mar_data['locale']),
|
||||
]
|
||||
with ddstats.timer('generate_partial.time', tags=metric_tags):
|
||||
await generate_partial(work_env, from_path, to_path, dest_mar,
|
||||
mar_data, use_old_format)
|
||||
await generate_partial(work_env, from_path, to_path, dest_mar,
|
||||
mar_data, use_old_format)
|
||||
|
||||
mar_data["size"] = os.path.getsize(dest_mar)
|
||||
|
||||
metric_tags.append("unit:bytes")
|
||||
# Allows us to find out how many releases there were between the two,
|
||||
# making buckets of the file sizes easier.
|
||||
metric_tags.append("update_number:{}".format(mar_data.get('update_number', 0)))
|
||||
ddstats.gauge('partial_mar_size', mar_data['size'], tags=metric_tags)
|
||||
|
||||
mar_data["hash"] = get_hash(dest_mar)
|
||||
|
||||
shutil.copy(dest_mar, artifacts_dir)
|
||||
@ -470,8 +433,6 @@ async def async_main(args, signing_certs):
|
||||
|
||||
def main():
|
||||
|
||||
start = time.time()
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--artifacts-dir", required=True)
|
||||
parser.add_argument("--sha1-signing-cert", required=True)
|
||||
@ -504,24 +465,6 @@ def main():
|
||||
assert(get_keysize(signing_certs['sha1']) == 2048)
|
||||
assert(get_keysize(signing_certs['sha384']) == 4096)
|
||||
|
||||
# Intended for local testing.
|
||||
dd_api_key = os.environ.get('DATADOG_API_KEY')
|
||||
# Intended for Taskcluster.
|
||||
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
|
||||
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
|
||||
|
||||
if dd_api_key:
|
||||
dd_options = {
|
||||
'api_key': dd_api_key,
|
||||
}
|
||||
log.info("Starting metric collection")
|
||||
initialize(**dd_options)
|
||||
ddstats.start(flush_interval=1)
|
||||
# For use in shell scripts.
|
||||
write_dogrc(dd_api_key)
|
||||
else:
|
||||
log.info("No metric collection")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
manifest = loop.run_until_complete(async_main(args, signing_certs))
|
||||
loop.close()
|
||||
@ -532,21 +475,6 @@ def main():
|
||||
|
||||
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
|
||||
|
||||
# Warning: Assumption that one partials task will always be for one branch.
|
||||
metric_tags = [
|
||||
"branch:{}".format(manifest[0]['branch']),
|
||||
"platform:{}".format(manifest[0]['platform']),
|
||||
]
|
||||
|
||||
ddstats.timing('task_duration', time.time() - start,
|
||||
start, tags=metric_tags)
|
||||
|
||||
# Wait for all the metrics to flush. If the program ends before
|
||||
# they've been sent, they'll be dropped.
|
||||
# Should be more than the flush_interval for the ThreadStats object
|
||||
if dd_api_key:
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@ -63,22 +63,6 @@ Partials are made as part of the ``promote`` task group. The previous
|
||||
versions used to create the update are specified in ship-it by
|
||||
Release Management.
|
||||
|
||||
Data About Partials
|
||||
-------------------
|
||||
|
||||
Some metrics are collected in Datadog_ about partial update tasks.
|
||||
The prefix used is ``releng.releases.partials``, so the relevant metric names
|
||||
will all start with that.
|
||||
|
||||
Some dashboards in Datadog are public, some require a login. If you need
|
||||
access, file a bug under 'Cloud Services :: Operations: Metrics/Monitoring'
|
||||
|
||||
Some examples of potentially useful metrics:
|
||||
|
||||
* ``releng.releases.partials.partial_mar_size`` (tagged with branch, platform and update-number)
|
||||
* ``releng.releases.partials.task_duration`` - the time the task took, running partial generation concurrently.
|
||||
* ``releng.releases.partials.generate_partial.time`` - the time taken to make one partial update
|
||||
|
||||
Nightly Partials
|
||||
----------------
|
||||
|
||||
@ -137,5 +121,3 @@ Otherwise, it will depend on ``repackage-signing``.
|
||||
|
||||
This is not ideal, as it results in unclear logic in the task graph
|
||||
generation. It will be improved.
|
||||
|
||||
.. _Datadog: https://app.datadoghq.com/metric/explorer
|
||||
|
||||
@ -117,8 +117,6 @@ def make_task_description(config, jobs):
|
||||
'env': {
|
||||
'SHA1_SIGNING_CERT': 'nightly_sha1',
|
||||
'SHA384_SIGNING_CERT': 'nightly_sha384',
|
||||
'DATADOG_API_SECRET':
|
||||
'project/releng/gecko/build/level-{}/datadog-api-key'.format(level),
|
||||
'EXTRA_PARAMS': '--arch={}'.format(architecture(attributes['build_platform'])),
|
||||
'MAR_CHANNEL_ID': attributes['mar-channel-id']
|
||||
}
|
||||
@ -132,9 +130,7 @@ def make_task_description(config, jobs):
|
||||
dep_job.task["metadata"]["description"]),
|
||||
'worker-type': 'b-linux',
|
||||
'dependencies': dependencies,
|
||||
'scopes': [
|
||||
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key' % level
|
||||
],
|
||||
'scopes': [],
|
||||
'attributes': attributes,
|
||||
'run-on-projects': dep_job.attributes.get('run_on_projects'),
|
||||
'treeherder': treeherder,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user