Bug 1906899 - remove unused resource-monitor code. r=taskgraph-reviewers,bhearsum

Differential Revision: https://phabricator.services.mozilla.com/D216058

taskcluster/docs/attributes.rst

@@ -389,12 +389,6 @@ review bot, the task will ran for every new Phabricator diff.
 Any supported and detected issue will be automatically reported on the
 Phabricator revision.
 
-resource-monitor
-================
-If a task set this boolean attribute to `true`, it will collect CPU, memory, and
-- if available - Disk and Network IO by running the resource-monitor utility,
-provided through fetches.
-
 retrigger
 =========
 Whether the task can be retriggered, or if it needs to be re-run.

taskcluster/gecko_taskgraph/transforms/job/__init__.py

@@ -161,47 +161,6 @@ def set_label(config, jobs):
         yield job
 
 
-@transforms.add
-def add_resource_monitor(config, jobs):
-    for job in jobs:
-        if job.get("attributes", {}).get("resource-monitor"):
-            worker_implementation, worker_os = worker_type_implementation(
-                config.graph_config, config.params, job["worker-type"]
-            )
-            # Normalise worker os so that linux-bitbar and similar use linux tools.
-            worker_os = worker_os.split("-")[0]
-            # We don't currently support an Arm worker, due to gopsutil's indirect
-            # dependencies (go-ole)
-            if "aarch64" in job["worker-type"]:
-                yield job
-                continue
-            elif "win7" in job["worker-type"]:
-                arch = "32"
-            else:
-                arch = "64"
-            job.setdefault("fetches", {})
-            job["fetches"].setdefault("toolchain", [])
-            job["fetches"]["toolchain"].append(f"{worker_os}{arch}-resource-monitor")
-
-            if worker_implementation == "docker-worker":
-                artifact_source = "/builds/worker/monitoring/resource-monitor.json"
-            else:
-                artifact_source = "monitoring/resource-monitor.json"
-            job["worker"].setdefault("artifacts", [])
-            job["worker"]["artifacts"].append(
-                {
-                    "name": "public/monitoring/resource-monitor.json",
-                    "type": "file",
-                    "path": artifact_source,
-                }
-            )
-            # Set env for output file
-            job["worker"].setdefault("env", {})
-            job["worker"]["env"]["RESOURCE_MONITOR_OUTPUT"] = artifact_source
-
-        yield job
-
-
 @transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""

taskcluster/kinds/fetch/kind.yml

@@ -15,7 +15,6 @@ tasks-from:
     - browsertime.yml
     - chromium-fetch.yml
     - cft-chromedriver-fetch.yml
-    - resource-monitor.yml
     - toolchain-clang-tidy-external.yml
     - toolchains.yml
     - updatebot.yml

taskcluster/kinds/fetch/resource-monitor.yml

@@ -1,10 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-resource-monitor:
-    description: Resource monitoring tools
-    fetch:
-        type: git
-        repo: https://github.com/mozilla-releng/resource-monitor
-        revision: 17371502a3b04579375d707844e6bf08dee95d22

taskcluster/kinds/toolchain/kind.yml

@@ -47,7 +47,6 @@ tasks-from:
     - node.yml
     - pkgconf.yml
     - python.yml
-    - resourcemonitor.yml
     - rust.yml
     - rust-size.yml
     - sccache.yml

taskcluster/kinds/toolchain/resourcemonitor.yml

@@ -1,44 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-task-defaults:
-    worker-type: b-linux-gcp
-    worker:
-        max-run-time: 1800
-    fetches:
-        fetch:
-            - go-1.14.4
-            - resource-monitor
-    run:
-        script: build-resourcemonitor.sh
-        sparse-profile: null
-        toolchain-artifact: public/build/resource-monitor.tar.zst
-
-linux64-resource-monitor:
-    description: "linux64 resourcemonitor toolchain build"
-    treeherder:
-        symbol: TL(rs-monitor)
-    run:
-        arguments: ['linux64']
-
-macosx64-resource-monitor:
-    description: "macos64 resourcemonitor toolchain build"
-    treeherder:
-        symbol: TM(rs-monitor)
-    run:
-        arguments: ['macos64']
-
-windows32-resource-monitor:
-    description: "windows32 resourcemonitor toolchain build"
-    treeherder:
-        symbol: TW32(rs-monitor)
-    run:
-        arguments: ['windows32']
-
-windows64-resource-monitor:
-    description: "windows64 resourcemonitor toolchain build"
-    treeherder:
-        symbol: TW64(rs-monitor)
-    run:
-        arguments: ['windows64']

taskcluster/scripts/misc/build-resourcemonitor.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-set -x -e -v
-
-cd "$MOZ_FETCHES_DIR"/resource-monitor/ || exit 1
-COMPRESS_EXT=zst
-
-PATH="$MOZ_FETCHES_DIR/go/bin:$PATH"
-export PATH
-
-EXE_SUFFIX=""
-
-case "$1" in
-    linux64)   GOOS=linux; GOARCH=amd64 ;;
-    macos64)   GOOS=darwin; GOARCH=amd64 ;;
-    windows64) GOOS=windows; GOARCH=amd64; EXE_SUFFIX=".exe" ;;
-    windows32) GOOS=windows; GOARCH=386;   EXE_SUFFIX=".exe" ;;
-    *)
-        echo "Unknown architecture $1 not recognized in build-resourcemonitor.sh" >&2
-        exit 1
-    ;;
-esac
-
-export GOOS
-export GOARCH
-export EXE_SUFFIX
-
-echo "GOOS=$GOOS"
-echo "GOARCH=$GOARCH"
-
-go build .
-
-STAGING_DIR="resource-monitor"
-mv "resource-monitor${EXE_SUFFIX}" resource-monitor.tmp
-mkdir "${STAGING_DIR}"
-
-cp resource-monitor.tmp "${STAGING_DIR}/resource-monitor${EXE_SUFFIX}"
-
-tar -acf "resource-monitor.tar.$COMPRESS_EXT" "${STAGING_DIR}"/
-mkdir -p "$UPLOAD_DIR"
-cp "resource-monitor.tar.$COMPRESS_EXT" "$UPLOAD_DIR"

taskcluster/scripts/run-task

@@ -684,57 +684,6 @@ def vcs_checkout_from_args(args, project):
         sparse_profile=options['sparse-profile'])
 
 
-def maybe_run_resource_monitoring():
-    """Run the resource monitor if available.
-
-    Discussion in https://github.com/taskcluster/taskcluster-rfcs/pull/160
-    and https://bugzil.la/1648051
-    """
-    if 'MOZ_FETCHES' not in os.environ:
-        return
-    if 'RESOURCE_MONITOR_OUTPUT' not in os.environ:
-        return
-
-    prefix = b'resource_monitor'
-
-    executable = '{}/resource-monitor/resource-monitor{}'.format(
-        os.environ.get('MOZ_FETCHES_DIR'), '.exe' if IS_WINDOWS else '')
-
-    if not os.path.exists(executable) or not os.access(executable, os.X_OK):
-        print_line(prefix, b"%s not executable\n" % executable.encode('utf-8'))
-        return
-    args = [
-        executable,
-        '-process',
-        str(os.getpid()),
-        '-output',
-        os.environ["RESOURCE_MONITOR_OUTPUT"],
-    ]
-    print_line(prefix, b"Resource monitor starting: %s\n" % str(args).encode('utf-8'))
-    # Avoid environment variables the payload doesn't need.
-    del os.environ['RESOURCE_MONITOR_OUTPUT']
-
-    # Without CREATE_NEW_PROCESS_GROUP Windows signals will attempt to kill run-task, too.
-    process = subprocess.Popen(args,
-                               bufsize=0,
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.STDOUT,
-                               creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if IS_WINDOWS else 0,
-                               cwd=os.getcwd())
-
-    def capture_output():
-        fh = io.TextIOWrapper(process.stdout, encoding='latin1')
-        while True:
-            data = fh.readline().encode('latin1')
-            if data == b'':
-                break
-            print_line(prefix, data)
-
-    monitor_process = Thread(target=capture_output)
-    monitor_process.start()
-    return process
-
-
 def main(args):
     print_line(b'setup', b'run-task started in %s\n' % os.getcwd().encode('utf-8'))
     running_as_root = IS_POSIX and os.getuid() == 0
@@ -943,85 +892,67 @@ def main(args):
     vcs_checkout_from_args(args, 'gecko')
     vcs_checkout_from_args(args, 'comm')
 
-    resource_process = None
+    for k in ('GECKO_PATH', 'MOZ_FETCHES_DIR', 'UPLOAD_DIR', 'MOZ_PYTHON_HOME'):
+        if k in os.environ:
+            # Normalize paths to use forward slashes. Some shell scripts
+            # tolerate that better on Windows.
+            os.environ[k] = os.path.abspath(os.environ[k]).replace(os.sep, '/')
+            print_line(b'setup', b'%s is %s\n' % (
+                k.encode('utf-8'),
+                os.environ[k].encode('utf-8')))
 
-    try:
-        for k in ('GECKO_PATH', 'MOZ_FETCHES_DIR', 'UPLOAD_DIR', 'MOZ_PYTHON_HOME'):
-            if k in os.environ:
-                # Normalize paths to use forward slashes. Some shell scripts
-                # tolerate that better on Windows.
-                os.environ[k] = os.path.abspath(os.environ[k]).replace(os.sep, '/')
-                print_line(b'setup', b'%s is %s\n' % (
-                    k.encode('utf-8'),
-                    os.environ[k].encode('utf-8')))
+    if 'MOZ_FETCHES' in os.environ:
+        fetch_artifacts()
 
-        if 'MOZ_FETCHES' in os.environ:
-            fetch_artifacts()
+        # If Python is a fetch dependency, add it to the PATH and setting
+        # the mozilla-specific MOZ_PYTHON_HOME to relocate binaries.
+        if 'MOZ_PYTHON_HOME' in os.environ:
 
-            # If Python is a fetch dependency, add it to the PATH and setting
-            # the mozilla-specific MOZ_PYTHON_HOME to relocate binaries.
-            if 'MOZ_PYTHON_HOME' in os.environ:
+            print_line(b'setup',
+                       b'Setting up local python environment\n')
+            prev = [os.environ['PATH']] if 'PATH' in os.environ else []
 
-                print_line(b'setup',
-                           b'Setting up local python environment\n')
-                prev = [os.environ['PATH']] if 'PATH' in os.environ else []
-
-                moz_python_home = os.environ['MOZ_PYTHON_HOME']
-                if IS_WINDOWS:
-                    ext = '.exe'
-                    moz_python_bindir = moz_python_home
-                else:
-                    ext = ''
-                    moz_python_bindir = moz_python_home + '/bin'
-
-
-                new = os.environ['PATH'] = os.pathsep.join([moz_python_bindir]
-                                                           + prev)
-
-                # Relocate the python binary. Standard way uses PYTHONHOME, but
-                # this conflicts with system python (e.g. used by hg) so we
-                # maintain a small patch to use MOZPYTHONHOME instead.
-                os.environ['MOZPYTHONHOME'] = moz_python_home
-
-                pyinterp = os.path.join(moz_python_bindir, f'python3{ext}')
-                # just a sanity check
-                if not os.path.exists(pyinterp):
-                    raise RuntimeError("Inconsistent Python installation: "
-                                       "archive found, but no python3 binary "
-                                       "detected")
-
-                if IS_MACOSX:
-                    # On OSX, we may not have access to the system certificate,
-                    # so use the certifi ones.
-                    certifi_cert_file = subprocess.check_output(
-                    [pyinterp, '-c',
-                     'import certifi; print(certifi.where())'],
-                    text=True
-                    )
-                    os.environ['SSL_CERT_FILE'] = certifi_cert_file.strip()
-                    print_line(b'setup',
-                               b'patching ssl certificate\n')
-
-                print_line(b'setup',
-                           b'updated PATH with python artifact: '
-                           + new.encode() + b'\n')
-
-
-        resource_process = maybe_run_resource_monitoring()
-
-        return run_and_prefix_output(b'task', task_args, cwd=args.task_cwd)
-    finally:
-        if resource_process:
-            print_line(b'resource_monitor', b'terminating\n')
+            moz_python_home = os.environ['MOZ_PYTHON_HOME']
             if IS_WINDOWS:
-                # .terminate() on Windows is not a graceful shutdown, due to
-                # differences in signals. CTRL_BREAK_EVENT will work provided
-                # the subprocess is in a different process group, so this script
-                # isn't also killed.
-                os.kill(resource_process.pid, signal.CTRL_BREAK_EVENT)
+                ext = '.exe'
+                moz_python_bindir = moz_python_home
             else:
-                resource_process.terminate()
-            resource_process.wait()
+                ext = ''
+                moz_python_bindir = moz_python_home + '/bin'
+
+
+            new = os.environ['PATH'] = os.pathsep.join([moz_python_bindir]
+                                                       + prev)
+
+            # Relocate the python binary. Standard way uses PYTHONHOME, but
+            # this conflicts with system python (e.g. used by hg) so we
+            # maintain a small patch to use MOZPYTHONHOME instead.
+            os.environ['MOZPYTHONHOME'] = moz_python_home
+
+            pyinterp = os.path.join(moz_python_bindir, f'python3{ext}')
+            # just a sanity check
+            if not os.path.exists(pyinterp):
+                raise RuntimeError("Inconsistent Python installation: "
+                                   "archive found, but no python3 binary "
+                                   "detected")
+
+            if IS_MACOSX:
+                # On OSX, we may not have access to the system certificate,
+                # so use the certifi ones.
+                certifi_cert_file = subprocess.check_output(
+                [pyinterp, '-c',
+                 'import certifi; print(certifi.where())'],
+                text=True
+                )
+                os.environ['SSL_CERT_FILE'] = certifi_cert_file.strip()
+                print_line(b'setup',
+                           b'patching ssl certificate\n')
+
+            print_line(b'setup',
+                       b'updated PATH with python artifact: '
+                       + new.encode() + b'\n')
+
+    return run_and_prefix_output(b'task', task_args, cwd=args.task_cwd)
 
 
 if __name__ == '__main__':