Bug 1473951: Add infer to taskcluster and build. r=gps

MozReview-Commit-ID: BHi3E6J3nuH

--HG--
extra : rebase_source : a59180efe4fed56222d2847d60133739f38c8ca8

.flake8

@@ -21,3 +21,4 @@ exclude =
     security/nss/,
     testing/mochitest/pywebsocket,
     tools/lint/test/files,
+    build/build-infer/build-infer.py,

build/build-infer/README

@@ -0,0 +1,36 @@
+build-infer.py
+==============
+
+A script to build infer from source.
+
+```
+usage: build-infer.py [-h] -c CONFIG [--clean]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c CONFIG, --config CONFIG
+                        infer configuration file
+  --clean               Clean the build directory
+```
+
+Pre-requisites
+--------------
+* Working build toolchain.
+* ocam
+* git
+* autoconf
+* libsqlite-dev
+* CMake
+* Ninja
+* Python 2.7
+
+Please use the latest available CMake for your platform to avoid surprises.
+
+Config file format
+------------------
+
+build-clang.py accepts a JSON config format with the following fields:
+
+* infer_revision: The infer revision to build.
+* infer_repo: git repository for infer.
+* patches: Optional list of patches to apply.

build/build-infer/build-infer.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import subprocess
+import json
+import argparse
+import sys
+import shutil
+from functools import reduce
+
+
+def check_run(args, path):
+    print(' '.join(args) + ' in ' + path, file=sys.stderr)
+    subprocess.run(args, cwd=path, check=True)
+
+
+def run_in(path, args, extra_env=None):
+    '''
+    Runs the given commands in the directory specified by <path>.
+    '''
+    env = dict(os.environ)
+    env.update(extra_env or {})
+    check_run(args, path)
+    subprocess.run(args, cwd=path)
+
+
+def build_tar_package(tar, name, base, directory):
+    name = os.path.realpath(name)
+    run_in(base, [tar,
+                  '-c',
+                  '-%s' % ('J' if '.xz' in name else 'j'),
+                  '-f',
+                  name, directory])
+
+
+def is_git_repo(dir):
+    '''Check whether the given directory is a git repository.'''
+    from subprocess import CalledProcessError
+    try:
+        check_run(['git', 'rev-parse'], dir)
+        return True
+    except CalledProcessError:
+        return False
+
+
+def git_clone(main_dir, url, clone_dir, commit):
+    '''
+    Clones the repository from <url> into <clone_dir>, and brings the
+    repository to the state of <commit>.
+    '''
+    run_in(main_dir, ['git', 'clone', url, clone_dir])
+    run_in(clone_dir, ['git', 'checkout', commit])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', required=True,
+                        type=argparse.FileType('r'),
+                        help='Infer configuration file')
+    parser.add_argument('-b', '--base-dir',
+                        help="Base directory for code and build artifacts")
+    parser.add_argument('--clean', action='store_true',
+                        help='Clean the build directory')
+    parser.add_argument('--skip-tar', action='store_true',
+                        help='Skip tar packaging stage')
+
+    args = parser.parse_args()
+
+    # The directories end up in the debug info, so the easy way of getting
+    # a reproducible build is to run it in a know absolute directory.
+    # We use a directory that is registered as a volume in the Docker image.
+    if args.base_dir:
+        base_dir = args.base_dir
+    else:
+        base_dir = reduce(os.path.join,
+                          [os.sep + 'builds', 'worker',
+                           'workspace', 'moz-toolchain'])
+    infer_dir = os.path.join(base_dir, 'infer')
+    source_dir = os.path.join(infer_dir, 'src')
+    build_dir = os.path.join(infer_dir, 'build')
+
+    if args.clean:
+        shutil.rmtree(build_dir)
+        os.sys.exit(0)
+
+    config = json.load(args.config)
+    infer_revision = config['infer_revision']
+    infer_repo = config['infer_repo']
+
+    for folder in [infer_dir, source_dir, build_dir]:
+        os.makedirs(folder, exist_ok=True)
+
+    # clone infer
+    if not is_git_repo(source_dir):
+        # git doesn't like cloning into a non-empty folder. If src is not a git
+        # repo then just remove it in order to reclone
+        shutil.rmtree(source_dir)
+        git_clone(infer_dir, infer_repo, source_dir, infer_revision)
+    # apply a few patches
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    # clean the git directory by reseting all changes
+    git_commands = [['clean', '-f'], ['reset', '--hard']]
+    for command in git_commands:
+        run_in(source_dir, ['git']+command)
+    for p in config.get('patches', []):
+        run_in(source_dir, ['git', 'apply', os.path.join(dir_path, p)])
+    # build infer
+    run_in(source_dir, ['./build-infer.sh', 'java'],
+           extra_env={'NO_CMAKE_STRIP': '1'})
+    package_name = 'infer'
+    if not args.skip_tar:
+        build_tar_package('tar', '%s.tar.xz' % (package_name),
+                          source_dir, os.path.join('infer', 'bin'))

build/build-infer/infer-linux64.json

@@ -0,0 +1,5 @@
+{
+    "infer_repo": "https://github.com/facebook/infer",
+    "infer_revision": "14aa1edbf532675f48ba0a35a2e55d07b79b1f3c",
+    "patches": []
+}

taskcluster/ci/docker-image/kind.yml

@@ -86,6 +86,9 @@ jobs:
   fetch:
     symbol: I(fetch)
     parent: debian9-base
+  infer-build:
+    symbol: I(infer)
+    parent: debian9-base
   mingw32-build:
     symbol: I(mingw)
     parent: debian9-base

taskcluster/ci/static-analysis-autotest/kind.yml

@@ -48,6 +48,7 @@ jobs:
         toolchains:
             - linux64-clang
             - linux64-clang-tidy
+            - linux64-infer
             - linux64-rust
             - linux64-sccache
 

taskcluster/ci/toolchain/linux.yml

@@ -125,6 +125,28 @@ linux64-clang-tidy:
     toolchains:
         - linux64-gcc-4.9
 
+linux64-infer:
+    description: "infer build"
+    index:
+        product: static-analysis
+        job-name: linux64-infer
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(infer)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: infer-build}
+        max-run-time: 3600
+    run:
+        using: toolchain-script
+        script: build-infer-linux.sh
+        resources:
+            - 'build/build-infer/build-infer.py'
+            - 'build/build-infer/infer-linux64.json'
+        toolchain-artifact: public/build/infer.tar.xz
+
 linux64-gcc-4.9:
     description: "GCC 4.9 toolchain build"
     treeherder:

taskcluster/docker/infer-build/Dockerfile

@@ -0,0 +1,30 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Mike Hommey <mhommey@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+ENV XZ_OPT=-T0
+
+RUN apt-get update && \
+    apt-get install \
+      autoconf \
+      bison \
+      bzip2 \
+      flex \
+      curl \
+      git \
+      opam \
+      libsqlite3-dev \
+      autoconf \
+      automake \
+      cmake \
+      libc6-dev \
+      openjdk-8-jdk-headless \
+      pkg-config \
+      patch \
+      tar \
+      unzip \
+      zlib1g-dev

taskcluster/scripts/misc/build-infer-linux.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building infer for Linux.
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-infer
+./build-infer.py -c infer-linux64.json
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp infer.tar.* $UPLOAD_DIR