Bug 1418188 - Run Searchfox indexing in TaskCluster on a daily basis (r=dustin,chmanchester)

MozReview-Commit-ID: FKrA8ejwuic

.cron.yml

@@ -98,6 +98,18 @@ jobs:
               mozilla-central: [{hour: 10, minute: 0}]
               # No default
 
+    - name: searchfox-index
+      job:
+          type: decision-task
+          treeherder-symbol: Searchfox
+          target-tasks-method: searchfox_index
+      run-on-projects:
+          - mozilla-central
+      when:
+          by-project:
+              mozilla-central: [{hour: 10, minute: 30}]
+              # No default
+
     - name: periodic-update
       job:
           type: decision-task

browser/config/mozconfigs/linux64/debug-searchfox-clang

@@ -0,0 +1,25 @@
+MOZ_AUTOMATION_BUILD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_TESTS=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
+. "$topsrcdir/build/mozconfig.common"
+
+ac_add_options --enable-debug
+ac_add_options --enable-dmd
+
+. $topsrcdir/build/mozconfig.stylo
+
+# Use Clang as specified in manifest
+export CC="$topsrcdir/clang/bin/clang"
+export CXX="$topsrcdir/clang/bin/clang++"
+
+# Add the static checker
+ac_add_options --enable-clang-plugin
+ac_add_options --enable-mozsearch-plugin
+
+. "$topsrcdir/build/unix/mozconfig.stdcxx"
+
+export PKG_CONFIG_LIBDIR=/usr/lib64/pkgconfig:/usr/share/pkgconfig
+. $topsrcdir/build/unix/mozconfig.gtk
+
+. "$topsrcdir/build/mozconfig.common.override"

browser/config/mozconfigs/macosx64/debug-searchfox

@@ -0,0 +1,13 @@
+MOZ_AUTOMATION_BUILD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_TESTS=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
+. $topsrcdir/build/macosx/mozconfig.common
+
+ac_add_options --enable-debug
+ac_add_options --enable-dmd
+
+ac_add_options --enable-clang-plugin
+ac_add_options --enable-mozsearch-plugin
+
+. "$topsrcdir/build/mozconfig.common.override"

browser/config/mozconfigs/win32/debug-searchfox

@@ -0,0 +1,17 @@
+MOZ_AUTOMATION_BUILD_SYMBOLS=0
+MOZ_AUTOMATION_PACKAGE_TESTS=0
+MOZ_AUTOMATION_L10N_CHECK=0
+
+. "$topsrcdir/build/mozconfig.win-common"
+. "$topsrcdir/browser/config/mozconfigs/common"
+
+ac_add_options --enable-optimize
+ac_add_options --enable-debug
+
+ac_add_options --enable-clang-plugin
+ac_add_options --enable-mozsearch-plugin
+
+. $topsrcdir/build/win32/mozconfig.vs-latest
+
+. "$topsrcdir/build/mozconfig.common.override"
+. "$topsrcdir/build/mozconfig.clang-cl"

taskcluster/ci/config.yml

@@ -48,6 +48,7 @@ treeherder:
         'TMW': 'Toolchain builds for Windows MinGW'
         'TW32': 'Toolchain builds for Windows 32-bits'
         'TW64': 'Toolchain builds for Windows 64-bits'
+        'Searchfox': 'Searchfox builds'
         'SM-tc': 'Spidermonkey builds'
         'pub': 'APK publishing'
         'p': 'Partial generation'

taskcluster/ci/searchfox/kind.yml

@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+   - toolchain
+
+transforms:
+   - taskgraph.transforms.build_attrs:transforms
+   - taskgraph.transforms.use_toolchains:transforms
+   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.task:transforms
+
+job-defaults:
+    index:
+        product: firefox
+    treeherder:
+        symbol: Searchfox(idx)
+        kind: build
+        tier: 2
+    run-on-projects: []
+
+jobs:
+    linux64-searchfox/debug:
+        description: "Linux64 Debug Searchfox"
+        index:
+            job-name: linux64-searchfox-debug
+        treeherder:
+            platform: linux64/debug
+        worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+        worker:
+            docker-image: {in-tree: desktop-build}
+            max-run-time: 36000
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+        run:
+            using: mozharness
+            actions: [clobber build]
+            config:
+                - builds/releng_base_firefox.py
+                - builds/releng_sub_linux_configs/64_searchfox_and_debug.py
+                - balrog/production.py
+            script: "mozharness/scripts/fx_desktop_build.py"
+            tooltool-downloads: public
+            keep-artifacts: false
+        toolchains:
+            - linux64-clang
+            - linux64-rust
+
+    macosx64-searchfox/debug:
+        description: "MacOS X x64 Debug Cross-compile Searchfox"
+        index:
+            job-name: macosx64-searchfox-debug
+        treeherder:
+            platform: osx-cross/debug
+        worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+        worker:
+            docker-image: {in-tree: desktop-build}
+            max-run-time: 36000
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+        run:
+            using: mozharness
+            actions: [clobber get-secrets build update]
+            config:
+                - builds/releng_base_firefox.py
+                - builds/releng_base_mac_64_cross_builds.py
+                - balrog/production.py
+            script: "mozharness/scripts/fx_desktop_build.py"
+            custom-build-variant-cfg: cross-debug-searchfox
+            secrets: true
+            tooltool-downloads: internal
+            keep-artifacts: false
+        toolchains:
+            - linux64-cctools-port
+            - linux64-clang
+            - linux64-hfsplus
+            - linux64-libdmg
+            - linux64-rust-macos
+
+    # NOTE: Windows Searchfox jobs aren't working quite yet (bug 1418415).
+
+    win32-searchfox/debug:
+        description: "Win32 Searchfox Debug (clang-cl)"
+        index:
+            product: firefox
+            job-name: win32-searchfox-debug
+        treeherder:
+            platform: windows2012-32/debug
+        worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+        worker:
+            max-run-time: 7200
+            env:
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
+        run:
+            using: mozharness
+            script: mozharness/scripts/fx_desktop_build.py
+            config:
+                - builds/releng_base_firefox.py
+                - builds/taskcluster_firefox_win32_clang_searchfox_debug.py
+        toolchains:
+            - win32-clang-cl
+            - win32-rust

taskcluster/docs/kinds.rst

@@ -68,7 +68,7 @@ Upload-symbols tasks run after builds and upload the symbols files generated by
 build tasks to Socorro for later use in crash analysis.
 
 upload-generated-sources
---------------
+------------------------
 
 Upload-generated-sources tasks run after builds and upload source files that were generated as part of the build process to an s3 bucket for later use in links from crash reports or when debugging shipped builds.
 
@@ -77,6 +77,11 @@ valgrind
 
 Valgrind tasks produce builds instrumented by valgrind.
 
+searchfox
+---------
+
+Searchfox builds generate C++ index data for Searchfox.
+
 static-analysis
 ---------------
 

taskcluster/taskgraph/target_tasks.py

@@ -505,6 +505,15 @@ def target_tasks_dmd(full_task_graph, parameters, graph_config):
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
 
 
+# Run Searchfox analysis once daily.
+@_target_task('searchfox_index')
+def target_tasks_searchfox(full_task_graph, parameters, graph_config):
+    """Select tasks required for indexing Firefox for Searchfox web site each day"""
+    # For now we only do Linux debug builds. Windows and Mac builds
+    # are currently broken (bug 1418415).
+    return ['searchfox-linux64-searchfox/debug']
+
+
 @_target_task('file_update')
 def target_tasks_file_update(full_task_graph, parameters, graph_config):
     """Select the set of tasks required to perform nightly in-tree file updates

taskcluster/taskgraph/transforms/gecko_v2_whitelist.py

@@ -61,6 +61,7 @@ JOB_NAME_WHITELIST = set([
     'linux64-pgo',
     'linux64-rusttests-opt',
     'linux64-rusttests-debug',
+    'linux64-searchfox-debug',
     'linux64-st-an-debug',
     'linux64-st-an-opt',
     'linux64-valgrind-opt',
@@ -76,6 +77,7 @@ JOB_NAME_WHITELIST = set([
     'macosx64-devedition-nightly-repackage-signing',
     'macosx64-st-an-debug',
     'macosx64-st-an-opt',
+    'macosx64-searchfox-debug',
     'macosx64-dmd-opt',
     'shell-haz-debug',
     'sm-arm-sim-debug',
@@ -108,6 +110,7 @@ JOB_NAME_WHITELIST = set([
     'win32-opt',
     'win32-pgo',
     'win32-rusttests-opt',
+    'win32-searchfox-debug',
     'win32-st-an-debug',
     'win32-st-an-opt',
     'win32-dmd-opt',

testing/mozharness/configs/builds/releng_sub_linux_configs/64_searchfox_and_debug.py

@@ -0,0 +1,86 @@
+import os
+
+config = {
+    # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
+    'default_actions': [
+        'clobber',
+        'clone-tools',
+        'checkout-sources',
+        'setup-mock',
+        'build',
+    ],
+    "buildbot_json_path": "buildprops.json",
+    'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
+    # decides whether we want to use moz_sign_cmd in env
+    'enable_signing': False,
+    'vcs_share_base': '/builds/hg-shared',
+    'tooltool_script': ["/builds/tooltool.py"],
+    'tooltool_bootstrap': "setup.sh",
+    'enable_count_ctors': True,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
+    'perfherder_extra_options': ['static-analysis'],
+    #########################################################################
+
+
+    #########################################################################
+    ###### 64 bit specific ######
+    'base_name': 'Linux_x86-64_%(branch)s_Searchfox',
+    'platform': 'linux64',
+    'stage_platform': 'linux64-searchfox-opt',
+    'publish_nightly_en_US_routes': False,
+    'env': {
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'DISPLAY': ':2',
+        'HG_SHARE_BASE_DIR': '/builds/hg-shared',
+        'MOZ_OBJDIR': '%(abs_obj_dir)s',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': '/builds/worker/tooltool-cache',
+        'TOOLTOOL_HOME': '/builds',
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'CCACHE_DIR': '/builds/ccache',
+        'CCACHE_COMPRESS': '1',
+        'CCACHE_UMASK': '002',
+        'LC_ALL': 'C',
+        # Disable sccache because otherwise we won't index the files that
+        # sccache optimizes away compilation for
+        'SCCACHE_DISABLE': '1',
+        ## 64 bit specific
+        'PATH': '/usr/local/bin:/usr/lib64/ccache:/bin:\
+/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
+/tools/python27-mercurial/bin:/home/cltbld/bin',
+        'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64",
+        ##
+    },
+    'mock_packages': [
+        'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
+        'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
+        'yasm', 'wget',
+        'mpfr',  # required for system compiler
+        'xorg-x11-font*',  # fonts required for PGO
+        'imake',  # required for makedepend!?!
+        ### <-- from releng repo
+        'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
+        'yasm', 'ccache',
+        ###
+        'valgrind', 'dbus-x11',
+        ######## 64 bit specific ###########
+        'glibc-static', 'libstdc++-static',
+        'gtk2-devel', 'libnotify-devel',
+        'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
+        'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
+        'GConf2-devel',
+        ### from releng repo
+        'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
+        'yasm', 'ccache',
+        ###
+        'pulseaudio-libs-devel', 'gstreamer-devel',
+        'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
+        'freetype-devel-2.3.11-6.el6_1.8.x86_64'
+    ],
+    # This doesn't actually inherit from anything.
+    'mozconfig_platform': 'linux64',
+    'mozconfig_variant': 'debug-searchfox-clang',
+    #######################
+    'artifact_flag_build_variant_in_try': None,
+}

testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug_searchfox.py

@@ -0,0 +1,45 @@
+import os
+
+config = {
+    'default_actions': [
+        'clobber',
+        'clone-tools',
+        'checkout-sources',
+        # 'setup-mock',
+        'build',
+        'upload-files',
+        'sendchange',
+        'update',  # decided by query_is_nightly()
+    ],
+    'stage_platform': 'macosx64-searchfox-debug',
+    'debug_build': True,
+    'enable_talos_sendchange': False,
+    'enable_signing': False,
+    'enable_talos_sendchange': False,
+    'enable_unittest_sendchange': False,
+    'perfherder_extra_options': ['static-analysis'],
+    #### 64 bit build specific #####
+    'env': {
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'HG_SHARE_BASE_DIR': '/builds/hg-shared',
+        'MOZ_OBJDIR': '%(abs_obj_dir)s',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': '/builds/tooltool_cache',
+        'TOOLTOOL_HOME': '/builds',
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'CCACHE_DIR': '/builds/ccache',
+        'CCACHE_COMPRESS': '1',
+        'CCACHE_UMASK': '002',
+        'LC_ALL': 'C',
+        'XPCOM_DEBUG_BREAK': 'stack-and-abort',
+        # Disable sccache because otherwise we won't index the files that
+        # sccache optimizes away compilation for
+        'SCCACHE_DISABLE': '1',
+        # 64 bit specific
+        'PATH': '/tools/python/bin:/opt/local/bin:/usr/bin:'
+                '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
+    },
+    'mozconfig_variant': 'debug-searchfox',
+    'artifact_flag_build_variant_in_try': None,
+    #######################
+}

testing/mozharness/configs/builds/taskcluster_firefox_win32_clang_searchfox_debug.py

@@ -0,0 +1,76 @@
+import os
+import sys
+
+config = {
+    #########################################################################
+    ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
+    # if you are updating this with custom 32 bit keys/values please add them
+    # below under the '32 bit specific' code block otherwise, update in this
+    # code block and also make sure this is synced between:
+    # - taskcluster_firefox_win32_debug
+    # - taskcluster_firefox_win32_opt
+    # - taskcluster_firefox_win64_debug
+    # - taskcluster_firefox_win64_opt
+    # - taskcluster_firefox_win32_clang
+    # - taskcluster_firefox_win32_clang_debug
+    # - taskcluster_firefox_win64_clang
+    # - taskcluster_firefox_win64_clang_debug
+
+    'default_actions': [
+        'clone-tools',
+        'build',
+        'check-test',
+    ],
+    'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
+    # decides whether we want to use moz_sign_cmd in env
+    'enable_signing': True,
+    'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
+    'tooltool_script': [
+      sys.executable,
+      os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')
+    ],
+    'tooltool_bootstrap': 'setup.sh',
+    'enable_count_ctors': False,
+    'max_build_output_timeout': 60 * 80,
+    'perfherder_extra_options': ['static-analysis'],
+    #########################################################################
+
+
+     #########################################################################
+     ###### 32 bit specific ######
+    'base_name': 'WINNT_5.2_%(branch)s',
+    'platform': 'win32',
+    'stage_platform': 'win32-st-an-debug',
+    'debug_build': True,
+    'publish_nightly_en_US_routes': True,
+    'env': {
+        'BINSCOPE': os.path.join(
+            os.environ['ProgramFiles(x86)'], 'Microsoft', 'SDL BinScope', 'BinScope.exe'
+        ),
+        'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
+        'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
+        'MOZ_CRASHREPORTER_NO_REPORT': '1',
+        'MOZ_OBJDIR': '%(abs_obj_dir)s',
+        'PDBSTR_PATH': 'C:/Program Files (x86)/Windows Kits/10/Debuggers/x86/srcsrv/pdbstr.exe',
+        'TINDERBOX_OUTPUT': '1',
+        'TOOLTOOL_CACHE': 'c:/builds/tooltool_cache',
+        'TOOLTOOL_HOME': '/c/builds',
+        'XPCOM_DEBUG_BREAK': 'stack-and-abort',
+        'MSYSTEM': 'MINGW32',
+        # Disable sccache because otherwise we won't index the files that
+        # sccache optimizes away compilation for
+        'SCCACHE_DISABLE': '1',
+    },
+    'upload_env': {
+        'UPLOAD_HOST': 'localhost',
+        'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
+    },
+    "check_test_env": {
+        'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win32\\minidump_stackwalk.exe',
+        'MINIDUMP_SAVE_PATH': os.path.join(os.getcwd(), 'public', 'build'),
+    },
+    'mozconfig_platform': 'win32',
+    'mozconfig_variant': 'debug-searchfox',
+    'artifact_flag_build_variant_in_try': None,
+    #########################################################################
+}

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -414,6 +414,7 @@ class BuildOptionParser(object):
         'fuzzing-asan-tc': 'builds/releng_sub_%s_configs/%s_fuzzing_asan_tc.py',
         'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py',
         'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py',
+        'cross-debug-searchfox': 'builds/releng_sub_%s_configs/%s_cross_debug_searchfox.py',
         'cross-debug-artifact': 'builds/releng_sub_%s_configs/%s_cross_debug_artifact.py',
         'cross-noopt-debug': 'builds/releng_sub_%s_configs/%s_cross_noopt_debug.py',
         'cross-artifact': 'builds/releng_sub_%s_configs/%s_cross_artifact.py',