diff --git a/.cron.yml b/.cron.yml index 2c19c2b7d6df..e610ea687b4b 100644 --- a/.cron.yml +++ b/.cron.yml @@ -98,6 +98,18 @@ jobs: mozilla-central: [{hour: 10, minute: 0}] # No default + - name: searchfox-index + job: + type: decision-task + treeherder-symbol: Searchfox + target-tasks-method: searchfox_index + run-on-projects: + - mozilla-central + when: + by-project: + mozilla-central: [{hour: 10, minute: 30}] + # No default + - name: periodic-update job: type: decision-task diff --git a/browser/config/mozconfigs/linux64/debug-searchfox-clang b/browser/config/mozconfigs/linux64/debug-searchfox-clang new file mode 100644 index 000000000000..da12067c770e --- /dev/null +++ b/browser/config/mozconfigs/linux64/debug-searchfox-clang @@ -0,0 +1,25 @@ +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE_TESTS=0 +MOZ_AUTOMATION_L10N_CHECK=0 + +. "$topsrcdir/build/mozconfig.common" + +ac_add_options --enable-debug +ac_add_options --enable-dmd + +. $topsrcdir/build/mozconfig.stylo + +# Use Clang as specified in manifest +export CC="$topsrcdir/clang/bin/clang" +export CXX="$topsrcdir/clang/bin/clang++" + +# Add the static checker +ac_add_options --enable-clang-plugin +ac_add_options --enable-mozsearch-plugin + +. "$topsrcdir/build/unix/mozconfig.stdcxx" + +export PKG_CONFIG_LIBDIR=/usr/lib64/pkgconfig:/usr/share/pkgconfig +. $topsrcdir/build/unix/mozconfig.gtk + +. "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/macosx64/debug-searchfox b/browser/config/mozconfigs/macosx64/debug-searchfox new file mode 100644 index 000000000000..526fd941f330 --- /dev/null +++ b/browser/config/mozconfigs/macosx64/debug-searchfox @@ -0,0 +1,13 @@ +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE_TESTS=0 +MOZ_AUTOMATION_L10N_CHECK=0 + +. $topsrcdir/build/macosx/mozconfig.common + +ac_add_options --enable-debug +ac_add_options --enable-dmd + +ac_add_options --enable-clang-plugin +ac_add_options --enable-mozsearch-plugin + +. "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/win32/debug-searchfox b/browser/config/mozconfigs/win32/debug-searchfox new file mode 100644 index 000000000000..c7d71470d608 --- /dev/null +++ b/browser/config/mozconfigs/win32/debug-searchfox @@ -0,0 +1,17 @@ +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE_TESTS=0 +MOZ_AUTOMATION_L10N_CHECK=0 + +. "$topsrcdir/build/mozconfig.win-common" +. "$topsrcdir/browser/config/mozconfigs/common" + +ac_add_options --enable-optimize +ac_add_options --enable-debug + +ac_add_options --enable-clang-plugin +ac_add_options --enable-mozsearch-plugin + +. $topsrcdir/build/win32/mozconfig.vs-latest + +. "$topsrcdir/build/mozconfig.common.override" +. "$topsrcdir/build/mozconfig.clang-cl" diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 6282202e3f0d..b06c390ede6d 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -48,6 +48,7 @@ treeherder: 'TMW': 'Toolchain builds for Windows MinGW' 'TW32': 'Toolchain builds for Windows 32-bits' 'TW64': 'Toolchain builds for Windows 64-bits' + 'Searchfox': 'Searchfox builds' 'SM-tc': 'Spidermonkey builds' 'pub': 'APK publishing' 'p': 'Partial generation' diff --git a/taskcluster/ci/searchfox/kind.yml b/taskcluster/ci/searchfox/kind.yml new file mode 100644 index 000000000000..4f3e3accda99 --- /dev/null +++ b/taskcluster/ci/searchfox/kind.yml @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - toolchain + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.use_toolchains:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + index: + product: firefox + treeherder: + symbol: Searchfox(idx) + kind: build + tier: 2 + run-on-projects: [] + +jobs: + linux64-searchfox/debug: + description: "Linux64 Debug Searchfox" + index: + job-name: linux64-searchfox-debug + treeherder: + platform: linux64/debug + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest" + run: + using: mozharness + actions: [clobber build] + config: + - builds/releng_base_firefox.py + - builds/releng_sub_linux_configs/64_searchfox_and_debug.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + tooltool-downloads: public + keep-artifacts: false + toolchains: + - linux64-clang + - linux64-rust + + macosx64-searchfox/debug: + description: "MacOS X x64 Debug Cross-compile Searchfox" + index: + job-name: macosx64-searchfox-debug + treeherder: + platform: osx-cross/debug + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker: + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest" + run: + using: mozharness + actions: [clobber get-secrets build update] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: cross-debug-searchfox + secrets: true + tooltool-downloads: internal + keep-artifacts: false + toolchains: + - linux64-cctools-port + - linux64-clang + - linux64-hfsplus + - linux64-libdmg + - linux64-rust-macos + + # NOTE: Windows Searchfox jobs aren't working quite yet (bug 1418415). + + win32-searchfox/debug: + description: "Win32 Searchfox Debug (clang-cl)" + index: + product: firefox + job-name: win32-searchfox-debug + treeherder: + platform: windows2012-32/debug + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + max-run-time: 7200 + env: + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest" + run: + using: mozharness + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/releng_base_firefox.py + - builds/taskcluster_firefox_win32_clang_searchfox_debug.py + toolchains: + - win32-clang-cl + - win32-rust diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index ce2b3fcf7b9a..be7516209704 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -68,7 +68,7 @@ Upload-symbols tasks run after builds and upload the symbols files generated by build tasks to Socorro for later use in crash analysis. upload-generated-sources --------------- +------------------------ Upload-generated-sources tasks run after builds and upload source files that were generated as part of the build process to an s3 bucket for later use in links from crash reports or when debugging shipped builds. @@ -77,6 +77,11 @@ valgrind Valgrind tasks produce builds instrumented by valgrind. +searchfox +--------- + +Searchfox builds generate C++ index data for Searchfox. + static-analysis --------------- diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 19b0e98c8ca3..5895758838cd 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -505,6 +505,15 @@ def target_tasks_dmd(full_task_graph, parameters, graph_config): return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] +# Run Searchfox analysis once daily. +@_target_task('searchfox_index') +def target_tasks_searchfox(full_task_graph, parameters, graph_config): + """Select tasks required for indexing Firefox for Searchfox web site each day""" + # For now we only do Linux debug builds. Windows and Mac builds + # are currently broken (bug 1418415). + return ['searchfox-linux64-searchfox/debug'] + + @_target_task('file_update') def target_tasks_file_update(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform nightly in-tree file updates diff --git a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py index cd4785761dae..c1fb67031187 100644 --- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py +++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py @@ -61,6 +61,7 @@ JOB_NAME_WHITELIST = set([ 'linux64-pgo', 'linux64-rusttests-opt', 'linux64-rusttests-debug', + 'linux64-searchfox-debug', 'linux64-st-an-debug', 'linux64-st-an-opt', 'linux64-valgrind-opt', @@ -76,6 +77,7 @@ JOB_NAME_WHITELIST = set([ 'macosx64-devedition-nightly-repackage-signing', 'macosx64-st-an-debug', 'macosx64-st-an-opt', + 'macosx64-searchfox-debug', 'macosx64-dmd-opt', 'shell-haz-debug', 'sm-arm-sim-debug', @@ -108,6 +110,7 @@ JOB_NAME_WHITELIST = set([ 'win32-opt', 'win32-pgo', 'win32-rusttests-opt', + 'win32-searchfox-debug', 'win32-st-an-debug', 'win32-st-an-opt', 'win32-dmd-opt', diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_searchfox_and_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_searchfox_and_debug.py new file mode 100644 index 000000000000..adbafb437fa9 --- /dev/null +++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_searchfox_and_debug.py @@ -0,0 +1,86 @@ +import os + +config = { + # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks + 'default_actions': [ + 'clobber', + 'clone-tools', + 'checkout-sources', + 'setup-mock', + 'build', + ], + "buildbot_json_path": "buildprops.json", + 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini', + # decides whether we want to use moz_sign_cmd in env + 'enable_signing': False, + 'vcs_share_base': '/builds/hg-shared', + 'tooltool_script': ["/builds/tooltool.py"], + 'tooltool_bootstrap': "setup.sh", + 'enable_count_ctors': True, + 'enable_talos_sendchange': False, + 'enable_unittest_sendchange': False, + 'perfherder_extra_options': ['static-analysis'], + ######################################################################### + + + ######################################################################### + ###### 64 bit specific ###### + 'base_name': 'Linux_x86-64_%(branch)s_Searchfox', + 'platform': 'linux64', + 'stage_platform': 'linux64-searchfox-opt', + 'publish_nightly_en_US_routes': False, + 'env': { + 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'), + 'DISPLAY': ':2', + 'HG_SHARE_BASE_DIR': '/builds/hg-shared', + 'MOZ_OBJDIR': '%(abs_obj_dir)s', + 'TINDERBOX_OUTPUT': '1', + 'TOOLTOOL_CACHE': '/builds/worker/tooltool-cache', + 'TOOLTOOL_HOME': '/builds', + 'MOZ_CRASHREPORTER_NO_REPORT': '1', + 'CCACHE_DIR': '/builds/ccache', + 'CCACHE_COMPRESS': '1', + 'CCACHE_UMASK': '002', + 'LC_ALL': 'C', + # Disable sccache because otherwise we won't index the files that + # sccache optimizes away compilation for + 'SCCACHE_DISABLE': '1', + ## 64 bit specific + 'PATH': '/usr/local/bin:/usr/lib64/ccache:/bin:\ +/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\ +/tools/python27-mercurial/bin:/home/cltbld/bin', + 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64", + ## + }, + 'mock_packages': [ + 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial', + 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General', + 'yasm', 'wget', + 'mpfr', # required for system compiler + 'xorg-x11-font*', # fonts required for PGO + 'imake', # required for makedepend!?! + ### <-- from releng repo + 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1', + 'yasm', 'ccache', + ### + 'valgrind', 'dbus-x11', + ######## 64 bit specific ########### + 'glibc-static', 'libstdc++-static', + 'gtk2-devel', 'libnotify-devel', + 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel', + 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel', + 'GConf2-devel', + ### from releng repo + 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1', + 'yasm', 'ccache', + ### + 'pulseaudio-libs-devel', 'gstreamer-devel', + 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64', + 'freetype-devel-2.3.11-6.el6_1.8.x86_64' + ], + # This doesn't actually inherit from anything. + 'mozconfig_platform': 'linux64', + 'mozconfig_variant': 'debug-searchfox-clang', + ####################### + 'artifact_flag_build_variant_in_try': None, +} diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug_searchfox.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug_searchfox.py new file mode 100644 index 000000000000..29eda5144686 --- /dev/null +++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug_searchfox.py @@ -0,0 +1,45 @@ +import os + +config = { + 'default_actions': [ + 'clobber', + 'clone-tools', + 'checkout-sources', + # 'setup-mock', + 'build', + 'upload-files', + 'sendchange', + 'update', # decided by query_is_nightly() + ], + 'stage_platform': 'macosx64-searchfox-debug', + 'debug_build': True, + 'enable_talos_sendchange': False, + 'enable_signing': False, + 'enable_talos_sendchange': False, + 'enable_unittest_sendchange': False, + 'perfherder_extra_options': ['static-analysis'], + #### 64 bit build specific ##### + 'env': { + 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'), + 'HG_SHARE_BASE_DIR': '/builds/hg-shared', + 'MOZ_OBJDIR': '%(abs_obj_dir)s', + 'TINDERBOX_OUTPUT': '1', + 'TOOLTOOL_CACHE': '/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/builds', + 'MOZ_CRASHREPORTER_NO_REPORT': '1', + 'CCACHE_DIR': '/builds/ccache', + 'CCACHE_COMPRESS': '1', + 'CCACHE_UMASK': '002', + 'LC_ALL': 'C', + 'XPCOM_DEBUG_BREAK': 'stack-and-abort', + # Disable sccache because otherwise we won't index the files that + # sccache optimizes away compilation for + 'SCCACHE_DISABLE': '1', + # 64 bit specific + 'PATH': '/tools/python/bin:/opt/local/bin:/usr/bin:' + '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin', + }, + 'mozconfig_variant': 'debug-searchfox', + 'artifact_flag_build_variant_in_try': None, + ####################### +} diff --git a/testing/mozharness/configs/builds/taskcluster_firefox_win32_clang_searchfox_debug.py b/testing/mozharness/configs/builds/taskcluster_firefox_win32_clang_searchfox_debug.py new file mode 100644 index 000000000000..7bb58eb370d2 --- /dev/null +++ b/testing/mozharness/configs/builds/taskcluster_firefox_win32_clang_searchfox_debug.py @@ -0,0 +1,76 @@ +import os +import sys + +config = { + ######################################################################### + ######## WINDOWS GENERIC CONFIG KEYS/VAlUES + # if you are updating this with custom 32 bit keys/values please add them + # below under the '32 bit specific' code block otherwise, update in this + # code block and also make sure this is synced between: + # - taskcluster_firefox_win32_debug + # - taskcluster_firefox_win32_opt + # - taskcluster_firefox_win64_debug + # - taskcluster_firefox_win64_opt + # - taskcluster_firefox_win32_clang + # - taskcluster_firefox_win32_clang_debug + # - taskcluster_firefox_win64_clang + # - taskcluster_firefox_win64_clang_debug + + 'default_actions': [ + 'clone-tools', + 'build', + 'check-test', + ], + 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini', + # decides whether we want to use moz_sign_cmd in env + 'enable_signing': True, + 'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'), + 'tooltool_script': [ + sys.executable, + os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py') + ], + 'tooltool_bootstrap': 'setup.sh', + 'enable_count_ctors': False, + 'max_build_output_timeout': 60 * 80, + 'perfherder_extra_options': ['static-analysis'], + ######################################################################### + + + ######################################################################### + ###### 32 bit specific ###### + 'base_name': 'WINNT_5.2_%(branch)s', + 'platform': 'win32', + 'stage_platform': 'win32-st-an-debug', + 'debug_build': True, + 'publish_nightly_en_US_routes': True, + 'env': { + 'BINSCOPE': os.path.join( + os.environ['ProgramFiles(x86)'], 'Microsoft', 'SDL BinScope', 'BinScope.exe' + ), + 'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'), + 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'), + 'MOZ_CRASHREPORTER_NO_REPORT': '1', + 'MOZ_OBJDIR': '%(abs_obj_dir)s', + 'PDBSTR_PATH': 'C:/Program Files (x86)/Windows Kits/10/Debuggers/x86/srcsrv/pdbstr.exe', + 'TINDERBOX_OUTPUT': '1', + 'TOOLTOOL_CACHE': 'c:/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/c/builds', + 'XPCOM_DEBUG_BREAK': 'stack-and-abort', + 'MSYSTEM': 'MINGW32', + # Disable sccache because otherwise we won't index the files that + # sccache optimizes away compilation for + 'SCCACHE_DISABLE': '1', + }, + 'upload_env': { + 'UPLOAD_HOST': 'localhost', + 'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'), + }, + "check_test_env": { + 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win32\\minidump_stackwalk.exe', + 'MINIDUMP_SAVE_PATH': os.path.join(os.getcwd(), 'public', 'build'), + }, + 'mozconfig_platform': 'win32', + 'mozconfig_variant': 'debug-searchfox', + 'artifact_flag_build_variant_in_try': None, + ######################################################################### +} diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index cc1cd4edb61b..850294e1fba8 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -414,6 +414,7 @@ class BuildOptionParser(object): 'fuzzing-asan-tc': 'builds/releng_sub_%s_configs/%s_fuzzing_asan_tc.py', 'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py', 'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py', + 'cross-debug-searchfox': 'builds/releng_sub_%s_configs/%s_cross_debug_searchfox.py', 'cross-debug-artifact': 'builds/releng_sub_%s_configs/%s_cross_debug_artifact.py', 'cross-noopt-debug': 'builds/releng_sub_%s_configs/%s_cross_noopt_debug.py', 'cross-artifact': 'builds/releng_sub_%s_configs/%s_cross_artifact.py',