Bug 1444171 - Add pgo-generate-only source functionality; r=glandium

For clang-cl, we want to add code to libxul that only exists during the
PGO generation phase, so we can collect data.  The most expedient way to
do that is to enable certain files in SOURCES to be marked as to only be
compiled during the PGO generation step.

config/rules.mk

@@ -186,6 +186,10 @@ ifndef TARGETS
 TARGETS			= $(LIBRARY) $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS) $(HOST_LIBRARY) $(HOST_PROGRAM) $(HOST_SIMPLE_PROGRAMS) $(HOST_SHARED_LIBRARY)
 endif
 
+ifdef MOZ_PROFILE_GENERATE
+CPPSRCS := $(CPPSRCS) $(PGO_GEN_ONLY_CPPSRCS)
+endif
+
 COBJS = $(notdir $(CSRCS:.c=.$(OBJ_SUFFIX)))
 SOBJS = $(notdir $(SSRCS:.S=.$(OBJ_SUFFIX)))
 # CPPSRCS can have different extensions (eg: .cpp, .cc)

mozglue/build/cygprofile.cpp

@@ -0,0 +1,121 @@
+// Copyright (c) 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+// Copied from Chromium's /src/tools/cygprofile_win/cygprofile.cc.
+
+#include <stdio.h>
+#include <atomic>
+#include <string>
+#include <unordered_set>
+
+#include <windows.h>  // Needs to be included before the others.
+
+#include <dbghelp.h>
+#include <process.h>
+
+#include "mozilla/Sprintf.h"
+#include "mozilla/Types.h"
+
+namespace {
+
+// The main purpose of the order file is to optimize startup time,
+// so capturing the first N function calls is enough.
+static constexpr int kSamplesCapacity = 25 * 1024 * 1024;
+
+void* samples[kSamplesCapacity];
+std::atomic_int num_samples;
+std::atomic_int done;
+
+
+// Symbolize the samples and write them to disk.
+void dump(void*) {
+  HMODULE dbghelp = LoadLibraryA("dbghelp.dll");
+  auto sym_from_addr = reinterpret_cast<decltype(::SymFromAddr)*>(
+      ::GetProcAddress(dbghelp, "SymFromAddr"));
+  auto sym_initialize = reinterpret_cast<decltype(::SymInitialize)*>(
+      ::GetProcAddress(dbghelp, "SymInitialize"));
+  auto sym_set_options = reinterpret_cast<decltype(::SymSetOptions)*>(
+      ::GetProcAddress(dbghelp, "SymSetOptions"));
+
+  // Path to the dump file. %s will be substituted by objdir path.
+  static const char kDumpFile[] = "%s/cygprofile.txt";
+
+  char filename[MAX_PATH];
+  const char* objdir = ::getenv("MOZ_OBJDIR");
+
+  if (!objdir) {
+    fprintf(stderr, "ERROR: cannot determine objdir\n");
+    return;
+  }
+
+  SprintfLiteral(filename, kDumpFile, objdir);
+
+  FILE* f = fopen(filename, "w");
+  if (!f) {
+    fprintf(stderr, "ERROR: Cannot open %s\n", filename);
+    return;
+  }
+
+  sym_initialize(::GetCurrentProcess(), NULL, TRUE);
+  sym_set_options(SYMOPT_DEFERRED_LOADS | SYMOPT_PUBLICS_ONLY);
+  char sym_buf[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
+
+  std::unordered_set<void*> seen;
+  std::unordered_set<std::string> seen_names;
+
+  for (void* sample : samples) {
+    // Only print the first call of a function.
+    if (seen.count(sample))
+      continue;
+    seen.insert(sample);
+
+    SYMBOL_INFO* symbol = reinterpret_cast<SYMBOL_INFO*>(sym_buf);
+    symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+    symbol->MaxNameLen = MAX_SYM_NAME;
+    DWORD64 offset = 0;
+
+    if (sym_from_addr(::GetCurrentProcess(), reinterpret_cast<DWORD64>(sample),
+                      &offset, symbol)) {
+      const char* name = symbol->Name;
+      if (name[0] == '_')
+        name++;
+      if (seen_names.count(name))
+        continue;
+      seen_names.insert(name);
+
+      fprintf(f, "%s\n", name);
+    }
+  }
+
+  fclose(f);
+}
+
+}  // namespace
+
+extern "C" {
+
+MOZ_EXPORT void
+__cyg_profile_func_enter(void* this_fn, void* call_site_unused) {
+  if (done)
+    return;
+
+  // Get our index for the samples array atomically.
+  int n = num_samples++;
+
+  if (n < kSamplesCapacity) {
+    samples[n] = this_fn;
+
+    if (n + 1 == kSamplesCapacity) {
+      // This is the final sample; start dumping the samples to a file (on a
+      // separate thread so as not to disturb the main program).
+      done = 1;
+      _beginthread(dump, 0, nullptr);
+    }
+  }
+}
+
+MOZ_EXPORT void
+__cyg_profile_func_exit(void* this_fn, void* call_site) {}
+
+}  // extern "C"

mozglue/build/moz.build

@@ -31,6 +31,10 @@ if CONFIG['OS_TARGET'] == 'WINNT':
         'user32.dll',
     ]
 
+    if CONFIG['MOZ_PGO'] and CONFIG['CC_TYPE'] == 'clang-cl':
+        SOURCES += ['cygprofile.cpp']
+        SOURCES['cygprofile.cpp'].pgo_generate_only = True
+
     if CONFIG['CC_TYPE'] == "msvc":
         CFLAGS += ['-guard:cf']
         CXXFLAGS += ['-guard:cf']

python/mozbuild/mozbuild/backend/common.py

@@ -209,18 +209,27 @@ class CommonBackend(BuildBackend):
         no_pgo_objs = []
 
         seen_objs = set()
+        seen_pgo_gen_only_objs = set()
         seen_libs = set()
 
         def add_objs(lib):
+            seen_pgo_gen_only_objs.update(lib.pgo_gen_only_objs)
+
             for o in lib.objs:
-                if o not in seen_objs:
-                    seen_objs.add(o)
-                    objs.append(o)
-                    # This is slightly odd, buf for consistency with the
-                    # recursivemake backend we don't replace OBJ_SUFFIX if any
-                    # object in a library has `no_pgo` set.
-                    if lib.no_pgo_objs or lib.no_pgo:
-                        no_pgo_objs.append(o)
+                if o in seen_objs:
+                    continue
+
+                # The front end should keep pgo generate-only objects and
+                # normal objects separate.
+                assert o not in seen_pgo_gen_only_objs
+
+                seen_objs.add(o)
+                objs.append(o)
+                # This is slightly odd, but for consistency with the
+                # recursivemake backend we don't replace OBJ_SUFFIX if any
+                # object in a library has `no_pgo` set.
+                if lib.no_pgo_objs or lib.no_pgo:
+                    no_pgo_objs.append(o)
 
         def expand(lib, recurse_objs, system_libs):
             if isinstance(lib, StaticLibrary):
@@ -262,7 +271,8 @@ class CommonBackend(BuildBackend):
                 seen_libs.add(lib)
                 os_libs.append(lib)
 
-        return objs, no_pgo_objs, shared_libs, os_libs, static_libs
+        return (objs, sorted(seen_pgo_gen_only_objs), no_pgo_objs, \
+                shared_libs, os_libs, static_libs)
 
     def _make_list_file(self, objdir, objs, name):
         if not objs:

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -59,6 +59,7 @@ from ..frontend.data import (
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
+    PgoGenerateOnlySources,
     Program,
     RustLibrary,
     HostSharedLibrary,
@@ -479,6 +480,10 @@ class RecursiveMakeBackend(CommonBackend):
                 f = mozpath.relpath(f, base)
                 for var in variables:
                     backend_file.write('%s += %s\n' % (var, f))
+        elif isinstance(obj, PgoGenerateOnlySources):
+            assert obj.canonical_suffix == '.cpp'
+            for f in sorted(obj.files):
+                backend_file.write('PGO_GEN_ONLY_CPPSRCS += %s\n' % f)
         elif isinstance(obj, (HostSources, HostGeneratedSources)):
             suffix_map = {
                 '.c': 'HOST_CSRCS',
@@ -1299,7 +1304,7 @@ class RecursiveMakeBackend(CommonBackend):
         build_target = self._build_target_for_obj(obj)
         self._compile_graph[build_target]
 
-        objs, no_pgo_objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
+        objs, pgo_gen_objs, no_pgo_objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
 
         if obj.KIND == 'target':
             obj_target = obj.name
@@ -1309,13 +1314,19 @@ class RecursiveMakeBackend(CommonBackend):
             is_unit_test = isinstance(obj, BaseProgram) and obj.is_unit_test
             profile_gen_objs = []
 
-            if (self.environment.substs.get('MOZ_PGO') and
-                self.environment.substs.get('GNU_CC')):
+            doing_pgo = self.environment.substs.get('MOZ_PGO')
+            obj_suffix_change_needed = (self.environment.substs.get('GNU_CC') or
+                                        self.environment.substs.get('CLANG_CL'))
+            if doing_pgo and obj_suffix_change_needed:
                 # We use a different OBJ_SUFFIX for the profile generate phase on
-                # linux. These get picked up via OBJS_VAR_SUFFIX in config.mk.
+                # systems where the pgo generate phase requires instrumentation
+                # that can only be removed by recompiling objects. These get
+                # picked up via OBJS_VAR_SUFFIX in config.mk.
                 if not is_unit_test and not isinstance(obj, SimpleProgram):
                     profile_gen_objs = [o if o in no_pgo_objs else '%s.%s' %
                                         (mozpath.splitext(o)[0], 'i_o') for o in objs]
+                    profile_gen_objs += ['%s.%s' % (mozpath.splitext(o)[0], 'i_o')
+                                         for o in pgo_gen_objs]
 
             def write_obj_deps(target, objs_ref, pgo_objs_ref):
                 if pgo_objs_ref:

python/mozbuild/mozbuild/backend/tup.py

@@ -347,7 +347,7 @@ class TupBackend(CommonBackend):
             ['-o', shlib.lib_name]
         )
 
-        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(shlib)
+        objs, _, _, shared_libs, os_libs, static_libs = self._expand_libs(shlib)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
         shared_libs = self._lib_paths(backend_file.objdir, shared_libs)
 
@@ -402,7 +402,7 @@ class TupBackend(CommonBackend):
 
     def _gen_program(self, backend_file, prog):
         cc_or_cxx = 'CXX' if prog.cxx_link else 'CC'
-        objs, _, shared_libs, os_libs, static_libs = self._expand_libs(prog)
+        objs, _, _, shared_libs, os_libs, static_libs = self._expand_libs(prog)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
         shared_libs = self._lib_paths(backend_file.objdir, shared_libs)
 
@@ -462,7 +462,7 @@ class TupBackend(CommonBackend):
 
 
     def _gen_host_program(self, backend_file, prog):
-        _, _, _, extra_libs, _ = self._expand_libs(prog)
+        _, _, _, _, extra_libs, _ = self._expand_libs(prog)
         objs = prog.objs
 
         if isinstance(prog, HostSimpleProgram):
@@ -501,7 +501,7 @@ class TupBackend(CommonBackend):
             backend_file.environment.substs['AR_FLAGS'].replace('$@', '%o')
         ]
 
-        objs, _, shared_libs, _, static_libs = self._expand_libs(backend_file.static_lib)
+        objs, _, _, shared_libs, _, static_libs = self._expand_libs(backend_file.static_lib)
         static_libs = self._lib_paths(backend_file.objdir, static_libs)
         shared_libs = self._lib_paths(backend_file.objdir, shared_libs)
 

python/mozbuild/mozbuild/frontend/context.py

@@ -1206,7 +1206,7 @@ SUBCONTEXTS = {cls.__name__: cls for cls in SUBCONTEXTS}
 #   (storage_type, input_types, docs)
 
 VARIABLES = {
-    'SOURCES': (ContextDerivedTypedListWithItems(Path, StrictOrderingOnAppendListWithFlagsFactory({'no_pgo': bool, 'flags': List})), list,
+    'SOURCES': (ContextDerivedTypedListWithItems(Path, StrictOrderingOnAppendListWithFlagsFactory({'no_pgo': bool, 'flags': List, 'pgo_generate_only': bool})), list,
         """Source code files.
 
         This variable contains a list of source code files to compile.

python/mozbuild/mozbuild/frontend/data.py

@@ -387,6 +387,7 @@ class Linkable(ContextDerived):
         'linked_libraries',
         'linked_system_libs',
         'no_pgo_sources',
+        'pgo_gen_only_sources',
         'no_pgo',
         'sources',
     )
@@ -399,6 +400,7 @@ class Linkable(ContextDerived):
         self.lib_defines = Defines(context, {})
         self.sources = defaultdict(list)
         self.no_pgo_sources = []
+        self.pgo_gen_only_sources = set()
         self.no_pgo = False
 
     def link_library(self, obj):
@@ -457,6 +459,10 @@ class Linkable(ContextDerived):
     def objs(self):
         return self._get_objs(self.source_files())
 
+    @property
+    def pgo_gen_only_objs(self):
+        return self._get_objs(self.pgo_gen_only_sources)
+
 
 class BaseProgram(Linkable):
     """Context derived container object for programs, which is a unicode
@@ -951,6 +957,15 @@ class Sources(BaseSources):
         BaseSources.__init__(self, context, files, canonical_suffix)
 
 
+class PgoGenerateOnlySources(BaseSources):
+    """Represents files to be compiled during the build.
+
+    These files are only used during the PGO generation phase."""
+
+    def __init__(self, context, files):
+        BaseSources.__init__(self, context, files, '.cpp')
+
+
 class GeneratedSources(BaseSources):
     """Represents generated files to be compiled during the build."""
 

python/mozbuild/mozbuild/frontend/emitter.py

@@ -57,6 +57,7 @@ from .data import (
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
+    PgoGenerateOnlySources,
     WebIDLCollection,
     Program,
     RustLibrary,
@@ -842,6 +843,7 @@ class TreeMetadataEmitter(LoggingMixin):
 
         sources = defaultdict(list)
         gen_sources = defaultdict(list)
+        pgo_generate_only = set()
         all_flags = {}
         for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'):
             srcs = sources[symbol]
@@ -858,6 +860,19 @@ class TreeMetadataEmitter(LoggingMixin):
                     flags = context_srcs[f]
                     if flags:
                         all_flags[full_path] = flags
+                    # Files for the generation phase of PGO are unusual, so
+                    # it's not unreasonable to require them to be special.
+                    if flags.pgo_generate_only:
+                        if not isinstance(f, Path):
+                            raise SandboxValidationError('pgo_generate_only file'
+                                'must not be a generated file: %s' % f, context)
+                        if mozpath.splitext(f)[1] != '.cpp':
+                            raise SandboxValidationError('pgo_generate_only file'
+                                'must be a .cpp file: %s' % f, context)
+                        if flags.no_pgo:
+                            raise SandboxValidationError('pgo_generate_only files'
+                                'cannot be marked no_pgo: %s' % f, context)
+                        pgo_generate_only.add(f)
 
                 if isinstance(f, SourcePath) and not os.path.exists(full_path):
                     raise SandboxValidationError('File listed in %s does not '
@@ -870,6 +885,8 @@ class TreeMetadataEmitter(LoggingMixin):
         no_pgo = context.get('NO_PGO')
         no_pgo_sources = [f for f, flags in all_flags.iteritems()
                           if flags.no_pgo]
+        pgo_gen_only_sources = set(f for f, flags in all_flags.iteritems()
+                                   if flags.pgo_generate_only)
         if no_pgo:
             if no_pgo_sources:
                 raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo '
@@ -932,6 +949,8 @@ class TreeMetadataEmitter(LoggingMixin):
 
             for srcs, cls in ((sources[variable], klass),
                               (gen_sources[variable], gen_klass)):
+                if variable == 'SOURCES' and pgo_gen_only_sources:
+                    srcs = [s for s in srcs if s not in pgo_gen_only_sources]
                 # Now sort the files to let groupby work.
                 sorted_files = sorted(srcs, key=canonical_suffix_for_file)
                 for canonical_suffix, files in itertools.groupby(
@@ -955,6 +974,8 @@ class TreeMetadataEmitter(LoggingMixin):
                 for target_var in ('SOURCES', 'UNIFIED_SOURCES'):
                     for suffix, srcs in ctxt_sources[target_var].items():
                         linkable.sources[suffix] += srcs
+                if pgo_gen_only_sources:
+                    linkable.pgo_gen_only_sources = pgo_gen_only_sources
                 if no_pgo_sources:
                     linkable.no_pgo_sources = no_pgo_sources
                 elif no_pgo:
@@ -968,6 +989,9 @@ class TreeMetadataEmitter(LoggingMixin):
                 ext = mozpath.splitext(f)[1]
                 yield PerSourceFlag(context, f, flags.flags)
 
+        if pgo_generate_only:
+            yield PgoGenerateOnlySources(context, pgo_generate_only)
+
         # If there are any C++ sources, set all the linkables defined here
         # to require the C++ linker.
         for vars, linkable_items in ((('SOURCES', 'UNIFIED_SOURCES'), linkables),