Bug 1528150 - Build gn from https://gn.googlesource.com/gn/. r=froydnj

Make some adjustments to make the build work:
- On Linux, a newer GCC is needed, and -lrt is missing for
  clock_gettime.
- On Mac, ninja expect AR be accept ar arguments, which clang doesn't,
  so use llvm-ar.
- On Windows, remove the /WX flag that upstream sets (warnings as
  errors) because there _are_ warnings in the code, and remove the
  explicit /MACHINE:x64 linker flag because we're building for x86.
  (we could build for x64, but I'd rather leave that to a followup)

Differential Revision: https://phabricator.services.mozilla.com/D19911

taskcluster/ci/toolchain/gn.yml

@@ -23,7 +23,7 @@ linux64-gn:
     run:
         script: build-gn-linux.sh
     toolchains:
-        - linux64-gcc-4.9
+        - linux64-gcc-6
 
 macosx64-gn:
     treeherder:

taskcluster/scripts/misc/build-gn-common.sh

@@ -2,37 +2,48 @@
 set -e -v
 
 # This is shared code for building GN.
-
-# Each is a recent commit from chromium's master branch.
-: CHROMIUM_REV           ${CHROMIUM_REV:=e6ba81e00ae835946e069e5bd80bd533b11d8442}
-: GTEST_REV              ${GTEST_REV:=6c5116014ce51ef3273d800cbf75fcef99e798c6}
-: CHROMIUM_SRC_REV       ${CHROMIUM_SRC_REV:=c338d43f49c0d72e69cd6e40eeaf4c0597dbdda1}
+: GN_REV                 ${GN_REV:=d69a9c3765dee2e650bcccebbadf72c5d42d92b1}
 
 
-git clone --no-checkout https://chromium.googlesource.com/chromium/src $WORKSPACE/gn-standalone
+git clone --no-checkout https://gn.googlesource.com/gn $WORKSPACE/gn-standalone
 cd $WORKSPACE/gn-standalone
-git checkout $CHROMIUM_SRC_REV
+git checkout $GN_REV
 
-git clone --no-checkout https://chromium.googlesource.com/chromium/chromium chromium_checkout
-cd chromium_checkout
-git checkout $CHROMIUM_REV
-mkdir -p ../third_party
-mv third_party/libevent ../third_party
-cd ..
+# We remove /WC because of https://bugs.chromium.org/p/gn/issues/detail?id=51
+# And /MACHINE:x64 because we just let the PATH decide what cl and link are
+# used, and if cl is targetting x86, we don't want linkage to fail because of
+# /MACHINE:x64.
+patch -p1 <<'EOF'
+diff --git a/build/gen.py b/build/gen.py
+index a7142fab..78d0fd56 100755
+--- a/build/gen.py
++++ b/build/gen.py
+@@ -357,7 +357,6 @@ def WriteGNNinja(path, platform, host, options):
+         '/D_WIN32_WINNT=0x0A00',
+         '/FS',
+         '/W4',
+-        '/WX',
+         '/Zi',
+         '/wd4099',
+         '/wd4100',
+@@ -373,7 +372,7 @@ def WriteGNNinja(path, platform, host, options):
+         '/D_HAS_EXCEPTIONS=0',
+     ])
+ 
+-    ldflags.extend(['/DEBUG', '/MACHINE:x64'])
++    ldflags.extend(['/DEBUG'])
+ 
+   static_libraries = {
+       'base': {'sources': [
+EOF
 
-rm -rf testing
-mkdir testing
-cd testing
-git clone https://chromium.googlesource.com/chromium/testing/gtest
-cd gtest
-git checkout $GTEST_REV
-cd ../..
+if test -n "$MAC_CROSS"; then
+    python build/gen.py --platform darwin
+else
+    python build/gen.py
+fi
 
-cd tools/gn
-patch -p1 < $WORKSPACE/build/src/taskcluster/scripts/misc/gn.patch
-
-./bootstrap/bootstrap.py -s
-cd ../..
+ninja -C out -v
 
 STAGE=gn
 mkdir -p $UPLOAD_DIR $STAGE
@@ -40,9 +51,9 @@ mkdir -p $UPLOAD_DIR $STAGE
 # At this point, the resulting binary is at:
 # $WORKSPACE/out/Release/gn
 if test "$MAC_CROSS" = "" -a "$(uname)" = "Linux"; then
-    strip out/Release/gn
+    strip out/gn
 fi
-cp out/Release/gn $STAGE
+cp out/gn $STAGE
 
 tar -acf gn.tar.$COMPRESS_EXT $STAGE
 cp gn.tar.$COMPRESS_EXT $UPLOAD_DIR

taskcluster/scripts/misc/build-gn-linux.sh

@@ -8,6 +8,7 @@ UPLOAD_DIR=$HOME/artifacts
 COMPRESS_EXT=xz
 export CC=$WORKSPACE/build/src/gcc/bin/gcc
 export CXX=$WORKSPACE/build/src/gcc/bin/g++
+export LDFLAGS=-lrt
 
 # Gn build scripts use #!/usr/bin/env python, which will be python 2.6 on
 # the worker and cause failures. Work around this by putting python2.7

taskcluster/scripts/misc/build-gn-macosx.sh

@@ -13,7 +13,7 @@ CROSS_SYSROOT=$WORKSPACE/build/src/MacOSX10.11.sdk
 export LD_LIBRARY_PATH=$WORKSPACE/build/src/clang/lib
 export CC=$WORKSPACE/build/src/clang/bin/clang
 export CXX=$WORKSPACE/build/src/clang/bin/clang++
-export AR=$WORKSPACE/build/src/clang/bin/clang
+export AR=$WORKSPACE/build/src/clang/bin/llvm-ar
 export CFLAGS="-target x86_64-darwin11 -mlinker-version=137 -B ${CROSS_CCTOOLS_PATH}/bin -isysroot ${CROSS_SYSROOT} -I${CROSS_SYSROOT}/usr/include -iframework ${CROSS_SYSROOT}/System/Library/Frameworks"
 export CXXFLAGS="-stdlib=libc++ ${CFLAGS}"
 export LDFLAGS="${CXXFLAGS} -Wl,-syslibroot,${CROSS_SYSROOT} -Wl,-dead_strip"

taskcluster/scripts/misc/gn.patch

@@ -1,78 +0,0 @@
-diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py
-index ff2ae57..7e12436 100755
---- a/bootstrap/bootstrap.py
-+++ b/bootstrap/bootstrap.py
-@@ -29,9 +29,10 @@ BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
- GN_ROOT = os.path.dirname(BOOTSTRAP_DIR)
- SRC_ROOT = os.path.dirname(os.path.dirname(GN_ROOT))
- 
-+is_mac_cross = os.environ.get('MAC_CROSS', False)
- is_win = sys.platform.startswith('win')
--is_linux = sys.platform.startswith('linux')
--is_mac = sys.platform.startswith('darwin')
-+is_linux = sys.platform.startswith('linux') and not is_mac_cross
-+is_mac = sys.platform.startswith('darwin') or is_mac_cross
- is_aix = sys.platform.startswith('aix')
- is_posix = is_linux or is_mac or is_aix
- 
-@@ -595,7 +596,6 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'base/synchronization/condition_variable_posix.cc',
-         'base/synchronization/lock_impl_posix.cc',
-         'base/synchronization/read_write_lock_posix.cc',
--        'base/synchronization/waitable_event_posix.cc',
-         'base/sys_info_posix.cc',
-         'base/task_scheduler/task_tracker_posix.cc',
-         'base/threading/platform_thread_internal_posix.cc',
-@@ -603,10 +603,19 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'base/threading/thread_local_storage_posix.cc',
-         'base/threading/worker_pool_posix.cc',
-         'base/time/time_conversion_posix.cc',
-+    ])
-+
-+    if not is_mac:
-+      static_libraries['base']['sources'].extend([
-+        'base/synchronization/waitable_event_posix.cc',
-         'base/time/time_exploded_posix.cc',
-         'base/time/time_now_posix.cc',
--        'base/trace_event/heap_profiler_allocation_register_posix.cc',
-+      ])
-+
-+    static_libraries['base']['sources'].extend([
-+      'base/trace_event/heap_profiler_allocation_register_posix.cc',
-     ])
-+
-     static_libraries['libevent'] = {
-         'sources': [
-             'base/third_party/libevent/buffer.c',
-@@ -685,6 +694,7 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'base/mac/call_with_eh_frame.cc',
-         'base/mac/call_with_eh_frame_asm.S',
-         'base/mac/foundation_util.mm',
-+        'base/mac/mac_util.mm',
-         'base/mac/mach_logging.cc',
-         'base/mac/scoped_mach_port.cc',
-         'base/mac/scoped_mach_vm.cc',
-@@ -697,6 +707,7 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'base/process/process_iterator_mac.cc',
-         'base/process/process_metrics_mac.cc',
-         'base/strings/sys_string_conversions_mac.mm',
-+        'base/synchronization/waitable_event_mac.cc',
-         'base/sys_info_mac.mm',
-         'base/time/time_mac.cc',
-         'base/threading/platform_thread_mac.mm',
-@@ -735,6 +746,7 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'base/memory/memory_pressure_monitor_win.cc',
-         'base/memory/shared_memory_handle_win.cc',
-         'base/memory/shared_memory_win.cc',
-+        'base/memory/shared_memory_tracker.cc',
-         'base/message_loop/message_pump_win.cc',
-         'base/native_library_win.cc',
-         'base/power_monitor/power_monitor_device_source_win.cc',
-@@ -799,6 +811,7 @@ def write_gn_ninja(path, root_gen_dir, options):
-         'version.lib',
-         'winmm.lib',
-         'ws2_32.lib',
-+        'shlwapi.lib',
-     ])
- 
-   # we just build static libraries that GN needs