Bug 1239018 - add file watching and hot reloading for devtools files r=jryans

devtools/client/preferences/devtools.js

@@ -17,6 +17,9 @@ pref("devtools.devedition.promo.url", "https://www.mozilla.org/firefox/developer
 // Disable the error console
 pref("devtools.errorconsole.enabled", false);
 
+// DevTools development workflow
+pref("devtools.loader.hotreload", false);
+
 // Developer toolbar preferences
 pref("devtools.toolbar.enabled", true);
 pref("devtools.toolbar.visible", false);

devtools/client/shared/browser-loader.js

@@ -6,8 +6,9 @@
 var { classes: Cc, interfaces: Ci, utils: Cu } = Components;
 
 const loaders = Cu.import("resource://gre/modules/commonjs/toolkit/loader.js", {});
-const { devtools, DevToolsLoader } = Cu.import("resource://devtools/shared/Loader.jsm", {});
+const { devtools } = Cu.import("resource://devtools/shared/Loader.jsm", {});
 const { joinURI } = devtools.require("devtools/shared/path");
+const { Services } = devtools.require("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/AppConstants.jsm");
 
 const BROWSER_BASED_DIRS = [
@@ -17,6 +18,40 @@ const BROWSER_BASED_DIRS = [
   "resource://devtools/client/shared/redux"
 ];
 
+function clearCache() {
+  Services.obs.notifyObservers(null, "startupcache-invalidate", null);
+}
+
+function hotReloadFile(window, require, loader, componentProxies, fileURI) {
+  dump("Hot reloading: " + fileURI + "\n");
+
+  if (fileURI.match(/\.js$/)) {
+    // Test for React proxy components
+    const proxy = componentProxies.get(fileURI);
+    if (proxy) {
+      // Remove the old module and re-require the new one; the require
+      // hook in the loader will take care of the rest
+      delete loader.modules[fileURI];
+      clearCache();
+      require(fileURI);
+    }
+  } else if (fileURI.match(/\.css$/)) {
+    const links = [...window.document.getElementsByTagNameNS("http://www.w3.org/1999/xhtml", "link")];
+    links.forEach(link => {
+      if (link.href.indexOf(fileURI) === 0) {
+        const parentNode = link.parentNode;
+        const newLink = window.document.createElementNS("http://www.w3.org/1999/xhtml", "link");
+        newLink.rel = "stylesheet";
+        newLink.type = "text/css";
+        newLink.href = fileURI + "?s=" + Math.random();
+
+        parentNode.insertBefore(newLink, link);
+        parentNode.removeChild(link);
+      }
+    });
+  }
+}
+
 /*
  * Create a loader to be used in a browser environment. This evaluates
  * modules in their own environment, but sets window (the normal
@@ -45,6 +80,8 @@ const BROWSER_BASED_DIRS = [
 function BrowserLoader(baseURI, window) {
   const loaderOptions = devtools.require("@loader/options");
   const dynamicPaths = {};
+  const componentProxies = new Map();
+  const hotReloadEnabled = Services.prefs.getBoolPref("devtools.loader.hotreload");
 
   if(AppConstants.DEBUG || AppConstants.DEBUG_JS_MODULES) {
     dynamicPaths["devtools/client/shared/vendor/react"] =
@@ -81,12 +118,47 @@ function BrowserLoader(baseURI, window) {
     }
   };
 
+  if(hotReloadEnabled) {
+    opts.loadModuleHook = (module, require) => {
+      const { uri, exports } = module;
+
+      if (exports.prototype &&
+          exports.prototype.isReactComponent) {
+        const { createProxy, getForceUpdate } =
+              require("devtools/client/shared/vendor/react-proxy");
+        const React = require("devtools/client/shared/vendor/react");
+
+        if (!componentProxies.get(uri)) {
+          const proxy = createProxy(exports);
+          componentProxies.set(uri, proxy);
+          module.exports = proxy.get();
+        }
+        else {
+          const proxy = componentProxies.get(uri);
+          const instances = proxy.update(exports);
+          instances.forEach(getForceUpdate(React));
+          module.exports = proxy.get();
+        }
+      }
+      return exports;
+    }
+  }
+
+
   const mainModule = loaders.Module(baseURI, joinURI(baseURI, "main.js"));
   const mainLoader = loaders.Loader(opts);
+  const require = loaders.Require(mainLoader, mainModule);
+
+  if(hotReloadEnabled) {
+    const watcher = devtools.require("devtools/client/shared/file-watcher");
+    watcher.on("file-changed", (_, fileURI) => {
+      hotReloadFile(window, require, mainLoader, componentProxies, fileURI);
+    });
+  }
 
   return {
     loader: mainLoader,
-    require: loaders.Require(mainLoader, mainModule)
+    require: require
   };
 }
 

devtools/client/shared/file-watcher-worker.js

@@ -0,0 +1,78 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+/* eslint-env worker */
+/* global OS */
+importScripts("resource://gre/modules/osfile.jsm");
+
+const modifiedTimes = new Map();
+
+function gatherFiles(path, fileRegex) {
+  let files = [];
+  const iterator = new OS.File.DirectoryIterator(path);
+
+  try {
+    for (let child in iterator) {
+      // Don't descend into test directories. Saves us some time and
+      // there's no reason to.
+      if (child.isDir && !child.path.endsWith("/test")) {
+        files = files.concat(gatherFiles(child.path, fileRegex));
+      } else if (child.path.match(fileRegex)) {
+        let info;
+        try {
+          info = OS.File.stat(child.path);
+        } catch (e) {
+          // Just ignore it.
+          continue;
+        }
+
+        files.push(child.path);
+        modifiedTimes.set(child.path, info.lastModificationDate.getTime());
+      }
+    }
+  } finally {
+    iterator.close();
+  }
+
+  return files;
+}
+
+function scanFiles(files, onChangedFile) {
+  files.forEach(file => {
+    let info;
+    try {
+      info = OS.File.stat(file);
+    } catch (e) {
+      // Just ignore it. It was probably deleted.
+      return;
+    }
+
+    const lastTime = modifiedTimes.get(file);
+
+    if (info.lastModificationDate.getTime() > lastTime) {
+      modifiedTimes.set(file, info.lastModificationDate.getTime());
+      onChangedFile(file);
+    }
+  });
+}
+
+onmessage = function(event) {
+  const { path, fileRegex } = event.data;
+  let info = OS.File.stat(path);
+  if (!info.isDir) {
+    throw new Error("watcher expects a directory as root path");
+  }
+
+  // We get a list of all the files upfront, which means we don't
+  // support adding new files. But you need to rebuild Firefox when
+  // adding a new file anyway.
+  const files = gatherFiles(path, fileRegex || /.*/);
+
+  // Every second, scan for file changes by stat-ing each of them and
+  // comparing modification time.
+  setInterval(() => {
+    scanFiles(files, changedFile => postMessage(changedFile));
+  }, 1000);
+};

devtools/client/shared/file-watcher.js

@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { Ci, ChromeWorker } = require("chrome");
+const { Services } = require("resource://gre/modules/Services.jsm");
+const EventEmitter = require("devtools/shared/event-emitter");
+
+const HOTRELOAD_PREF = "devtools.loader.hotreload";
+
+function resolveResourceURI(uri) {
+  const handler = Services.io.getProtocolHandler("resource")
+        .QueryInterface(Ci.nsIResProtocolHandler);
+  return handler.resolveURI(Services.io.newURI(uri, null, null));
+}
+
+function watchFiles(path, onFileChanged) {
+  if (!path.startsWith("devtools/")) {
+    throw new Error("`watchFiles` expects a devtools path");
+  }
+
+  // We need to figure out a local path to watch. We start with
+  // whatever devtools points to.
+  let resolvedRootURI = resolveResourceURI("resource://devtools");
+  if (resolvedRootURI.match(/\/obj\-.*/)) {
+    // Move from the built directory to the user's local files
+    resolvedRootURI = resolvedRootURI.replace(/\/obj\-.*/, "") + "/devtools";
+  }
+  resolvedRootURI = resolvedRootURI.replace(/^file:\/\//, "");
+  const localURI = resolvedRootURI + "/" + path.replace(/^devtools\//, "");
+
+  const watchWorker = new ChromeWorker(
+    "resource://devtools/client/shared/file-watcher-worker.js"
+  );
+
+  watchWorker.onmessage = event => {
+    // We need to turn a local path back into a resource URI (or
+    // chrome). This means that this system will only work when built
+    // files are symlinked, so that these URIs actually read from
+    // local sources. There might be a better way to do this.
+    const relativePath = event.data.replace(resolvedRootURI + "/", "");
+    if (relativePath.startsWith("client/themes")) {
+      onFileChanged(relativePath.replace("client/themes",
+                                         "chrome://devtools/skin"));
+    }
+    onFileChanged("resource://devtools/" + relativePath);
+  };
+
+  watchWorker.postMessage({ path: localURI, fileRegex: /\.(js|css)$/ });
+  return watchWorker;
+}
+
+EventEmitter.decorate(module.exports);
+
+let watchWorker;
+function onPrefChange() {
+  if (Services.prefs.getBoolPref(HOTRELOAD_PREF) && !watchWorker) {
+    watchWorker = watchFiles("devtools/client", changedFile => {
+      module.exports.emit("file-changed", changedFile);
+    });
+  }
+  else if(watchWorker) {
+    watchWorker.terminate();
+    watchWorker = null;
+  }
+}
+
+Services.prefs.addObserver(HOTRELOAD_PREF, {
+  observe: onPrefChange
+}, false);
+
+onPrefChange();

devtools/client/shared/moz.build

@@ -25,6 +25,8 @@ DevToolsModules(
     'devices.js',
     'DOMHelpers.jsm',
     'doorhanger.js',
+    'file-watcher-worker.js',
+    'file-watcher.js',
     'frame-script-utils.js',
     'getjson.js',
     'inplace-editor.js',

devtools/client/shared/vendor/moz.build

@@ -6,11 +6,14 @@
 
 modules = []
 
+# react-dev is used if either debug mode is enabled,
+# so include it for both
 if CONFIG['DEBUG_JS_MODULES'] or CONFIG['MOZ_DEBUG']:
     modules += ['react-dev.js']
 
 modules += [
     'react-dom.js',
+    'react-proxy.js',
     'react-redux.js',
     'react.js',
     'redux.js',