Bug 972712 (part 4) - Report script sources in more detail. r=till.

--HG--
extra : rebase_source : b28fc8f4ff791966cb784e1c12def58927d3e3d3

js/public/MemoryMetrics.h

@@ -77,6 +77,13 @@ struct InefficientNonFlatteningStringHashPolicy
     static bool match(const JSString *const &k, const Lookup &l);
 };
 
+struct CStringHashPolicy
+{
+    typedef const char *Lookup;
+    static HashNumber hash(const Lookup &l);
+    static bool match(const char *const &k, const Lookup &l);
+};
+
 // This file features many classes with numerous size_t fields, and each such
 // class has one or more methods that need to operate on all of these fields.
 // Writing these individually is error-prone -- it's easy to add a new field
@@ -268,6 +275,67 @@ struct NotableStringInfo : public StringInfo
     NotableStringInfo(const NotableStringInfo& info) MOZ_DELETE;
 };
 
+// This class holds information about the memory taken up by script sources
+// from a particular file.
+struct ScriptSourceInfo
+{
+#define FOR_EACH_SIZE(macro) \
+    macro(_, _, compressed) \
+    macro(_, _, uncompressed) \
+    macro(_, _, misc)
+
+    ScriptSourceInfo()
+      : FOR_EACH_SIZE(ZERO_SIZE)
+        numScripts(0)
+    {}
+
+    void add(const ScriptSourceInfo &other) {
+        FOR_EACH_SIZE(ADD_OTHER_SIZE)
+        numScripts++;
+    }
+
+    void subtract(const ScriptSourceInfo &other) {
+        FOR_EACH_SIZE(SUB_OTHER_SIZE)
+        numScripts--;
+    }
+
+    bool isNotable() const {
+        static const size_t NotabilityThreshold = 16 * 1024;
+        size_t n = 0;
+        FOR_EACH_SIZE(ADD_SIZE_TO_N)
+        return n >= NotabilityThreshold;
+    }
+
+    FOR_EACH_SIZE(DECL_SIZE)
+    uint32_t numScripts;    // How many ScriptSources come from this file? (It
+                            // can be more than one in XML files that have
+                            // multiple scripts in CDATA sections.)
+#undef FOR_EACH_SIZE
+};
+
+// Holds data about a notable script source file (one whose combined
+// script sources use more than a certain amount of memory) so we can report it
+// individually.
+//
+// The only difference between this class and ScriptSourceInfo is that this
+// class holds a copy of the filename.
+struct NotableScriptSourceInfo : public ScriptSourceInfo
+{
+    NotableScriptSourceInfo();
+    NotableScriptSourceInfo(const char *filename, const ScriptSourceInfo &info);
+    NotableScriptSourceInfo(NotableScriptSourceInfo &&info);
+    NotableScriptSourceInfo &operator=(NotableScriptSourceInfo &&info);
+
+    ~NotableScriptSourceInfo() {
+        js_free(filename_);
+    }
+
+    char *filename_;
+
+  private:
+    NotableScriptSourceInfo(const NotableScriptSourceInfo& info) MOZ_DELETE;
+};
+
 // These measurements relate directly to the JSRuntime, and not to zones and
 // compartments within it.
 struct RuntimeSizes
@@ -283,17 +351,45 @@ struct RuntimeSizes
     macro(_, _, mathCache) \
     macro(_, _, sourceDataCache) \
     macro(_, _, scriptData) \
-    macro(_, _, scriptSources)
 
     RuntimeSizes()
       : FOR_EACH_SIZE(ZERO_SIZE)
+        scriptSourceInfo(),
         code(),
-        gc()
-    {}
+        gc(),
+        notableScriptSources()
+    {
+        allScriptSources = js_new<ScriptSourcesHashMap>();
+        if (!allScriptSources || !allScriptSources->init())
+            MOZ_CRASH("oom");
+    }
 
+    ~RuntimeSizes() {
+        // |allScriptSources| is usually deleted and set to nullptr before this
+        // destructor runs. But there are failure cases due to OOMs that may
+        // prevent that, so it doesn't hurt to try again here.
+        js_delete(allScriptSources);
+    }
+
+    // The script source measurements in |scriptSourceInfo| are initially for
+    // all script sources.  At the end, if the measurement granularity is
+    // FineGrained, we subtract the measurements of the notable script sources
+    // and move them into |notableScriptSources|.
     FOR_EACH_SIZE(DECL_SIZE)
-    CodeSizes code;
-    GCSizes   gc;
+    ScriptSourceInfo    scriptSourceInfo;
+    CodeSizes           code;
+    GCSizes             gc;
+
+    typedef js::HashMap<const char*, ScriptSourceInfo,
+                        js::CStringHashPolicy,
+                        js::SystemAllocPolicy> ScriptSourcesHashMap;
+
+    // |allScriptSources| is only used transiently.  During the reporting phase
+    // it is filled with info about every script source in the runtime.  It's
+    // then used to fill in |notableScriptSources| (which actually gets
+    // reported), and immediately discarded afterwards.
+    ScriptSourcesHashMap *allScriptSources;
+    js::Vector<NotableScriptSourceInfo, 0, js::SystemAllocPolicy> notableScriptSources;
 
 #undef FOR_EACH_SIZE
 };

js/src/jsscript.cpp

@@ -1654,17 +1654,18 @@ ScriptSource::destroy()
     js_free(this);
 }
 
-size_t
-ScriptSource::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf)
+void
+ScriptSource::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+                                     JS::ScriptSourceInfo *info) const
 {
-    // |data| is a union, but both members are pointers to allocated memory,
-    // |emptySource|, or nullptr, so just using |data.compressed| will work.
-    size_t n = mallocSizeOf(this);
-    n += (ready() && data.compressed != emptySource)
-       ? mallocSizeOf(data.compressed)
-       : 0;
-    n += mallocSizeOf(filename_);
-    return n;
+    if (ready() && data.compressed != emptySource) {
+        if (compressed())
+            info->compressed += mallocSizeOf(data.compressed);
+        else
+            info->uncompressed += mallocSizeOf(data.source);
+    }
+    info->misc += mallocSizeOf(this) + mallocSizeOf(filename_);
+    info->numScripts++;
 }
 
 template<XDRMode mode>

js/src/jsscript.h

@@ -25,6 +25,10 @@
 #include "jit/IonCode.h"
 #include "vm/Shape.h"
 
+namespace JS {
+struct ScriptSourceInfo;
+}
+
 namespace js {
 
 namespace jit {
@@ -482,7 +486,8 @@ class ScriptSource
     }
     const jschar *chars(JSContext *cx, const SourceDataCache::AutoSuppressPurge &asp);
     JSFlatString *substring(JSContext *cx, uint32_t start, uint32_t stop);
-    size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
+    void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+                                JS::ScriptSourceInfo *info) const;
 
     // XDR handling
     template <XDRMode mode>

js/src/vm/MemoryMetrics.cpp

@@ -25,6 +25,7 @@
 using mozilla::DebugOnly;
 using mozilla::MallocSizeOf;
 using mozilla::Move;
+using mozilla::PodCopy;
 using mozilla::PodEqual;
 
 using namespace js;
@@ -89,6 +90,18 @@ InefficientNonFlatteningStringHashPolicy::match(const JSString *const &k, const
     return PodEqual(c1, c2, k->length());
 }
 
+/* static */ HashNumber
+CStringHashPolicy::hash(const Lookup &l)
+{
+    return mozilla::HashString(l);
+}
+
+/* static */ bool
+CStringHashPolicy::match(const char *const &k, const Lookup &l)
+{
+    return strcmp(k, l) == 0;
+}
+
 } // namespace js
 
 namespace JS {
@@ -142,6 +155,38 @@ NotableStringInfo &NotableStringInfo::operator=(NotableStringInfo &&info)
     return *this;
 }
 
+NotableScriptSourceInfo::NotableScriptSourceInfo()
+  : ScriptSourceInfo(),
+    filename_(nullptr)
+{
+}
+
+NotableScriptSourceInfo::NotableScriptSourceInfo(const char *filename, const ScriptSourceInfo &info)
+  : ScriptSourceInfo(info)
+{
+    size_t bytes = strlen(filename) + 1;
+    filename_ = js_pod_malloc<char>(bytes);
+    if (!filename_)
+        MOZ_CRASH("oom");
+    PodCopy(filename_, filename, bytes);
+}
+
+NotableScriptSourceInfo::NotableScriptSourceInfo(NotableScriptSourceInfo &&info)
+  : ScriptSourceInfo(Move(info))
+{
+    filename_ = info.filename_;
+    info.filename_ = nullptr;
+}
+
+NotableScriptSourceInfo &NotableScriptSourceInfo::operator=(NotableScriptSourceInfo &&info)
+{
+    MOZ_ASSERT(this != &info, "self-move assignment is prohibited");
+    this->~NotableScriptSourceInfo();
+    new (this) NotableScriptSourceInfo(Move(info));
+    return *this;
+}
+
+
 } // namespace JS
 
 typedef HashSet<ScriptSource *, DefaultHasher<ScriptSource *>, SystemAllocPolicy> SourceSet;
@@ -346,9 +391,30 @@ StatsCellCallback(JSRuntime *rt, void *data, void *thing, JSGCTraceKind traceKin
         ScriptSource *ss = script->scriptSource();
         SourceSet::AddPtr entry = closure->seenSources.lookupForAdd(ss);
         if (!entry) {
-            closure->seenSources.add(entry, ss); // Not much to be done on failure.
-            rtStats->runtime.scriptSources += ss->sizeOfIncludingThis(rtStats->mallocSizeOf_);
+            (void)closure->seenSources.add(entry, ss); // Not much to be done on failure.
+
+            JS::ScriptSourceInfo info;  // This zeroes all the sizes.
+            ss->addSizeOfIncludingThis(rtStats->mallocSizeOf_, &info);
+            MOZ_ASSERT(info.compressed == 0 || info.uncompressed == 0);
+
+            rtStats->runtime.scriptSourceInfo.add(info);
+
+            if (granularity == FineGrained) {
+                const char* filename = ss->filename();
+                if (!filename)
+                    filename = "<no filename>";
+
+                JS::RuntimeSizes::ScriptSourcesHashMap::AddPtr p =
+                    rtStats->runtime.allScriptSources->lookupForAdd(filename);
+                if (!p) {
+                    // Ignore failure -- we just won't record the script source as notable.
+                    (void)rtStats->runtime.allScriptSources->add(p, filename, info);
+                } else {
+                    p->value().add(info);
+                }
+            }
         }
+
         break;
       }
 
@@ -429,6 +495,40 @@ ZoneStats::initStrings(JSRuntime *rt)
     return true;
 }
 
+static bool
+FindNotableScriptSources(JS::RuntimeSizes &runtime)
+{
+    using namespace JS;
+
+    // We should only run FindNotableScriptSources once per RuntimeSizes.
+    MOZ_ASSERT(runtime.notableScriptSources.empty());
+
+    for (RuntimeSizes::ScriptSourcesHashMap::Range r = runtime.allScriptSources->all();
+         !r.empty();
+         r.popFront())
+    {
+        const char *filename = r.front().key();
+        ScriptSourceInfo &info = r.front().value();
+
+        if (!info.isNotable())
+            continue;
+
+        if (!runtime.notableScriptSources.growBy(1))
+            return false;
+
+        runtime.notableScriptSources.back() = NotableScriptSourceInfo(filename, info);
+
+        // We're moving this script source from a non-notable to a notable
+        // bucket, so subtract its sizes from the non-notable tallies.
+        runtime.scriptSourceInfo.subtract(info);
+    }
+    // Delete |allScriptSources| now, rather than waiting for zStats's
+    // destruction, to reduce peak memory consumption during reporting.
+    js_delete(runtime.allScriptSources);
+    runtime.allScriptSources = nullptr;
+    return true;
+}
+
 JS_PUBLIC_API(bool)
 JS::CollectRuntimeStats(JSRuntime *rt, RuntimeStats *rtStats, ObjectPrivateVisitor *opv)
 {
@@ -457,6 +557,9 @@ JS::CollectRuntimeStats(JSRuntime *rt, RuntimeStats *rtStats, ObjectPrivateVisit
     // Take the "explicit/js/runtime/" measurements.
     rt->addSizeOfIncludingThis(rtStats->mallocSizeOf_, &rtStats->runtime);
 
+    if (!FindNotableScriptSources(rtStats->runtime))
+        return false;
+
     ZoneStatsVector &zs = rtStats->zoneStatsVector;
     ZoneStats &zTotals = rtStats->zTotals;
 

js/xpconnect/src/XPCJSRuntime.cpp

@@ -2139,6 +2139,33 @@ ReportCompartmentStats(const JS::CompartmentStats &cStats,
     return NS_OK;
 }
 
+static nsresult
+ReportScriptSourceStats(const ScriptSourceInfo &scriptSourceInfo,
+                        const nsACString &path,
+                        nsIHandleReportCallback *cb, nsISupports *closure,
+                        size_t &rtTotal)
+{
+    if (scriptSourceInfo.compressed > 0) {
+        RREPORT_BYTES(path + NS_LITERAL_CSTRING("compressed"),
+            KIND_HEAP, scriptSourceInfo.compressed,
+            "Compressed JavaScript source code.");
+    }
+
+    if (scriptSourceInfo.uncompressed > 0) {
+        RREPORT_BYTES(path + NS_LITERAL_CSTRING("uncompressed"),
+            KIND_HEAP, scriptSourceInfo.uncompressed,
+            "Uncompressed JavaScript source code.");
+    }
+
+    if (scriptSourceInfo.misc > 0) {
+        RREPORT_BYTES(path + NS_LITERAL_CSTRING("misc"),
+            KIND_HEAP, scriptSourceInfo.misc,
+            "Miscellaneous data relating to JavaScript source code.");
+    }
+
+    return NS_OK;
+}
+
 static nsresult
 ReportJSRuntimeExplicitTreeStats(const JS::RuntimeStats &rtStats,
                                  const nsACString &rtPath,
@@ -2214,9 +2241,35 @@ ReportJSRuntimeExplicitTreeStats(const JS::RuntimeStats &rtStats,
         KIND_HEAP, rtStats.runtime.scriptData,
         "The table holding script data shared in the runtime.");
 
-    RREPORT_BYTES(rtPath + NS_LITERAL_CSTRING("runtime/script-sources"),
-        KIND_HEAP, rtStats.runtime.scriptSources,
-        "JavaScript source code (possibly compressed) and filenames.");
+    nsCString nonNotablePath =
+        rtPath + nsPrintfCString("runtime/script-sources/source(scripts=%d, <non-notable files>)/",
+                                 rtStats.runtime.scriptSourceInfo.numScripts);
+
+    rv = ReportScriptSourceStats(rtStats.runtime.scriptSourceInfo,
+                                 nonNotablePath, cb, closure, rtTotal);
+    NS_ENSURE_SUCCESS(rv, rv);
+
+    for (size_t i = 0; i < rtStats.runtime.notableScriptSources.length(); i++) {
+        const JS::NotableScriptSourceInfo& scriptSourceInfo =
+            rtStats.runtime.notableScriptSources[i];
+
+        // Escape / to \ before we put the filename into the memory reporter
+        // path, because we don't want any forward slashes in the string to
+        // count as path separators. Consumers of memory reporters (e.g.
+        // about:memory) will convert them back to / after doing path
+        // splitting.
+        nsDependentCString filename(scriptSourceInfo.filename_);
+        nsCString escapedFilename(filename);
+        escapedFilename.ReplaceSubstring("/", "\\");
+
+        nsCString notablePath = rtPath +
+            nsPrintfCString("runtime/script-sources/source(scripts=%d, %s)/",
+                            scriptSourceInfo.numScripts, escapedFilename.get());
+
+        rv = ReportScriptSourceStats(scriptSourceInfo, notablePath,
+                                     cb, closure, rtTotal);
+        NS_ENSURE_SUCCESS(rv, rv);
+    }
 
     RREPORT_BYTES(rtPath + NS_LITERAL_CSTRING("runtime/code/ion"),
         KIND_NONHEAP, rtStats.runtime.code.ion,