Memory/Perf: For link-mode, isolate file I/O so API can be looped over.

Separating file I/O from compile/link lets the compile/link be done
repeatedly in a loop for testing and measuring of performance and
memory footprint, including seeing memory growth over time for
functional-level memory-leak testing.

While the older compile-only mode already had this functionality,
and typically showed no memory leaks, SPIR-V uses the link path,
has pending "TODO" for memory freeing, and this shows several
kilobytes of leaking per compile-link. Most likely, pending
merge request 131 will address much of this.

StandAlone/StandAlone.cpp

@@ -658,17 +658,27 @@ void StderrIfNonEmpty(const char* str)
     }
 }
 
+// Simple bundling of what makes a compilation unit for ease in passing around,
+// and separation of handling file IO versus API (programmatic) compilation.
+struct ShaderCompUnit {
+    EShLanguage stage;
+    std::string fileName;
+    char** text;           // memory owned/managed externally
+};
+
 //
-// For linking mode: Will independently parse each item in the worklist, but then put them
+// For linking mode: Will independently parse each compilation unit, but then put them
-// in the same program and link them together.
+// in the same program and link them together, making at most one linked module per
+// pipeline stage.
 //
 // Uses the new C++ interface instead of the old handle-based interface.
 //
-void CompileAndLinkShaders()
+
+void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
 {
     // keep track of what to free
     std::list<glslang::TShader*> shaders;
-    
+
     EShMessages messages = EShMsgDefault;
     SetMessageOptions(messages);
 
@@ -677,22 +687,13 @@ void CompileAndLinkShaders()
     //
 
     glslang::TProgram& program = *new glslang::TProgram;
-    glslang::TWorkItem* workItem;
+    for (auto compUnit : compUnits) {
-    while (Worklist.remove(workItem)) {
+        glslang::TShader* shader = new glslang::TShader(compUnit.stage);
-        EShLanguage stage = FindLanguage(workItem->name);
+        shader->setStrings(compUnit.text, 1);
-        glslang::TShader* shader = new glslang::TShader(stage);
         shaders.push_back(shader);
-    
-        char** shaderStrings = ReadFileData(workItem->name.c_str());
-        if (! shaderStrings) {
-            usage();
-            delete &program;
 
-            return;
-        }
         const int defaultVersion = Options & EOptionDefaultDesktop? 110: 100;
 
-        shader->setStrings(shaderStrings, 1);
         if (Options & EOptionOutputPreprocessed) {
             std::string str;
             if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false,
@@ -703,7 +704,6 @@ void CompileAndLinkShaders()
             }
             StderrIfNonEmpty(shader->getInfoLog());
             StderrIfNonEmpty(shader->getInfoDebugLog());
-            FreeFileData(shaderStrings);
             continue;
         }
         if (! shader->parse(&Resources, defaultVersion, false, messages))
@@ -711,13 +711,12 @@ void CompileAndLinkShaders()
 
         program.addShader(shader);
 
-        if (! (Options & EOptionSuppressInfolog)) {
+        if (! (Options & EOptionSuppressInfolog) &&
-            PutsIfNonEmpty(workItem->name.c_str());
+            ! (Options & EOptionMemoryLeakMode)) {
+            PutsIfNonEmpty(compUnit.fileName.c_str());
             PutsIfNonEmpty(shader->getInfoLog());
             PutsIfNonEmpty(shader->getInfoDebugLog());
         }
-
-        FreeFileData(shaderStrings);
     }
 
     //
@@ -727,7 +726,8 @@ void CompileAndLinkShaders()
     if (! (Options & EOptionOutputPreprocessed) && ! program.link(messages))
         LinkFailed = true;
 
-    if (! (Options & EOptionSuppressInfolog)) {
+    if (! (Options & EOptionSuppressInfolog) &&
+        ! (Options & EOptionMemoryLeakMode)) {
         PutsIfNonEmpty(program.getInfoLog());
         PutsIfNonEmpty(program.getInfoDebugLog());
     }
@@ -745,10 +745,15 @@ void CompileAndLinkShaders()
                 if (program.getIntermediate((EShLanguage)stage)) {
                     std::vector<unsigned int> spirv;
                     glslang::GlslangToSpv(*program.getIntermediate((EShLanguage)stage), spirv);
-                    glslang::OutputSpv(spirv, GetBinaryName((EShLanguage)stage));
+
-                    if (Options & EOptionHumanReadableSpv) {
+                    // Dump the spv to a file or stdout, etc., but only if not doing
-                        spv::Parameterize();
+                    // memory/perf testing, as it's not internal to programmatic use.
-                        spv::Disassemble(std::cout, spirv);
+                    if (! (Options & EOptionMemoryLeakMode)) {
+                        glslang::OutputSpv(spirv, GetBinaryName((EShLanguage)stage));
+                        if (Options & EOptionHumanReadableSpv) {
+                            spv::Parameterize();
+                            spv::Disassemble(std::cout, spirv);
+                        }
                     }
                 }
             }
@@ -766,6 +771,59 @@ void CompileAndLinkShaders()
     }
 }
 
+//
+// Do file IO part of compile and link, handing off the pure
+// API/programmatic mode to CompileAndLinkShaderUnits(), which can
+// be put in a loop for testing memory footprint and performance.
+//
+// This is just for linking mode: meaning all the shaders will be put into the
+// the same program linked together.
+//
+// This means there are a limited number of work items (not multi-threading mode)
+// and that the point is testing at the linking level. Hence, to enable
+// performance and memory testing, the actual compile/link can be put in
+// a loop, independent of processing the work items and file IO.
+//
+void CompileAndLinkShaderFiles()
+{
+    std::vector<ShaderCompUnit> compUnits;
+
+    // Transfer all the work items from to a simple list of
+    // of compilation units.  (We don't care about the thread
+    // work-item distribution properties in this path, which
+    // is okay due to the limited number of shaders, know since
+    // they are all getting linked together.)
+    glslang::TWorkItem* workItem;
+    while (Worklist.remove(workItem)) {
+        ShaderCompUnit compUnit = {
+            FindLanguage(workItem->name),
+            workItem->name,
+            ReadFileData(workItem->name.c_str())
+        };
+
+        if (! compUnit.text) {
+            usage();
+            return;
+        }
+
+        compUnits.push_back(compUnit);
+    }
+
+    // Actual call to programmatic processing of compile and link,
+    // in a loop for testing memory and performance.  This part contains
+    // all the perf/memory that a programmatic consumer will care about.
+    for (int i = 0; i < ((Options & EOptionMemoryLeakMode) ? 100 : 1); ++i) {
+        for (int j = 0; j < ((Options & EOptionMemoryLeakMode) ? 100 : 1); ++j)
+           CompileAndLinkShaderUnits(compUnits);
+
+        if (Options & EOptionMemoryLeakMode)
+            glslang::OS_DumpMemoryCounters();
+    }
+
+    for (auto c : compUnits)
+        FreeFileData(c.text);
+}
+
 int C_DECL main(int argc, char* argv[])
 {
     ProcessArguments(argc, argv);
@@ -803,7 +861,7 @@ int C_DECL main(int argc, char* argv[])
     if (Options & EOptionLinkProgram ||
         Options & EOptionOutputPreprocessed) {
         glslang::InitializeProcess();
-        CompileAndLinkShaders();
+        CompileAndLinkShaderFiles();
         glslang::FinalizeProcess();
     } else {
         ShInitialize();

glslang/OSDependent/Windows/ossource.cpp

@@ -147,6 +147,8 @@ void OS_Sleep(int milliseconds)
     Sleep(milliseconds);
 }
 
+//#define DUMP_COUNTERS
+
 void OS_DumpMemoryCounters()
 {
 #ifdef DUMP_COUNTERS