SPIR-V compression: Add stripping and remapping tools for compressibility of generated SPIR-V.

git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@31180 e7fa87d3-cd2b-0410-9028-fcbf551c1848

CMakeLists.txt

@@ -13,6 +13,10 @@ else(WIN32)
     message("unkown platform")
 endif(WIN32)
 
+if(CMAKE_COMPILER_IS_GNUCXX)
+    add_definitions(-std=c++11)
+endif()
+
 add_subdirectory(glslang)
 add_subdirectory(OGLCompilersDLL)
 add_subdirectory(StandAlone)

SPIRV/CMakeLists.txt

@@ -5,6 +5,7 @@ include_directories(.. ${CMAKE_CURRENT_BINARY_DIR})
 set(SOURCES
     GlslangToSpv.cpp
     SpvBuilder.cpp
+    SPVRemapper.cpp
     doc.cpp
     disassemble.cpp)
 
@@ -12,6 +13,7 @@ set(HEADERS
     spirv.h
     GlslangToSpv.h
     SpvBuilder.h
+    SPVRemapper.h
     spvIR.h
     doc.h
     disassemble.h)

SPIRV/SPVRemapper.h

@@ -0,0 +1,281 @@
+
+#ifndef SPIRVREMAPPER_H
+#define SPIRVREMAPPER_H
+
+#include <string>
+#include <vector>
+
+namespace spv {
+
+// MSVC defines __cplusplus as an older value, even when it supports almost all of 11.
+// We handle that here by making our own symbol.
+#if __cplusplus >= 201103L || _MSC_VER >= 1800
+#   define use_cpp11 1
+#endif
+
+class spirvbin_base_t
+{
+public:
+   enum Options {
+      NONE          = 0,
+      STRIP         = (1<<0),
+      MAP_TYPES     = (1<<1),
+      MAP_NAMES     = (1<<2),
+      MAP_FUNCS     = (1<<3),
+      DCE_FUNCS     = (1<<4),
+      DCE_VARS      = (1<<5),
+      DCE_TYPES     = (1<<6),
+      OPT_LOADSTORE = (1<<7),
+      OPT_FWD_LS    = (1<<8), // EXPERIMENTAL: PRODUCES INVALID SCHEMA-0 SPIRV
+      MAP_ALL       = (MAP_TYPES | MAP_NAMES | MAP_FUNCS),
+      DCE_ALL       = (DCE_FUNCS | DCE_VARS | DCE_TYPES),
+      OPT_ALL       = (OPT_LOADSTORE),
+
+      ALL_BUT_STRIP = (MAP_ALL | DCE_ALL | OPT_ALL),
+      DO_EVERYTHING = (STRIP | ALL_BUT_STRIP)
+   };
+   
+// OS dependent path separator (avoiding boost::filesystem dependency)
+#if defined(_WIN32)
+   static const char path_sep_char() { return '\\'; }
+#else
+   static const char path_sep_char() { return '/';  }
+#endif
+
+   // Poor man's basename, to avoid external dependencies
+   static const std::string basename(const std::string& filename);
+};
+
+} // namespace SPV
+
+#if !defined (use_cpp11)
+#include <stdio.h>
+
+namespace spv {
+
+class spirvbin_t : public spirvbin_base_t
+{
+public:
+    spirvbin_t(int verbose = 0) { }
+
+    void remap(std::vector<unsigned int>& spv, unsigned int opts = 0)
+    {
+        printf("Tool not compiled for C++11, which is required for SPIR-V remapping.\n");
+    }
+
+   void remap(const std::string& filename, const std::string& outputDir, unsigned int opts = 0)
+   {
+       printf("Tool not compiled for C++11, which is required for SPIR-V remapping.\n");
+   }
+};
+
+} // namespace SPV
+
+#else // defined (use_cpp11)
+
+#include <functional>
+#include <cstdint>
+#include <unordered_map>
+#include <unordered_set>
+#include <map>
+#include <set>
+#include <cassert>
+
+#include "../../glslang/SPIRV/spirv.h"
+#include "../../glslang/SPIRV/spvIR.h"
+
+namespace spv {
+
+// class to hold SPIRV binary data for remapping, DCE, and debug stripping
+class spirvbin_t : public spirvbin_base_t
+{
+public:
+   spirvbin_t(int verbose = 0) : entryPoint(spv::NoResult), largestNewId(0), verbose(verbose) { }
+   
+   // remap on an existing binary in memory
+   void remap(std::vector<std::uint32_t>& spv, std::uint32_t opts = Options::DO_EVERYTHING);
+
+   // load binary from disk file, and remap that.
+   void remap(const std::string& filename, const std::string& outputDir,
+              std::uint32_t opts = Options::DO_EVERYTHING);
+
+   // Type for error/log handler functions
+   typedef std::function<void(const std::string&)> errorfn_t;
+   typedef std::function<void(const std::string&)> logfn_t;
+
+   // Register error/log handling functions (can be lambda fn / functor / etc)
+   static void registerErrorHandler(errorfn_t handler) { errorHandler = handler; }
+   static void registerLogHandler(logfn_t handler)     { logHandler   = handler; }
+
+protected:
+   // This can be overridden to provide other message behavior if needed
+   virtual void msg(int minVerbosity, int indent, const std::string& txt) const;
+
+private:
+   // write SPV to given directory using filename passed to remap(filename...)
+   void write(const std::string& outputDir) const;
+
+   // Local to global, or global to local ID map
+   typedef std::unordered_map<spv::Id, spv::Id> idmap_t;
+   typedef std::unordered_set<spv::Id>          idset_t;
+
+   void read(const std::string& filename);          // read SPV from disk file
+   void remap(std::uint32_t opts = Options::DO_EVERYTHING);
+
+   // Map of names to IDs
+   typedef std::unordered_map<std::string, spv::Id> namemap_t;
+
+   typedef std::uint32_t spirword_t;
+
+   typedef std::pair<int, int> range_t;
+   typedef std::function<void(spv::Id&)>           idfn_t;
+   typedef std::function<bool(spv::Op, int start)> instfn_t;
+
+   // Special Values for ID map:
+   static const spv::Id unmapped;     // unchanged from default value
+   static const spv::Id unused;       // unused ID
+   static const int     header_size;  // SPIR header = 5 words
+
+   class id_iterator_t;
+
+   // For mapping type entries between different shaders
+   typedef std::vector<spirword_t>        typeentry_t;
+   typedef std::map<spv::Id, typeentry_t> globaltypes_t;
+
+   // A set that preserves position order, and a reverse map
+   typedef std::set<int>                    posmap_t;
+   typedef std::unordered_map<spv::Id, int> posmap_rev_t;
+
+   // handle error
+   void error(const std::string& txt) const { errorHandler(txt); }
+   // handle error with our filename appended to the string
+   void ferror(const std::string& txt) const {
+      error(std::string("\nERROR processing file ") + filename + ":\n" + txt);
+   }
+
+   bool    isConstOp(spv::Op opCode)       const;
+   bool    isTypeOp(spv::Op opCode)        const;
+   bool    isStripOp(spv::Op opCode)       const;
+   bool    isFlowCtrlOpen(spv::Op opCode)  const;
+   bool    isFlowCtrlClose(spv::Op opCode) const;
+   range_t literalRange(spv::Op opCode)    const;
+   range_t typeRange(spv::Op opCode)       const;
+   range_t constRange(spv::Op opCode)      const;
+   
+   spv::Id&        asId(int word)                { return spv[word]; }
+   const spv::Id&  asId(int word)          const { return spv[word]; }
+   spv::Op         asOpCode(int word)      const { return opOpCode(spv[word]); }
+   std::uint32_t   asOpCodeHash(int word);
+   spv::Decoration asDecoration(int word)  const { return spv::Decoration(spv[word]); }
+   unsigned        asWordCount(int word)   const { return opWordCount(spv[word]); }
+   spv::Id         asTypeConstId(int word) const { return asId(word + (isTypeOp(asOpCode(word)) ? 1 : 2)); }
+   int             typePos(spv::Id id)     const;
+
+   static unsigned    opWordCount(spirword_t data) { return data >> spv::WordCountShift; }
+   static spv::Op     opOpCode(spirword_t data)    { return spv::Op(data & spv::OpCodeMask); }
+
+   // Header access & set methods
+   spirword_t  magic()    const       { return spv[0]; } // return magic number
+   spirword_t  bound()    const       { return spv[3]; } // return Id bound from header
+   spirword_t  bound(spirword_t b)    { return spv[3] = b; };
+   spirword_t  genmagic() const       { return spv[2]; } // generator magic
+   spirword_t  genmagic(spirword_t m) { return spv[2] = m; }
+   spirword_t  schemaNum() const      { return spv[4]; } // schema number from header
+
+   // Mapping fns: get
+   spv::Id     localId(spv::Id id) const { return idMapL[id]; }
+
+   // Mapping fns: set
+   inline spv::Id   localId(spv::Id id, spv::Id newId);
+   void             countIds(spv::Id id);
+
+   // Return next unused new local ID.
+   // NOTE: boost::dynamic_bitset would be more efficient due to find_next(),
+   // which std::vector<bool> doens't have.
+   inline spv::Id   nextUnusedId(spv::Id id);
+
+   void buildLocalMaps();
+   std::string literalString(int word) const; // Return literal as a std::string
+   int literalStringWords(const std::string& str) const { return (int(str.size())+4)/4; }
+
+   bool isNewIdMapped(spv::Id newId)   const { return isMapped(newId);            }
+   bool isOldIdUnmapped(spv::Id oldId) const { return localId(oldId) == unmapped; }
+   bool isOldIdUnused(spv::Id oldId)   const { return localId(oldId) == unused;   }
+   bool isOldIdMapped(spv::Id oldId)   const { return !isOldIdUnused(oldId) && !isOldIdUnmapped(oldId); }
+   bool isFunction(spv::Id oldId)      const { return fnPos.find(oldId) != fnPos.end(); }
+
+   // bool    matchType(const globaltypes_t& globalTypes, spv::Id lt, spv::Id gt) const;
+   // spv::Id findType(const globaltypes_t& globalTypes, spv::Id lt) const;
+   std::uint32_t hashType(int typeStart) const;
+
+   spirvbin_t& process(instfn_t, idfn_t, int begin = 0, int end = 0);
+   int         processInstruction(int word, instfn_t, idfn_t);
+
+   void        validate() const;
+   void        mapTypeConst();
+   void        mapFnBodies();
+   void        optLoadStore();
+   void        dceFuncs();
+   void        dceVars();
+   void        dceTypes();
+   void        mapNames();
+   void        foldIds();  // fold IDs to smallest space
+   void        forwardLoadStores(); // load store forwarding (EXPERIMENTAL)
+   void        offsetIds(); // create relative offset IDs
+
+   void        applyMap();            // remap per local name map
+   void        mapRemainder();        // map any IDs we haven't touched yet
+   void        stripDebug();          // strip debug info
+   void        strip();               // remove debug symbols
+   
+   std::vector<spirword_t> spv;      // SPIR words
+   std::string             filename; // the file this came from
+
+   namemap_t               nameMap;  // ID names from OpName
+
+   // Since we want to also do binary ops, we can't use std::vector<bool>.  we could use
+   // boost::dynamic_bitset, but we're trying to avoid a boost dependency.
+   typedef std::uint64_t bits_t;
+   std::vector<bits_t> mapped; // which new IDs have been mapped
+   static const int mBits = sizeof(bits_t) * 4;
+
+   bool isMapped(spv::Id id) const  { return id < maxMappedId() && ((mapped[id/mBits] & (1LL<<(id%mBits))) != 0); }
+   void setMapped(spv::Id id) { resizeMapped(id); mapped[id/mBits] |= (1LL<<(id%mBits)); }
+   void resizeMapped(spv::Id id) { if (id >= maxMappedId()) mapped.resize(id/mBits+1, 0); }
+   size_t maxMappedId() const { return mapped.size() * mBits; }
+
+   // Add a strip range for a given instruction starting at 'start'
+   // Note: avoiding brace initializers to please older versions os MSVC.
+   void stripInst(int start) { stripRange.push_back(std::pair<unsigned, unsigned>(start, start + asWordCount(start))); }
+
+   // Function start and end.  use unordered_map because we'll have
+   // many fewer functions than IDs.
+   std::unordered_map<spv::Id, std::pair<int, int>> fnPos;
+   std::unordered_map<spv::Id, std::pair<int, int>> fnPosDCE; // deleted functions
+
+   // Which functions are called, anywhere in the module, with a call count
+   std::unordered_map<spv::Id, int> fnCalls;
+   
+   posmap_t     typeConstPos;   // word positions that define types & consts (ordered)
+   posmap_rev_t typeConstPosR;  // reverse map from IDs to positions
+   
+   std::vector<spv::Id>  idMapL;   // ID {M}ap from {L}ocal to {G}lobal IDs
+
+   spv::Id entryPoint;      // module entry point
+   spv::Id largestNewId;    // biggest new ID we have mapped anything to
+
+   // Sections of the binary to strip, given as [begin,end)
+   std::vector<std::pair<unsigned, unsigned>> stripRange;
+
+   // processing options:
+   std::uint32_t options;
+   int           verbose;     // verbosity level
+
+   static errorfn_t errorHandler;
+   static logfn_t   logHandler;
+};
+
+} // namespace SPV
+
+#endif // defined (use_cpp11)
+#endif // SPIRVREMAPPER_H

SPIRV/doc.cpp

@@ -871,6 +871,14 @@ EnumParameters KernelProfilingInfoParams[KernelProfilingInfoCeiling];
 // Set up all the parameterizing descriptions of the opcodes, operands, etc.
 void Parameterize()
 {
+    static bool initialized = false;
+
+    // only do this once.
+    if (initialized)
+        return;
+
+    initialized = true;
+
     // Exceptions to having a result <id> and a resulting type <id>.
     // (Everything is initialized to have both).
 

StandAlone/StandAlone.cpp

@@ -44,6 +44,7 @@
 #include "../SPIRV/GLSL450Lib.h"
 #include "../SPIRV/doc.h"
 #include "../SPIRV/disassemble.h"
+#include "../SPIRV/SPVRemapper.h"
 #include <string.h>
 #include <stdlib.h>
 #include <math.h>
@@ -71,6 +72,8 @@ enum TOptions {
     EOptionSpv                = 0x0800,
     EOptionHumanReadableSpv   = 0x1000,
     EOptionDefaultDesktop     = 0x2000,
+    EOptionCanonicalizeSpv    = 0x4000,
+    EOptionStripSpv           = 0x8000,
 };
 
 //
@@ -481,11 +484,17 @@ bool ProcessArguments(int argc, char* argv[])
     for (; argc >= 1; argc--, argv++) {
         Work[argc] = 0;
         if (argv[0][0] == '-') {
-            switch (argv[0][1]) {
-            case 'H':
-                Options |= EOptionHumanReadableSpv;
-                // fall through to -V
+            const char optLetter = argv[0][1];
+
+            switch (optLetter) {
+            case 'S': // fall through to -V
+            case 'C': // fall through to -V
+            case 'H': // fall through to -V
             case 'V':
+                if (optLetter == 'H') Options |= EOptionHumanReadableSpv;
+                if (optLetter == 'S') Options |= EOptionStripSpv;
+                if (optLetter == 'C') Options |= EOptionCanonicalizeSpv;
+                
                 Options |= EOptionSpv;
                 Options |= EOptionLinkProgram;
                 break;
@@ -660,7 +669,15 @@ void CompileAndLinkShaders()
                     case EShLangCompute:         name = "comp";    break;
                     default:                     name = "unknown"; break;
                     }
-                    glslang::OutputSpv(spirv, name);
+                    if (Options & (EOptionCanonicalizeSpv | EOptionStripSpv)) {
+                        const unsigned int remapOpts =
+                            ((Options & EOptionCanonicalizeSpv) ? (spv::spirvbin_t::ALL_BUT_STRIP) : 0) |
+                            ((Options & EOptionStripSpv)        ? (spv::spirvbin_t::STRIP) : 0);
+
+                        spv::Parameterize();
+                        spv::spirvbin_t().remap(spirv, remapOpts);
+                    }
+
                     if (Options & EOptionHumanReadableSpv) {
                         spv::Parameterize();
                         GLSL_STD_450::GetDebugNames(GlslStd450DebugNames);
@@ -867,6 +884,8 @@ void usage()
            "To get other information, use one of the following options:\n"
            "(Each option must be specified separately, but can go anywhere in the command line.)\n"
            "  -V  create SPIR-V in file <stage>.spv\n"
+           "  -C  canonicalize generated SPIR-V: turns on -V\n"
+           "  -S  debug-strip SPIR-V: turns on -V\n"
            "  -H  print human readable form of SPIR-V; turns on -V\n"
            "  -c  configuration dump; use to create default configuration file (redirect to a .conf file)\n"
            "  -d  default to desktop (#version 110) when there is no version in the shader (default is ES version 100)\n"