Splitting pprint.lua

MSVC suddenly didn't like the too-big string...

src/lua/extra.cc

@@ -38,9 +38,12 @@ void PCSX::LuaFFI::open_extra(Lua L) {
     luaopen_pb_unsafe(L.getState());
     L.setfield("pb.unsafe");
 
-    static int lualoader = 6;
+    static int lualoader = 7;
     static const char* pprint = (
 #include "pprint.lua/pprint.lua"
+    );
+    static const char* pprint_internals = (
+#include "pprint.lua/pprint-internals.lua"
     );
     static const char* reflectFFI = (
 #include "ffi-reflect/reflect.lua"
@@ -58,6 +61,7 @@ void PCSX::LuaFFI::open_extra(Lua L) {
 #include "lua-protobuf/protoc.lua"
     );
     L.load(pprint, "internal:pprinter.lua/pprint.lua");
+    L.load(pprint_internals, "internal:pprinter.lua/pprint-internals.lua");
     L.load(reflectFFI, "internal:ffi-reflect/reflect.lua");
 
     L.load(protobufLexer, "internal:lua-protobuf/lexer.lua");

third_party/pprint.lua/pprint-internals.lua

@@ -0,0 +1,127 @@
+--lualoader, R"EOF(--
+
+-- seems this is the only way to escape these, as lua don't know how to map char '\a' to 'a'
+local ESCAPE_MAP = {
+    ['\a'] = '\\a', ['\b'] = '\\b', ['\f'] = '\\f', ['\n'] = '\\n', ['\r'] = '\\r',
+    ['\t'] = '\\t', ['\v'] = '\\v', ['\\'] = '\\\\',
+}
+
+-- generic utilities
+pprint._internals.tokenize_string = function(s)
+    local t = {}
+    for i = 1, #s do
+        local c = s:sub(i, i)
+        local b = c:byte()
+        local e = ESCAPE_MAP[c]
+        if (b >= 0x20 and b < 0x80) or e then
+            local s = e or c
+            t[i] = { char = s, len = #s }
+        else
+            t[i] = { char = string.format('\\x%02x', b), len = 4 }
+        end
+        if c == '"' then
+            t.has_double_quote = true
+        elseif c == "'" then
+            t.has_single_quote = true
+        end
+    end
+    return t
+end
+pprint._internals.tokenize_utf8_string = tokenize_string
+
+local has_lpeg, lpeg = pcall(require, 'lpeg')
+
+if has_lpeg then
+    local function utf8_valid_char(c)
+        return { char = c, len = 1 }
+    end
+
+    local function utf8_invalid_char(c)
+        local b = c:byte()
+        local e = ESCAPE_MAP[c]
+        if (b >= 0x20 and b < 0x80) or e then
+            local s = e or c
+            return { char = s, len = #s }
+        else
+            return { char = string.format('\\x%02x', b), len = 4 }
+        end
+    end
+
+    local cont = lpeg.R('\x80\xbf')
+    local utf8_char =
+        lpeg.R('\x20\x7f') +
+        lpeg.R('\xc0\xdf') * cont +
+        lpeg.R('\xe0\xef') * cont * cont +
+        lpeg.R('\xf0\xf7') * cont * cont * cont
+
+    local utf8_capture = (((utf8_char / utf8_valid_char) + (lpeg.P(1) / utf8_invalid_char)) ^ 0) * -1
+
+    pprint._internals.tokenize_utf8_string = function(s)
+        local dq = s:find('"')
+        local sq = s:find("'")
+        local t = table.pack(utf8_capture:match(s))
+        t.has_double_quote = not not dq
+        t.has_single_quote = not not sq
+        return t
+    end
+end
+
+local CACHE_TYPES = {
+    ['table'] = true, ['function'] = true, ['thread'] = true,
+    ['userdata'] = true, ['cdata'] = true,
+}
+
+-- cache would be populated to be like:
+-- {
+--     function = { `fun1` = 1, _cnt = 1 }, -- object id
+--     table = { `table1` = 1, `table2` = 2, _cnt = 2 },
+--     visited_tables = { `table1` = 7, `table2` = 8  }, -- visit count
+-- }
+-- use weakrefs to avoid accidentall adding refcount
+pprint._internals.cache_apperance = function(obj, cache, option)
+    if not cache.visited_tables then
+        cache.visited_tables = setmetatable({}, {__mode = 'k'})
+    end
+    local t = type(obj)
+
+    -- TODO can't test filter_function here as we don't have the ix and key,
+    -- might cause different results?
+    -- respect show_xxx and filter_function to be consistent with print results
+    if (not pprint._internals.TYPES[t] and not option.show_table)
+        or (pprint._internals.TYPES[t] and not option['show_'..t]) then
+        return
+    end
+
+    if CACHE_TYPES[t] or pprint._internals.TYPES[t] == nil then
+        if not cache[t] then
+            cache[t] = setmetatable({}, {__mode = 'k'})
+            cache[t]._cnt = 0
+        end
+        if not cache[t][obj] then
+            cache[t]._cnt = cache[t]._cnt + 1
+            cache[t][obj] = cache[t]._cnt
+        end
+    end
+    if t == 'table' or pprint._internals.TYPES[t] == nil then
+        if cache.visited_tables[obj] == false then
+            -- already printed, no need to mark this and its children anymore
+            return
+        elseif cache.visited_tables[obj] == nil then
+            cache.visited_tables[obj] = 1
+        else
+            -- visited already, increment and continue
+            cache.visited_tables[obj] = cache.visited_tables[obj] + 1
+            return
+        end
+        for k, v in pairs(obj) do
+            pprint._internals.cache_apperance(k, cache, option)
+            pprint._internals.cache_apperance(v, cache, option)
+        end
+        local mt = getmetatable(obj)
+        if mt and option.show_metatable then
+            pprint._internals.cache_apperance(mt, cache, option)
+        end
+    end
+end
+
+-- )EOF"

third_party/pprint.lua/pprint.lua

@@ -4,6 +4,8 @@ pprint = { VERSION = '0.1' }
 
 local depth = 1
 
+pprint._internals = {}
+
 pprint.defaults = {
     -- If set to number N, then limit table recursion to N deep.
     depth_limit = false,
@@ -34,140 +36,16 @@ pprint.defaults = {
     sort_keys = true,           -- sort table keys
 }
 
-local TYPES = {
+pprint._internals.TYPES = {
     ['nil'] = 1, ['boolean'] = 2, ['number'] = 3, ['string'] = 4, 
     ['table'] = 5, ['function'] = 6, ['thread'] = 7, ['userdata'] = 8,
     ['cdata'] = 9,
 }
 
--- seems this is the only way to escape these, as lua don't know how to map char '\a' to 'a'
-local ESCAPE_MAP = {
-    ['\a'] = '\\a', ['\b'] = '\\b', ['\f'] = '\\f', ['\n'] = '\\n', ['\r'] = '\\r',
-    ['\t'] = '\\t', ['\v'] = '\\v', ['\\'] = '\\\\',
-}
-
--- generic utilities
-local tokenize_string = function(s)
-    local t = {}
-    for i = 1, #s do
-        local c = s:sub(i, i)
-        local b = c:byte()
-        local e = ESCAPE_MAP[c]
-        if (b >= 0x20 and b < 0x80) or e then
-            local s = e or c
-            t[i] = { char = s, len = #s }
-        else
-            t[i] = { char = string.format('\\x%02x', b), len = 4 }
-        end
-        if c == '"' then
-            t.has_double_quote = true
-        elseif c == "'" then
-            t.has_single_quote = true
-        end
-    end
-    return t
-end
-local tokenize_utf8_string = tokenize_string
-
-local has_lpeg, lpeg = pcall(require, 'lpeg')
-
-if has_lpeg then
-    local function utf8_valid_char(c)
-        return { char = c, len = 1 }
-    end
-
-    local function utf8_invalid_char(c)
-        local b = c:byte()
-        local e = ESCAPE_MAP[c]
-        if (b >= 0x20 and b < 0x80) or e then
-            local s = e or c
-            return { char = s, len = #s }
-        else
-            return { char = string.format('\\x%02x', b), len = 4 }
-        end
-    end
-
-    local cont = lpeg.R('\x80\xbf')
-    local utf8_char =
-        lpeg.R('\x20\x7f') +
-        lpeg.R('\xc0\xdf') * cont +
-        lpeg.R('\xe0\xef') * cont * cont +
-        lpeg.R('\xf0\xf7') * cont * cont * cont
-
-    local utf8_capture = (((utf8_char / utf8_valid_char) + (lpeg.P(1) / utf8_invalid_char)) ^ 0) * -1
-
-    tokenize_utf8_string = function(s)
-        local dq = s:find('"')
-        local sq = s:find("'")
-        local t = table.pack(utf8_capture:match(s))
-        t.has_double_quote = not not dq
-        t.has_single_quote = not not sq
-        return t
-    end
-end
-
 local function is_plain_key(key)
     return type(key) == 'string' and key:match('^[%a_][%a%d_]*$')
 end
 
-local CACHE_TYPES = {
-    ['table'] = true, ['function'] = true, ['thread'] = true,
-    ['userdata'] = true, ['cdata'] = true,
-}
-
--- cache would be populated to be like:
--- {
---     function = { `fun1` = 1, _cnt = 1 }, -- object id
---     table = { `table1` = 1, `table2` = 2, _cnt = 2 },
---     visited_tables = { `table1` = 7, `table2` = 8  }, -- visit count
--- }
--- use weakrefs to avoid accidentall adding refcount
-local function cache_apperance(obj, cache, option)
-    if not cache.visited_tables then
-        cache.visited_tables = setmetatable({}, {__mode = 'k'})
-    end
-    local t = type(obj)
-
-    -- TODO can't test filter_function here as we don't have the ix and key,
-    -- might cause different results?
-    -- respect show_xxx and filter_function to be consistent with print results
-    if (not TYPES[t] and not option.show_table)
-        or (TYPES[t] and not option['show_'..t]) then
-        return
-    end
-
-    if CACHE_TYPES[t] or TYPES[t] == nil then
-        if not cache[t] then
-            cache[t] = setmetatable({}, {__mode = 'k'})
-            cache[t]._cnt = 0
-        end
-        if not cache[t][obj] then
-            cache[t]._cnt = cache[t]._cnt + 1
-            cache[t][obj] = cache[t]._cnt
-        end
-    end
-    if t == 'table' or TYPES[t] == nil then
-        if cache.visited_tables[obj] == false then
-            -- already printed, no need to mark this and its children anymore
-            return
-        elseif cache.visited_tables[obj] == nil then
-            cache.visited_tables[obj] = 1
-        else
-            -- visited already, increment and continue
-            cache.visited_tables[obj] = cache.visited_tables[obj] + 1
-            return
-        end
-        for k, v in pairs(obj) do
-            cache_apperance(k, cache, option)
-            cache_apperance(v, cache, option)
-        end
-        local mt = getmetatable(obj)
-        if mt and option.show_metatable then
-            cache_apperance(mt, cache, option)
-        end
-    end
-end
-
 -- makes 'foo2' < 'foo100000'. string.sub makes substring anyway, no need to use index based method
 local function str_natural_cmp(lhs, rhs)
     while #lhs > 0 and #rhs > 0 do
@@ -201,8 +79,8 @@ local function cmp(lhs, rhs)
     if tleft == tright then return str_natural_cmp(tostring(lhs), tostring(rhs)) end
 
     -- allow custom types
-    local oleft = TYPES[tleft] or 10
-    local oright = TYPES[tright] or 10
+    local oleft = pprint._internals.TYPES[tleft] or 10
+    local oright = pprint._internals.TYPES[tright] or 10
     return oleft < oright
 end
 
@@ -216,7 +94,7 @@ local function make_option(option)
             option[k] = v
         end
         if option.show_all then
-            for t, _ in pairs(TYPES) do
+            for t, _ in pairs(pprint._internals.TYPES) do
                 option['show_'..t] = true
             end
             option.show_metatable = true
@@ -324,7 +202,7 @@ function pprint.pformat(obj, option, printer)
     end
 
     local function string_formatter(s, force_long_quote)
-        local tokens = option.string_is_utf8 and tokenize_utf8_string(s) or tokenize_string(s)
+        local tokens = option.string_is_utf8 and pprint._internals.tokenize_utf8_string(s) or pprint._internals.tokenize_string(s)
         local string_len = 0
         local escape_quotes = tokens.has_double_quote and tokens.has_single_quote
         for _, token in ipairs(tokens) do
@@ -555,7 +433,7 @@ function pprint.pformat(obj, option, printer)
 
     if option.object_cache then
         -- needs to visit the table before start printing
-        cache_apperance(obj, cache, option)
+        pprint._internals.cache_apperance(obj, cache, option)
     end
 
     _p(format(obj))

vsprojects/Lua/Lua.vcxproj

@@ -68,6 +68,7 @@
     <None Include="..\..\third_party\lua-protobuf\lexer.lua" />
     <None Include="..\..\third_party\lua-protobuf\protoc.lua" />
     <None Include="..\..\third_party\lua-protobuf\toplevel.lua" />
+    <None Include="..\..\third_party\pprint.lua\pprint-internals.lua" />
     <None Include="..\..\third_party\pprint.lua\pprint.lua" />
     <None Include="..\..\third_party\zlibffi\zlibffi.lua" />
     <None Include="packages.config" />

vsprojects/Lua/Lua.vcxproj.filters

@@ -56,7 +56,7 @@
     <ClInclude Include="..\..\third_party\lua-protobuf\pb.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\third_party\luafilesystem\lfs.h">
+    <ClInclude Include="..\..\third_party\luafilesystem\src\lfs.h">
       <Filter>Header Files</Filter>
     </ClInclude>
   </ItemGroup>
@@ -113,5 +113,8 @@
       <Filter>Source Files</Filter>
     </None>
     <None Include="packages.config" />
+    <None Include="..\..\third_party\pprint.lua\pprint-internals.lua">
+      <Filter>Source Files</Filter>
+    </None>
   </ItemGroup>
 </Project>