Start handling pypy 2.7

Need to understand whether we care compiling pypy.
Pypy 2.7 list comprehensions are different and use
its own opcode.

__pkginfo__.py

@@ -37,7 +37,7 @@ entry_points={
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.4.0',
-                      'xdis >= 1.1.7']
+                      'xdis >= 1.2.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

test/test_pyenvlib.py

@@ -27,7 +27,8 @@ from fnmatch import fnmatch
 
 #----- configure this for your needs
 
-TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9', '2.7.10', '2.7.11',
+TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9', 'pypy-2.6.1',
+               '2.7.10', '2.7.11',
                '3.2.6', '3.3.5', '3.4.2', '3.5.1')
 
 target_base = '/tmp/py-dis/'
@@ -45,9 +46,14 @@ test_options = {
     }
 
 for vers in TEST_VERSIONS:
-    short_vers = vers[:3]
-    test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),
-                          PYC, 'python-lib'+short_vers)
+    if vers.startswith('pypy-'):
+        short_vers = vers[0:-2]
+        test_options[vers] = (os.path.join(lib_prefix, vers, 'lib_pypy'),
+                              PYC, 'python-lib'+short_vers)
+    else:
+        short_vers = vers[:3]
+        test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),
+                              PYC, 'python-lib'+short_vers)
 
 def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
 

uncompyle6/disas.py

@@ -77,7 +77,7 @@ def disassemble_file(filename, outstream=None, native=False):
         return
 
     filename = check_object_path(filename)
-    version, timestamp, magic_int, co = load_module(filename)
+    version, timestamp, magic_int, co, is_pypy = load_module(filename)
     if type(co) == list:
         for con in co:
             disco(version, con, outstream)

uncompyle6/main.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 import datetime, os, sys
 
-from uncompyle6 import verify, PYTHON_VERSION
+from uncompyle6 import verify, PYTHON_VERSION, IS_PYPY
 from xdis.code import iscode
 from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
@@ -9,8 +9,10 @@ from uncompyle6.parser import ParserError
 
 from xdis.load import load_module
 
-def uncompyle(version, co, out=None, showasm=False, showast=False,
-              timestamp=None, showgrammar=False, code_objects={}):
+def uncompyle(
+        version, co, out=None, showasm=False, showast=False,
+        timestamp=None, showgrammar=False, code_objects={},
+        is_pypy=False):
     """
     disassembles and deparses a given code block 'co'
     """
@@ -19,7 +21,10 @@ def uncompyle(version, co, out=None, showasm=False, showast=False,
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python bytecode %s (decompiled from Python %s)' % (version, PYTHON_VERSION),
+    co_pypy_str = 'PyPy ' if is_pypy else ''
+    run_pypy_str = 'PyPy ' if IS_PYPY else ''
+    print('# %sPython bytecode %s (disassembled from %sPython %s)\n' %
+              (co_pypy_str, version, run_pypy_str, PYTHON_VERSION),
           file=real_out)
     if co.co_filename:
         print('# Embedded file name: %s' % co.co_filename,
@@ -30,7 +35,7 @@ def uncompyle(version, co, out=None, showasm=False, showast=False,
 
     try:
         pysource.deparse_code(version, co, out, showasm, showast, showgrammar,
-                              code_objects=code_objects)
+                              code_objects=code_objects, is_pypy=is_pypy)
     except pysource.SourceWalkerError as e:
         # deparsing failed
         print("\n")
@@ -49,16 +54,18 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False,
 
     filename = check_object_path(filename)
     code_objects = {}
-    version, timestamp, magic_int, co = load_module(filename, code_objects)
+    version, timestamp, magic_int, co, is_pypy = load_module(filename, code_objects)
 
 
     if type(co) == list:
         for con in co:
             uncompyle(version, con, outstream, showasm, showast,
-                      timestamp, showgrammar, code_objects=code_objects)
+                      timestamp, showgrammar, code_objects=code_objects,
+                      is_pypy=is_pypy)
     else:
         uncompyle(version, co, outstream, showasm, showast,
-                  timestamp, showgrammar, code_objects=code_objects)
+                  timestamp, showgrammar, code_objects=code_objects,
+                  is_pypy=is_pypy)
     co = None
 
 # FIXME: combine into an options parameter

uncompyle6/parser.py

@@ -570,7 +570,9 @@ def parse(p, tokens, customize):
     return ast
 
 
-def get_python_parser(version, debug_parser, compile_mode='exec'):
+def get_python_parser(
+        version, debug_parser, compile_mode='exec',
+        is_pypy = False):
     """Returns parser object for Python version 2 or 3, 3.2, 3.5on,
     etc., depending on the parameters passed.  *compile_mode* is either
     'exec', 'eval', or 'single'. See
@@ -662,7 +664,7 @@ class PythonParserSingle(PythonParser):
 
 
 def python_parser(version, co, out=sys.stdout, showasm=False,
-                  parser_debug=PARSER_DEFAULT_DEBUG):
+                  parser_debug=PARSER_DEFAULT_DEBUG, is_pypy=False):
     """
     Parse a code object to an abstract syntax tree representation.
 
@@ -681,7 +683,7 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
 
     assert iscode(co)
     from uncompyle6.scanner import get_scanner
-    scanner = get_scanner(version)
+    scanner = get_scanner(version, is_pypy)
     tokens, customize = scanner.disassemble(co)
     maybe_show_asm(showasm, tokens)
 
@@ -693,8 +695,8 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
 
 if __name__ == '__main__':
     def parse_test(co):
-        from uncompyle6 import PYTHON_VERSION
-        ast = python_parser(PYTHON_VERSION, co, showasm=True)
+        from uncompyle6 import PYTHON_VERSION, IS_PYPY
+        ast = python_parser(PYTHON_VERSION, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)
         return
     parse_test(parse_test.__code__)

uncompyle6/parsers/parse27.py

@@ -15,6 +15,8 @@ class Python27Parser(Python2Parser):
     def p_list_comprehension27(self, args):
         """
         list_for ::= expr _for designator list_iter JUMP_BACK
+
+        list_compr ::= expr  BUILD_LIST_FROM_ARG _for designator list_iter JUMP_BACK
         """
 
     def p_try27(self, args):

uncompyle6/scanner.py

@@ -48,12 +48,15 @@ class Code(object):
 
 class Scanner(object):
 
-    def __init__(self, version, show_asm=None):
+    def __init__(self, version, show_asm=None, is_pypy=False):
         self.version = version
         self.show_asm = show_asm
 
         if version in PYTHON_VERSIONS:
-            v_str = "opcode_%s" % (int(version * 10))
+            if is_pypy:
+                v_str = "opcode_pypy%s" % (int(version * 10))
+            else:
+                v_str = "opcode_%s" % (int(version * 10))
             exec("from xdis.opcodes import %s" % v_str)
             exec("self.opc = %s" % v_str)
         else:
@@ -251,18 +254,26 @@ class Scanner(object):
 def parse_fn_counts(argc):
     return ((argc & 0xFF), (argc >> 8) & 0xFF, (argc >> 16) & 0x7FFF)
 
-def get_scanner(version, show_asm=None):
+def get_scanner(version, show_asm=None, is_pypy=False):
     # Pick up appropriate scanner
     if version in PYTHON_VERSIONS:
         v_str = "%s" % (int(version * 10))
-        exec("import uncompyle6.scanners.scanner%s as scan" % v_str)
         if PYTHON3:
             import importlib
-            scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str)
+            if is_pypy:
+                scan = importlib.import_module("uncompyle6.scanners.pypy%s" % v_str)
+            else:
+                scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str)
             if False: print(scan)  # Avoid unused scan
         else:
-            exec("import uncompyle6.scanners.scanner%s as scan" % v_str)
-        scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str)
+            if is_pypy:
+                exec("import uncompyle6.scanners.pypy%s as scan" % v_str)
+            else:
+                exec("import uncompyle6.scanners.scanner%s as scan" % v_str)
+        if is_pypy:
+            scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str)
+        else:
+            scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str)
     else:
         raise RuntimeError("Unsupported Python version %s" % version)
     return scanner

uncompyle6/scanners/scanner2.py

@@ -32,8 +32,8 @@ from xdis.bytecode import findlinestarts
 import uncompyle6.scanner as scan
 
 class Scanner2(scan.Scanner):
-    def __init__(self, version, show_asm=None):
-        scan.Scanner.__init__(self, version, show_asm)
+    def __init__(self, version, show_asm=None, is_pypy=False):
+        scan.Scanner.__init__(self, version, show_asm, is_pypy)
         self.pop_jump_if = frozenset([self.opc.PJIF, self.opc.PJIT])
         self.jump_forward = frozenset([self.opc.JUMP_ABSOLUTE, self.opc.JUMP_FORWARD])
         # This is the 2.5+ default

uncompyle6/scanners/scanner27.py

@@ -17,8 +17,8 @@ from xdis.opcodes import opcode_27
 JUMP_OPs = opcode_27.JUMP_OPs
 
 class Scanner27(Scanner2):
-    def __init__(self, show_asm=False):
-        super(Scanner27, self).__init__(2.7, show_asm)
+    def __init__(self, show_asm=False, is_pypy=False):
+        super(Scanner27, self).__init__(2.7, show_asm, is_pypy)
 
         # opcodes that start statements
         self.stmt_opcodes = frozenset([

uncompyle6/semantics/fragments.py

@@ -118,7 +118,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
     stacked_params = ('f', 'indent', 'isLambda', '_globals')
 
     def __init__(self, version, scanner, showast=False,
-                 debug_parser=PARSER_DEFAULT_DEBUG):
+                 debug_parser=PARSER_DEFAULT_DEBUG,
+                 compile_mode='exec', is_pypy=False):
         GenericASTTraversal.__init__(self, ast=None)
         self.scanner = scanner
         params = {
@@ -126,7 +127,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
             'indent': '',
             }
         self.version = version
-        self.p = get_python_parser(version, dict(debug_parser))
+        self.p = get_python_parser(
+            version, dict(debug_parser),
+            compile_mode=compile_mode, is_pypy=is_pypy
+        )
         self.showast = showast
         self.params = params
         self.param_stack = []

uncompyle6/semantics/pysource.py

@@ -526,7 +526,7 @@ class SourceWalker(GenericASTTraversal, object):
 
     def __init__(self, version, out, scanner, showast=False,
                  debug_parser=PARSER_DEFAULT_DEBUG,
-                 compile_mode='exec'):
+                 compile_mode='exec', is_pypy=False):
         GenericASTTraversal.__init__(self, ast=None)
         self.scanner = scanner
         params = {
@@ -555,6 +555,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.hide_internal = True
         self.name = None
         self.version = version
+        self.is_pypy = is_pypy
 
         if 2.0 <= version <= 2.3:
                 TABLE_DIRECT['tryfinallystmt'] = (
@@ -1022,6 +1023,9 @@ class SourceWalker(GenericASTTraversal, object):
         p = self.prec
         self.prec = 27
         if self.version >= 2.7:
+            if self.is_pypy:
+                self.n_list_compr_pypy27(node)
+                return
             n = node[-1]
         elif node[-1] == 'del_stmt':
             n = node[-3] if node[-2] == 'JUMP_BACK' else node[-2]
@@ -1053,6 +1057,42 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = p
         self.prune() # stop recursing
 
+    def n_list_compr_pypy27(self, node):
+        """List comprehensions the way they are done in PYPY Python 2.7.
+        """
+        p = self.prec
+        self.prec = 27
+        n = node[-2] if self.is_pypy and node[-1] == 'JUMP_BACK' else node[-1]
+        list_expr = node[0]
+        designator = node[3]
+
+        assert n == 'list_iter'
+        assert designator == 'designator'
+
+        # find innermost node
+        while n == 'list_iter':
+            n = n[0] # recurse one step
+            if   n == 'list_for':	n = n[3]
+            elif n == 'list_if':	n = n[2]
+            elif n == 'list_if_not': n= n[2]
+        assert n == 'lc_body'
+        self.write( '[ ')
+
+        expr = n[0]
+        list_iter = node[-2] if self.is_pypy and node[-1] == 'JUMP_BACK' else node[-1]
+
+        assert expr == 'expr'
+        assert list_iter == 'list_iter'
+
+        self.preorder(expr)
+        self.write( ' for ')
+        self.preorder(designator)
+        self.write( ' in ')
+        self.preorder(list_expr)
+        self.write( ' ]')
+        self.prec = p
+        self.prune() # stop recursing
+
     def comprehension_walk(self, node, iter_index, code_index=-5):
         p = self.prec
         self.prec = 27
@@ -2127,14 +2167,14 @@ class SourceWalker(GenericASTTraversal, object):
 
 
 def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False,
-                 showgrammar=False, code_objects={}, compile_mode='exec'):
+                 showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False):
     """
     disassembles and deparses a given code block 'co'
     """
 
     assert iscode(co)
     # store final output stream for case of error
-    scanner = get_scanner(version)
+    scanner = get_scanner(version, is_pypy=is_pypy)
 
     tokens, customize = scanner.disassemble(co, code_objects=code_objects)
     maybe_show_asm(showasm, tokens)
@@ -2146,7 +2186,8 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False,
 
     #  Build AST from disassembly.
     deparsed = SourceWalker(version, out, scanner, showast=showast,
-                            debug_parser=debug_parser, compile_mode=compile_mode)
+                            debug_parser=debug_parser, compile_mode=compile_mode,
+                            is_pypy = is_pypy)
 
     isTopLevel = co.co_name == '<module>'
     deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)

uncompyle6/verify.py

@@ -364,7 +364,7 @@ class Token(scanner.Token):
 
 def compare_code_with_srcfile(pyc_filename, src_filename):
     """Compare a .pyc with a source code file."""
-    version, timestamp, magic_int, code_obj1 = load_module(pyc_filename)
+    version, timestamp, magic_int, code_obj1, is_pypy = load_module(pyc_filename)
     if magic_int != PYTHON_MAGIC_INT:
         msg = ("Can't compare code - Python is running with magic %s, but code is magic %s "
                % (PYTHON_MAGIC_INT, magic_int))
@@ -375,8 +375,8 @@ def compare_code_with_srcfile(pyc_filename, src_filename):
 
 def compare_files(pyc_filename1, pyc_filename2):
     """Compare two .pyc files."""
-    version, timestamp, magic_int1, code_obj1 = uncompyle6.load_module(pyc_filename1)
-    version, timestamp, magic_int2, code_obj2 = uncompyle6.load_module(pyc_filename2)
+    version, timestamp, magic_int1, code_obj1, is_pypy = uncompyle6.load_module(pyc_filename1)
+    version, timestamp, magic_int2, code_obj2, is_pypy = uncompyle6.load_module(pyc_filename2)
     cmp_code_objects(version, code_obj1, code_obj2)
 
 if __name__ == '__main__':