Magic-Mirror/python-uncompyle6
1
0
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2024-12-03 18:41:03 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Handle 3.3 MAKE_FUNCTION annotation args properly

Browse Source
This commit is contained in:
rocky 2024-02-11 11:50:25 -05:00
parent 147155e1d5
commit 454fac4adb
6 changed files with 123 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
uncompyle6/bin/uncompile.py
View File

@ -159,10 +159,10 @@ def main_bin(
"""
version_tuple = sys.version_info[0:2]
if version_tuple < (3, 7):
if version_tuple < (3, 6):
print(
f"Error: This version of the {program} runs from Python 3.7 or greater."
f"You need another branch of this code for Python before 3.7."
f"Error: This version of the {program} runs from Python 3.6 or greater."
f"You need another branch of this code for Python before 3.6."
f""" \n\tYou have version: {version_tuple_to_str()}."""
)
sys.exit(-1)

38
uncompyle6/main.py
View File

@ -13,10 +13,12 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import ast
import datetime
import os
import os.path as osp
import py_compile
import subprocess
import sys
import tempfile
from typing import Any, Optional, TextIO, Tuple
@ -51,6 +53,17 @@ def _get_outstream(outfile: str) -> Any:
return open(outfile, mode="w", encoding="utf-8")
def syntax_check(filename: str) -> bool:
with open(filename) as f:
source = f.read()
valid = True
try:
ast.parse(source)
except SyntaxError:
valid = False
return valid
def decompile(
co,
bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE,
@ -368,15 +381,22 @@ def main(
check_type = "syntax check"
if do_verify == "run":
check_type = "run"
result = subprocess.run(
[sys.executable, deparsed_object.f.name],
capture_output=True,
)
valid = result.returncode == 0
output = result.stdout.decode()
if output:
print(output)
pass
if PYTHON_VERSION_TRIPLE >= (3, 7):
result = subprocess.run(
[sys.executable, deparsed_object.f.name],
capture_output=True,
)
valid = result.returncode == 0
output = result.stdout.decode()
if output:
print(output)
pass
else:
result = subprocess.run(
[sys.executable, deparsed_object.f.name],
)
valid = result.returncode == 0
pass
if not valid:
print(result.stderr.decode())

98
uncompyle6/parsers/parse3.py
View File

@ -27,22 +27,24 @@ that a later phase can turn into a sequence of ASCII text.
"""
import re
from uncompyle6.scanners.tok import Token
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import (
and_invalid,
except_handler_else,
ifelsestmt,
ifstmt,
iflaststmt,
ifstmt,
or_check,
testtrue,
tryelsestmtl3,
tryexcept,
while1stmt
while1stmt,
)
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.scanners.tok import Token
class Python3Parser(PythonParser):
@ -98,7 +100,7 @@ class Python3Parser(PythonParser):
"""
def p_dict_comp3(self, args):
""""
""" "
expr ::= dict_comp
stmt ::= dict_comp_func
dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
@ -519,7 +521,7 @@ class Python3Parser(PythonParser):
expr
call
CALL_FUNCTION_3
"""
"""
# FIXME: I bet this can be simplified
# look for next MAKE_FUNCTION
for i in range(i + 1, len(tokens)):
@ -625,7 +627,11 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, token.kind, uniq_param, customize)
if "LOAD_BUILD_CLASS" in self.seen_ops:
if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1:
if (
next_token == "CALL_FUNCTION"
and next_token.attr == 1
and pos_args_count > 1
):
rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
("expr " * (pos_args_count - 1)),
opname,
@ -764,18 +770,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT":
rule = """
rule = (
"""
add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s
dict ::= const_list
expr ::= dict
""" % opname
"""
% opname
)
else:
rule = """
rule = (
"""
add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s
expr ::= const_list
""" % opname
"""
% opname
)
self.addRule(rule, nop_func)
elif opname.startswith("BUILD_DICT_OLDER"):
@ -854,18 +866,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT":
rule = """
rule = (
"""
add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s
dict ::= const_list
expr ::= dict
""" % opname
"""
% opname
)
else:
rule = """
rule = (
"""
add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s
expr ::= const_list
""" % opname
"""
% opname
)
self.addRule(rule, nop_func)
elif opname_base in (
@ -946,7 +964,6 @@ class Python3Parser(PythonParser):
"CALL_FUNCTION_VAR_KW",
)
) or opname.startswith("CALL_FUNCTION_KW"):
if opname == "CALL_FUNCTION" and token.attr == 1:
rule = """
dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
@ -1122,7 +1139,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1:
rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
)
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@ -1190,6 +1208,8 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, opname, token.attr, customize)
elif (3, 3) <= self.version < (3, 6):
# FIXME move this into version-specific custom rules.
# In fact, some of this has been done for 3.3.
if annotate_args > 0:
rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s"
@ -1208,7 +1228,6 @@ class Python3Parser(PythonParser):
)
self.add_unique_rule(rule, opname, token.attr, customize)
if self.version >= (3, 4):
if not self.is_pypy:
load_op = "LOAD_STR"
@ -1292,14 +1311,16 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1:
rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
)
self.add_make_function_rule(
rule_pat, opname, token.attr, customize
)
rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
)
self.add_make_function_rule(
rule_pat, opname, token.attr, customize
@ -1351,7 +1372,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1:
rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
)
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@ -1363,7 +1385,8 @@ class Python3Parser(PythonParser):
# Todo: For Pypy we need to modify this slightly
rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname)
"GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname)
)
self.add_make_function_rule(
rule_pat, opname, token.attr, customize
@ -1450,9 +1473,6 @@ class Python3Parser(PythonParser):
)
)
if self.version >= (3, 3):
# Normally we remove EXTENDED_ARG from the opcodes, but in the case of
# annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
# Yes this is a little hacky
if self.version == (3, 3):
# 3.3 puts kwargs before pos_arg
pos_kw_tuple = (
@ -1466,17 +1486,17 @@ class Python3Parser(PythonParser):
("kwargs " * kw_args_count),
)
rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% (
pos_kw_tuple[0],
pos_kw_tuple[1],
("call " * annotate_args),
("annotate_arg " * annotate_args),
opname,
)
)
self.add_unique_rule(rule, opname, token.attr, customize)
rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% (
pos_kw_tuple[0],
pos_kw_tuple[1],
@ -1485,9 +1505,8 @@ class Python3Parser(PythonParser):
)
)
else:
# See above comment about use of EXTENDED_ARG
rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% (
("kwargs " * kw_args_count),
("pos_arg " * (pos_args_count)),
@ -1497,7 +1516,7 @@ class Python3Parser(PythonParser):
)
self.add_unique_rule(rule, opname, token.attr, customize)
rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% (
("kwargs " * kw_args_count),
("pos_arg " * pos_args_count),
@ -1594,7 +1613,7 @@ class Python3Parser(PythonParser):
}
if self.version == (3, 6):
self.reduce_check_table["and"] = and_invalid
self.reduce_check_table["and"] = and_invalid
self.check_reduce["and"] = "AST"
self.check_reduce["annotate_tuple"] = "noAST"
@ -1624,7 +1643,7 @@ class Python3Parser(PythonParser):
def reduce_is_invalid(self, rule, ast, tokens, first, last):
lhs = rule[0]
n = len(tokens)
last = min(last, n-1)
last = min(last, n - 1)
fn = self.reduce_check_table.get(lhs, None)
if fn:
if fn(self, lhs, n, rule, ast, tokens, first, last):
@ -1650,13 +1669,18 @@ class Python3Parser(PythonParser):
condition_jump2 = tokens[min(last - 1, len(tokens) - 1)]
# If there are two *distinct* condition jumps, they should not jump to the
# same place. Otherwise we have some sort of "and"/"or".
if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2:
if (
condition_jump2.kind.startswith("POP_JUMP_IF")
and condition_jump != condition_jump2
):
return condition_jump.attr == condition_jump2.attr
if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr:
if (
tokens[last] == "COME_FROM"
and tokens[last].off2int() != condition_jump.attr
):
return False
# if condition_jump.attr < condition_jump2.off2int():
# print("XXX", first, last)
# for t in range(first, last): print(tokens[t])
@ -1678,7 +1702,6 @@ class Python3Parser(PythonParser):
< tokens[last].off2int()
)
elif lhs == "while1stmt":
if while1stmt(self, lhs, n, rule, ast, tokens, first, last):
return True
@ -1700,7 +1723,6 @@ class Python3Parser(PythonParser):
return True
return False
elif lhs == "while1elsestmt":
n = len(tokens)
if last == n:
# Adjust for fuzziness in parsing

17
uncompyle6/parsers/parse33.py
View File

@ -1,15 +1,13 @@
# Copyright (c) 2016 Rocky Bernstein
# Copyright (c) 2016, 2024 Rocky Bernstein
"""
spark grammar differences over Python 3.2 for Python 3.3.
"""
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle
from uncompyle6.parsers.parse32 import Python32Parser
class Python33Parser(Python32Parser):
def p_33on(self, args):
"""
# Python 3.3+ adds yield from.
@ -19,13 +17,22 @@ class Python33Parser(Python32Parser):
"""
def customize_grammar_rules(self, tokens, customize):
self.remove_rules("""
self.remove_rules(
"""
# 3.3+ adds POP_BLOCKS
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
""")
"""
)
super(Python33Parser, self).customize_grammar_rules(tokens, customize)
# FIXME: move 3.3 stuff out of parse3.py and put it here.
# for i, token in enumerate(tokens):
# opname = token.kind
# opname_base = opname[: opname.rfind("_")]
return
class Python33ParserSingle(Python33Parser, PythonParserSingle):
pass

25
uncompyle6/scanners/scanner3.py
View File

@ -35,22 +35,19 @@ Finally we save token information.
from __future__ import print_function
import sys
from typing import Optional, Tuple
from xdis import iscode, instruction_size, Instruction
from xdis.bytecode import _get_const_info
from uncompyle6.scanners.tok import Token
from uncompyle6.scanner import parse_fn_counts_30_35
from uncompyle6.util import get_code_name
import xdis
# Get all the opcodes into globals
import xdis.opcodes.opcode_33 as op3
from xdis import Instruction, instruction_size, iscode
from xdis.bytecode import _get_const_info
from uncompyle6.scanner import Scanner, CONST_COLLECTIONS
import sys
from uncompyle6.scanner import CONST_COLLECTIONS, Scanner, parse_fn_counts_30_35
from uncompyle6.scanners.tok import Token
from uncompyle6.util import get_code_name
intern = sys.intern
@ -485,6 +482,7 @@ class Scanner3(Scanner):
last_op_was_break = False
new_tokens = []
operand_value = 0
for i, inst in enumerate(self.insts):
opname = inst.opname
@ -536,10 +534,11 @@ class Scanner3(Scanner):
op = inst.opcode
if opname == "EXTENDED_ARG":
# FIXME: The EXTENDED_ARG is used to signal annotation
# parameters
if i + 1 < n and self.insts[i + 1].opcode != self.opc.MAKE_FUNCTION:
if i + 1 < n:
operand_value = argval << 16
continue
else:
operand_value = 0
if inst.offset in jump_targets:
jump_idx = 0
@ -646,7 +645,7 @@ class Scanner3(Scanner):
attr = attr[:4] # remove last value: attr[5] == False
else:
pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35(
inst.argval
inst.argval + operand_value
)
pattr = f"{pos_args} positional, {name_pair_args} keyword only, {annotate_args} annotated"

12
uncompyle6/scanners/scanner33.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein
# Copyright (c) 2015-2019, 2021-2022, 2024 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -19,20 +19,22 @@ This sets up opcodes Python's 3.3 and calls a generalized
scanner routine for Python 3.
"""
from __future__ import print_function
# bytecode verification, verify(), uses JUMP_OPs from here
from xdis.opcodes import opcode_33 as opc
JUMP_OPS = opc.JUMP_OPS
from uncompyle6.scanners.scanner3 import Scanner3
class Scanner33(Scanner3):
JUMP_OPS = opc.JUMP_OPS
class Scanner33(Scanner3):
def __init__(self, show_asm=False, is_pypy=False):
Scanner3.__init__(self, (3, 3), show_asm)
return
pass
if __name__ == "__main__":
from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str