Magic-Mirror/python-uncompyle6
1
0
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2024-11-23 13:09:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
d0dc879b37
python-uncompyle6/uncompyle6/parsers/parse37.py
rocky 156188f8bb withasstmt -> with_as 
This matches Python's AST naamae better. Some linting and
sorting of dictionary keys done as well.
2024-03-08 04:10:33 -05:00

1781 lines
69 KiB
Python
Raw Blame History

# Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Python 3.7 grammar for the spark Earley-algorithm parser.
"""
from __future__ import print_function
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse37base import Python37BaseParser
from uncompyle6.scanners.tok import Token
class Python37Parser(Python37BaseParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python37Parser, self).__init__(debug_parser)
self.customized = {}
###############################################
# Python 3.7 grammar rules
###############################################
def p_start(self, args):
"""
# The start or goal symbol
stmts ::= sstmt+
"""
def p_call_stmt(self, args):
"""
# eval-mode compilation. Single-mode interactive compilation
# adds another rule.
call_stmt ::= expr POP_TOP
"""
def p_eval_mode(self, args):
"""
# eval-mode compilation. Single-mode interactive compilation
# adds another rule.
expr_stmt ::= expr POP_TOP
"""
def p_stmt(self, args):
"""
pass ::=
_stmts ::= stmt+
# statements with continue and break
c_stmts ::= _stmts
c_stmts ::= _stmts lastc_stmt
c_stmts ::= lastc_stmt
c_stmts ::= continues
ending_return ::= RETURN_VALUE RETURN_LAST
ending_return ::= RETURN_VALUE_LAMBDA LAMBDA_MARKER
lastc_stmt ::= iflaststmt
lastc_stmt ::= forelselaststmt
lastc_stmt ::= ifelsestmtc
# Statements in a loop
lstmt ::= stmt
l_stmts ::= lstmt+
c_stmts_opt ::= c_stmts
c_stmts_opt ::= pass
# statements inside a loop
l_stmts ::= _stmts
l_stmts ::= returns
l_stmts ::= continues
l_stmts ::= _stmts lastl_stmt
l_stmts ::= lastl_stmt
lastl_stmt ::= iflaststmtl
lastl_stmt ::= ifelsestmtl
lastl_stmt ::= forelselaststmtl
lastl_stmt ::= tryelsestmtl
l_stmts_opt ::= l_stmts
l_stmts_opt ::= pass
suite_stmts ::= _stmts
suite_stmts ::= returns
suite_stmts ::= continues
suite_stmts_opt ::= suite_stmts
# passtmt is needed for semantic actions to add "pass"
suite_stmts_opt ::= pass
else_suite ::= suite_stmts
else_suitel ::= l_stmts
else_suitec ::= c_stmts
else_suitec ::= returns
else_suite_opt ::= else_suite
else_suite_opt ::= pass
stmt ::= classdef
stmt ::= expr_stmt
stmt ::= call_stmt
stmt ::= ifstmt
stmt ::= ifelsestmt
stmt ::= whilestmt
stmt ::= while1stmt
stmt ::= whileelsestmt
stmt ::= while1elsestmt
stmt ::= for
stmt ::= forelsestmt
stmt ::= try_except
stmt ::= tryelsestmt
stmt ::= tryfinallystmt
stmt ::= delete
delete ::= DELETE_FAST
delete ::= DELETE_NAME
delete ::= DELETE_GLOBAL
stmt ::= return
return ::= return_expr RETURN_VALUE
# "returns" nonterminal is a sequence of statements that ends in a
# RETURN statement.
# In later Python versions with jump optimization, this can cause JUMPs
# that would normally appear to be omitted.
returns ::= return
returns ::= _stmts return
stmt ::= genexpr_func
genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter
_come_froms JUMP_BACK _come_froms
"""
pass
def p_expr(self, args):
"""
expr ::= LOAD_CODE
expr ::= LOAD_CONST
expr ::= LOAD_DEREF
expr ::= LOAD_FAST
expr ::= LOAD_GLOBAL
expr ::= LOAD_NAME
expr ::= LOAD_STR
expr ::= _lambda_body
expr ::= and
expr ::= attribute37
expr ::= bin_op
expr ::= call
expr ::= compare
expr ::= dict
expr ::= generator_exp
expr ::= list
expr ::= or
expr ::= subscript
expr ::= subscript2
expr ::= unary_not
expr ::= unary_op
expr ::= yield
# bin_op (formerly "binary_expr") is the Python AST BinOp
bin_op ::= expr expr binary_operator
binary_operator ::= BINARY_ADD
binary_operator ::= BINARY_MULTIPLY
binary_operator ::= BINARY_AND
binary_operator ::= BINARY_OR
binary_operator ::= BINARY_XOR
binary_operator ::= BINARY_SUBTRACT
binary_operator ::= BINARY_TRUE_DIVIDE
binary_operator ::= BINARY_FLOOR_DIVIDE
binary_operator ::= BINARY_MODULO
binary_operator ::= BINARY_LSHIFT
binary_operator ::= BINARY_RSHIFT
binary_operator ::= BINARY_POWER
# unary_op (formerly "unary_expr") is the Python AST UnaryOp
unary_op ::= expr unary_operator
unary_operator ::= UNARY_POSITIVE
unary_operator ::= UNARY_NEGATIVE
unary_operator ::= UNARY_INVERT
unary_not ::= expr UNARY_NOT
subscript ::= expr expr BINARY_SUBSCR
get_iter ::= expr GET_ITER
yield ::= expr YIELD_VALUE
_lambda_body ::= lambda_body
expr ::= if_exp
return_expr ::= expr
return_expr ::= ret_and
return_expr ::= ret_or
return_expr_or_cond ::= return_expr
return_expr_or_cond ::= if_exp_ret
stmt ::= return_expr_lambda
return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
compare ::= compare_chained
compare ::= compare_single
compare_single ::= expr expr COMPARE_OP
# A compare_chained is two comparisons like x <= y <= z
compare_chained ::= expr compared_chained_middle ROT_TWO POP_TOP _come_froms
compare_chained_right ::= expr COMPARE_OP JUMP_FORWARD
# Non-null kvlist items are broken out in the individual grammars
kvlist ::=
# Positional arguments in make_function
pos_arg ::= expr
"""
def p_function_def(self, args):
"""
stmt ::= function_def
function_def ::= mkfunc store
stmt ::= function_def_deco
function_def_deco ::= mkfuncdeco store
mkfuncdeco ::= expr mkfuncdeco CALL_FUNCTION_1
mkfuncdeco ::= expr mkfuncdeco0 CALL_FUNCTION_1
mkfuncdeco0 ::= mkfunc
load_closure ::= load_closure LOAD_CLOSURE
load_closure ::= LOAD_CLOSURE
"""
def p_generator_exp(self, args):
""" """
def p_jump(self, args):
"""
_jump ::= JUMP_ABSOLUTE
_jump ::= JUMP_FORWARD
_jump ::= JUMP_BACK
# Zero or more COME_FROMs - loops can have this
_come_froms ::= COME_FROM*
_come_froms ::= _come_froms COME_FROM_LOOP
# One or more COME_FROMs - joins of tryelse's have this
come_froms ::= COME_FROM+
# Zero or one COME_FROM
# And/or expressions have this
come_from_opt ::= COME_FROM?
"""
def p_augmented_assign(self, args):
"""
stmt ::= aug_assign1
stmt ::= aug_assign2
# This is odd in that other aug_assign1's have only 3 slots
# The store isn't used as that's supposed to be also
# indicated in the first expr
aug_assign1 ::= expr expr
inplace_op store
aug_assign1 ::= expr expr
inplace_op ROT_THREE STORE_SUBSCR
aug_assign2 ::= expr DUP_TOP LOAD_ATTR expr
inplace_op ROT_TWO STORE_ATTR
inplace_op ::= INPLACE_ADD
inplace_op ::= INPLACE_SUBTRACT
inplace_op ::= INPLACE_MULTIPLY
inplace_op ::= INPLACE_TRUE_DIVIDE
inplace_op ::= INPLACE_FLOOR_DIVIDE
inplace_op ::= INPLACE_MODULO
inplace_op ::= INPLACE_POWER
inplace_op ::= INPLACE_LSHIFT
inplace_op ::= INPLACE_RSHIFT
inplace_op ::= INPLACE_AND
inplace_op ::= INPLACE_XOR
inplace_op ::= INPLACE_OR
"""
def p_assign(self, args):
"""
stmt ::= assign
assign ::= expr DUP_TOP designList
assign ::= expr store
stmt ::= assign2
stmt ::= assign3
assign2 ::= expr expr ROT_TWO store store
assign3 ::= expr expr expr ROT_THREE ROT_TWO store store store
"""
def p_forstmt(self, args):
"""
get_for_iter ::= GET_ITER _come_froms FOR_ITER
for_block ::= l_stmts_opt _come_froms JUMP_BACK
forelsestmt ::= SETUP_LOOP expr get_for_iter store
for_block POP_BLOCK else_suite _come_froms
forelselaststmt ::= SETUP_LOOP expr get_for_iter store
for_block POP_BLOCK else_suitec _come_froms
forelselaststmtl ::= SETUP_LOOP expr get_for_iter store
for_block POP_BLOCK else_suitel _come_froms
"""
def p_import20(self, args):
"""
stmt ::= import
stmt ::= import_from
stmt ::= import_from_star
stmt ::= importmultiple
importlist ::= importlist alias
importlist ::= alias
alias ::= IMPORT_NAME store
alias ::= IMPORT_FROM store
alias ::= IMPORT_NAME attributes store
import ::= LOAD_CONST LOAD_CONST alias
import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME IMPORT_STAR
import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR IMPORT_STAR
import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP
importmultiple ::= LOAD_CONST LOAD_CONST alias imports_cont
imports_cont ::= import_cont+
import_cont ::= LOAD_CONST LOAD_CONST alias
attributes ::= LOAD_ATTR+
"""
def p_import37(self, args):
"""
# The 3.7base scanner adds IMPORT_NAME_ATTR
alias ::= IMPORT_NAME_ATTR attributes store
alias ::= IMPORT_NAME_ATTR store
alias37 ::= IMPORT_NAME store
alias37 ::= IMPORT_FROM store
attribute37 ::= expr LOAD_METHOD
import_as37 ::= LOAD_CONST LOAD_CONST importlist37 store POP_TOP
import_from ::= LOAD_CONST LOAD_CONST importlist POP_TOP
import_from37 ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist37 POP_TOP
import_from_as37 ::= LOAD_CONST LOAD_CONST import_from_attr37 store POP_TOP
# A single entry in a dotted import a.b.c.d
import_one ::= importlists ROT_TWO IMPORT_FROM
import_one ::= importlists ROT_TWO POP_TOP IMPORT_FROM
# Semantic checks distinguish importattr37 from import_from_attr37
# in the former the "from" slot in a prior LOAD_CONST is null.
# Used in: import .. as ..
importattr37 ::= IMPORT_NAME_ATTR IMPORT_FROM
# Used in: from xx import .. as ..
import_from_attr37 ::= IMPORT_NAME_ATTR IMPORT_FROM
importlist37 ::= import_one
importlist37 ::= importattr37
importlist37 ::= alias37+
importlists ::= importlist37+
stmt ::= import_as37
stmt ::= import_from37
stmt ::= import_from_as37
"""
def p_list_comprehension(self, args):
"""
expr ::= list_comp
list_iter ::= list_for
list_iter ::= list_if
list_iter ::= list_if_not
list_iter ::= lc_body
list_if ::= expr jmp_false list_iter
list_if_not ::= expr jmp_true list_iter
"""
def p_gen_comp37(self, args):
"""
comp_iter ::= comp_for
comp_body ::= gen_comp_body
gen_comp_body ::= expr YIELD_VALUE POP_TOP
comp_if ::= expr jmp_false comp_iter
"""
def p_store(self, args):
"""
# Note. The below is right-recursive:
designList ::= store store
designList ::= store DUP_TOP designList
## Can we replace with left-recursive, and redo with:
##
## designList ::= designLists store store
## designLists ::= designLists store DUP_TOP
## designLists ::=
## Will need to redo semantic actiion
store ::= STORE_FAST
store ::= STORE_NAME
store ::= STORE_GLOBAL
store ::= STORE_DEREF
store ::= expr STORE_ATTR
store ::= store_subscript
store_subscript ::= expr expr STORE_SUBSCR
store ::= unpack
"""
def p_32on(self, args):
"""
if_exp::= expr jmp_false expr jump_forward_else expr COME_FROM
# compare_chained_right is used in a "chained_compare": x <= y <= z
# used exclusively in compare_chained
compare_chained_right ::= expr COMPARE_OP RETURN_VALUE
compare_chained_right ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
# Python < 3.5 no POP BLOCK
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP
# Python 3.5+ has jump optimization to remove the redundant
# jump_excepts. But in 3.3 we need them added
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
END_FINALLY
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suite
jump_excepts come_from_except_clauses
jump_excepts ::= jump_except+
subscript2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
# FIXME: The below rule was in uncompyle6.
# In decompyle6 though "_ifstmts_jump" is part of an "ifstmt"
# where as the below rule is appropriate for an "ifelsesmt"
# Investigate and reconcile
# _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_froms
kv3 ::= expr expr STORE_MAP
"""
return
def p_33on(self, args):
"""
# Python 3.3+ adds yield from.
expr ::= yield_from
yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
# We do the grammar hackery below for semantics
# actions that want c_stmts_opt at index 1
# Python 3.5+ has jump optimization to remove the redundant
# jump_excepts. But in 3.3 we need them added
try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler
jump_excepts come_from_except_clauses
"""
def p_34on(self, args):
"""
whilestmt ::= setup_loop testexpr returns come_froms POP_BLOCK COME_FROM_LOOP
# Seems to be needed starting 3.4.4 or so
while1stmt ::= setup_loop l_stmts
COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP
while1stmt ::= setup_loop l_stmts
POP_BLOCK COME_FROM_LOOP
# FIXME the below masks a bug in not detecting COME_FROM_LOOP
# grammar rules with COME_FROM -> COME_FROM_LOOP already exist
whileelsestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suitel COME_FROM
while1elsestmt ::= setup_loop l_stmts JUMP_BACK _come_froms POP_BLOCK else_suitel
COME_FROM_LOOP
# Python 3.4+ optimizes the trailing two JUMPS away
_ifstmts_jump ::= c_stmts_opt JUMP_ABSOLUTE JUMP_FORWARD _come_froms
"""
def p_35_on(self, args):
"""
while1elsestmt ::= setup_loop l_stmts JUMP_BACK
POP_BLOCK else_suite COME_FROM_LOOP
# The following rule is for Python 3.5+ where we can have stuff like
# while ..
# if
# ...
# the end of the if will jump back to the loop and there will be a COME_FROM
# after the jump
l_stmts ::= lastl_stmt come_froms l_stmts
# Python 3.5+ Await statement
expr ::= await_expr
await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
stmt ::= await_stmt
await_stmt ::= await_expr POP_TOP
# Python 3.5+ async additions
inplace_op ::= INPLACE_MATRIX_MULTIPLY
binary_operator ::= BINARY_MATRIX_MULTIPLY
# Python 3.5+ does jump optimization
# In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
return_if_stmt ::= return_expr RETURN_END_IF POP_BLOCK
return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM
jb_else ::= JUMP_BACK ELSE
jb_else ::= JUMP_BACK COME_FROM
ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel
# We want to keep the positions of the "then" and
# "else" statements in "ifelstmtl" similar to others of this ilk.
testexpr_cf ::= testexpr come_froms
ifelsestmtl ::= testexpr_cf c_stmts_opt jb_else else_suitel
# 3.5 Has jump optimization which can route the end of an
# "if/then" back to a loop just before an else.
jump_absolute_else ::= jb_else
jump_absolute_else ::= CONTINUE ELSE
# Our hacky "ELSE" determination doesn't do a good job and really
# determine the start of an "else". It could also be the end of an
# "if-then" which ends in a "continue". Perhaps with real control-flow
# analysis we'll sort this out. Or call "ELSE" something more appropriate.
_ifstmts_jump ::= c_stmts_opt ELSE
# ifstmt ::= testexpr c_stmts_opt
iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
"""
def p_37_async(self, args):
"""
stmt ::= async_for_stmt37
stmt ::= async_for_stmt
stmt ::= async_forelse_stmt
async_for_stmt ::= setup_loop expr
GET_AITER
SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY COME_FROM
for_block
COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
COME_FROM_LOOP
# Order of LOAD_CONST YIELD_FROM is switched from 3.6 to save a LOAD_CONST
async_for_stmt37 ::= setup_loop expr
GET_AITER
_come_froms
SETUP_EXCEPT GET_ANEXT
LOAD_CONST YIELD_FROM
store
POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY for_block COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT
POP_TOP POP_BLOCK
COME_FROM_LOOP
async_forelse_stmt ::= setup_loop expr
GET_AITER
_come_froms
SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY COME_FROM
for_block
COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
else_suite COME_FROM_LOOP
"""
def p_37_chained(self, args):
"""
testtrue ::= compare_chained37
testfalse ::= compare_chained37_false
compare_chained ::= compare_chained37
compare_chained ::= compare_chained37_false
compare_chained37 ::= expr compared_chained_middlea_37
compare_chained37 ::= expr compared_chained_middlec_37
compare_chained37_false ::= expr compared_chained_middle_false_37
compare_chained37_false ::= expr compared_chained_middleb_false_37
compare_chained37_false ::= expr compare_chained_right_false_37
compared_chained_middlea_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compared_chained_middlea_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_righta_37 COME_FROM POP_TOP COME_FROM
compared_chained_middleb_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_rightb_false_37 POP_TOP _jump COME_FROM
compared_chained_middlec_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_righta_37 POP_TOP
compared_chained_middle_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_rightc_37 POP_TOP JUMP_FORWARD COME_FROM
compared_chained_middle_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_rightb_false_37 POP_TOP _jump COME_FROM
compare_chained_right_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained_righta_false_37 POP_TOP JUMP_BACK COME_FROM
compare_chained_righta_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_FORWARD
compare_chained_righta_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_BACK
compare_chained_righta_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE jf_cfs
compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD COME_FROM
compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD
compare_chained_rightc_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
compare_chained_righta_false_37 ELSE
compare_chained_rightc_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
compare_chained_righta_false_37
"""
def p_37_conditionals(self, args):
"""
expr ::= if_exp37
if_exp37 ::= expr expr jf_cfs expr COME_FROM
jf_cfs ::= JUMP_FORWARD _come_froms
ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite
opt_come_from_except
# This is probably more realistically an "ifstmt" (with a null else)
# see _cmp() of python3.8/distutils/__pycache__/version.cpython-38.opt-1.pyc
ifelsestmt ::= testexpr stmts jf_cfs else_suite_opt
opt_come_from_except
expr_pjit ::= expr POP_JUMP_IF_TRUE
expr_jit ::= expr JUMP_IF_TRUE
expr_jt ::= expr jmp_true
jmp_false37 ::= POP_JUMP_IF_FALSE COME_FROM
list_if ::= expr jmp_false37 list_iter
list_iter ::= list_if37
list_iter ::= list_if37_not
list_if37 ::= compare_chained37_false list_iter
list_if37_not ::= compare_chained37 list_iter
_ifstmts_jump ::= c_stmts_opt come_froms
_ifstmts_jump ::= COME_FROM c_stmts come_froms
and_not ::= expr jmp_false expr POP_JUMP_IF_TRUE
testfalse ::= and_not
expr ::= if_exp_37a
expr ::= if_exp_37b
if_exp_37a ::= and_not expr JUMP_FORWARD come_froms expr COME_FROM
if_exp_37b ::= expr jmp_false expr POP_JUMP_IF_FALSE
jump_forward_else expr
jmp_false_cf ::= POP_JUMP_IF_FALSE COME_FROM
comp_if ::= or jmp_false_cf comp_iter
"""
def p_comprehension3(self, args):
"""
# Python3 scanner adds LOAD_LISTCOMP. Python3 does list comprehension like
# other comprehensions (set, dictionary).
# Our "continue" heuristic - in two successive JUMP_BACKS, the first
# one may be a continue - sometimes classifies a JUMP_BACK
# as a CONTINUE. The two are kind of the same in a comprehension.
comp_for ::= expr get_for_iter store comp_iter CONTINUE
comp_for ::= expr get_for_iter store comp_iter JUMP_BACK
for_iter ::= _come_froms FOR_ITER
list_comp ::= BUILD_LIST_0 list_iter
lc_body ::= expr LIST_APPEND
list_for ::= expr_or_arg
for_iter
store list_iter
jb_or_c _come_froms
set_for ::= expr_or_arg
for_iter
store set_iter
jb_or_c _come_froms
# This is seen in PyPy, but possibly it appears on other Python 3?
list_if ::= expr jmp_false list_iter COME_FROM
list_if_not ::= expr jmp_true list_iter COME_FROM
jb_or_c ::= JUMP_BACK
jb_or_c ::= CONTINUE
stmt ::= set_comp_func
# TODO: simplify this
set_comp_func ::= BUILD_SET_0 LOAD_ARG for_iter store comp_iter
JUMP_BACK ending_return
set_comp_func ::= BUILD_SET_0 LOAD_ARG for_iter store comp_iter
COME_FROM JUMP_BACK ending_return
comp_body ::= dict_comp_body
comp_body ::= set_comp_body
dict_comp_body ::= expr expr MAP_ADD
set_comp_body ::= expr SET_ADD
# See also common Python p_list_comprehension
"""
def p_dict_comp3(self, args):
""" "
expr ::= dict_comp
stmt ::= dict_comp_func
dict_comp_func ::= BUILD_MAP_0 LOAD_ARG for_iter store
comp_iter JUMP_BACK ending_return
comp_iter ::= comp_if
comp_iter ::= comp_if_not
comp_if_not ::= expr jmp_true comp_iter
comp_iter ::= comp_body
expr_or_arg ::= LOAD_ARG
expr_or_arg ::= expr
"""
def p_expr3(self, args):
"""
expr ::= if_exp_not
if_exp_not ::= expr jmp_true expr jump_forward_else expr COME_FROM
# a JUMP_FORWARD to another JUMP_FORWARD can get turned into
# a JUMP_ABSOLUTE with no COME_FROM
if_exp ::= expr jmp_false expr jump_absolute_else expr
# if_exp_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately
expr ::= if_exp_true
if_exp_true ::= expr JUMP_FORWARD expr COME_FROM
"""
def p_generator_exp3(self, args):
"""
load_genexpr ::= LOAD_GENEXPR
load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
"""
def p_grammar(self, args):
"""
sstmt ::= stmt
sstmt ::= ifelsestmtr
sstmt ::= return RETURN_LAST
return_if_stmts ::= return_if_stmt come_from_opt
return_if_stmts ::= _stmts return_if_stmt _come_froms
return_if_stmt ::= return_expr RETURN_END_IF
returns ::= _stmts return_if_stmt
stmt ::= break
break ::= BREAK_LOOP
stmt ::= continue
continue ::= CONTINUE
continues ::= _stmts lastl_stmt continue
continues ::= lastl_stmt continue
continues ::= continue
kwarg ::= LOAD_STR expr
kwargs ::= kwarg+
classdef ::= build_class store
# FIXME: we need to add these because don't detect this properly
# in custom rules. Specifically if one of the exprs is CALL_FUNCTION
# then we'll mistake that for the final CALL_FUNCTION.
# We can fix by triggering on the CALL_FUNCTION op
# Python3 introduced LOAD_BUILD_CLASS
# Other definitions are in a custom rule
build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
stmt ::= classdefdeco
classdefdeco ::= classdefdeco1 store
# In 3.7 there are some LOAD_GLOBALs we don't convert to LOAD_ASSERT
stmt ::= assert2
assert2 ::= expr jmp_true LOAD_GLOBAL expr CALL_FUNCTION_1 RAISE_VARARGS_1
# "assert_invert" tests on the negative of the condition given
stmt ::= assert_invert
assert_invert ::= testtrue LOAD_GLOBAL RAISE_VARARGS_1
expr ::= LOAD_ASSERT
# FIXME: add this:
# expr ::= assert_expr_or
ifstmt ::= testexpr _ifstmts_jump
testexpr ::= testfalse
testexpr ::= testtrue
testfalse ::= expr jmp_false
testtrue ::= expr jmp_true
_ifstmts_jump ::= return_if_stmts
_ifstmts_jump ::= c_stmts_opt COME_FROM
iflaststmt ::= testexpr c_stmts
iflaststmt ::= testexpr c_stmts JUMP_ABSOLUTE
iflaststmtl ::= testexpr c_stmts JUMP_BACK
iflaststmtl ::= testexpr c_stmts JUMP_BACK COME_FROM_LOOP
iflaststmtl ::= testexpr c_stmts JUMP_BACK POP_BLOCK
# These are used to keep parse tree indices the same
jump_forward_else ::= JUMP_FORWARD
jump_forward_else ::= JUMP_FORWARD ELSE
jump_forward_else ::= JUMP_FORWARD COME_FROM
jump_absolute_else ::= JUMP_ABSOLUTE ELSE
jump_absolute_else ::= JUMP_ABSOLUTE _come_froms
jump_absolute_else ::= come_froms _jump COME_FROM
# Note: in if/else kinds of statements, we err on the side
# of missing "else" clauses. Therefore we include grammar
# rules with and without ELSE.
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD
else_suite opt_come_from_except
ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
else_suite _come_froms
# This handles the case where a "JUMP_ABSOLUTE" is part
# of an inner if in c_stmts_opt
ifelsestmt ::= testexpr c_stmts come_froms
else_suite come_froms
# ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
# pass _come_froms
ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
ifelsestmtr ::= testexpr return_if_stmts returns
ifelsestmtl ::= testexpr c_stmts_opt cf_jump_back else_suitel
cf_jump_back ::= COME_FROM JUMP_BACK
# FIXME: this feels like a hack. Is it just 1 or two
# COME_FROMs? the parsed tree for this and even with just the
# one COME_FROM for Python 2.7 seems to associate the
# COME_FROM targets from the wrong places
# this is nested inside a try_except
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
except_handler ::= jmp_abs COME_FROM except_stmts
_come_froms END_FINALLY
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
_come_froms END_FINALLY
# FIXME: remove this
except_handler ::= JUMP_FORWARD COME_FROM except_stmts
come_froms END_FINALLY come_from_opt
except_stmts ::= except_stmt+
except_stmt ::= except_cond1 except_suite come_from_opt
except_stmt ::= except_cond2 except_suite come_from_opt
except_stmt ::= except_cond2 except_suite_finalize
except_stmt ::= except
## FIXME: what's except_pop_except?
except_stmt ::= except_pop_except
# Python3 introduced POP_EXCEPT
except_suite ::= c_stmts_opt POP_EXCEPT jump_except
jump_except ::= JUMP_ABSOLUTE
jump_except ::= JUMP_BACK
jump_except ::= JUMP_FORWARD
jump_except ::= CONTINUE
# This is used in Python 3 in
# "except ... as e" to remove 'e' after the c_stmts_opt finishes
except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize
END_FINALLY _jump
except_var_finalize ::= POP_BLOCK POP_EXCEPT LOAD_CONST COME_FROM_FINALLY
LOAD_CONST store delete
except_suite ::= returns
except_cond1 ::= DUP_TOP expr COMPARE_OP
jmp_false POP_TOP POP_TOP POP_TOP
except_cond2 ::= DUP_TOP expr COMPARE_OP
jmp_false POP_TOP store POP_TOP come_from_opt
except ::= POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT _jump
except ::= POP_TOP POP_TOP POP_TOP returns
jmp_abs ::= JUMP_ABSOLUTE
jmp_abs ::= JUMP_BACK
"""
def p_misc3(self, args):
"""
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
come_froms END_FINALLY
for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK
for_block ::= l_stmts
for_block ::= l_stmts JUMP_BACK
iflaststmtl ::= testexpr c_stmts
"""
def p_come_from3(self, args):
"""
opt_come_from_except ::= COME_FROM_EXCEPT
opt_come_from_except ::= _come_froms
opt_come_from_except ::= come_from_except_clauses
come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+
"""
def p_jump3(self, args):
"""
jmp_false ::= POP_JUMP_IF_FALSE
jmp_true ::= POP_JUMP_IF_TRUE
# FIXME: Common with 2.7
ret_and ::= expr JUMP_IF_FALSE_OR_POP return_expr_or_cond COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP return_expr_or_cond COME_FROM
if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM return_expr_or_cond
jitop_come_from_expr ::= JUMP_IF_TRUE_OR_POP come_froms expr
jifop_come_from ::= JUMP_IF_FALSE_OR_POP come_froms
expr_jitop ::= expr JUMP_IF_TRUE_OR_POP
or ::= and jitop_come_from_expr COME_FROM
or ::= expr_jitop expr COME_FROM
or ::= expr_jit expr COME_FROM
or ::= expr_pjit expr POP_JUMP_IF_FALSE COME_FROM
testfalse_not_or ::= expr jmp_false expr jmp_false COME_FROM
testfalse_not_and ::= and jmp_true come_froms
testfalse_not_and ::= expr jmp_false expr jmp_true COME_FROM
testfalse ::= testfalse_not_or
testfalse ::= testfalse_not_and
testfalse ::= or jmp_false COME_FROM
iflaststmtl ::= testexprl c_stmts JUMP_BACK
iflaststmtl ::= testexprl c_stmts JUMP_BACK COME_FROM_LOOP
iflaststmtl ::= testexprl c_stmts JUMP_BACK POP_BLOCK
testexprl ::= testfalsel
testfalsel ::= expr jmp_true
or ::= expr_jt expr
and ::= expr JUMP_IF_FALSE_OR_POP expr come_from_opt
and ::= expr jifop_come_from expr
expr_pjit_come_from ::= expr POP_JUMP_IF_TRUE COME_FROM
or ::= expr_pjit_come_from expr
## Note that "jmp_false" is what we check on in the "and" reduce rule.
and ::= expr jmp_false expr COME_FROM
or ::= expr_jt expr COME_FROM
# compared_chained_middle is used exclusively in chained_compare
compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compared_chained_middle COME_FROM
compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained_right COME_FROM
"""
def p_stmt3(self, args):
"""
stmt ::= if_exp_lambda
stmt ::= if_exp_not_lambda
# If statement inside a loop:
stmt ::= ifstmtl
if_exp_lambda ::= expr jmp_false expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
if_exp_not_lambda
::= expr jmp_true expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
return_stmt_lambda ::= return_expr RETURN_VALUE_LAMBDA
return_if_lambda ::= RETURN_END_IF_LAMBDA
stmt ::= return_closure
return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
stmt ::= whileTruestmt
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms
ifelsestmtl ::= testexpr c_stmts_opt jump_forward_else else_suitec
ifstmtl ::= testexpr _ifstmts_jumpl
_ifstmts_jumpl ::= c_stmts JUMP_BACK
_ifstmts_jumpl ::= _ifstmts_jump
# The following can happen when the jump offset is large and
# Python is looking to do a small jump to a larger jump to get
# around the problem that the offset can't be represented in
# the size allowed for the jump offset. This is more likely to
# happen in wordcode Python since the offset range has been
# reduced. FIXME: We should add a reduction check that the
# final jump goes to another jump.
_ifstmts_jumpl ::= COME_FROM c_stmts JUMP_BACK
_ifstmts_jumpl ::= COME_FROM c_stmts JUMP_FORWARD
"""
def p_loop_stmt3(self, args):
"""
setup_loop ::= SETUP_LOOP _come_froms
for ::= setup_loop expr get_for_iter store for_block POP_BLOCK
for ::= setup_loop expr get_for_iter store for_block POP_BLOCK
COME_FROM_LOOP
forelsestmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite
COME_FROM_LOOP
forelselaststmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
COME_FROM_LOOP
forelselaststmtl ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitel
COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= setup_loop testexpr returns POP_BLOCK
COME_FROM_LOOP
# We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else
# so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
# This is undoubtedly some sort of JUMP optimization going on.
whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms
POP_BLOCK
while1elsestmt ::= setup_loop l_stmts JUMP_BACK
else_suitel
whileelsestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suitel COME_FROM_LOOP
whileTruestmt ::= setup_loop l_stmts_opt JUMP_BACK POP_BLOCK
_come_froms
# FIXME: Python 3.? starts adding branch optimization? Put this starting there.
while1stmt ::= setup_loop l_stmts COME_FROM_LOOP
while1stmt ::= setup_loop l_stmts COME_FROM_LOOP JUMP_BACK POP_BLOCK COME_FROM_LOOP
while1stmt ::= setup_loop l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
while1elsestmt ::= setup_loop l_stmts JUMP_BACK
else_suite COME_FROM_LOOP
# FIXME: investigate - can code really produce a NOP?
for ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP
COME_FROM_LOOP
"""
def p_36misc(self, args):
"""
sstmt ::= sstmt RETURN_LAST
# 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
for_block ::= l_stmts_opt come_from_loops JUMP_BACK
come_from_loops ::= COME_FROM_LOOP*
whilestmt ::= setup_loop testexpr l_stmts_opt
JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt
come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
# 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
# RETURN_VALUE is meant. Specifically this can happen in
# ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
return ::= return_expr RETURN_END_IF
return ::= return_expr RETURN_VALUE COME_FROM
return_stmt_lambda ::= return_expr RETURN_VALUE_LAMBDA COME_FROM
# A COME_FROM is dropped off because of JUMP-to-JUMP optimization
and ::= expr jmp_false expr
and ::= expr jmp_false expr jmp_false
jf_cf ::= JUMP_FORWARD COME_FROM
cf_jf_else ::= come_froms JUMP_FORWARD ELSE
if_exp ::= expr jmp_false expr jf_cf expr COME_FROM
async_for_stmt ::= setup_loop expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block POP_BLOCK
COME_FROM_LOOP
# Adds a COME_FROM_ASYNC_WITH over 3.5
# FIXME: remove corresponding rule for 3.5?
except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
jb_cfs ::= come_from_opt JUMP_BACK come_froms
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
# In 3.6+, A sequence of statements ending in a RETURN can cause
# JUMP_FORWARD END_FINALLY to be omitted from try middle
except_return ::= POP_TOP POP_TOP POP_TOP returns
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
# Try middle following a returns
except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
stmt ::= try_except36
try_except36 ::= SETUP_EXCEPT returns except_handler36
opt_come_from_except
try_except36 ::= SETUP_EXCEPT suite_stmts
try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler36 come_from_opt
# 3.6 omits END_FINALLY sometimes
except_handler36 ::= COME_FROM_EXCEPT except_stmts
except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
stmt ::= tryfinally36
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
except_suite_finalize ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
stmt ::= tryfinally_return_stmt
tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_FINALLY
compare_chained_right ::= expr COMPARE_OP come_froms JUMP_FORWARD
"""
def p_37_misc(self, args):
"""
# long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a
# JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional
# ELSE added to the except_suite. With better flow control perhaps we can
# sort this out better.
except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE
# FIXME: the below is to work around test_grammar expecting a "call" to be
# on the LHS because it is also somewhere on in a rule.
call ::= expr CALL_METHOD_0
"""
def customize_grammar_rules(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
self.check_reduce["call_kw"] = "AST"
# Opcode names in the custom_ops_processed set have rules that get added
# unconditionally and the rules are constant. So they need to be done
# only once and if we see the opcode a second we don't have to consider
# adding more rules.
#
# Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by
# default because it starts with BUILD. So we'll set to ignore it from
# the start.
custom_ops_processed = set()
for i, token in enumerate(tokens):
opname = token.kind
if opname == "LOAD_ASSERT":
if "PyPy" in customize:
rules_str = """
stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "FORMAT_VALUE":
rules_str = """
expr ::= formatted_value1
formatted_value1 ::= expr FORMAT_VALUE
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "FORMAT_VALUE_ATTR":
rules_str = """
expr ::= formatted_value2
formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "MAKE_FUNCTION_CLOSURE":
if "LOAD_DICTCOMP" in self.seen_ops:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif "LOAD_SETCOMP" in self.seen_ops:
rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_CLOSURE expr
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif opname == "BEFORE_ASYNC_WITH":
rules_str = """
stmt ::= async_with_stmt SETUP_ASYNC_WITH
async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
async_with_post ::= COME_FROM_ASYNC_WITH
WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_with_as_stmt
async_with_as_stmt ::= expr
async_with_pre
store
suite_stmts_opt
POP_BLOCK LOAD_CONST
async_with_post
async_with_stmt ::= expr
async_with_pre
POP_TOP
suite_stmts_opt
POP_BLOCK LOAD_CONST
async_with_post
async_with_stmt ::= expr
async_with_pre
POP_TOP
suite_stmts_opt
async_with_post
"""
self.addRule(rules_str, nop_func)
elif opname.startswith("BUILD_STRING"):
v = token.attr
rules_str = """
expr ::= joined_str
joined_str ::= %sBUILD_STRING_%d
""" % (
"expr " * v,
v,
)
self.add_unique_doc_rules(rules_str, customize)
if "FORMAT_VALUE_ATTR" in self.seen_ops:
rules_str = """
formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
expr ::= formatted_value_attr
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
v = token.attr
rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
self.addRule(rule, nop_func)
elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
v = token.attr
rule = (
"build_tuple_unpack_with_call ::= "
+ "expr1024 " * int(v // 1024)
+ "expr32 " * int((v // 32) % 32)
+ "expr " * (v % 32)
+ opname
)
self.addRule(rule, nop_func)
rule = "starred ::= %s %s" % ("expr " * v, opname)
self.addRule(rule, nop_func)
elif opname == "GET_AITER":
self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
if not {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} in self.seen_ops:
self.addRule(
"""
expr ::= dict_comp_async
expr ::= generator_exp_async
expr ::= list_comp_async
dict_comp_async ::= LOAD_DICTCOMP
LOAD_STR
MAKE_FUNCTION_0
get_aiter
CALL_FUNCTION_1
dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
dict_comp_async
func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY COME_FROM
func_async_prefix ::= _come_froms SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
get_aiter CALL_FUNCTION_1
genexpr_func_async ::= LOAD_ARG func_async_prefix
store func_async_middle comp_iter
JUMP_BACK COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
# FIXME this is a workaround for probably some bug in the Earley parser
# if we use get_aiter, then list_comp_async doesn't match, and I don't
# understand why.
expr_get_aiter ::= expr GET_AITER
list_afor ::= get_aiter list_afor2
list_afor2 ::= func_async_prefix
store func_async_middle list_iter
JUMP_BACK COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
list_comp_async ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
expr_get_aiter CALL_FUNCTION_1
GET_AWAITABLE LOAD_CONST
YIELD_FROM
list_iter ::= list_afor
set_comp_async ::= LOAD_SETCOMP
LOAD_STR
MAKE_FUNCTION_0
get_aiter
CALL_FUNCTION_1
set_comp_async ::= LOAD_CLOSURE
BUILD_TUPLE_1
LOAD_SETCOMP
LOAD_STR MAKE_FUNCTION_CLOSURE
get_aiter CALL_FUNCTION_1
await
""",
nop_func,
)
custom_ops_processed.add(opname)
self.addRule(
"""
dict_comp_async ::= BUILD_MAP_0 LOAD_ARG
dict_comp_async
expr ::= dict_comp_async
expr ::= generator_exp_async
expr ::= list_comp_async
expr ::= set_comp_async
func_async_middle ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY _come_froms
# async_iter ::= block_break SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
get_aiter ::= expr GET_AITER
list_afor ::= get_aiter list_afor2
list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
list_iter ::= list_afor
set_afor ::= get_aiter set_afor2
set_iter ::= set_afor
set_iter ::= set_for
set_comp_async ::= BUILD_SET_0 LOAD_ARG
set_comp_async
""",
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "GET_ANEXT":
self.addRule(
"""
expr ::= genexpr_func_async
expr ::= BUILD_MAP_0 genexpr_func_async
expr ::= list_comp_async
dict_comp_async ::= BUILD_MAP_0 genexpr_func_async
async_iter ::= _come_froms
SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
store_async_iter_end ::= store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY COME_FROM
# We use store_async_iter_end to make comp_iter come out in the right position,
# (after the logical "store")
genexpr_func_async ::= LOAD_ARG async_iter
store_async_iter_end
comp_iter
JUMP_BACK COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
list_afor2 ::= async_iter
store
list_iter
JUMP_BACK
COME_FROM_FINALLY
END_ASYNC_FOR
list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2
set_afor2 ::= async_iter
store
func_async_middle
set_iter
JUMP_BACK COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
set_afor2 ::= expr_or_arg
set_iter_async
set_comp_async ::= BUILD_SET_0 set_afor2
set_iter_async ::= async_iter
store
set_iter
JUMP_BACK
_come_froms
END_ASYNC_FOR
return_expr_lambda ::= genexpr_func_async
LOAD_CONST RETURN_VALUE
RETURN_VALUE_LAMBDA
return_expr_lambda ::= BUILD_SET_0 genexpr_func_async
RETURN_VALUE_LAMBDA LAMBDA_MARKER
""",
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "GET_AWAITABLE":
rule_str = """
await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
expr ::= await_expr
"""
self.add_unique_doc_rules(rule_str, customize)
elif opname == "GET_ITER":
self.addRule(
"""
expr ::= get_iter
get_iter ::= expr GET_ITER
""",
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "LOAD_ASSERT":
if "PyPy" in customize:
rules_str = """
stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == "LOAD_ATTR":
self.addRule(
"""
expr ::= attribute
attribute ::= expr LOAD_ATTR
""",
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "SETUP_WITH":
rules_str = """
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
# Removes POP_BLOCK LOAD_CONST from 3.6-
with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
if self.version < (3, 8):
rules_str += """
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
LOAD_CONST
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
else:
rules_str += """
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH
END_FINALLY
"""
self.addRule(rules_str, nop_func)
pass
pass
def custom_classfunc_rule(self, opname, token, customize, next_token):
args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args:
# 0 if neither
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky.
nak = (len(opname) - len("CALL_FUNCTION")) // 3
uniq_param = args_kw + args_pos
if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
rule = (
"""
await ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
await_expr ::= expr await
expr ::= await_expr
async_call ::= expr """
+ ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ "expr " * nak
+ token.kind
+ " GET_AWAITABLE LOAD_CONST YIELD_FROM"
)
self.add_unique_doc_rules(rule, customize)
self.add_unique_rule(
"expr ::= async_call", token.kind, uniq_param, customize
)
if opname.startswith("CALL_FUNCTION_KW"):
self.addRule("expr ::= call_kw36", nop_func)
values = "expr " * token.attr
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
self.add_unique_rule(rule, token.kind, token.attr, customize)
elif opname == "CALL_FUNCTION_EX_KW":
# Note: this doesn't exist in 3.7 and later
self.addRule(
"""expr ::= call_ex_kw4
call_ex_kw4 ::= expr
expr
expr
CALL_FUNCTION_EX_KW
""",
nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
self.addRule(
"""expr ::= call_ex_kw
call_ex_kw ::= expr expr build_map_unpack_with_call
CALL_FUNCTION_EX_KW
""",
nop_func,
)
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value?
self.addRule(
"""expr ::= call_ex_kw3
call_ex_kw3 ::= expr
build_tuple_unpack_with_call
expr
CALL_FUNCTION_EX_KW
""",
nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value?
self.addRule(
"""expr ::= call_ex_kw2
call_ex_kw2 ::= expr
build_tuple_unpack_with_call
build_map_unpack_with_call
CALL_FUNCTION_EX_KW
""",
nop_func,
)
elif opname == "CALL_FUNCTION_EX":
self.addRule(
"""
expr ::= call_ex
starred ::= expr
call_ex ::= expr starred CALL_FUNCTION_EX
""",
nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(
"""
expr ::= call_ex_kw
call_ex_kw ::= expr expr
build_map_unpack_with_call CALL_FUNCTION_EX
""",
nop_func,
)
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(
"""
expr ::= call_ex_kw3
call_ex_kw3 ::= expr
build_tuple_unpack_with_call
%s
CALL_FUNCTION_EX
"""
% "expr "
* token.attr,
nop_func,
)
pass
# FIXME: Is this right?
self.addRule(
"""
expr ::= call_ex_kw4
call_ex_kw4 ::= expr
expr
expr
CALL_FUNCTION_EX
""",
nop_func,
)
pass
else:
super(Python37Parser, self).custom_classfunc_rule(
opname, token, customize, next_token
)
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python37Parser, self).reduce_is_invalid(
rule, ast, tokens, first, last
)
if invalid:
return invalid
if rule[0] == "call_kw":
# Make sure we don't derive call_kw
nt = ast[0]
while not isinstance(nt, Token):
if nt[0] == "call_kw":
return True
nt = nt[0]
pass
pass
return False
def info(args):
# Check grammar
p = Python37Parser()
if len(args) > 0:
arg = args[0]
if arg == "3.7":
from uncompyle6.parser.parse37 import Python37Parser
p = Python37Parser()
elif arg == "3.8":
from uncompyle6.parser.parse38 import Python38Parser
p = Python38Parser()
else:
raise RuntimeError("Only 3.7 and 3.8 supported")
p.check_grammar()
if len(sys.argv) > 1 and sys.argv[1] == "dump":
print("-" * 50)
p.dump_grammar()
class Python37ParserSingle(Python37Parser, PythonParserSingle):
pass
if __name__ == "__main__":
# Check grammar
# FIXME: DRY this with other parseXX.py routines
p = Python37Parser()
p.check_grammar()
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 7):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
opcode_set = set(s.opc.opname).union(
set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()
)
)
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
import sys
if len(sys.argv) > 1:
from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0]))
Reference in New Issue View Git Blame Copy Permalink