Magic-Mirror/python-uncompyle6
1
0
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2024-11-30 16:40:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Start to segregate and clean up grammar

Browse Source
This commit is contained in:
rocky 2016-07-27 17:35:21 -04:00
parent f33f425692
commit 3f7c4209d9
6 changed files with 79 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pytest/test_grammar.py
View File

@ -1,6 +1,7 @@
import pytest
from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
from uncompyle6.parser import get_python_parser
from uncompyle6.scanner import get_scanner
def test_grammar():
p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
@ -19,6 +20,17 @@ def test_grammar():
assert expect_lhs == set(lhs)
assert unused_rhs == set(rhs)
assert expect_right_recursive == right_recursive
expect_tokens =
# FIXME: check that tokens are in list of opcodes
# print(tokens)
s = get_scanner(PYTHON_VERSION, IS_PYPY)
if PYTHON_VERSION == 2.7:
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
assert remain_tokens == set([]), \
"Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())

15
uncompyle6/parser.py
View File

@ -273,7 +273,9 @@ class PythonParser(GenericASTBuilder):
def p_forstmt(self, args):
"""
_for ::= GET_ITER FOR_ITER
_for ::= LOAD_CONST FOR_LOOP
# Possibly before Python 2.3
# _for ::= LOAD_CONST FOR_LOOP
for_block ::= l_stmts_opt _come_from JUMP_BACK
for_block ::= return_stmts _come_from
@ -339,8 +341,6 @@ class PythonParser(GenericASTBuilder):
imports_cont ::= imports_cont import_cont
imports_cont ::= import_cont
import_cont ::= LOAD_CONST LOAD_CONST import_as_cont
import_as_cont ::= IMPORT_NAME_CONT designator
import_as_cont ::= IMPORT_NAME_CONT load_attrs designator
import_as_cont ::= IMPORT_FROM designator
load_attrs ::= LOAD_ATTR
@ -371,9 +371,6 @@ class PythonParser(GenericASTBuilder):
stmt ::= setcomp_func
setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
JUMP_BACK RETURN_VALUE RETURN_LAST
comp_iter ::= comp_if
comp_iter ::= comp_ifnot
comp_iter ::= comp_for
@ -381,9 +378,7 @@ class PythonParser(GenericASTBuilder):
comp_body ::= set_comp_body
comp_body ::= gen_comp_body
comp_body ::= dict_comp_body
set_comp_body ::= expr SET_ADD
gen_comp_body ::= expr YIELD_VALUE POP_TOP
dict_comp_body ::= expr expr MAP_ADD
comp_if ::= expr jmp_false comp_iter
comp_ifnot ::= expr jmp_true comp_iter
@ -394,7 +389,6 @@ class PythonParser(GenericASTBuilder):
def p_expr(self, args):
'''
expr ::= _mklambda
expr ::= SET_LINENO
expr ::= LOAD_FAST
expr ::= LOAD_NAME
expr ::= LOAD_CONST
@ -424,6 +418,9 @@ class PythonParser(GenericASTBuilder):
expr ::= buildslice3
expr ::= yield
# Possibly Python < 2.3
# expr ::= SET_LINENO
binary_expr ::= expr expr binary_op
binary_op ::= BINARY_ADD
binary_op ::= BINARY_MULTIPLY

17
uncompyle6/parsers/parse2.py
View File

@ -207,13 +207,14 @@ class Python2Parser(PythonParser):
genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
'''
def p_import2(self, args):
'''
# These might be relevant for only Python 2.0 or so.
# Not relevant for Python 3.
importstar ::= LOAD_CONST LOAD_CONST IMPORT_NAME_CONT IMPORT_STAR
importfrom ::= LOAD_CONST LOAD_CONST IMPORT_NAME_CONT importlist2 POP_TOP
'''
# def p_import2(self, args):
# '''
# # These might be relevant for only Python 2.0 or so.
# importstar ::= LOAD_CONST LOAD_CONST IMPORT_NAME_CONT IMPORT_STAR
# importfrom ::= LOAD_CONST LOAD_CONST IMPORT_NAME_CONT importlist2 POP_TOP
# import_as_cont ::= IMPORT_NAME_CONT designator
# import_as_cont ::= IMPORT_NAME_CONT load_attrs designator
# '''
def p_expr2(self, args):
@ -262,6 +263,8 @@ class Python2Parser(PythonParser):
stmt ::= assign2_pypy
assign3_pypy ::= expr expr expr designator designator designator
assign2_pypy ::= expr expr designator designator
list_compr ::= expr BUILD_LIST_FROM_ARG _for designator list_iter
JUMP_BACK
""", nop_func)
continue
elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):

17
uncompyle6/parsers/parse26.py
View File

@ -226,3 +226,20 @@ if __name__ == '__main__':
# Check grammar
p = Python26Parser()
p.checkGrammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 2.6:
lhs, rhs, tokens, right_recursive = p.checkSets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))

27
uncompyle6/parsers/parse27.py
View File

@ -12,11 +12,17 @@ class Python27Parser(Python2Parser):
super(Python27Parser, self).__init__(debug_parser)
self.customized = {}
def p_list_comprehension27(self, args):
def p_comprehension27(self, args):
"""
list_for ::= expr _for designator list_iter JUMP_BACK
list_compr ::= expr BUILD_LIST_FROM_ARG _for designator list_iter JUMP_BACK
setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
JUMP_BACK RETURN_VALUE RETURN_LAST
dict_comp_body ::= expr expr MAP_ADD
set_comp_body ::= expr SET_ADD
# See also common Python p_list_comprehension
"""
def p_try27(self, args):
@ -75,3 +81,20 @@ if __name__ == '__main__':
# Check grammar
p = Python27Parser()
p.checkGrammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 2.7:
lhs, rhs, tokens, right_recursive = p.checkSets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# p.dumpGrammar()

7
uncompyle6/parsers/parse3.py
View File

@ -28,7 +28,7 @@ class Python3Parser(PythonParser):
super(Python3Parser, self).__init__(AST, 'stmts', debug=debug_parser)
self.new_rules = set()
def p_list_comprehension3(self, args):
def p_comprehension3(self, args):
"""
# Python3 scanner adds LOAD_LISTCOMP. Python3 does list comprehension like
# other comprehensions (set, dictionary).
@ -47,6 +47,11 @@ class Python3Parser(PythonParser):
jb_or_c ::= JUMP_BACK
jb_or_c ::= CONTINUE
setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
JUMP_BACK RETURN_VALUE RETURN_LAST
dict_comp_body ::= expr expr MAP_ADD
set_comp_body ::= expr SET_ADD
# See also common Python p_list_comprehension
"""