python-uncompyle6/pytest/test_grammar.py

import re
from uncompyle6.parser import get_python_parser, python_parser
from uncompyle6.scanner import get_scanner
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY


def test_grammar():
    def check_tokens(tokens, opcode_set):
        remain_tokens = set(tokens) - opcode_set
        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
        remain_tokens = set(remain_tokens) - opcode_set
        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
            remain_tokens,
            p.dump_grammar(),
        )

    p = get_python_parser(PYTHON_VERSION_TRIPLE, is_pypy=IS_PYPY)
    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()

    # We have custom rules that create the below
    expect_lhs = set(["pos_arg"])

    if PYTHON_VERSION_TRIPLE < (3, 8):
        if PYTHON_VERSION_TRIPLE < (3, 7):
            expect_lhs.add("attribute")

        expect_lhs.add("get_iter")

        if PYTHON_VERSION_TRIPLE >= (3, 8) or PYTHON_VERSION_TRIPLE < (3, 0):
            expect_lhs.add("stmts_opt")
    else:
        expect_lhs.add("async_with_as_stmt")
        expect_lhs.add("async_with_stmt")

    unused_rhs = set(["list", "mkfunc", "lambda_body", "unpack"])

    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])

    if PYTHON_VERSION_TRIPLE[:2] <= (3, 6):
        unused_rhs.add("call")

    if PYTHON_VERSION_TRIPLE >= (2, 7):
        expect_lhs.add("kvlist")
        expect_lhs.add("kv3")
        unused_rhs.add("dict")

        if PYTHON_VERSION_TRIPLE < (3, 7) and PYTHON_VERSION_TRIPLE[:2] != (2, 7):
            # NOTE: this may disappear
            expect_lhs.add("except_handler_else")

    expect_lhs.add("load_genexpr")

    unused_rhs = unused_rhs.union(
        set(
            """
    except_pop_except generator_exp
    """.split()
        )
    )
    if PYTHON_VERSION_TRIPLE < (3, 7):
        expect_lhs.add("annotate_arg")
        expect_lhs.add("annotate_tuple")
        unused_rhs.add("mkfunc_annotate")

    unused_rhs.add("dict_comp")
    unused_rhs.add("classdefdeco1")
    unused_rhs.add("tryelsestmtl")
    if PYTHON_VERSION_TRIPLE >= (3, 5):
        expect_right_recursive.add(
            (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
        )
        pass
    pass

    if PYTHON_VERSION_TRIPLE >= (3, 7):
        expect_lhs.add("set_for")
        unused_rhs.add("set_iter")
        pass
    pass
    # FIXME
    if PYTHON_VERSION_TRIPLE < (3, 8):
        assert expect_lhs == set(lhs)
        assert unused_rhs == set(rhs)

    assert expect_right_recursive == right_recursive

    expect_dup_rhs = frozenset(
        [
            ("COME_FROM",),
            ("CONTINUE",),
            ("JUMP_ABSOLUTE",),
            ("LOAD_CONST",),
            ("JUMP_BACK",),
            ("JUMP_FORWARD",),
        ]
    )
    reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
    if reduced_dup_rhs:
        print("\nPossible duplicate RHS that might be folded, into one of the LHS symbols")
        for k in reduced_dup_rhs:
            print(k, reduced_dup_rhs[k])
    # assert not reduced_dup_rhs, reduced_dup_rhs

    s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
    ignore_set = set(
        """
            JUMP_BACK CONTINUE
            COME_FROM COME_FROM_EXCEPT
            COME_FROM_EXCEPT_CLAUSE
            COME_FROM_LOOP COME_FROM_WITH
            COME_FROM_FINALLY ELSE
            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
            LAMBDA_MARKER
            RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
            """.split()
    )

    if (2, 6) <= PYTHON_VERSION_TRIPLE <= (2, 7):
        opcode_set = set(s.opc.opname).union(ignore_set)
        if PYTHON_VERSION_TRIPLE[:2] == (2, 6):
            opcode_set.add("THEN")
        check_tokens(tokens, opcode_set)
    elif PYTHON_VERSION_TRIPLE[:2] == (3, 4):
        ignore_set.add("LOAD_CLASSNAME")
        ignore_set.add("STORE_LOCALS")
        opcode_set = set(s.opc.opname).union(ignore_set)
        check_tokens(tokens, opcode_set)


def test_dup_rule():
    import inspect

    python_parser(
        PYTHON_VERSION_TRIPLE,
        inspect.currentframe().f_code,
        is_pypy=IS_PYPY,
        parser_debug={
            "dups": True,
            "transition": False,
            "reduce": False,
            "rules": False,
            "errorstack": None,
            "context": True,
        },
    )