Update treesitter to 0.26.0 (#3138)

Update query API
The argument tree in check_patching_result() is now of type Tree
Corrected the predicates in BitCastStdArray
Sorted imports and fixed some linter warnings

suite/auto-sync/pyproject.toml

@@ -7,7 +7,7 @@ name = "autosync"
 version = "0.1.0"
 dependencies = [
   "termcolor >= 2.3.0",
-  "tree_sitter == 0.24.0",
+  "tree_sitter == 0.26.0",
   "tree-sitter-cpp == 0.23.4",
   "black >= 24.3.0",
   "usort >= 1.0.8",

suite/auto-sync/src/autosync/cpptranslator/CppTranslator.py

@@ -5,8 +5,8 @@
 
 import argparse
 import logging as log
-import sys
 import re
+import sys
 from pathlib import Path
 
 import termcolor
@@ -16,6 +16,7 @@ from autosync.cpptranslator.Configurator import Configurator
 from autosync.cpptranslator.patches.AddCSDetail import AddCSDetail
 from autosync.cpptranslator.patches.AddOperand import AddOperand
 from autosync.cpptranslator.patches.Assert import Assert
+from autosync.cpptranslator.patches.BadConditionCode import BadConditionCode
 from autosync.cpptranslator.patches.BitCastStdArray import BitCastStdArray
 from autosync.cpptranslator.patches.CheckDecoderStatus import CheckDecoderStatus
 from autosync.cpptranslator.patches.ClassConstructorDef import ClassConstructorDef
@@ -48,10 +49,10 @@ from autosync.cpptranslator.patches.InlineToStaticInline import InlineToStaticIn
 from autosync.cpptranslator.patches.IsOptionalDef import IsOptionalDef
 from autosync.cpptranslator.patches.IsPredicate import IsPredicate
 from autosync.cpptranslator.patches.IsRegImm import IsOperandRegImm
+from autosync.cpptranslator.patches.isUInt import IsUInt
+from autosync.cpptranslator.patches.LLVM_DEBUG import LLVM_DEBUG
 from autosync.cpptranslator.patches.LLVMFallThrough import LLVMFallThrough
 from autosync.cpptranslator.patches.LLVMunreachable import LLVMUnreachable
-from autosync.cpptranslator.patches.BadConditionCode import BadConditionCode
-from autosync.cpptranslator.patches.LLVM_DEBUG import LLVM_DEBUG
 from autosync.cpptranslator.patches.MethodToFunctions import MethodToFunction
 from autosync.cpptranslator.patches.MethodTypeQualifier import MethodTypeQualifier
 from autosync.cpptranslator.patches.NamespaceAnon import NamespaceAnon
@@ -83,6 +84,7 @@ from autosync.cpptranslator.patches.TemplateRefs import TemplateRefs
 from autosync.cpptranslator.patches.UseMarkup import UseMarkup
 from autosync.cpptranslator.patches.UsingDeclaration import UsingDeclaration
 from autosync.cpptranslator.TemplateCollector import TemplateCollector
+from autosync.cpptranslator.tree_sitter_compatibility import query_captures_22_3
 from autosync.Helper import (
     convert_loglevel,
     get_header,
@@ -90,8 +92,6 @@ from autosync.Helper import (
     print_prominent_warning,
     run_clang_format,
 )
-from autosync.cpptranslator.patches.isUInt import IsUInt
-from autosync.cpptranslator.tree_sitter_compatibility import query_captures_22_3
 
 
 class Translator:
@@ -434,7 +434,7 @@ class Translator:
                 # Each patch has a capture which includes the whole subtree searched for.
                 # Additionally, it can include captures within this subtree.
                 # Here we bundle these captures together.
-                query: Query = self.ts_cpp_lang.query(pattern)
+                query: Query = Query(self.ts_cpp_lang, pattern)
                 captures_bundle: [[(Node, str)]] = list()
                 for q in query_captures_22_3(query, self.tree.root_node):
                     if q[1] == patch.get_main_capture_name():

suite/auto-sync/src/autosync/cpptranslator/TemplateCollector.py

@@ -3,6 +3,7 @@
 
 import logging as log
 import re
+import sys
 from pathlib import Path
 
 from tree_sitter import Language, Node, Parser, Query
@@ -20,7 +21,7 @@ class TemplateRefInstance:
     name: bytes
     args: bytes
     args_list: list
-    dependent_calls = list()
+    dependent_calls = []
 
     # Holds the indices of the caller template parameters which set the templ. parameters
     # of this TemplateCallInstance.
@@ -28,7 +29,7 @@ class TemplateRefInstance:
     #
     # Only used if this is an incomplete TemplateInstance
     # (parameters are set by the template parameters of the calling function).
-    caller_param_indices: [{str: int}] = list()
+    caller_param_indices: list[dict[str, int]] = []
 
     def __init__(
         self, name: bytes, args: bytes, start_point, start_byte, end_point, end_byte
@@ -47,12 +48,8 @@ class TemplateRefInstance:
             self.name == other.name
             and self.args == other.args
             and any(
-                [
-                    a == b
-                    for a, b in zip(
-                        self.caller_param_indices, other.caller_param_indices
-                    )
-                ]
+                a == b
+                for a, b in zip(self.caller_param_indices, other.caller_param_indices)
             )
             and self.start_byte == other.start_byte
             and self.start_point == other.start_point
@@ -82,17 +79,17 @@ class TemplateCollector:
 
     # List of completed template instances indexed by their name.
     # One function can have multiple template instances. Depending on the template arguments
-    template_refs: {bytes: [TemplateRefInstance]} = dict()
+    template_refs: dict[bytes, list[TemplateRefInstance]] = {}
     # List of incomplete template instances indexed by the **function name they depend on**!
-    incomplete_template_refs: {bytes: [TemplateRefInstance]} = dict()
-    sources: [{str: bytes}] = list()
+    incomplete_template_refs: dict[bytes, list[TemplateRefInstance]] = {}
+    sources: list[dict[str, bytes]] = []
 
     def __init__(
         self,
         ts_parser: Parser,
         ts_cpp: Language,
-        searchable_files: [Path],
-        temp_arg_deduction: [bytes],
+        searchable_files: list[Path],
+        temp_arg_deduction: list[bytes],
     ):
         self.parser = ts_parser
         self.lang_cpp = ts_cpp
@@ -107,7 +104,7 @@ class TemplateCollector:
             log.debug(f"Search for template references in {path}")
 
             tree = self.parser.parse(src)
-            query: Query = self.lang_cpp.query(self.get_template_pattern())
+            query: Query = Query(self.lang_cpp, self.get_template_pattern())
             capture_bundles = self.get_capture_bundles(query, tree)
 
             for cb in capture_bundles:
@@ -131,7 +128,7 @@ class TemplateCollector:
 
                 if not self.contains_template_dependent_param(src, ti, cb[0]):
                     if name not in self.template_refs:
-                        self.template_refs[name] = list()
+                        self.template_refs[name] = []
                     # The template function has no parameter which is part of a previous
                     # template definition. So all template parameters are well-defined.
                     # Add it to the well-defined list.
@@ -148,8 +145,8 @@ class TemplateCollector:
         ):
             # Dict with new template calls which were previously incomplete
             # because one or more parameters were unknown.
-            new_completed_tcs: {str: list} = dict()
-            tc_instance_list: [TemplateRefInstance]
+            new_completed_tcs: dict[str, list] = {}
+            tc_instance_list: list[TemplateRefInstance]
             for caller_name, tc_instance_list in self.template_refs.items():
                 # Check if this caller has a dependent template call.
                 # In other words: If a template parameter of this caller is given
@@ -166,7 +163,7 @@ class TemplateCollector:
                         )
                         callee_name = new_tc.name
                         if callee_name not in new_completed_tcs:
-                            new_completed_tcs[callee_name] = list()
+                            new_completed_tcs[callee_name] = []
                         if new_tc not in new_completed_tcs[callee_name]:
                             new_completed_tcs[callee_name].append(new_tc)
                 del self.incomplete_template_refs[caller_name]
@@ -206,7 +203,7 @@ class TemplateCollector:
         return new_tc
 
     def contains_template_dependent_param(
-        self, src, ti: TemplateRefInstance, parse_tree: (Node, str)
+        self, src, ti: TemplateRefInstance, parse_tree: tuple[Node, str]
     ) -> bool:
         """Here we check if one of the template parameters of the given template call,
         is a parameter of the callers template definition.
@@ -237,7 +234,7 @@ class TemplateCollector:
         node: Node = parse_tree[0]
         while node.type != "function_definition":
             node = node.parent
-        if not node.prev_named_sibling.type == "template_parameter_list":
+        if node.prev_named_sibling.type != "template_parameter_list":
             # Caller is a normal function definition.
             # Nothing to do here.
             return False
@@ -262,7 +259,7 @@ class TemplateCollector:
             return False
 
         if caller_fcn_name not in self.incomplete_template_refs:
-            self.incomplete_template_refs[caller_fcn_name] = list()
+            self.incomplete_template_refs[caller_fcn_name] = []
         if ti not in self.incomplete_template_refs[caller_fcn_name]:
             self.incomplete_template_refs[caller_fcn_name].append(ti)
         return True
@@ -271,7 +268,7 @@ class TemplateCollector:
         for sf in self.searchable_files:
             if not Path.exists(sf):
                 log.fatal(f"TemplateCollector: Could not find '{sf}' for search.")
-                exit(1)
+                sys.exit(1)
             log.debug(f"TemplateCollector: Read {sf}")
             with open(sf) as f:
                 file = {"path": sf, "content": bytes(f.read(), "utf8")}
@@ -279,7 +276,7 @@ class TemplateCollector:
 
     @staticmethod
     def get_capture_bundles(query, tree):
-        captures_bundle: list[list[tuple[Node, str]]] = list()
+        captures_bundle: list[list[tuple[Node, str]]] = []
         for q in query_captures_22_3(query, tree.root_node):
             if q[1] == "templ_ref":
                 captures_bundle.append([q])
@@ -302,11 +299,11 @@ class TemplateCollector:
     @staticmethod
     def templ_params_to_list(templ_params: bytes) -> list[bytes]:
         if not templ_params:
-            return list()
+            return []
 
         params = templ_params.strip(b"<>").split(b",")
         params = [p.strip() for p in params]
-        res = list()
+        res = []
         for p in params:
             if len(p.split(b" ")) == 2:
                 # Typename specified for parameter. Remove it.
@@ -325,7 +322,9 @@ class TemplateCollector:
         return b"<" + b", ".join(temp_param_list) + b">"
 
     @staticmethod
-    def get_macro_c_call(name: bytes, arg_param_list: [bytes], fcn_args: bytes = b""):
+    def get_macro_c_call(
+        name: bytes, arg_param_list: list[bytes], fcn_args: bytes = b""
+    ):
         res = b""
         fa = [name] + arg_param_list
         for x in fa[:-1]:
@@ -343,4 +342,4 @@ class TemplateCollector:
             "\t\t\t- The function is a template with argument deduction and has no `template<...>` preamble. "
             "Add it in the config as exception in this case."
         )
-        exit(1)
+        sys.exit(1)

suite/auto-sync/src/autosync/cpptranslator/Tests/test_patches.py

@@ -5,16 +5,16 @@
 # SPDX-License-Identifier: BSD-3
 
 import unittest
-
 from pathlib import Path
-from tree_sitter import Node, Query
+
+from tree_sitter import Node, Query, Tree
 
 from autosync.cpptranslator import CppTranslator
-
 from autosync.cpptranslator.Configurator import Configurator
 from autosync.cpptranslator.patches.AddCSDetail import AddCSDetail
 from autosync.cpptranslator.patches.AddOperand import AddOperand
 from autosync.cpptranslator.patches.Assert import Assert
+from autosync.cpptranslator.patches.BadConditionCode import BadConditionCode
 from autosync.cpptranslator.patches.BitCastStdArray import BitCastStdArray
 from autosync.cpptranslator.patches.CheckDecoderStatus import CheckDecoderStatus
 from autosync.cpptranslator.patches.ClassesDef import ClassesDef
@@ -46,16 +46,17 @@ from autosync.cpptranslator.patches.InlineToStaticInline import InlineToStaticIn
 from autosync.cpptranslator.patches.IsOptionalDef import IsOptionalDef
 from autosync.cpptranslator.patches.IsPredicate import IsPredicate
 from autosync.cpptranslator.patches.IsRegImm import IsOperandRegImm
-from autosync.cpptranslator.patches.LLVMFallThrough import LLVMFallThrough
+from autosync.cpptranslator.patches.isUInt import IsUInt
 from autosync.cpptranslator.patches.LLVM_DEBUG import LLVM_DEBUG
+from autosync.cpptranslator.patches.LLVMFallThrough import LLVMFallThrough
 from autosync.cpptranslator.patches.LLVMunreachable import LLVMUnreachable
-from autosync.cpptranslator.patches.Override import Override
 from autosync.cpptranslator.patches.MethodToFunctions import MethodToFunction
 from autosync.cpptranslator.patches.MethodTypeQualifier import MethodTypeQualifier
 from autosync.cpptranslator.patches.NamespaceAnon import NamespaceAnon
 from autosync.cpptranslator.patches.NamespaceArch import NamespaceArch
 from autosync.cpptranslator.patches.NamespaceLLVM import NamespaceLLVM
 from autosync.cpptranslator.patches.OutStreamParam import OutStreamParam
+from autosync.cpptranslator.patches.Override import Override
 from autosync.cpptranslator.patches.PredicateBlockFunctions import (
     PredicateBlockFunctions,
 )
@@ -79,10 +80,8 @@ from autosync.cpptranslator.patches.TemplateRefs import TemplateRefs
 from autosync.cpptranslator.patches.UseMarkup import UseMarkup
 from autosync.cpptranslator.patches.UsingDeclaration import UsingDeclaration
 from autosync.cpptranslator.TemplateCollector import TemplateCollector
-from autosync.cpptranslator.patches.BadConditionCode import BadConditionCode
-from autosync.Helper import get_path
-from autosync.cpptranslator.patches.isUInt import IsUInt
 from autosync.cpptranslator.tree_sitter_compatibility import query_captures_22_3
+from autosync.Helper import get_path
 
 
 class TestPatches(unittest.TestCase):
@@ -97,16 +96,21 @@ class TestPatches(unittest.TestCase):
         )
 
     def check_patching_result(
-        self, patch, syntax, expected, filename: Path = None, tree: dict = None
+        self,
+        patch,
+        syntax: bytes,
+        expected,
+        filename: Path | None = None,
+        tree: Tree | None = None,
     ):
         kwargs = self.translator.get_patch_kwargs(patch)
         if filename:
             kwargs["filename"] = filename
 
-        query: Query = self.ts_cpp_lang.query(patch.get_search_pattern())
+        query: Query = Query(self.ts_cpp_lang, patch.get_search_pattern())
         tree = self.parser.parse(syntax)
         kwargs["tree"] = tree
-        captures_bundle: [[(Node, str)]] = list()
+        captures_bundle: list[list[tuple[Node, str]]] = []
         for q in query_captures_22_3(query, tree.root_node):
             if q[1] == patch.get_main_capture_name():
                 captures_bundle.append([q])
@@ -485,7 +489,7 @@ public:
             b"VPTBlock_instrInVPTBlock(&(MI->csh->VPTBlock))",
         )
 
-    def test_predicateblockfunctions(self):
+    def test_printAnnotation(self):
         patch = PrintAnnotation(0)
         syntax = b"printAnnotation();"
         self.check_patching_result(patch, syntax, b"")

suite/auto-sync/src/autosync/cpptranslator/patches/BitCastStdArray.py

@@ -31,9 +31,9 @@ class BitCastStdArray(Patch):
             "       (identifier) @arr_name"
             "       (call_expression"
             "           (template_function"
-            '               ((identifier) @tfid (#eq @tfid "bit_cast"))'
+            '               ((identifier) @tfid (#eq? @tfid "bit_cast"))'
             "               (template_argument_list"
-            '                   ((type_descriptor) @td (#match @td "std::array<.*>"))'
+            '                   ((type_descriptor) @td (#match? @td "std::array<.*>"))'
             "                )"
             "           )"
             "           (argument_list) @cast_target"

suite/auto-sync/src/autosync/cpptranslator/patches/StreamOperation.py

@@ -1,7 +1,7 @@
 # Copyright © 2022 Rot127 <[email protected]>
 # SPDX-License-Identifier: BSD-3
 
-from tree_sitter import Node
+from tree_sitter import Node, Query, QueryCursor
 
 from autosync.cpptranslator.patches.Helper import get_text, get_text_from_node
 from autosync.cpptranslator.patches.Patch import Patch
@@ -33,7 +33,7 @@ class StreamOperations(Patch):
     def get_patch(self, captures: [(Node, str)], src: bytes, **kwargs) -> bytes:
         bin_expr = captures[1][0]
         # Extract operands passed to the stream into a list.
-        ops = list()
+        ops = []
         while bin_expr.type == "binary_expression":
             ops.append(bin_expr.named_children[1])
             bin_expr = bin_expr.named_children[0]
@@ -49,7 +49,7 @@ class StreamOperations(Patch):
         # SStream_concat0 - Copies a string and null terminates the buffer.
         last_op: Node = ops[-1]
         op: Node = ops[0]
-        string_ops = list()
+        string_ops = []
         i = 0
         while op != last_op:
             if op.type == "char_literal":
@@ -114,12 +114,13 @@ class StreamOperations(Patch):
                     )
                 )) @decl
 """
-            query = kwargs["ts_cpp_lang"].query(queue_str)
+            query = Query(kwargs["ts_cpp_lang"], queue_str)
             root_node = kwargs["tree"].root_node
+            cursor = QueryCursor(query)
             query_result = list(
                 filter(
                     lambda x: "typ" in x[1],
-                    query.matches(root_node),
+                    cursor.matches(root_node),
                 )
             )
             if len(query_result) == 0:

suite/auto-sync/src/autosync/cpptranslator/tree_sitter_compatibility.py

@@ -1,14 +1,15 @@
-from tree_sitter import Node, Query
+from tree_sitter import Node, Query, QueryCursor
 
 
 # Queries for the given pattern and converts the query back to the tree-siter v22.3 format.
 # Which is: A list of tuples where the first element is the
 # Node of the capture and the second one is the name.
 def query_captures_22_3(query: Query, node: Node) -> list[tuple[Node, str]]:
-    result = list()
-    captures = query.captures(node)
+    result = []
+    cursor = QueryCursor(query)
+    captures = cursor.captures(node)
     # Captures are no longer sorted by start point.
-    captures_sorted = dict()
+    captures_sorted = {}
     nodes: list[Node]
     for name, nodes in captures.items():
         captures_sorted[name] = sorted(nodes, key=lambda n: n.start_point)