!551 精准补充

Merge pull request !551 from 糖宝儿/master

ci/generate_accurate_targets.py

@@ -18,7 +18,7 @@
 import os
 import sys
 import json
-from targetUtils import ChangeFileEntity, ComponentUtils, XTSUtils, XTSTargetUtils, PathUtils
+from targetUtils import ChangeFileEntity, ComponentUtils, XTSUtils, XTSTargetUtils, PathUtils, HOME
 
 
 class AccurateTarget:
@@ -52,10 +52,14 @@ class AccurateTarget:
 
         for item in data:
             changeFileEntity = ChangeFileEntity(name=data[item]["name"], path=item)
-            changeFileEntity.addAddPaths(data[item]["change_file_list"]["add"])
-            changeFileEntity.addModifiedPaths(data[item]["change_file_list"]["modified"])
-            changeFileEntity.addRenamePathsto(data[item]["change_file_list"]["rename"])
-            changeFileEntity.addDeletePaths(data[item]["change_file_list"]["delete"])
+            if "added" in data[item]["changed_file_list"]:
+                changeFileEntity.addAddPaths(data[item]["changed_file_list"]["added"])
+            if "modified" in data[item]["changed_file_list"]:
+                changeFileEntity.addModifiedPaths(data[item]["changed_file_list"]["modified"])
+            if "rename" in data[item]["changed_file_list"]:
+                changeFileEntity.addRenamePathsto(data[item]["changed_file_list"]["rename"])
+            if "deleted" in data[item]["changed_file_list"]:
+                changeFileEntity.addDeletePaths(data[item]["changed_file_list"]["deleted"])
             change_list.append(changeFileEntity)
         self._change_list = change_list
         return 0
@@ -66,12 +70,17 @@ class AccurateTarget:
         targets = []
         xts_u = XTSUtils(self._xts_root_dir, self._code_root_dir)
         for changeFileEntity in self._change_list:
+            # tools仓修改，编译全量
+            if changeFileEntity.path == "test/xts/tools":
+                xts_u._need_all = True
             if changeFileEntity.path in self.XTS_PATH_LIST and changeFileEntity.path in self._xts_root_dir:
                 # 只有当前编译的xts仓修改参与计算
                 ret = xts_u.getTargstsPaths(changeFileEntity)
                 if ret == 1:
                     print(f"{changeFileEntity.name}仓修改解析失败")
                     return 1, []
+        if xts_u._need_all:
+            xts_u._build_paths.append(self._xts_root_dir)
         return 0, xts_u._build_paths
 
     # 部件仓修改
@@ -96,7 +105,10 @@ class AccurateTarget:
         if ret == 1:
             # changeinfo读取失败-全量编译
             print (f"未获取到修改文件列表,编译全量代码")
-            target_paths = [self._xts_root_dir]
+            xts_suite = os.path.basename(self._xts_root_dir)
+            relative_path = os.path.relpath(self._xts_root_dir, HOME)
+            targets = [f"{relative_path}:xts_{xts_suite}"]
+
         else:
             func_list = [
                 self._get_targets_from_testcase_change
@@ -113,15 +125,15 @@ class AccurateTarget:
                     break
                 target_paths = target_paths.union(m)
 
-        # 去除子目录\重复目录
-        sum_path = PathUtils.removeSubandDumpPath(list(target_paths))
+            # 去除子目录\重复目录
+            sum_path = PathUtils.removeSubandDumpPath(list(target_paths))
 
-        targets = []
-        # 每个目录获取 target
-        for path in sum_path:
-            targets += XTSTargetUtils.getTargetfromPath(self._xts_root_dir, path)
+            targets = []
+            # 每个目录获取 target
+            for path in sum_path:
+                targets += XTSTargetUtils.getTargetfromPath(self._xts_root_dir, path)
 
-        return 0, list(targets)
+        return 0, targets
 
 
 def generate(xts_root_dir, change_info_file, build_target):

ci/targetUtils.py

@@ -23,6 +23,8 @@ import json
 HOME = os.path.dirname(os.path.dirname(os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
 
+MACTH_CONFIG = os.path.join(HOME,"test", "xts", "tools", "config", "ci_match_config.json")
+
 class ChangeFileEntity:
     def __init__(self, name, path):
         self.name = name
@@ -64,6 +66,58 @@ class ChangeFileEntity:
                 f"  delete: [\n    {delete_str}\n  ]\n"
                 f")")
 
+class MatchConfig:
+
+    exception_path = {}
+    all_com_path = {}
+    skip_judge_build_path = {}
+    temple_list = []
+    acts_All_template_ex_list = []
+
+    @classmethod
+    def initialization(cls, config_path):
+        if cls.exception_path == {} :
+            print("MatchConfig 开始初始化")
+            if not os.path.exists(config_path):
+                print(f"{config_path} 不存在,读取配置文件异常")
+            with open(config_path, 'r') as file:
+                rules_data = json.load(file)
+                cls.exception_path = rules_data['exception_path']
+                cls.all_com_path = rules_data['all_com_path']
+                cls.skip_judge_build_path = rules_data['skip_judge_build_path']
+                cls.temple_list = rules_data['temple_list']
+                cls.acts_All_template_ex_list = rules_data['acts_All_template_ex']
+        print("MatchConfig 已完成初始化")
+
+    @classmethod
+    def get_exception_path(cls, config_path):
+        if cls.exception_path == {}:
+            cls.initialization(config_path)
+        return cls.exception_path
+
+    @classmethod
+    def get_all_com_path(cls, config_path):
+        if cls.all_com_path == {}:
+            cls.initialization(config_path)
+        return cls.all_com_path
+
+    @classmethod
+    def get_skip_judge_build_path(cls, config_path):
+        if cls.skip_judge_build_path == {}:
+            cls.initialization(config_path)
+        return cls.skip_judge_build_path
+
+    @classmethod
+    def get_temple_list(cls, config_path):
+        if cls.temple_list == []:
+            cls.initialization(config_path)
+        return cls.temple_list
+
+    @classmethod
+    def get_acts_All_template_ex_list(cls, config_path):
+        if cls.acts_All_template_ex_list == []:
+            cls.initialization(config_path)
+        return cls.acts_All_template_ex_list
 
 class ComponentUtils:
     
@@ -95,6 +149,7 @@ class XTSUtils:
         self._xts_root_dir = xts_root_dir
         self._code_root_dir = code_root_dir
         self._build_paths = []
+        self._need_all = False
 
 
     # 获取path接口
@@ -104,7 +159,7 @@ class XTSUtils:
             # file转为绝对路径
             file = os.path.join(self._code_root_dir, file)
             # 筛选掉例外的目录
-            if PathUtils.isContainsKeywords(file, XTSTargetUtils.EXCEPTION_PATH_LIST):
+            if PathUtils.isMatchRules(file, MatchConfig.get_exception_path(MACTH_CONFIG)):
                 continue
             # 当前文件路径或父已存在,跳过
             if PathUtils.isTargetContains(self._build_paths, file):
@@ -112,72 +167,39 @@ class XTSUtils:
             # 当前file对应BUILD.gn路径
             build_File = XTSTargetUtils.get_current_Build(self._xts_root_dir, file)
             # 计算到根目录或指定目录,直接编译全量
-            print(f"file: {file}")
-            print(f"ALL_COM_PATH_LIST: {XTSTargetUtils.ALL_COM_PATH_LIST}")
-            print(f"HOME: {HOME}")
             if (os.path.dirname(build_File) == self._xts_root_dir or 
-                PathUtils.isContainsKeywords(file, XTSTargetUtils.ALL_COM_PATH_LIST)):
-                self._build_paths = [self._xts_root_dir]
-                print (f"文件: {file} 修改需编译全量代码")
-                return 0
-            self._build_paths.append(os.path.dirname(build_File))
+                PathUtils.isMatchRules(file, MatchConfig.get_all_com_path(MACTH_CONFIG))):
+                self._need_all = True
+            else:
+                self._build_paths.append(os.path.dirname(build_File))
         # 删除
         for file in changeFileEntity.delete:
             # file转为绝对路径
             file = os.path.join(self._code_root_dir, file)
             # 筛选掉例外的目录
-            if PathUtils.isContainsKeywords(file, XTSTargetUtils.EXCEPTION_PATH_LIST):
+            if PathUtils.isMatchRules(file, MatchConfig.get_exception_path(MACTH_CONFIG)):
                 continue
             # 当前文件路径或父已存在,跳过
             if PathUtils.isTargetContains(self._build_paths, file):
                 continue
             # 当前存在的最外层路径
-            exist_path = PathUtils.get_current_exist(os.path.dirname(file))
+            exist_path = PathUtils.get_current_exist(self._xts_root_dir, os.path.dirname(file))
             build_File = XTSTargetUtils.get_current_Build(self._xts_root_dir, exist_path)
             # 计算到根目录或指定目录,直接编译全量
             if (os.path.dirname(build_File) == self._xts_root_dir or 
-                PathUtils.isContainsKeywords(file, XTSTargetUtils.ALL_COM_PATH_LIST)):
-                self._build_paths = [self._xts_root_dir]
-                print (f"文件: {file} 删除需编译全量代码")
-                return 0
-            self._build_paths.append(os.path.dirname(build_File))
+                PathUtils.isMatchRules(file, MatchConfig.get_all_com_path(MACTH_CONFIG))):
+                self._need_all = True
+            else:
+                self._build_paths.append(os.path.dirname(build_File))
         return 0
 
 class XTSTargetUtils:
-    # 不需要编译的目录,包括deploy_testtools(所有的都编),lite
-    EXCEPTION_PATH_LIST = [
-        "/acts/testtools/"
-        "_lite/",
-        "/acts/applications/kitframework_ipcamera/",
-        "/acts/applications/kitframework/",
-        "/acts/open_posix_testsuite/"
-    ]
-
-    ALL_COM_PATH_LIST = [
-        "/acts/build/"
-    ]
-
-    SKIP_JUDGE_PATH_LIST = [
-        "/cpp"
-    ]
-
-    TEMPLATE_LIST = [
-        "ohos_moduletest_suite",
-        "ohos_js_hap_suite",
-        "ohos_js_app_suite",
-        "ohos_shell_app_suite",
-        "ohos_hap_assist_suite",
-        "ohos_app_assist_suite",
-        "ohos_sh_assist_suite",
-        "ohos_static_library",
-        "group"
-    ]
 
     @staticmethod
     def get_current_Build(xts_root_dir, current_dir):
         while PathUtils.is_parent_path(xts_root_dir, current_dir):
             # 当前目录是否包含需跳过的keywords
-            if PathUtils.isContainsKeywords(current_dir, XTSTargetUtils.SKIP_JUDGE_PATH_LIST):
+            if PathUtils.isMatchRules(current_dir, MatchConfig.get_skip_judge_build_path(MACTH_CONFIG)):
                 current_dir = os.path.dirname(current_dir)
                 continue
             # 检查当前目录下是否存在BUILD.gn文件
@@ -193,6 +215,8 @@ class XTSTargetUtils:
     @staticmethod
     def getTargetfromPath(xts_root_dir, path) -> list:
         if path == xts_root_dir:
+            if path.endswith("acts"):
+                return MatchConfig.get_acts_All_template_ex_list(MACTH_CONFIG)
             xts_suite = os.path.basename(xts_root_dir)
             relative_path = os.path.relpath(xts_root_dir, HOME)
             return [f"{relative_path}:xts_{xts_suite}"]
@@ -204,7 +228,7 @@ class XTSTargetUtils:
 
     @staticmethod
     def getTargetFromBuild(build_File) -> list:
-        pattern = re.compile(r'(\b(?:' + '|'.join(re.escape(word) for word in XTSTargetUtils.TEMPLATE_LIST) + r')\b)\("([^"]*)"\)')
+        pattern = re.compile(r'(\b(?:' + '|'.join(re.escape(word) for word in MatchConfig.get_temple_list(MACTH_CONFIG)) + r')\b)\("([^"]*)"\)')
         with open(build_File, 'r', encoding='utf-8') as file:
             content = file.read()
         matches = pattern.findall(content)
@@ -314,9 +338,14 @@ class PathUtils:
         return common_path == parent_path
 
     @staticmethod
-    def isContainsKeywords(file, keywords):
-        for keyword in keywords:
-            if keyword in file:
+    def isMatchRules(file, rules):
+        string_rules = rules["string_rules"]
+        re_rules = rules["re_rules"]
+        for rule in string_rules:
+            if rule in file:
+                return True
+        for rule in re_rules:
+            if re.compile(rule).search(file):
                 return True
         return False
 

config/ci_match_config.json

@@ -0,0 +1,46 @@
+{
+    "exception_path":{
+        "string_rules":[
+            "_lite/",
+            "/acts/applications/kitframework_ipcamera/",
+            "/acts/applications/kitframework/",
+            "/acts/open_posix_testsuite/"
+        ],
+        "re_rules":[
+            "/[a-z]{3}s/testtools/"
+        ]
+    },
+
+    "all_com_path":{
+        "string_rules":[
+        ],
+        "re_rules":[
+            "/[a-z]{3}s/build/"
+        ]
+    },
+    "skip_judge_build_path":{
+        "string_rules":[
+            "/cpp"
+        ],
+        "re_rules":[
+        ]
+    },
+    "temple_list":[
+        "ohos_moduletest_suite",
+        "ohos_js_hap_suite",
+        "ohos_js_app_suite",
+        "ohos_shell_app_suite",
+        "ohos_hap_assist_suite",
+        "ohos_app_assist_suite",
+        "ohos_sh_assist_suite",
+        "group"
+    ],
+    "acts_All_template_ex":[
+        "ActsMindSporeTest",
+        "ActsBmsAccessTokenTest",
+        "AACommandImplicitStartTest",
+        "StageSupplement",
+        "AACommand07",
+        "cpLibuv"
+    ]
+}