!112 [rom_ram_analzyer]change the process configuration file from xml to json

Merge pull request !112 from aodongbiao/master

tools/rom_ram_analyzer/lite_small/README.md

@@ -15,7 +15,7 @@
 
 ## 代码思路
 
-1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中
+1. 扫描BUILD.gn文件,收集各个target的编译产物及其对应的component_name, subsystem_name信息,并存储到config.yaml中的gn_info_file字段指定的json文件中。如果BUILD.gn中没有查找到，则直接使用get_subsytem_component.py中预先收集好的数据（根据bundle.json）
 2. 根据配置文件config.yaml扫描产品的编译产物目录,得到真实的编译产物信息(主要是大小)
 3. 用真实的编译产物与从BUILD.gn中收集的信息进行匹配,从而得到编译产物-大小-所属部件的对应信息
 4. 如果匹配失败,会直接利用grep到项目路径下进行模糊搜索,取出现次数top1的BUILD.gn,并根据该BUILD.gn文件去查找子系统和部件

tools/rom_ram_analyzer/lite_small/src/rom_analysis.py

@@ -88,7 +88,7 @@ class RomAnalysisTool:
         if (not sub_path) or (os.sep not in sub_path):
             return
         # 将其他目录添加到dir_list
-        t, sub_sub_path = sub_path.split(os.sep, 1) # 如果是c/e,分割成c,e
+        t, sub_sub_path = sub_path.split(os.sep, 1)  # 如果是c/e,分割成c,e
         t = os.path.join(rela_path, t)
         if t in dir_list:
             dir_list.remove(t)
@@ -107,7 +107,7 @@ class RomAnalysisTool:
             logging.error(
                 f"product_name '{product_name}' not found in the config.yaml")
             exit(1)
-        product_path_dit: Dict[str, str] = dict() # 存储编译产物的类型及目录
+        product_path_dit: Dict[str, str] = dict()  # 存储编译产物的类型及目录
         root_dir = product_dir.get("root")
         root_dir = os.path.join(project_path, root_dir)
         relative_dir: Dict[str, str] = product_dir.get("relative")
@@ -120,14 +120,14 @@ class RomAnalysisTool:
             product_path_dit[k] = os.path.join(root_dir, v)
         # 查找编译产物信息
         # product_dict格式: {"so": ["a.so", "b.so"]}
-        product_dict: Dict[str, List[str]] = dict() # 存储编译产物的名称
+        product_dict: Dict[str, List[str]] = dict()  # 存储编译产物的名称
         for k, v in product_path_dit.items():
             if not os.path.exists(v):
                 logging.warning(f"dir '{v}' not exist")
-            product_dict[k] = BasicTool.find_files_with_pattern(v) # v是全路径
+            product_dict[k] = BasicTool.find_files_with_pattern(v)  # v是全路径
         if product_dir.get("rest"):
             rest_dir_list: List[str] = os.listdir(
-                root_dir) # 除了配置在relative下之外的所有剩余目录,全部归到etc下
+                root_dir)  # 除了配置在relative下之外的所有剩余目录,全部归到etc下
             for v in relative_dir.values():
                 if v in rest_dir_list:
                     rest_dir_list.remove(v)
@@ -152,14 +152,14 @@ class RomAnalysisTool:
     @classmethod
     def _put(cls, sub: str, com: str, unit: Dict, rom_size_dict: Dict, com_size_baseline: str = str()):
         size = unit.get("size")
-        if not rom_size_dict.get("size"): # 总大小
+        if not rom_size_dict.get("size"):  # 总大小
             rom_size_dict["size"] = 0
-        if not rom_size_dict.get(sub): # 子系统大小
+        if not rom_size_dict.get(sub):  # 子系统大小
             rom_size_dict[sub]: Dict[str, Dict] = dict()
             rom_size_dict[sub]["size"] = 0
             rom_size_dict[sub]["count"] = 0
 
-        if not rom_size_dict.get(sub).get(com): # 部件
+        if not rom_size_dict.get(sub).get(com):  # 部件
             rom_size_dict.get(sub)[com] = dict()
             rom_size_dict[sub][com]["filelist"] = list()
             rom_size_dict[sub][com]["size"] = 0
@@ -329,7 +329,70 @@ class RomAnalysisTool:
                 del v[index]
                 break
 
-    @ classmethod
+    @classmethod
+    def _iterate_all_template_type(cls, type_list: List[str], gn_info: Dict, gn_info_file: str, base_name: str, rom_ram_baseline: Dict, rom_size_dict: Dict, f: str, size: int):
+        find_flag = False
+        component_rom_baseline = None
+        for tn in type_list:  # tn example: ohos_shared_library
+            if find_flag:  # 如果已经在前面的template中找到了,后面的就不必再查找
+                break
+            output_dict: Dict[str, Dict] = gn_info.get(
+                tn)  # 这个模板对应的所有可能编译产物
+            if not output_dict:
+                logging.warning(
+                    f"'{tn}' not found in the {gn_info_file}")
+                continue
+            d = output_dict.get(base_name)
+            if not d:
+                continue
+            d["size"] = size
+            d["file_name"] = f.replace(project_path, "")
+            if rom_ram_baseline.get(d["subsystem_name"]) and rom_ram_baseline.get(d["subsystem_name"]).get(d["component_name"]):
+                component_rom_baseline = rom_ram_baseline.get(
+                    d["subsystem_name"]).get(d["component_name"]).get("rom")
+            cls._put(d["subsystem_name"],
+                     d["component_name"], d, rom_size_dict, component_rom_baseline)
+            find_flag = True
+        if not find_flag:  # 如果指定序列中的template都没有查找到,则模糊匹配
+            # fuzzy match
+            psesudo_gn, sub, com = cls._fuzzy_match(f)
+            if sub and com:
+                if rom_ram_baseline.get(sub) and rom_ram_baseline.get(sub).get(com):
+                    component_rom_baseline = rom_ram_baseline.get(
+                        sub).get(com).get("baseline")
+                cls._put(sub, com, {
+                    "subsystem_name": sub,
+                    "component_name": com,
+                    "psesudo_gn_path": psesudo_gn,
+                    "description": "fuzzy match",
+                    "file_name": f.replace(project_path, ""),
+                    "size": size,
+                }, rom_size_dict, component_rom_baseline)
+                find_flag = True
+        if not find_flag:  # 模糊匹配都没有匹配到的,归属到NOTFOUND
+            cls._put("NOTFOUND", "NOTFOUND", {
+                "file_name": f.replace(project_path, ""),
+                "size": size,
+            }, rom_size_dict)
+
+    @classmethod
+    def _subsystem_component_for_all_product_file(cls, product_dict: Dict[str, List[str]], query_order: Dict[str, List[str]],
+                                                  gn_info: Dict, gn_info_file: str, rom_ram_baseline: Dict, rom_size_dict: Dict):
+        for t, l in product_dict.items():
+            for f in l:  # 遍历所有文件
+                if os.path.isdir(f):
+                    continue
+                type_list = query_order.get(t)
+                _, base_name = os.path.split(f)
+                size = os.path.getsize(f)
+                if not type_list:
+                    logging.warning(
+                        f"'{t}' not found in query_order of the config.yaml")
+                    break
+                cls._iterate_all_template_type(
+                    type_list, gn_info, gn_info_file, base_name, rom_ram_baseline, rom_size_dict, f, size)
+
+    @classmethod
     def analysis(cls, product_name: str, product_dict: Dict[str, List[str]]):
         """analysis the rom of lite/small product
 
@@ -346,69 +409,18 @@ class RomAnalysisTool:
             project_path)
         with open("rom_ram_baseline.json", 'w', encoding='utf-8') as f:
             json.dump(rom_ram_baseline, f, indent=4)
-        gn_info_file = configs["gn_info_file"] # filename to save gn_info
+        gn_info_file = configs["gn_info_file"]  # filename to save gn_info
         with open(gn_info_file, 'r', encoding='utf-8') as f:
             gn_info = json.load(f)
         query_order: Dict[str, List[str]
-                          ] = configs[product_name]["query_order"] # query order of the gn template to be matched
-        query_order["etc"] = configs["target_type"] # etc会查找所有的template
+                          ] = configs[product_name]["query_order"]  # query order of the gn template to be matched
+        query_order["etc"] = configs["target_type"]  # etc会查找所有的template
         rom_size_dict: Dict = dict()
         if "manual_config" in configs[product_name].keys():
             cls._match_manual_configured(
                 configs[product_name]["manual_config"], product_dict, configs[product_name]["product_dir"]["root"], rom_size_dict)
-        for t, l in product_dict.items():
-            for f in l: # 遍历所有文件
-                if os.path.isdir(f):
-                    continue
-                find_flag = False
-                type_list = query_order.get(t)
-                _, base_name = os.path.split(f)
-                size = os.path.getsize(f)
-                if not type_list:
-                    logging.warning(
-                        f"'{t}' not found in query_order of the config.yaml")
-                    break
-                for tn in type_list: # tn example: ohos_shared_library
-                    if find_flag: # 如果已经在前面的template中找到了,后面的就不必再查找
-                        break
-                    output_dict: Dict[str, Dict] = gn_info.get(
-                        tn) # 这个模板对应的所有可能编译产物
-                    if not output_dict:
-                        logging.warning(
-                            f"'{tn}' not found in the {gn_info_file}")
-                        continue
-                    d = output_dict.get(base_name)
-                    if not d:
-                        continue
-                    d["size"] = size
-                    d["file_name"] = f.replace(project_path, "")
-                    if rom_ram_baseline.get(d["subsystem_name"]) and rom_ram_baseline.get(d["subsystem_name"]).get(d["component_name"]):
-                        component_rom_baseline = rom_ram_baseline.get(
-                            d["subsystem_name"]).get(d["component_name"]).get("rom")
-                    cls._put(d["subsystem_name"],
-                             d["component_name"], d, rom_size_dict, component_rom_baseline)
-                    find_flag = True
-                if not find_flag: # 如果指定序列中的template都没有查找到,则模糊匹配
-                    # fuzzy match
-                    psesudo_gn, sub, com = cls._fuzzy_match(f)
-                    if sub and com:
-                        if rom_ram_baseline.get(sub) and rom_ram_baseline.get(sub).get(com):
-                            component_rom_baseline = rom_ram_baseline.get(
-                                sub).get(com).get("baseline")
-                        cls._put(sub, com, {
-                            "subsystem_name": sub,
-                            "component_name": com,
-                            "psesudo_gn_path": psesudo_gn,
-                            "description": "fuzzy match",
-                            "file_name": f.replace(project_path, ""),
-                            "size": size,
-                        }, rom_size_dict, component_rom_baseline)
-                        find_flag = True
-                if not find_flag: # 模糊匹配都没有匹配到的,归属到NOTFOUND
-                    cls._put("NOTFOUND", "NOTFOUND", {
-                        "file_name": f.replace(project_path, ""),
-                        "size": size,
-                    }, rom_size_dict)
+        cls._subsystem_component_for_all_product_file(
+            product_dict, query_order, gn_info, gn_info_file, rom_ram_baseline, rom_size_dict)
         if unit_adapt:
             cls._result_unit_adaptive(rom_size_dict)
         with open(configs[product_name]["output_name"], 'w', encoding='utf-8') as f:

tools/rom_ram_analyzer/standard/README.md

@@ -12,6 +12,10 @@
 
 主要是rk3568系列,已测试产品包括rk3568、rk3568_mini_system
 
+## 实现思路
+
+利用编译构建自动生成的out/rk3568/packages/phone/system_module_info.json中已有的信息重新组织，对于其中没有子系统和部件的文件，手动查找。目前已知ohos_sa_profile没有
+
 ## 使用说明
 
 前置条件：
@@ -30,7 +34,7 @@
 1. `-h`或`--help`命令查看帮助
    ```shell
    > python3 rom_analyzer.py -h
-   usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-b] [-o OUTPUT_FILE] [-e EXCEL]
+   usage: rom_analyzer.py [-h] [-v] -p PROJECT_PATH -j MODULE_INFO_JSON -n PRODUCT_NAME -d PRODUCT_DIR [-b] [-o OUTPUT_FILE] [-u] [-e EXCEL]
 
    analyze rom size of component.
 
@@ -48,6 +52,7 @@
    -b, --baseline        add baseline of component to the result(-b) or not.
    -o OUTPUT_FILE, --output_file OUTPUT_FILE
                            basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result
+   -u, --unit_adaptive   unit adaptive
    -e EXCEL, --excel EXCEL
                            if output result as excel, default: False. eg: -e True
    ```
@@ -55,7 +60,7 @@
    ```shell
    python3 rom_analyzer.py -p ~/oh/ -j ~/oh/out/rk3568/packages/phone/system_module_info.json -n rk3568 -d system -d vendor -d updater -e True -b
    # oh：rootpath of oh
-   # -b: add baseline of to the result
+   # -b: add baseline info to the result
    # -e True：output result in excel format additionally
    ```
 
@@ -88,9 +93,9 @@
 
 ## 功能介绍
 
-基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有xml文件，rom的分析结果，（rom_ram_baseline.json——可以在rom分析阶段通过-b参数生成）分析各进程及对应部件的ram占用（默认取Pss）
+基于out/{product_name}/packages/phone下所有cfg文件、out/{product_name}/packages/phone/system/profile下所有json文件，rom的分析结果，（rom_ram_baseline.json——可以在rom分析阶段通过-b参数生成）分析各进程及对应部件的ram占用（默认取Pss）
 
-收集cfg、xml文件的可供参考命令：`mkdir xml && cp $(find ~/oh/out/rk3568/packages/phone/system/profile -name *.xml | xargs) xml`
+收集cfg、json文件的可供参考命令：`mkdir cfgs && cp $(find ~/oh/out/rk3568/packages/phone -name *.cfg | xargs) cfgs`
 
 结果以json与xls格式存储，其中，json格式是必输出的，xls格式需要-e参数控制。
 
@@ -109,7 +114,7 @@
    ```
 5. 准备好相关数据：
    1. out/{product_name}/packages/phone下所有cfg文件，并将其放置于同一个目录中（ps：同名文件仅保存一份即可）
-   1. out/{product_name}/packages/phone/system/profile下所有xml文件
+   1. out/{product_name}/packages/phone/system/profile下所有json 文件
    1. rom_ram_baseline.json——如果需要在结果中生成基线信息
 6. 运行rom_analyzer.py产生的json结果一份（即-o参数对应的文件，默认rom_analysis_result.json）
 
@@ -118,16 +123,16 @@
 1. 使用`-h`或`--help`查看帮助
    ```shell
    > python .\ram_analyzer.py -h
-   usage: ram_analyzer.py [-h] [-v] -x XML_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-b BASELINE_FILE]
-                       [-o OUTPUT_FILENAME] [-e EXCEL]
+   usage: ram_analyzer.py [-h] [-v] -s JSON_PATH -c CFG_PATH [-j ROM_RESULT] -n DEVICE_NUM [-b BASELINE_FILE]
+                       [-o OUTPUT_FILENAME] [-u] [-e EXCEL]
 
    analyze ram size of component
 
    optional arguments:
    -h, --help            show this help message and exit
    -v, -version          show program\'s version number and exit
-   -x XML_PATH, --xml_path XML_PATH
-                           path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile
+   -s JSON_PATH, --json_path JSON_PATH
+                           path of sa json file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile
    -c CFG_PATH, --cfg_path CFG_PATH
                            path of cfg files. eg: -c ./cfgs/
    -j ROM_RESULT, --rom_result ROM_RESULT
@@ -139,13 +144,15 @@
                            baseline file of rom and ram generated by rom analysis.
    -o OUTPUT_FILENAME, --output_filename OUTPUT_FILENAME
                            base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result
+   -u, --unit_adaptive   unit adaptive
    -e EXCEL, --excel EXCEL
                            if output result as excel, default: False. eg: -e True
    ```
 2. 使用示例：
    ```shell
-    python ram_analyzer.py -x xml -c cfg -j rom_analysis_result.json -n 7001005458323933328a59a140913900 -b rom_ram_baseline.json -e True
-   # -x xml：指定xml文件的存放路径为xml目录，-c cfg类似
+    python  .\ram_analyzer.py -s .\profile\ -c .\cfgs -j .\rom_analysis_result.json -n 7001005458323933328a59a140913900 -e True
+   # -s profile：copy到本地的out/rk3568/package/phone/system/profile目录
+   # -c .\cfgs：copy到本地的out/rk3568/packages/phone下所有cfg
    # -b rom_ram_baseline.json：指定rom与ram的基线信息文件（可在rom统计阶段使用-b参数生成）
    # -e True：生成xls格式的结果文件
    ```

tools/rom_ram_analyzer/standard/pkgs/basic_tool.py

@@ -2,6 +2,7 @@ import sys
 import typing
 import os
 import glob
+import re
 from pathlib import Path
 from typing import *
 
@@ -17,6 +18,23 @@ def unit_adaptive(size: int) -> str:
     return str(round(size,2))+unit_list[index]
 
 class BasicTool:
+    @classmethod
+    def match_paragraph(cls, content: str, start_pattern: str = r"\w+\(\".*?\"\) *{", end_pattern: str = "\}") -> \
+            Iterator[re.Match]:
+        """
+        匹配代码段，支持单行
+        注意：ptrn中已经包含前面的空格，所以start_pattern中可以省略
+        :param content: 被匹配的字符串
+        :param start_pattern: 模式的开头
+        :param end_pattern: 模式的结尾
+        :return: 匹配到的段落的迭代器
+        """
+        ptrn = r'^( *){s}(?#匹配开头).*?(?#中间非贪婪)\1(?#如果开头前面有空格,则结尾的前面应该有相同数量的空格)?{e}$(?#匹配结尾)'.format(
+            s=start_pattern, e=end_pattern)
+        ptrn = re.compile(ptrn, re.M | re.S)
+        result = re.finditer(ptrn, content)
+        return result
+    
     @classmethod
     def find_all_files(cls, folder: str, real_path: bool = True, apply_abs: bool = True, de_duplicate: bool = True,
                        p_filter: typing.Callable = lambda x: True) -> list:
@@ -36,6 +54,22 @@ class BasicTool:
     def get_abs_path(cls, path: str) -> str:
         return os.path.abspath(os.path.expanduser(path))
     
+    @classmethod
+    def re_group_1(cls, content: str, pattern: str, **kwargs) -> str:
+        """
+        匹配正则表达式，如果有匹配到内容，返回group(1)的内容
+        :param content: 要被匹配的内容
+        :param pattern: 进行匹配的模式
+        :return: 匹配到的结果（group(1)）
+        TODO 对（）的检查应该更严格
+        """
+        if not (r'(' in pattern and r')' in pattern):
+            raise ValueError("parentheses'()' must in the pattern")
+        result = re.search(pattern, content, **kwargs)
+        if result:
+            return result.group(1)
+        return str()
+    
     @classmethod
     def execute(cls, cmd: str, post_processor: Callable[[Text], Text] = lambda x:x) -> Any:
         """

tools/rom_ram_analyzer/standard/pkgs/gn_common_tool.py

@@ -1,5 +1,12 @@
 import os
 import json
+import re
+from typing import *
+
+if __name__ == '__main__':
+    from basic_tool import BasicTool
+else:
+    from pkgs.basic_tool import BasicTool
 
 
 class GnCommonTool:
@@ -98,21 +105,8 @@ class GnCommonTool:
         return part_name, subsystem_name
 
     @classmethod
-    def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple:
-        """
-        查找gn_file对应的part_name和subsystem
-        如果在gn中找不到，就到bundle.json中去找
-        """
-        part_name = None
-        subsystem_name = None
-        part_var_flag = False  # 标识这个变量从gn中取出的原始值是不是变量
-        subsystem_var_flag = False
-        var_list = list()
-        part_name_pattern = r"part_name *=\s*\S*"
-        subsystem_pattern = r"subsystem_name *=\s*\S*"
-        meta_grep_pattern = "grep -E '{}' {} | head -n 1"
-        part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file)
-        subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file)
+    def _parse_part_subsystem(cls, part_var_flag: bool, subsystem_var_flag: bool, var_list: List[str], part_cmd: str, subsystem_cmd: str, gn_file: str, project_path: str) -> Tuple[str, str]:
+        part_name = subsystem_name = None
         part = os.popen(part_cmd).read().strip()
         if len(part) != 0:
             part = part.split('=')[-1].strip()
@@ -145,13 +139,63 @@ class GnCommonTool:
                 tuple(var_list), gn_file, project_path)[0]
             subsystem_name = t if t is not None and len(
                 t) != 0 else subsystem_name
-        if part_name is not None and subsystem_name is not None:
+        return part_name, subsystem_name
+
+    @classmethod
+    def find_part_subsystem(cls, gn_file: str, project_path: str) -> tuple:
+        """
+        查找gn_file对应的part_name和subsystem
+        如果在gn中找不到，就到bundle.json中去找
+        """
+        part_var_flag = False  # 标识这个变量从gn中取出的原始值是不是变量
+        subsystem_var_flag = False
+        var_list = list()
+        part_name_pattern = r"part_name *=\s*\S*"
+        subsystem_pattern = r"subsystem_name *=\s*\S*"
+        meta_grep_pattern = "grep -E '{}' {} | head -n 1"
+        part_cmd = meta_grep_pattern.format(part_name_pattern, gn_file)
+        subsystem_cmd = meta_grep_pattern.format(subsystem_pattern, gn_file)
+
+        part_name, subsystem_name = cls._parse_part_subsystem(part_var_flag, subsystem_var_flag,
+                                                              var_list, part_cmd, subsystem_cmd, gn_file, project_path)
+        if part_name and subsystem_name:
             return part_name, subsystem_name
         # 如果有一个没有找到，就要一层层去找bundle.json文件
         t_part_name, t_subsystem_name = cls.__find_part_subsystem_from_bundle(
             gn_file, stop_tail=project_path)
-        if t_part_name is not None:
+        if t_part_name:
             part_name = t_part_name
-        if t_subsystem_name is not None:
+        if t_subsystem_name:
             subsystem_name = t_subsystem_name
         return part_name, subsystem_name
+
+class GnVariableParser:
+    @classmethod
+    def string_parser(cls, var: str, content: str) -> str:
+        """
+        解析值为字符串的变量,没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配
+        :param content: 要进行解析的内容
+        :param var: 变量名
+        :return: 变量值[str]
+        """
+        result = BasicTool.re_group_1(
+            content, r"{} *= *[\n]?(\".*?\")".format(var), flags=re.S | re.M)
+        return result
+
+    @classmethod
+    def list_parser(cls, var: str, content: str) -> List[str]:
+        """
+        解析值为列表的变量，list的元素必须全为数字或字符串,且没有对引号进行去除,如果是a = b这种b为变量的,则无法匹配
+        :param var: 变量名
+        :param content: 要进行
+        :return: 变量值[List]
+        """
+        result = BasicTool.re_group_1(
+            content, r"{} *= *(\[.*?\])".format(var), flags=re.S | re.M)
+        result_list = list()
+        for item in result.lstrip('[').rstrip(']').split('\n'):
+            item = item.strip().strip(',"')
+            if not item:
+                continue
+            result_list.append(item)
+        return result_list

tools/rom_ram_analyzer/standard/ram_analyzer.py

@@ -43,7 +43,7 @@ class HDCTool:
         True：可用
         False：不可用
         """
-        cp = subprocess.run(["hdc"], capture_output=True)
+        cp = subprocess.run(["hdc", "--help"], capture_output=True)
         stdout = str(cp.stdout)
         stderr = str(cp.stderr)
         return verify_str in stdout or verify_str in stderr
@@ -139,9 +139,9 @@ class RamAnalyzer:
             # 如果第一列不是数字（pid），就过
             if not processed or not processed[0].isnumeric():
                 continue
-            name = processed[1] # 否则的话就取名字，和对应的size
+            name = processed[1]  # 否则的话就取名字，和对应的size
             size = int(processed[cls.__ss_dict.get(ss)]) * \
-                1024 # kilo byte to byte
+                1024  # kilo byte to byte
             full_process_name = find_full_process_name(name)
             if not full_process_name:
                 print(
@@ -172,26 +172,28 @@ class RamAnalyzer:
         return exec_once()
 
     @classmethod
-    def __parse_process_xml(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]):
+    def __parse_process_json(cls, file_path: str, result_dict: typing.Dict[str, typing.List[str]]):
         """
-        解析xml文件，结存存入 result_dict中，格式：{process_name: os_list}
+        解析json文件，结存存入 result_dict中，格式：{process_name: os_list}
         其中，so_list中是so的base_name
         """
-        if not (os.path.isfile(file_path) and file_path.endswith(".xml")):
-            print("warning: {} not exist or not a xml file".format(file_path))
+        if not (os.path.isfile(file_path) and file_path.endswith(".json")):
+            print("warning: {} not exist or not a json file".format(file_path))
             return
-        doc = dom.parse(file_path)
-        info = doc.getElementsByTagName("info")[0]
-        process = info.getElementsByTagName("process")[0]
-        process_name = process.childNodes[0].data
-        result_dict[process_name] = list()
-        libs = info.getElementsByTagName(
-            "loadlibs")[0].getElementsByTagName("libpath")
-        for lib in libs:
-            so = lib.childNodes[0].data
-            result_dict.get(process_name).append(os.path.split(so)[-1])
-            if debug:
-                print(process_name, " ", so)
+        with open(file_path, 'r', encoding='utf-8') as f:
+            j_content: typing.Dict[str, typing.Any] = json.load(f)
+        if "process" not in j_content.keys() or "systemability" not in j_content.keys():
+            print(
+                f"warning: {file_path} has no field 'process' or 'systemability'")
+            return
+        process_name: str = j_content.get("process")
+        elf_list: typing.List[str] = list()
+        for sa in j_content.get("systemability"):
+            libpath: str = sa.get("libpath")
+            if not libpath:
+                continue
+            elf_list.append(libpath)
+        result_dict[process_name] = elf_list
 
     @classmethod
     def get_elf_info_from_rom_result(cls, rom_result_json: str) -> typing.Dict[str, typing.Dict[str, str]]:
@@ -238,7 +240,7 @@ class RamAnalyzer:
             if first.endswith("sa_main"):
                 # 由sa_main去来起进程
                 xml_base_name = os.path.split(path_list[0])[-1]
-                cls.__parse_process_xml(os.path.join(
+                cls.__parse_process_json(os.path.join(
                     profile_path, xml_base_name), result_dict)
             else:
                 # 直接执行
@@ -247,25 +249,15 @@ class RamAnalyzer:
                 result_dict.get(process_name).append(os.path.split(first)[-1])
 
     @classmethod
-    def get_process_so_relationship(cls, xml_path: str, cfg_path: str, profile_path: str) -> typing.Dict[
+    def get_process_so_relationship(cls, cfg_path: str, profile_path: str) -> typing.Dict[
             str, typing.List[str]]:
         """
-        从out/{product_name}/packages/phone/sa_profile/merged_sa查找xml文件并处理得到进程与so的对应关系
+        parse the relationship between process and elf file
         """
         # 从merged_sa里面收集
-        xml_list = glob.glob(xml_path + os.sep + "*[.]xml", recursive=True)
+        # json_list = glob.glob(json_path + os.sep + "*[.]json", recursive=True)
         process_elf_dict: typing.Dict[str, typing.List[str]] = dict()
-        for xml in xml_list:
-            if debug:
-                print("parsing: ", xml)
-            try:
-                cls.__parse_process_xml(xml, process_elf_dict)
-            except:
-                print("parse '{}' failed".format(xml))
-            finally:
-                ...
-        # 从system/etc/init/*.cfg中收集，如果是sa_main拉起的，则从system/profile/*.xml中进行解析
-        cfg_list = glob.glob(cfg_path + os.sep + "*[.]cfg", recursive=True)
+        cfg_list = glob.glob(cfg_path + os.sep + "*.cfg", recursive=True)
         for cfg in cfg_list:
             if debug:
                 print("parsing: ", cfg)
@@ -278,7 +270,7 @@ class RamAnalyzer:
         return process_elf_dict
 
     @classmethod
-    def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str, unit_adapt:bool):
+    def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str, unit_adapt: bool):
         """
         保存结果到excel中
         子系统：{
@@ -411,8 +403,9 @@ class RamAnalyzer:
                     continue
                 component_info["baseline"] = baseline_dict[subsystem_name][component_name].get(
                     "ram")
+
     @classmethod
-    def refactored_result_unit_adaptive(cls, result_dict:Dict[str,Dict])->None:
+    def refactored_result_unit_adaptive(cls, result_dict: Dict[str, Dict]) -> None:
         for subsystem_name, subsystem_info in result_dict.items():
             sub_size = unit_adaptive(subsystem_info["size"])
             del subsystem_info["size"]
@@ -423,14 +416,14 @@ class RamAnalyzer:
                     pro_size = unit_adaptive(process_info["size"])
                     del process_info["size"]
                     for elf_name, elf_size in process_info["elf"].items():
-                        process_info["elf"][elf_name]=unit_adaptive(elf_size)
+                        process_info["elf"][elf_name] = unit_adaptive(elf_size)
                     process_info["size"] = pro_size
                 component_info["size"] = com_size
             subsystem_info["size"] = sub_size
-                
+
     @classmethod
-    def analysis(cls, cfg_path: str, xml_path: str, rom_result_json: str, device_num: str,
-                 output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt:bool):
+    def analysis(cls, cfg_path: str, json_path: str, rom_result_json: str, device_num: str,
+                 output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt: bool):
         """
         process size subsystem/component so so_size
         """
@@ -446,8 +439,8 @@ class RamAnalyzer:
         so_info_dict: typing.Dict[
             str, typing.Dict[str["component_name|subsystem_name|size"], str]] = cls.get_elf_info_from_rom_result(
             rom_result_json)
-        process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(xml_path, cfg_path,
-                                                                                               profile_path)
+        process_elf_dict: typing.Dict[str, typing.List[str]] = cls.get_process_so_relationship(cfg_path,
+                                                                                               json_path)
         process_size_dict: typing.Dict[str, int] = cls.process_hidumper_info(
             device_num, ss)
         result_dict: typing.Dict[str, typing.Dict[str, typing.Any]] = dict()
@@ -458,7 +451,7 @@ class RamAnalyzer:
                     # 要么uinput_inject的对应key为mmi_uinput_inject。对于此类特殊处理，即：如果service_name找不到，但是直接执行的bin等于这个名字，也认为找到
                     return v
 
-        for process_name, process_size in process_size_dict.items(): # 从进程出发
+        for process_name, process_size in process_size_dict.items():  # 从进程出发
             if not process_name:
                 print("warning: an empty 'process_name' has been found.")
                 continue
@@ -493,7 +486,7 @@ class RamAnalyzer:
             # 得到进程相关的elf文件list
             so_list: list = get(process_name, process_elf_dict)
             if so_list is None:
-                print("warning: process '{}' not found in .xml or .cfg".format(
+                print("warning: process '{}' not found in .json or .cfg".format(
                     process_name))
                 result_dict[process_name] = dict()
                 result_dict[process_name]["size"] = process_size
@@ -571,8 +564,8 @@ def get_args():
     )
     parser.add_argument("-v", "-version", action="version",
                         version=f"version {VERSION}")
-    parser.add_argument("-x", "--xml_path", type=str, required=True,
-                        help="path of xml file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile")
+    parser.add_argument("-s", "--json_path", type=str, required=True,
+                        help="path of sa json file. eg: -x ~/openharmony/out/rk3568/packages/phone/system/profile")
     parser.add_argument("-c", "--cfg_path", type=str, required=True,
                         help="path of cfg files. eg: -c ./cfgs/")
     parser.add_argument("-j", "--rom_result", type=str, default="./rom_analysis_result.json",
@@ -592,10 +585,14 @@ def get_args():
     return args
 
 
+def abspath(path: str) -> str:
+    return os.path.abspath(os.path.expanduser(path))
+
+
 if __name__ == '__main__':
     args = get_args()
-    cfg_path = args.cfg_path
-    profile_path = args.xml_path
+    cfg_path = abspath(args.cfg_path)
+    profile_path = abspath(args.json_path)
     rom_result = args.rom_result
     device_num = args.device_num
     output_filename = args.output_filename

tools/rom_ram_analyzer/standard/rom_analyzer.py

@@ -22,10 +22,11 @@ import sys
 import typing
 from copy import deepcopy
 from typing import *
-
+import re
+import subprocess
 from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector
 from pkgs.basic_tool import BasicTool, unit_adaptive
-from pkgs.gn_common_tool import GnCommonTool
+from pkgs.gn_common_tool import GnCommonTool, GnVariableParser
 from pkgs.simple_excel_writer import SimpleExcelWriter
 
 debug = bool(sys.gettrace())
@@ -33,27 +34,80 @@ debug = bool(sys.gettrace())
 NOTFOUND = "NOTFOUND"
 
 
+class PreCollector:
+    """
+    collect some info that system_module_info.json dosn't contains
+    """
+
+    def __init__(self, project_path: str) -> None:
+        self.info_dict: Dict[str, Any] = dict()
+        self.project_path = BasicTool.get_abs_path(project_path)
+        self.result_dict = dict()
+
+    def _process_single_sa(self, item: str, start_pattern: str):
+        gn, _, _ = item.split(':')
+        with open(gn, 'r', encoding='utf-8') as f:
+            content = f.read()
+        p_itr: Iterator[re.Match] = BasicTool.match_paragraph(
+            content=content, start_pattern=start_pattern)
+        for p in p_itr:
+            p_content = p.group()
+            files: List[str] = GnVariableParser.list_parser(
+                "sources", p_content)
+            component_name, subsystem_name = GnCommonTool.find_part_subsystem(
+                gn, self.project_path)
+            for f in files:
+                f = f.split('/')[-1]
+                self.result_dict[f] = {
+                    "subsystem_name": subsystem_name,
+                    "component_name": component_name,
+                    "gn_path": gn
+                }
+
+    def collect_sa_profile(self):
+        grep_kw = r"ohos_sa_profile"
+        grep_cmd = f"grep -rn '{grep_kw}' --include=BUILD.gn {self.project_path}"
+        content = BasicTool.execute(
+            grep_cmd, post_processor=lambda x: x.split('\n'))
+        for item in content:
+            if not item:
+                continue
+            self._process_single_sa(item, start_pattern=grep_kw)
+
+
 class RomAnalyzer:
 
     @classmethod
     def __collect_product_info(cls, system_module_info_json: Text,
-                               project_path: Text) -> Dict[Text, Dict[Text, Text]]:
+                               project_path: Text, extra_info: Dict[str, Dict]) -> Dict[Text, Dict[Text, Text]]:
         """
         根据system_module_info.json生成target字典
+        format:
+            {
+                "{file_name}":{
+                    "{subsytem_name}": abc,
+                    "{component_name}": def,
+                    "{gn_path}": ghi
+                }
+            }
+        if the unit of system_module_info.json has not field "label" and the "type" is "sa_profile",
+        find the subsystem_name and component_name in the BUILD.gn
         """
         with open(system_module_info_json, 'r', encoding='utf-8') as f:
             product_list = json.loads(f.read())
         project_path = BasicTool.get_abs_path(project_path)
         product_info_dict: Dict[Text, Dict[Text, Text]] = dict()
         for unit in product_list:
+            cs_flag = False
             dest: List = unit.get("dest")
-            if dest is None:
+            if not dest:
                 print("warning: keyword 'dest' not found in {}".format(
                     system_module_info_json))
                 continue
             label: Text = unit.get("label")
             gn_path = component_name = subsystem_name = None
             if label:
+                cs_flag = True
                 gn_path = os.path.join(project_path, label.split(':')[
                                        0].lstrip('/'), "BUILD.gn")
                 component_name = unit.get("part_name")
@@ -66,11 +120,18 @@ class RomAnalyzer:
             else:
                 print("warning: keyword 'label' not found in {}".format(unit))
             for target in dest:
-                product_info_dict[target] = {
-                    "component_name": component_name,
-                    "subsystem_name": subsystem_name,
-                    "gn_path": gn_path,
-                }
+                if cs_flag:
+                    product_info_dict[target] = {
+                        "component_name": component_name,
+                        "subsystem_name": subsystem_name,
+                        "gn_path": gn_path,
+                    }
+                else:
+                    tmp = target.split('/')[-1]
+                    pre_info = extra_info.get(tmp)
+                    if not pre_info:
+                        continue
+                    product_info_dict[target] = pre_info
         return product_info_dict
 
     @classmethod
@@ -142,12 +203,12 @@ class RomAnalyzer:
             return baseline_dict.get(subsystem_name).get(component_name).get("rom")
         size = unit.get("size")
         relative_filepath = unit.get("relative_filepath")
-        if result_dict.get(subsystem_name) is None: # 子系统
+        if result_dict.get(subsystem_name) is None:  # 子系统
             result_dict[subsystem_name] = dict()
             result_dict[subsystem_name]["size"] = 0
             result_dict[subsystem_name]["file_count"] = 0
 
-        if result_dict.get(subsystem_name).get(component_name) is None: # 部件
+        if result_dict.get(subsystem_name).get(component_name) is None:  # 部件
             result_dict[subsystem_name][component_name] = dict()
             result_dict[subsystem_name][component_name]["size"] = 0
             result_dict[subsystem_name][component_name]["file_count"] = 0
@@ -162,18 +223,18 @@ class RomAnalyzer:
         result_dict[subsystem_name][component_name][relative_filepath] = size
 
     @classmethod
-    def result_unit_adaptive(self, result_dict:Dict[str,Dict])->None:
+    def result_unit_adaptive(self, result_dict: Dict[str, Dict]) -> None:
         for subsystem_name, subsystem_info in result_dict.items():
-                size = unit_adaptive(subsystem_info["size"])
-                count = subsystem_info["file_count"]
-                if "size" in subsystem_info.keys():
-                    del subsystem_info["size"]
-                if "file_count" in subsystem_info.keys():
-                    del subsystem_info["file_count"]
-                for component_name, component_info in subsystem_info.items():
-                    component_info["size"] = unit_adaptive(component_info["size"])
-                subsystem_info["size"] = size
-                subsystem_info["file_count"] = count
+            size = unit_adaptive(subsystem_info["size"])
+            count = subsystem_info["file_count"]
+            if "size" in subsystem_info.keys():
+                del subsystem_info["size"]
+            if "file_count" in subsystem_info.keys():
+                del subsystem_info["file_count"]
+            for component_name, component_info in subsystem_info.items():
+                component_info["size"] = unit_adaptive(component_info["size"])
+            subsystem_info["size"] = size
+            subsystem_info["file_count"] = count
 
     @classmethod
     def analysis(cls, system_module_info_json: Text, product_dirs: List[str],
@@ -193,8 +254,11 @@ class RomAnalyzer:
         phone_dir = os.path.join(
             project_path, "out", product_name, "packages", "phone")
         product_dirs = [os.path.join(phone_dir, d) for d in product_dirs]
+        pre_collector = PreCollector(project_path)
+        pre_collector.collect_sa_profile()
+        extra_product_info_dict: Dict[str, Dict] = pre_collector.result_dict
         product_info_dict = cls.__collect_product_info(
-            system_module_info_json, project_path) # 所有产物信息
+            system_module_info_json, project_path, extra_info=extra_product_info_dict)  # collect product info from json file
         result_dict: Dict[Text:Dict] = dict()
         for d in product_dirs:
             file_list: List[Text] = BasicTool.find_all_files(d)
@@ -203,7 +267,10 @@ class RomAnalyzer:
                 relative_filepath = f.replace(phone_dir, "").lstrip(os.sep)
                 unit: Dict[Text, Any] = product_info_dict.get(
                     relative_filepath)
-                if unit is None:
+                if not unit:
+                    bf = f.split('/')[-1]
+                    unit: Dict[Text, Any] = product_info_dict.get(bf)
+                if not unit:
                     unit = dict()
                 unit["size"] = size
                 unit["relative_filepath"] = relative_filepath
@@ -257,4 +324,4 @@ if __name__ == '__main__':
     add_baseline = args.baseline
     unit_adapt = args.unit_adaptive
     RomAnalyzer.analysis(module_info_json, product_dirs,
-                         project_path, product_name, output_file, output_excel, add_baseline, unit_adapt)
+                         project_path, product_name, output_file, output_excel, add_baseline, unit_adapt)