!83 [rom_ram_analyzer]improve the accuracy

Merge pull request !83 from aodongbiao/master
This commit is contained in:
openharmony_ci
2023-05-16 06:09:57 +00:00
committed by Gitee
11 changed files with 244 additions and 39 deletions
+1 -7
View File
@@ -96,10 +96,4 @@ ipcamera_hispark_taurus: # 产品名称,需要和命令行参数中的-p参数
## 如何提高准确率
1. 如果已知编译产物不可能从某些目录下的BUILD.gn产生,则可以将其对应目录加入到config.yaml的black_list,从而不对该目录下的BUILD.gn进行扫描,以减少出错概率
## 后续工作
1. 对target(xxx,yyy)中,xxx/yyy为变量的情况可进一步优化
1. 重构
1. 在config.yaml中允许人为配置以进一步提高准确性
1. 单位自适应
1. 对于已知检测错误的或NOTFOUND的编译产物,如果知道其正确的部件和子系统,可在config.yaml中的manual_config进行配置
@@ -22,14 +22,26 @@ import glob
from typing import *
import unittest
__all__ = ["translate_str_unit", "BasicTool", "do_nothing", "get_unit"]
__all__ = ["translate_str_unit", "BasicTool", "do_nothing", "get_unit", "unit_adaptive"]
def unit_adaptive(size: int) -> str:
unit_list = ["Byte", "KB", "MB", "GB"]
index = 0
while index < len(unit_list) and size >= 1024:
size /= 1024
index += 1
if index == len(unit_list):
index = len(unit_list)-1
size *= 1024
return str(round(size))+unit_list[index]
def get_unit(x: str) -> str:
pattern = r"[a-z|A-Z]*$"
unit = re.search(pattern, x).group()
return unit
def translate_str_unit(x: str, dest: str, prefix: str = "~") -> float:
src_unit = get_unit(x)
trans_dict: Dict[str, int] = {
@@ -203,5 +203,10 @@ class GnVariableParser:
"""
result = BasicTool.re_group_1(
content, r"{} *= *(\[.*?\])".format(var), flags=re.S | re.M)
result = ast.literal_eval(result.strip())
return result
result_list = list()
for item in result.lstrip('[').rstrip(']').split('\n'):
item = item.strip().strip(',"')
if not item:
continue
result_list.append(item)
return result_list
@@ -37,6 +37,16 @@ class RomRamBaselineCollector:
@classmethod
def collect(cls, oh_path: str) -> Dict[str, Dict]:
"""
从bundle.json文件中收集rom、ram的baseline信息
返回结果:
"subsystem_name":{
"component_name":{
"rom":"123KB, # the value may be "" or null
"ram":"234KB"
}
}
"""
def post_handler(x:str)->list:
x = x.split("\n")
y = [item for item in x if item]
@@ -48,6 +48,8 @@ def parse_args():
help="recollect subsystem_component info or not(-s)")
parser.add_argument("-b", "--baseline", action="store_true",
help="add baseline of component to the result(-b) or not.")
parser.add_argument("-u", "--unit_adaptive",
action="store_true", help="unit adaptive")
args = parser.parse_args()
return args
@@ -58,11 +60,14 @@ _args = parse_args()
# # global variables
configs = SimpleYamlTool.read_yaml("config.yaml")
result_dict: Dict[str, Any] = dict()
project_path = BasicTool.abspath(_args.oh_path)
product_name = _args.product_name
if product_name not in configs.keys():
print(f"error: product_name '{product_name}' illegal")
exit(-1)
project_path = BasicTool.abspath(_args.oh_path)
recollect_gn = _args.recollect_gn
baseline = _args.baseline
unit_adapt = _args.unit_adaptive
_recollect_sc = _args.recollect_sc
_sc_json: Dict[Text, Text] = configs.get("subsystem_component")
_sc_save = _sc_json.get("save")
@@ -240,7 +245,41 @@ collector_config: Tuple[BaseProcessor] = (
},
unit_post_handler=DefaultPostHandler(),
ud_post_handler=TargetS2MPostHandler
)
),
DefaultProcessor(project_path=project_path,
result_dict=result_dict,
target_type=_target_type[14],
match_pattern=fr"^( *){_target_type[14]}\(.*?\)",
sub_com_dict=sub_com_dict,
target_name_parser=TargetNameParser.single_parser,
other_info_handlers={
"output_extension":extension_handler
},
unit_post_handler=UnittestPostHandler(),
),
DefaultProcessor(project_path=project_path,
result_dict=result_dict,
target_type=_target_type[15],
match_pattern=fr"^( *){_target_type[15]}\(.*?\)",
sub_com_dict=sub_com_dict,
target_name_parser=TargetNameParser.single_parser,
other_info_handlers={
"hap_name": hap_name_handler,
"mode": mod_handler,
},
unit_post_handler=HapPackPostHandler(),
),
ListResourceProcessor(project_path=project_path,
result_dict=result_dict,
target_type=_target_type[16],
match_pattern=fr"^( *){_target_type[16]}\(.*?\)",
sub_com_dict=sub_com_dict,
target_name_parser=TargetNameParser.single_parser,
other_info_handlers={
},
unit_post_handler=DefaultPostHandler(),
resource_field="sources"
),
)
__all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"]
__all__ = ["configs", "result_dict", "collector_config", "sub_com_dict"]
@@ -29,8 +29,10 @@ target_type:
- ohos_prebuilt_shared_library
- lite_component
- target
- unittest
- hap_pack
- copy
subsystem_component:
save: true
filename: sub_com_info.json
@@ -51,7 +53,6 @@ black_list:
- .repo
- .ccache
- doc
# - test
- build
# 排除out目录,为了避免排除其他路径下的out目录,这里详细列出了out下的所有目录
- out/gen
@@ -73,6 +74,7 @@ ipcamera_hispark_taurus:
bin: bin
so: usr/lib
etc: etc
hap: system/internal
rest: True # 是否将上面root目录下除了relative指定的目录归到etc并进行匹配
query_order: # 匹配顺序,key应当何relative字段中的key一致,value应当在上面的target_type字段中
so:
@@ -82,11 +84,20 @@ ipcamera_hispark_taurus:
- lite_library
- lite_component
- target
- unittest
bin:
- executable
- ohos_executable
- lite_component
- unittest
hap:
- hap_pack
# 对于脚本无法自动检测或误测部分,在这里手动配置
# manual_config:
# usr/lib/libplugin_demuxer_ffmpeg.so: # start with out
# subsystem: test_manual_config # [required]
# component: test_manual_config # [required]
# other_detail_info_key: other_detail_info_value # [optional]
ipcamera_hispark_taurus_linux:
@@ -98,6 +109,7 @@ ipcamera_hispark_taurus_linux:
bin: bin
so: usr/lib
etc: etc
hap: system/internal
rest: True
query_order:
so:
@@ -107,11 +119,15 @@ ipcamera_hispark_taurus_linux:
- lite_library
- lite_component
- target
- unittest
hap:
- hap_pack
bin:
- executable
- ohos_executable
- lite_component
- unittest
hispark_taurus_mini_system:
product_infofile: hispark_taurus_mini_system_product.json
@@ -131,11 +147,12 @@ hispark_taurus_mini_system:
- lite_library
- lite_component
- target
- unittest
bin:
- executable
- ohos_executable
- lite_component
- unittest
wifiiot_hispark_pegasus:
product_infofile: wifiiot_hispark_pegasus_product.json
@@ -169,7 +186,7 @@ hispark_pegasus_mini_system:
- lite_library
- target
rk3568_mini_system: # rk的目前从packages/phone/system_module_info.json中分析准确度更高,因为rk基本都使用的是ohos_xxx,而lite/small的更多的是使用的gn原生target template
rk3568: # rk的目前从packages/phone/system_module_info.json中分析准确度更高,因为rk基本都使用的是ohos_xxx,而lite/small的更多的是使用的gn原生target template
product_infofile: rk3568_mini_system_product.json
output_name: rk3568_mini_system_result.json
product_dir:
@@ -188,12 +205,13 @@ rk3568_mini_system: # rk的目前从packages/phone/system_module_info.json中分
- lite_library
- lite_component
- target
- unittest
bin:
- ohos_executable
- executable
- lite_component
- unittest
hap:
- ohos_hap
- hap_pack
@@ -34,6 +34,8 @@ _config = SimpleYamlTool.read_yaml("config.yaml")
"""
===============info handlers===============
"""
def extension_handler(paragraph: Text):
return GnVariableParser.string_parser("output_extension", paragraph).strip('"')
@@ -47,6 +49,10 @@ def target_type_handler(paragraph: Text):
return tt
def mod_handler(paragraph: Text):
return GnVariableParser.string_parser("mode", paragraph).strip('"')
"""
===============gn lineno collector===============
"""
@@ -132,9 +138,29 @@ def add_postfix(content: str, postfix: str) -> str:
class DefaultPostHandler(BasePostHandler):
def run(self, unit: Dict[str, AnyStr]):
if "extension" in unit.keys() and (not unit["output_name"].endswith(unit["extension"])):
out = unit["output_name"].rstrip(
".")+"."+unit["extension"].lstrip(".")
return out
return unit["output_name"]
class UnittestPostHandler(BasePostHandler):
def run(self, unit: Dict[str, AnyStr]):
if "output_extension" in unit.keys() and (not unit["output_name"].endswith(unit["output_extension"])):
out = unit["output_name"].rstrip(
".")+"."+unit["output_extension"].lstrip(".")
return out
return unit["output_name"]
class HapPackPostHandler(BasePostHandler):
def run(self, unit: Dict[str, AnyStr]):
hap_name = unit.get("hap_name")
mode = unit.get("mode")
return hap_name + "." + mode
class HAPPostHandler(BasePostHandler):
"""
for ohos_hap"""
@@ -29,8 +29,8 @@ from threading import RLock
import collections
from config import result_dict, collector_config, configs, \
project_path, sub_com_dict, product_name, recollect_gn, baseline
from pkgs.basic_tool import BasicTool
project_path, sub_com_dict, product_name, recollect_gn, baseline, unit_adapt
from pkgs.basic_tool import BasicTool, unit_adaptive
from pkgs.gn_common_tool import GnCommonTool
from pkgs.simple_excel_writer import SimpleExcelWriter
from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector
@@ -179,6 +179,7 @@ class RomAnalysisTool:
@classmethod
def _fuzzy_match(cls, file_name: str, filter_path_keyword: Tuple[str] = tuple()) -> Tuple[str, str, str]:
"""
TODO 应当先遍历gn_info进行匹配
直接grep,利用出现次数最多的BUILD.gn去定位subsystem_name和component_name"""
logging.info(f"fuzzy match: {file_name}")
_, base_name = os.path.split(file_name)
@@ -294,20 +295,67 @@ class RomAnalysisTool:
excel_writer.save(output_name)
logging.info("save as xls success.")
@classmethod
def _result_unit_adaptive(cls, result_dict: Dict[str, Dict]) -> None:
total_size = unit_adaptive(result_dict["size"])
del result_dict["size"]
for subsystem_name, subsystem_info in result_dict.items():
sub_size = unit_adaptive(subsystem_info["size"])
count = subsystem_info["count"]
del subsystem_info["size"]
del subsystem_info["count"]
for component_name, component_info in subsystem_info.items():
component_info["size"] = unit_adaptive(component_info["size"])
subsystem_info["size"] = sub_size
subsystem_info["count"] = count
result_dict["size"] = total_size
@classmethod
def _match_manual_configured(cls, manual_config_info: Dict[str, Dict], compiled_files: Dict[str, List], compiled_root_path: str, result_dict: Dict[str, Dict]) -> None:
for file_path, file_info in manual_config_info.items():
full_path = os.path.join(
project_path, compiled_root_path, file_path)
if not os.path.isfile(full_path):
logging.warning(f"config error: {file_path} is not a file.")
continue
file_info["size"] = os.path.getsize(full_path)
file_info["file_name"] = full_path
cls._put(file_info["subsystem"],
file_info["component"], file_info, result_dict)
for _, v in compiled_files.items():
if full_path not in v:
continue
index = v.index(full_path)
del v[index]
break
@ classmethod
def analysis(cls, product_name: str, product_dict: Dict[str, List[str]]):
"""analysis the rom of lite/small product
Args:
product_name (str): product name configured in the yaml
product_dict (Dict[str, List[str]]): result dict of compiled product file
format:
"bin":[...],
"so":[...]
...
"""
logging.info("start analyzing...")
rom_ram_baseline: Dict[str, Dict] = RomRamBaselineCollector.collect(
project_path)
with open("rom_ram_baseline.json", 'w', encoding='utf-8') as f:
json.dump(rom_ram_baseline, f, indent=4)
gn_info_file = configs["gn_info_file"]
gn_info_file = configs["gn_info_file"] # filename to save gn_info
with open(gn_info_file, 'r', encoding='utf-8') as f:
gn_info = json.load(f)
query_order: Dict[str, List[str]
] = configs[product_name]["query_order"]
] = configs[product_name]["query_order"] # query order of the gn template to be matched
query_order["etc"] = configs["target_type"] # etc会查找所有的template
rom_size_dict: Dict = dict()
if "manual_config" in configs[product_name].keys():
cls._match_manual_configured(
configs[product_name]["manual_config"], product_dict, configs[product_name]["product_dir"]["root"], rom_size_dict)
for t, l in product_dict.items():
for f in l: # 遍历所有文件
if os.path.isdir(f):
@@ -339,8 +387,6 @@ class RomAnalysisTool:
d["subsystem_name"]).get(d["component_name"]).get("rom")
cls._put(d["subsystem_name"],
d["component_name"], d, rom_size_dict, component_rom_baseline)
if d["component_name"] == "unionman_products":
print(d)
find_flag = True
if not find_flag: # 如果指定序列中的template都没有查找到,则模糊匹配
# fuzzy match
@@ -363,6 +409,8 @@ class RomAnalysisTool:
"file_name": f.replace(project_path, ""),
"size": size,
}, rom_size_dict)
if unit_adapt:
cls._result_unit_adaptive(rom_size_dict)
with open(configs[product_name]["output_name"], 'w', encoding='utf-8') as f:
json.dump(rom_size_dict, f, indent=4)
cls._save_as_xls(rom_size_dict, product_name, baseline)
@@ -5,6 +5,16 @@ import glob
from pathlib import Path
from typing import *
def unit_adaptive(size: int) -> str:
unit_list = ["Byte", "KB", "MB", "GB"]
index = 0
while index < len(unit_list) and size >= 1024:
size /= 1024
index += 1
if index == len(unit_list):
index = len(unit_list)-1
size *= 1024
return str(round(size,2))+unit_list[index]
class BasicTool:
@classmethod
@@ -28,6 +28,7 @@ import typing
import xml.dom.minidom as dom
from typing import Dict
from pprint import pprint
from pkgs.basic_tool import unit_adaptive
from pkgs.simple_excel_writer import SimpleExcelWriter
@@ -277,7 +278,7 @@ class RamAnalyzer:
return process_elf_dict
@classmethod
def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str):
def __save_result_as_excel(cls, data_dict: dict, filename: str, ss: str, baseline_file: str, unit_adapt:bool):
"""
保存结果到excel中
子系统:{
@@ -297,12 +298,13 @@ class RamAnalyzer:
"""
tmp_dict = copy.deepcopy(data_dict)
writer = SimpleExcelWriter("ram_info")
header_unit = "" if unit_adapt else ", Byte"
header = header = [
"subsystem_name", "component_name", "component_size(ram, Byte)", "process_name", f"process_size({ss}, Byte)", "elf", "elf_size(Byte)"
"subsystem_name", "component_name", f"component_size(ram{header_unit})", "process_name", f"process_size({ss}{header_unit})", "elf", f"elf_size{'' if unit_adapt else '(Byte)'}"
]
if baseline_file:
header = [
"subsystem_name", "component_name", "component_size(ram, Byte)", "baseline", "process_name", f"process_size({ss}, Byte)", "elf", "elf_size(Byte)"
"subsystem_name", "component_name", f"component_size(ram{header_unit})", "baseline", "process_name", f"process_size({ss}{header_unit})", "elf", f"elf_size{'' if unit_adapt else '(Byte)'}"
]
writer.set_sheet_header(header)
subsystem_c = 0
@@ -409,10 +411,26 @@ class RamAnalyzer:
continue
component_info["baseline"] = baseline_dict[subsystem_name][component_name].get(
"ram")
@classmethod
def refactored_result_unit_adaptive(cls, result_dict:Dict[str,Dict])->None:
for subsystem_name, subsystem_info in result_dict.items():
sub_size = unit_adaptive(subsystem_info["size"])
del subsystem_info["size"]
for component_name, component_info in subsystem_info.items():
com_size = unit_adaptive(component_info["size"])
del component_info["size"]
for process_name, process_info in component_info.items():
pro_size = unit_adaptive(process_info["size"])
del process_info["size"]
for elf_name, elf_size in process_info["elf"].items():
process_info["elf"][elf_name]=unit_adaptive(elf_size)
process_info["size"] = pro_size
component_info["size"] = com_size
subsystem_info["size"] = sub_size
@classmethod
def analysis(cls, cfg_path: str, xml_path: str, rom_result_json: str, device_num: str,
output_file: str, ss: str, output_excel: bool, baseline_file: str):
output_file: str, ss: str, output_excel: bool, baseline_file: str, unit_adapt:bool):
"""
process size subsystem/component so so_size
"""
@@ -505,16 +523,19 @@ class RamAnalyzer:
base_dir, _ = os.path.split(output_file)
if len(base_dir) != 0 and not os.path.isdir(base_dir):
os.makedirs(base_dir, exist_ok=True)
with open(output_file + ".json", 'w', encoding='utf-8') as f:
json.dump(result_dict, f, indent=4)
refactored_result: Dict[str, Dict] = refacotr_result(result_dict)
if unit_adapt:
cls.refactored_result_unit_adaptive(refactored_result)
if baseline_file:
cls.add_baseline(refactored_result, baseline_file)
with open(f"refactored_{output_file}.json", 'w', encoding='utf-8') as f:
json.dump(refactored_result, f, indent=4)
if output_excel:
cls.__save_result_as_excel(
refactored_result, output_file + ".xls", ss, baseline_file)
refactored_result, output_file + ".xls", ss, baseline_file, unit_adapt)
def refacotr_result(ram_result: Dict[str, Dict]) -> Dict[str, Dict]:
@@ -563,6 +584,8 @@ def get_args():
help="baseline file of rom and ram generated by rom analysis.")
parser.add_argument("-o", "--output_filename", default="ram_analysis_result", type=str,
help="base name of output file, default: ram_analysis_result. eg: -o ram_analysis_result")
parser.add_argument("-u", "--unit_adaptive",
action="store_true", help="unit adaptive")
parser.add_argument("-e", "--excel", type=bool, default=False,
help="if output result as excel, default: False. eg: -e True")
args = parser.parse_args()
@@ -578,5 +601,6 @@ if __name__ == '__main__':
output_filename = args.output_filename
baseline_file = args.baseline_file
output_excel = args.excel
unit_adapt = args.unit_adaptive
RamAnalyzer.analysis(cfg_path, profile_path, rom_result,
device_num=device_num, output_file=output_filename, ss="Pss", output_excel=output_excel, baseline_file=baseline_file)
device_num=device_num, output_file=output_filename, ss="Pss", output_excel=output_excel, baseline_file=baseline_file, unit_adapt=unit_adapt)
@@ -24,7 +24,7 @@ from copy import deepcopy
from typing import *
from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector
from pkgs.basic_tool import BasicTool
from pkgs.basic_tool import BasicTool, unit_adaptive
from pkgs.gn_common_tool import GnCommonTool
from pkgs.simple_excel_writer import SimpleExcelWriter
@@ -142,7 +142,7 @@ class RomAnalyzer:
return baseline_dict.get(subsystem_name).get(component_name).get("rom")
size = unit.get("size")
relative_filepath = unit.get("relative_filepath")
if result_dict.get(subsystem_name) is None: # 子系统
if result_dict.get(subsystem_name) is None: # 子系统
result_dict[subsystem_name] = dict()
result_dict[subsystem_name]["size"] = 0
result_dict[subsystem_name]["file_count"] = 0
@@ -161,9 +161,23 @@ class RomAnalyzer:
result_dict[subsystem_name][component_name]["file_count"] += 1
result_dict[subsystem_name][component_name][relative_filepath] = size
@classmethod
def result_unit_adaptive(self, result_dict:Dict[str,Dict])->None:
for subsystem_name, subsystem_info in result_dict.items():
size = unit_adaptive(subsystem_info["size"])
count = subsystem_info["file_count"]
if "size" in subsystem_info.keys():
del subsystem_info["size"]
if "file_count" in subsystem_info.keys():
del subsystem_info["file_count"]
for component_name, component_info in subsystem_info.items():
component_info["size"] = unit_adaptive(component_info["size"])
subsystem_info["size"] = size
subsystem_info["file_count"] = count
@classmethod
def analysis(cls, system_module_info_json: Text, product_dirs: List[str],
project_path: Text, product_name: Text, output_file: Text, output_execel: bool, add_baseline: bool):
project_path: Text, product_name: Text, output_file: Text, output_execel: bool, add_baseline: bool, unit_adapt: bool):
"""
system_module_info_json: json文件
product_dirs:要处理的产物的路径列表如["vendor", "system/"]
@@ -197,6 +211,8 @@ class RomAnalyzer:
output_dir, _ = os.path.split(output_file)
if len(output_dir) != 0:
os.makedirs(output_dir, exist_ok=True)
if unit_adapt:
cls.result_unit_adaptive(result_dict)
with open(output_file + ".json", 'w', encoding='utf-8') as f:
f.write(json.dumps(result_dict, indent=4))
if output_execel:
@@ -222,6 +238,8 @@ def get_args():
help="add baseline of component to the result(-b) or not.")
parser.add_argument("-o", "--output_file", type=str, default="rom_analysis_result",
help="basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result")
parser.add_argument("-u", "--unit_adaptive",
action="store_true", help="unit adaptive")
parser.add_argument("-e", "--excel", type=bool, default=False,
help="if output result as excel, default: False. eg: -e True")
args = parser.parse_args()
@@ -237,5 +255,6 @@ if __name__ == '__main__':
output_file = args.output_file
output_excel = args.excel
add_baseline = args.baseline
unit_adapt = args.unit_adaptive
RomAnalyzer.analysis(module_info_json, product_dirs,
project_path, product_name, output_file, output_excel, add_baseline)
project_path, product_name, output_file, output_excel, add_baseline, unit_adapt)