!20 deps_guard新增hdi和chipsetsdk检查

Merge pull request !20 from handy/0129

tools/deps_guard/deps_guard.py

@@ -11,7 +11,7 @@ def __createArgParser():
 	parser.add_argument('-i', '--input',
 						help='input asset files root directory', required=True)
 
-	parser.add_argument('-r', '--rules',
+	parser.add_argument('-r', '--rules', action='append',
 						help='rules directory', required=False)
 
 	parser.add_argument('-n', '--no-fail',

tools/deps_guard/elf_file_mgr/module_info/compile_info_loader.py

@@ -83,7 +83,9 @@ class CompileInfoLoader(object):
 			"innerapi": False,
 			"sa_id": 0,
 			"labelPath": "",
-			"version_script": ""
+			"version_script": "",
+			"shlib_type": "",
+			"innerapi_tags": ""
 		}
 
 		if info:
@@ -202,11 +204,11 @@ class CompileInfoLoader(object):
 				# For Chipset SDK modules detection
 				if callee["modGroup"] not in ("publicapi", "pentry"):
 					callee["modGroup"] = "innerapi_chc" # Cross high level component
-
-				dep["chipsetsdk"] = True
-				callee["chipsetsdk"] = True
-				if callee not in chipsetsdks:
-					chipsetsdks.append(callee)
+				if callee["hdiType"] != "hdi_proxy": # hdi proxy modules can be called by both system and chipset
+					dep["chipsetsdk"] = True
+					callee["chipsetsdk"] = True
+					if callee not in chipsetsdks:
+						chipsetsdks.append(callee)
 			elif dep["external"] == True:
 				if callee not in innerapi_ccs:
 					innerapi_ccs.append(callee)

tools/deps_guard/rules/ChipsetSDK/whitelist.json

@@ -1 +1,65 @@
-[]
+[
+	"libc.so",
+	"libc++.so",
+	"liblog.so",
+	"libhilog.so",
+	"libhisysevent.z.so",
+	"libservice_checker.z.so",
+	"libbegetutil.z.so",
+	"libbeget_proxy.z.so",
+	"libutils.z.so",
+	"libsec_shared.z.so",
+	"libhdi.z.so",
+	"libhdf_utils.z.so",
+	"libhdf_ipc_adapter.z.so",
+	"libipc_single.z.so",
+	"libdisplay_gralloc.z.so",
+
+	// Third party libraries
+	"libdrm.so",
+	"libpng.z.so",
+	"libxml2.z.so",
+	"libexif.z.so",
+	"libcjson.z.so",
+	"libjsoncpp.z.so",
+	"libnl_share.z.so",
+	"libprotobuf.z.so",
+	"libshared_libz.z.so",
+	"libcrypto_openssl.z.so",
+
+	"libudev.z.so",
+	"libsurface.z.so",
+	"libsamgr_proxy.z.so",
+	"libhril_innerkits.z.so",
+	"libpower_proxy_1.0.z.so",
+
+	"libbuffer_producer_sequenceable_1.0.z.so",
+	"libbuffer_handle_sequenceable_1.0.z.so",
+
+	// To be optimized
+	"libprotobuf_lite.z.so",
+	"libdmabufheap.z.so",
+	"libtinyxml2.z.so",
+	"libgralloc_priv.z.so",
+	"libsync_fence.z.so",
+
+	// chipset modules can be depended by system directly
+	"libosalbase.z.so",
+	"libmetadata.z.so",
+
+	"libdisplay_device.z.so",
+	"libdisplay_layer.z.so",
+
+	"libdispdev.z.so",
+	"libproperty.z.so",
+	"libhril_innerkits_ext.z.so",
+	"libhdi_input.z.so",
+
+	"libhitrace_meter.so",
+	"libinit_stub_empty.so",
+
+	// by libdisplay_layer_video.z.so for hispark taurus platform
+	"libhdi_video_layer_client.z.so",
+	// by libwpa.z.so for hispark taurus platform
+	"libwifi_driver_client.z.so"
+]

tools/deps_guard/rules/NO-Depends-On-HDI/README.md

@@ -0,0 +1,106 @@
+# NO-Depends-On-HDI规则说明
+
+
+
+## 1. HDI模块的定义
+
+HDI模块是指HDI服务对应的动态库模块，一般都在hcs文件中描述，示例如下：
+
+```json
+root {
+    module="sample";
+    ipp_algo_config {
+        algo1 {
+            name = "example";
+            description = "example algorithm";
+            path = "libcamera_ipp_algo_example.z.so";
+            mode = "IPP_ALGO_MODE_NORMAL";
+        }
+    }
+}
+```
+
+如上图所示，这些path里的动态库就是HDI模块，由HDF框架通过dlopen的方式动态加载。
+
+## 2. 规则解释
+
+NO-Depends-On-HDI规则有两个方面的含义：
+
+### 2.1 HDI模块需要在编译模板中标识
+
+如下图所示，每个HDI模块需要在对应的BUILD.gn中通过shlib_type字段来标识其类型：
+
+```go
+ohos_shared_library(sample_hdi_module) {
+    ...
+    shlib_type = "hdi"
+    ...
+}
+```
+
+同样，非HDI模块不要增加shlib_type="hdi"标识。
+
+### 2.2 不允许依赖HDI模块
+
+编译框架上ohos_shared_library模板会对所有的HDI模块（shlib_type为hdi）进行全局符号优化，使得所有的HDI模块默认都不对外暴露符号，减小HDI模块的大小，同时加快HDI模块的加载速度。
+
+```go
+# Set version_script for hdi service libraries
+if (defined(invoker.shlib_type) && invoker.shlib_type == "hdi") {
+  if (!defined(invoker.version_script)) {
+    _version_script = rebase_path("//build/templates/cxx/hdi.versionscript")
+    inputs += [ _version_script ]
+    ldflags += [ "-Wl,--version-script=${_version_script}" ]
+  }
+}
+```
+
+## 3. 违规场景及处理方案建议
+
+### 3.1 HDI模块没有添加shlib_type标识
+
+处理方式：参考2.2章节的描述增加shlib_type = "hdi"标识。
+
+### 3.2 其它模块确实使用到了SA模块中的符号
+
+**方案一**：把HDI模块中被使用到的符号下沉到对应的Inner API模块中供调用者使用，解除对HDI模块的依赖。
+
+
+
+**方案二**：对于支持插件扩展的HDI模块，需要提供API给插件调用。此场景可以通过version_script来显式申明HDI模块需要对外提供的符号，示例如下：
+
+```go
+ohos_shared_library(partly_exported_symbols) {
+    ...
+    version_script = "libbeget_proxy.versionscript"
+    ...
+}
+```
+
+示例的version_script文件如下：
+
+```apl
+1.0 {
+  global:
+    AclGetDevUdid;
+    AclGetSerial;
+    ServiceWatchForStatus;
+    SystemWatchParameter;
+    WatchParameter;
+    Remove*Watcher;
+  local:
+    *;
+};
+```
+
+> 提示：如果符号名称有规律，可以使用*作为通配符简化version_script编写的工作量；如Acl\*;。
+
+### 3.3 ut测试代码需使用HDI模块中的符号
+
+此场景可以为HDI模块增加静态库目标，ut测试代码deps静态库目标来完成测试。
+
+## 4. 例外说明
+
+HDI模块默认都不对外暴露符号；如需提供符号，通过version_script来申明，无其它例外选项。
+
+当前的白名单列表只用于归档存量待整改模块，整改完成后需清零。

tools/deps_guard/rules/NO-Depends-On-HDI/whitelist.json

@@ -0,0 +1,7 @@
+[
+	"libhril_hdf.z.so",
+	"libhril_hdf_ext.z.so",
+	"libusb_pnp_manager.z.so",
+	"libcamera_daemon.so",
+	"libril_driver.z.so"
+]

tools/deps_guard/rules/NO-Depends-On-SA/README.md

@@ -38,7 +38,7 @@ ohos_shared_library(sample_sa_module) {
 }
 ```
 
-同样，非SA模块不要增加shlib_type标识。
+同样，非SA模块不要增加shlib_type="sa"标识。
 
 ### 2.2 不允许依赖SA模块
 

tools/deps_guard/rules_checker/__init__.py

@@ -3,16 +3,21 @@
 
 from .napi_rule import NapiRule
 from .sa_rule import SaRule
+from .hdi_rule import HdiRule
+from .chipsetsdk import ChipsetSDKRule
 
 def check_all_rules(mgr, args):
 	rules = [
 		NapiRule,
-		SaRule
+		SaRule,
+		HdiRule,
+		ChipsetSDKRule
 	]
 
 	passed = True
 	for rule in rules:
 		r = rule(mgr, args)
+		r.log("Do %s rule checking now:" % rule.RULE_NAME)
 		if not r.check():
 			passed = False
 

tools/deps_guard/rules_checker/base_rule.py

@@ -9,18 +9,26 @@ class BaseRule(object):
 
 	def __init__(self, mgr, args):
 		self._mgr = mgr
-		self.__load_white_lists(args)
+		self._args = args
+		self.__white_lists = self.load_files("whitelist.json")
+
+	def load_files(self, name):
+		rules_dir = []
+		rules_dir.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../rules"))
+		if self._args and self._args.rules:
+			rules_dir = rules_dir + self._args.rules
 
-	def __load_white_lists(self, args):
 		res = []
-		if args and args.rules:
-			rules_path = args.rules
-		else:
-			rules_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../rules")
+		for d in rules_dir:
+			rules_file = os.path.join(d, self.__class__.RULE_NAME, name)
+			try:
+				with open(rules_file, "r") as f:
+					jsonstr = "".join([ line.strip() for line in f if not line.strip().startswith("//") ])
+					res = res + json.loads(jsonstr)
+			except:
+				pass
 
-		rules_file = os.path.join(rules_path, self.__class__.RULE_NAME, "whitelist.json")
-		with open(rules_file, "rb") as f:
-			self.__white_lists = json.load(f)
+		return res
 
 	def get_mgr(self):
 		return self._mgr

tools/deps_guard/rules_checker/chipsetsdk.py

@@ -0,0 +1,122 @@
+#! /usr/bin/env python
+#coding=utf-8
+
+import os
+import json
+
+from .base_rule import BaseRule
+
+class ChipsetSDKRule(BaseRule):
+	RULE_NAME = "ChipsetSDK"
+
+	def __is_chipsetsdk_tagged(self, mod):
+		if not "innerapi_tags" in mod:
+			return False
+		if "chipsetsdk" in mod["innerapi_tags"] or "csdk" in mod["innerapi_tags"]:
+			return True
+		return False
+
+	def __write_innerkits_header_files(self, chipsetsdks):
+		inner_kits_info = os.path.join(self.get_mgr().get_product_out_path(), "build_configs/parts_info/inner_kits_info.json")
+		with open(inner_kits_info, "r") as f:
+			info = json.load(f)
+
+		headers = []
+		for sdk in chipsetsdks:
+			path = sdk["labelPath"][:sdk["labelPath"].find(":")]
+			item = {"chipsetsdk": sdk["name"], "path": path, "headers": []}
+			if sdk["componentName"] not in info:
+				#print("%s:%s has no innerapi info" % (sdk["name"], sdk["componentName"]))
+				headers.append(item)
+				continue
+
+			for name, innerapi in info[sdk["componentName"]].items():
+				if innerapi["label"] != sdk["labelPath"]:
+					continue
+				gotHeaders = True
+				base = innerapi["header_base"]
+				for f in innerapi["header_files"]:
+					item["headers"].append(base + "/" + f)
+			headers.append(item)
+
+		try:
+			with open(os.path.join(self.get_mgr().get_product_images_path(), "chipsetsdk_info.json"), "w") as f:
+				json.dump(headers, f, indent = 4)
+		except:
+			pass
+
+		return headers
+
+	def __check_depends_on_chipsetsdk(self):
+		lists = self.get_white_lists()
+
+		passed = True
+
+		chipsetsdks = []
+		modules_with_chipsetsdk_tag = []
+
+		# Check if any napi modules has dependedBy
+		for mod in self.get_mgr().get_all():
+			if self.__is_chipsetsdk_tagged(mod):
+				modules_with_chipsetsdk_tag.append(mod)
+
+			# Check chipset modules only
+			if mod["path"].startswith("system"):
+				continue
+
+			# Check chipset modules depends
+			for dep in mod["deps"]:
+				callee = dep["callee"]
+
+				# If callee is chipset module, it is OK
+				if not callee["path"].startswith("system"):
+					continue
+
+				if callee not in chipsetsdks:
+					chipsetsdks.append(callee)
+				# If callee is in Chipset SDK white list module, it is OK
+				if callee["name"] in lists:
+					continue
+
+				# If callee is hdi proxy module, it is OK
+				if "hdiType" in callee and callee["hdiType"] == "hdi_proxy":
+					continue
+
+				# Not allowed
+				passed = False
+				self.error("chipset module %s depends on non Chipset SDK module %s in %s" % (mod["name"], callee["name"], mod["labelPath"]))
+
+			# Check chipset modules dependedBy
+			for dep in mod["dependedBy"]:
+				caller = dep["caller"]
+
+				# Called by chipset module, it is OK
+				if not caller["path"].startswith("system"):
+					continue
+
+				if mod not in chipsetsdks:
+					chipsetsdks.append(mod)
+
+				# If chipset module is in Chipset SDK white list module, it is OK
+				if mod["name"] in lists:
+					continue
+
+				# Not allowed
+				passed = False
+				self.error("system module %s depends on chipset module %s in %s" % (caller["name"], mod["name"], caller["labelPath"]))
+
+		for mod in chipsetsdks:
+			if not self.__is_chipsetsdk_tagged(mod):
+				self.warn('Chipset SDK module %s has no innerapi_tags with "chipsetsdk" or "csdk", add it in %s' % (mod["name"], mod["labelPath"]))
+
+		for mod in modules_with_chipsetsdk_tag:
+			if mod["name"] not in lists:
+				passed = False
+				self.error('non chipsetsdk module %s with innerapi_tags="chipsetsdk" or "csdk", %s' % (mod["name"], mod["labelPath"]))
+
+		self.__write_innerkits_header_files(chipsetsdks)
+
+		return passed
+
+	def check(self):
+		return self.__check_depends_on_chipsetsdk()

tools/deps_guard/rules_checker/hdi_rule.py

@@ -0,0 +1,67 @@
+#! /usr/bin/env python
+#coding=utf-8
+
+import json
+
+from .base_rule import BaseRule
+
+class HdiRule(BaseRule):
+	RULE_NAME = "NO-Depends-On-HDI"
+
+	def __check_depends_on_hdi(self):
+		lists = self.get_white_lists()
+
+		passed = True
+
+		hdi_without_shlib_type = []
+		non_hdi_with_hdi_shlib_type = []
+
+		# Check if any napi modules has dependedBy
+		for mod in self.get_mgr().get_all():
+			#print("Check %s now " % mod["path"])
+			is_hdi = False
+			if "hdiType" in mod and mod["hdiType"] == "hdi_service":
+				is_hdi = True
+			# Collect non HDI modules with shlib_type of value "hdi"
+			if not is_hdi and ("shlib_type" in mod and mod["shlib_type"] == "hdi"):
+				non_hdi_with_hdi_shlib_type.append(mod)
+
+			# Collect HDI modules without shlib_type with value of "hdi"
+			if is_hdi and ("shlib_type" not in mod or mod["shlib_type"] != "hdi"):
+				if mod["name"] not in lists:
+					hdi_without_shlib_type.append(mod)
+
+			if not is_hdi:
+				continue
+
+			if len(mod["dependedBy"]) == 0:
+				continue
+
+			if mod["name"] in lists:
+				continue
+
+			# If hdi module has version_script to specify exported symbols, it can be depended by others
+			if "version_script" in mod:
+				continue
+
+			# Check if HDI modules is depended by other modules
+			self.error("hdi module %s depended by:" % mod["name"])
+			for dep in mod["dependedBy"]:
+				caller = dep["caller"]
+				self.log("   module [%s] defined in [%s]" % (caller["name"], caller["labelPath"]))
+			passed = False
+
+		if len(hdi_without_shlib_type) > 0:
+			for mod in hdi_without_shlib_type:
+				if mod["name"] not in lists:
+					passed = False
+					self.error('hdi module %s has no shlib_type="hdi", add it in %s' % (mod["name"], mod["labelPath"]))
+
+		if len(non_hdi_with_hdi_shlib_type) > 0:
+			for mod in non_hdi_with_hdi_shlib_type:
+				self.warn('non hdi module %s with shlib_type="hdi", %s' % (mod["name"], mod["labelPath"]))
+
+		return passed
+
+	def check(self):
+		return self.__check_depends_on_hdi()

tools/deps_guard/rules_checker/napi_rule.py

@@ -35,5 +35,4 @@ class NapiRule(BaseRule):
 		return passed
 
 	def check(self):
-		self.log("Do %s rule checking now:" % self.__class__.RULE_NAME)
 		return self.__check_depends_on_napi()

tools/deps_guard/rules_checker/sa_rule.py

@@ -58,10 +58,9 @@ class SaRule(BaseRule):
 		if len(non_sa_with_sa_shlib_type) > 0:
 			passed = False
 			for mod in non_sa_with_sa_shlib_type:
-				self.error('\033[91m[NOT ALLOWED]\x1b[0m: non sa module %s with shlib_type="sa", %s' % (mod["name"], mod["labelPath"]))
+				self.error('non sa module %s with shlib_type="sa", %s' % (mod["name"], mod["labelPath"]))
 
 		return passed
 
 	def check(self):
-		self.log("Do %s rule checking now:" % self.__class__.RULE_NAME)
 		return self.__check_depends_on_sa()