From 2854114b28911ca0656578d63d907017641d54c2 Mon Sep 17 00:00:00 2001 From: Ajin Abraham Date: Mon, 14 Aug 2023 11:47:55 -0700 Subject: [PATCH] [HOTFIX][EFR06] Independent Shared Object (.so) Scan and Improved String search (#2228) * String extraction from APK, Source, AAR, JAR, SO * Strings sections to show source of strings extracted * Strings Refactor * Support for independent .SO scan * Android SCA rules update * Entropies scan support for strings * URLs/Email extraction refactor * Bug Fixes * iOS Source Report Fix * Frida APK Patcher (WIP) * Dynamic Analyzer identifier not available * Settings env var not working fix for enabled by default features * AppSec Score fix * Recent `scan not completed` fix for iOS zip --- mobsf/DynamicAnalyzer/tools/apk_patcher.py | 97 +++++++++ .../views/android/environment.py | 4 + .../views/MalwareDomainCheck.py | 3 +- mobsf/MalwareAnalyzer/views/apkid.py | 6 +- mobsf/MalwareAnalyzer/views/quark.py | 5 +- mobsf/MobSF/init.py | 2 +- mobsf/MobSF/settings.py | 15 +- mobsf/MobSF/utils.py | 36 +++- mobsf/MobSF/views/helpers.py | 12 +- mobsf/MobSF/views/home.py | 2 + mobsf/MobSF/views/scanning.py | 9 + mobsf/StaticAnalyzer/forms.py | 3 +- mobsf/StaticAnalyzer/models.py | 2 +- .../views/android/binary_analysis.py | 18 +- .../views/android/code_analysis.py | 16 +- .../StaticAnalyzer/views/android/converter.py | 3 +- .../views/android/db_interaction.py | 10 +- .../views/android/generate_downloads.py | 6 +- mobsf/StaticAnalyzer/views/android/jar_aar.py | 29 ++- .../views/android/rules/android_rules.yaml | 6 +- mobsf/StaticAnalyzer/views/android/so.py | 184 ++++++++++++++++++ .../views/android/static_analyzer.py | 55 ++++-- mobsf/StaticAnalyzer/views/android/strings.py | 160 +++++++++++---- .../views/android/view_source.py | 2 +- mobsf/StaticAnalyzer/views/common/appsec.py | 6 +- mobsf/StaticAnalyzer/views/common/entropy.py | 63 ++++++ .../views/common/shared_func.py | 66 +++++-- .../views/ios/plist_analysis.py | 6 +- .../views/ios/static_analyzer.py | 8 +- mobsf/StaticAnalyzer/views/sast_engine.py | 8 +- mobsf/templates/general/home.html | 4 +- mobsf/templates/general/recent.html | 5 +- mobsf/templates/pdf/android_report.html | 27 ++- .../android_binary_analysis.html | 141 ++++++++++---- .../android_source_analysis.html | 80 +++++--- .../static_analysis/ios_source_analysis.html | 33 +++- tox.ini | 2 + 37 files changed, 912 insertions(+), 222 deletions(-) create mode 100644 mobsf/DynamicAnalyzer/tools/apk_patcher.py create mode 100644 mobsf/StaticAnalyzer/views/android/so.py create mode 100644 mobsf/StaticAnalyzer/views/common/entropy.py diff --git a/mobsf/DynamicAnalyzer/tools/apk_patcher.py b/mobsf/DynamicAnalyzer/tools/apk_patcher.py new file mode 100644 index 00000000..feff9b59 --- /dev/null +++ b/mobsf/DynamicAnalyzer/tools/apk_patcher.py @@ -0,0 +1,97 @@ +"""Inject Frida Gadget to an APK.""" +import subprocess +from pathlib import Path +from lzma import LZMAFile +from shutil import copyfileobj +from tempfile import gettempdir +import logging + +import requests + +from django.conf import settings + +from mobsf.MobSF.utils import ( + find_java_binary, + is_file_exists, + is_internet_available, + upstream_proxy, +) + + +logger = logging.getLogger(__name__) + + +class GadgetPatcher: + + def __init__(self, app_dir, tools_dir, apk_file) -> None: + self.app_dir = app_dir + self.tools_dir = tools_dir + self.apk_file = apk_file + self.output_dir = Path(self.app_dir) / 'apktool_repack' + if (len(settings.APKTOOL_BINARY) > 0 + and is_file_exists(settings.APKTOOL_BINARY)): + self.apktool_path = settings.APKTOOL_BINARY + else: + self.apktool_path = Path(self.tools_dir) / 'apktool_2.7.0.jar' + + def decompile_apk(self): + """Decompile APK.""" + args = [find_java_binary(), + '-jar', + self.apktool_path, + '--frame-path', + gettempdir(), + '-f', '-r', '-s', 'd', + self.apk_file, + '-o', + self.output_dir] + manifest = self.output_dir / 'AndroidManifest.xml' + if not is_file_exists(manifest): + # Skip if already decompiled + subprocess.check_output(args) + + def recompile_apk(self): + """Recompile APK.""" + args = [find_java_binary(), + '-jar', + self.apktool_path, + '-f', 'b', + '-o', + '.patched.apk'.join( + self.apk_file.rsplit('.apk', 1)), + self.output_dir] + subprocess.check_output(args) + + def download_frida_gadget(self, frida_arch, aarch, version): + """Download Frida Gadget.""" + gadget_bin = self.output_dir / 'lib' / {aarch} / 'libfrida-gadget.so' + fgadget = f'frida-gadget-{version}-android-{frida_arch}.so' + if gadget_bin.is_file(): + return True + if not is_internet_available(): + return None + try: + proxies, verify = upstream_proxy('https') + except Exception: + logger.exception('[ERROR] Setting upstream proxy') + return None + try: + response = requests.get(f'{settings.FRIDA_SERVER}{version}', + timeout=3, + proxies=proxies, + verify=verify) + for item in response.json()['assets']: + if item['name'] == f'{fgadget}.xz': + url = item['browser_download_url'] + break + if not url: + return None + logger.info('Downloading frida-gadget %s', fgadget) + with requests.get(url, stream=True) as r: + with LZMAFile(r.raw) as f: + with open(gadget_bin, 'wb') as flip: + copyfileobj(f, flip) + return True + except Exception: + logger.exception('[ERROR] Fetching Frida Gadget Release') + return None diff --git a/mobsf/DynamicAnalyzer/views/android/environment.py b/mobsf/DynamicAnalyzer/views/android/environment.py index 888aa4af..c48b85a1 100644 --- a/mobsf/DynamicAnalyzer/views/android/environment.py +++ b/mobsf/DynamicAnalyzer/views/android/environment.py @@ -71,6 +71,8 @@ class Environment: def connect(self): """ADB Connect.""" + if not self.identifier: + return False logger.info('Connecting to Android %s', self.identifier) self.run_subprocess_verify_output([get_adb(), 'connect', @@ -78,6 +80,8 @@ class Environment: def connect_n_mount(self): """Test ADB Connection.""" + if not self.identifier: + return False self.adb_command(['kill-server']) self.adb_command(['start-server']) logger.info('ADB Restarted') diff --git a/mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py b/mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py index 971810c3..212e3e5e 100644 --- a/mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py +++ b/mobsf/MalwareAnalyzer/views/MalwareDomainCheck.py @@ -16,6 +16,7 @@ import IP2Location from mobsf.MobSF.utils import ( is_internet_available, + settings_enabled, update_local_db, ) @@ -178,7 +179,7 @@ class MalwareDomainCheck: 'Skipping Malware Database Update.') def scan(self, urls): - if not settings.DOMAIN_MALWARE_SCAN: + if not settings_enabled('DOMAIN_MALWARE_SCAN'): logger.info('Domain Malware Check disabled in settings') return self.result self.domainlist = get_domains(urls) diff --git a/mobsf/MalwareAnalyzer/views/apkid.py b/mobsf/MalwareAnalyzer/views/apkid.py index 3eead1a5..f1fc324a 100644 --- a/mobsf/MalwareAnalyzer/views/apkid.py +++ b/mobsf/MalwareAnalyzer/views/apkid.py @@ -4,12 +4,16 @@ import os from django.conf import settings +from mobsf.MobSF.utils import ( + settings_enabled, +) + logger = logging.getLogger(__name__) def apkid_analysis(app_dir, apk_file, apk_name): """APKID Analysis of DEX files.""" - if not settings.APKID_ENABLED: + if not settings_enabled('APKID_ENABLED'): return {} try: import apkid diff --git a/mobsf/MalwareAnalyzer/views/quark.py b/mobsf/MalwareAnalyzer/views/quark.py index cbc53ea5..976625fe 100644 --- a/mobsf/MalwareAnalyzer/views/quark.py +++ b/mobsf/MalwareAnalyzer/views/quark.py @@ -2,11 +2,10 @@ import logging from pathlib import Path -from django.conf import settings - from mobsf.MobSF.utils import ( disable_print, enable_print, + settings_enabled, ) logger = logging.getLogger(__name__) @@ -14,7 +13,7 @@ logger = logging.getLogger(__name__) def quark_analysis(app_dir, apk_file): """QUARK Analysis of APK files.""" - if not getattr(settings, 'QUARK_ENABLED', True): + if not settings_enabled('QUARK_ENABLED'): return [] try: import quark diff --git a/mobsf/MobSF/init.py b/mobsf/MobSF/init.py index 5ecdaa4a..3b0eced2 100644 --- a/mobsf/MobSF/init.py +++ b/mobsf/MobSF/init.py @@ -10,7 +10,7 @@ from mobsf.install.windows.setup import windows_config_local logger = logging.getLogger(__name__) -VERSION = '3.7.2' +VERSION = '3.7.3' BANNER = """ __ __ _ ____ _____ _____ _____ | \/ | ___ | |__/ ___|| ___|_ _|___ /|___ | diff --git a/mobsf/MobSF/settings.py b/mobsf/MobSF/settings.py index 250dec11..c64ea63b 100644 --- a/mobsf/MobSF/settings.py +++ b/mobsf/MobSF/settings.py @@ -71,6 +71,7 @@ ALLOWED_EXTENSIONS = { '.ipa': 'application/octet-stream', '.jar': 'application/java-archive', '.aar': 'application/octet-stream', + '.so': 'application/octet-stream', } # =============ALLOWED MIMETYPES================= APK_MIME = [ @@ -324,18 +325,18 @@ else: # Disable CVSSV2 Score by default CVSS_SCORE_ENABLED = bool(os.getenv('MOBSF_CVSS_SCORE_ENABLED', '')) # NIAP Scan - NIAP_ENABLED = bool(os.getenv('MOBSF_NIAP_ENABLED', '1')) + NIAP_ENABLED = os.getenv('MOBSF_NIAP_ENABLED', '1') # Dex 2 Smali Conversion - DEX2SMALI_ENABLED = bool(os.getenv('MOBSF_DEX2SMALI_ENABLED', '1')) + DEX2SMALI_ENABLED = os.getenv('MOBSF_DEX2SMALI_ENABLED', '1') # Android Shared Object Binary Analysis - SO_ANALYSIS_ENABLED = bool(os.getenv('MOBSF_SO_ANALYSIS_ENABLED', '1')) + SO_ANALYSIS_ENABLED = os.getenv('MOBSF_SO_ANALYSIS_ENABLED', '1') # ================================================= # -------------------------- # MALWARE ANALYZER SETTINGS # -------------------------- - DOMAIN_MALWARE_SCAN = bool(os.getenv('MOBSF_DOMAIN_MALWARE_SCAN', '1')) - APKID_ENABLED = bool(os.getenv('MOBSF_APKID_ENABLED', '1')) + DOMAIN_MALWARE_SCAN = os.getenv('MOBSF_DOMAIN_MALWARE_SCAN', '1') + APKID_ENABLED = os.getenv('MOBSF_APKID_ENABLED', '1') QUARK_ENABLED = bool(os.getenv('MOBSF_QUARK_ENABLED', '')) # ================================================== # ======WINDOWS STATIC ANALYSIS SETTINGS =========== @@ -402,8 +403,8 @@ else: # If you are behind a Proxy UPSTREAM_PROXY_ENABLED = bool(os.getenv( 'MOBSF_UPSTREAM_PROXY_ENABLED', '')) - UPSTREAM_PROXY_SSL_VERIFY = bool(os.getenv( - 'MOBSF_UPSTREAM_PROXY_SSL_VERIFY', '1')) + UPSTREAM_PROXY_SSL_VERIFY = os.getenv( + 'MOBSF_UPSTREAM_PROXY_SSL_VERIFY', '1') UPSTREAM_PROXY_TYPE = os.getenv('MOBSF_UPSTREAM_PROXY_TYPE', 'http') UPSTREAM_PROXY_IP = os.getenv('MOBSF_UPSTREAM_PROXY_IP', '127.0.0.1') UPSTREAM_PROXY_PORT = int(os.getenv('MOBSF_UPSTREAM_PROXY_PORT', '3128')) diff --git a/mobsf/MobSF/utils.py b/mobsf/MobSF/utils.py index 455d0774..629dbe6e 100755 --- a/mobsf/MobSF/utils.py +++ b/mobsf/MobSF/utils.py @@ -61,7 +61,7 @@ def upstream_proxy(flaw_type): proxies = {flaw_type: proxy_host} else: proxies = {flaw_type: None} - verify = bool(settings.UPSTREAM_PROXY_SSL_VERIFY) + verify = settings.UPSTREAM_PROXY_SSL_VERIFY in ('1', '"1"') return proxies, verify @@ -367,7 +367,8 @@ def get_device(): out = subprocess.check_output([get_adb(), 'devices']).splitlines() if len(out) > 2: dev_id = out[1].decode('utf-8').split('\t')[0] - return docker_translate_localhost(dev_id) + if 'daemon started successfully' not in dev_id: + return docker_translate_localhost(dev_id) logger.error(get_android_dm_exception_msg()) @@ -626,6 +627,13 @@ def is_zip_magic(file_obj): return bool(magic == b'\x50\x4B\x03\x04') +def is_elf_so_magic(file_obj): + magic = file_obj.read(4) + file_obj.seek(0, 0) + # ELF/SO Magic + return bool(magic == b'\x7F\x45\x4C\x46') + + def disable_print(): sys.stdout = open(os.devnull, 'w') @@ -680,3 +688,27 @@ def get_android_dm_exception_msg(): ' set ANALYZER_IDENTIFIER in ' f'{get_config_loc()} or via environment variable' ' MOBSF_ANALYZER_IDENTIFIER') + + +def get_android_src_dir(app_dir, typ): + """Get Android source code location.""" + if typ == 'apk': + src = app_dir / 'java_source' + elif typ == 'studio': + src = app_dir / 'app' / 'src' / 'main' / 'java' + kt = app_dir / 'app' / 'src' / 'main' / 'kotlin' + if not src.exists() and kt.exists(): + src = kt + elif typ == 'eclipse': + src = app_dir / 'src' + return src + + +def settings_enabled(attr): + """Get settings state if present.""" + disabled = ('', ' ', '""', '" "', '0', '"0"') + if not getattr(settings, attr, True): + return False + if getattr(settings, attr) in disabled: + return False + return True diff --git a/mobsf/MobSF/views/helpers.py b/mobsf/MobSF/views/helpers.py index 67b35572..31041022 100644 --- a/mobsf/MobSF/views/helpers.py +++ b/mobsf/MobSF/views/helpers.py @@ -1,7 +1,10 @@ """Helpers.""" import functools -from mobsf.MobSF.utils import is_zip_magic +from mobsf.MobSF.utils import ( + is_elf_so_magic, + is_zip_magic, +) from django.conf import settings from django.http import HttpRequest, HttpResponseNotAllowed @@ -16,6 +19,7 @@ class FileType(object): self.file_type = file_obj.content_type self.file_name_lower = file_obj.name.lower() self.zip = is_zip_magic(file_obj) + self.so = is_elf_so_magic(file_obj) def is_allow_file(self): """ @@ -23,6 +27,8 @@ class FileType(object): return bool """ + if self.so and self.is_so(): + return True if self.zip and ( self.is_apk() or self.is_xapk() @@ -55,6 +61,10 @@ class FileType(object): return (self.file_type in settings.APK_MIME and self.file_name_lower.endswith('.aar')) + def is_so(self): + return (self.file_type in settings.APK_MIME + and self.file_name_lower.endswith('.so')) + def is_zip(self): return (self.file_type in settings.ZIP_MIME and self.file_name_lower.endswith('.zip')) diff --git a/mobsf/MobSF/views/home.py b/mobsf/MobSF/views/home.py index 8a8dd690..e42e18f2 100755 --- a/mobsf/MobSF/views/home.py +++ b/mobsf/MobSF/views/home.py @@ -141,6 +141,8 @@ class Upload(object): return scanning.scan_jar() elif self.file_type.is_aar(): return scanning.scan_aar() + elif self.file_type.is_so(): + return scanning.scan_so() elif self.file_type.is_zip(): return scanning.scan_zip() elif self.file_type.is_ipa(): diff --git a/mobsf/MobSF/views/scanning.py b/mobsf/MobSF/views/scanning.py index 79fae435..11e8c980 100644 --- a/mobsf/MobSF/views/scanning.py +++ b/mobsf/MobSF/views/scanning.py @@ -116,6 +116,15 @@ class Scanning(object): logger.info('Performing Static Analysis of Android AAR') return self.data + def scan_so(self): + """Shared object file.""" + md5 = handle_uploaded_file(self.file, '.so') + self.data['hash'] = md5 + self.data['scan_type'] = 'so' + add_to_recent_scan(self.data) + logger.info('Performing Static Analysis of Shared Object') + return self.data + def scan_zip(self): """Android /iOS Zipped Source.""" md5 = handle_uploaded_file(self.file, '.zip') diff --git a/mobsf/StaticAnalyzer/forms.py b/mobsf/StaticAnalyzer/forms.py index f62e65f5..3cd606f1 100644 --- a/mobsf/StaticAnalyzer/forms.py +++ b/mobsf/StaticAnalyzer/forms.py @@ -66,7 +66,8 @@ class AndroidChecks(forms.Form): ('smali', 'smali'), ('apk', 'apk'), ('jar', 'jar'), - ('aar', 'aar'))) + ('aar', 'aar'), + ('so', 'so'))) def clean_file(self): """Safe Extension.""" diff --git a/mobsf/StaticAnalyzer/models.py b/mobsf/StaticAnalyzer/models.py index 910112da..5688a223 100755 --- a/mobsf/StaticAnalyzer/models.py +++ b/mobsf/StaticAnalyzer/models.py @@ -50,7 +50,7 @@ class StaticAnalyzerAndroid(models.Model): URLS = models.TextField(default=[]) DOMAINS = models.TextField(default={}) EMAILS = models.TextField(default=[]) - STRINGS = models.TextField(default=[]) + STRINGS = models.TextField(default={}) FIREBASE_URLS = models.TextField(default=[]) FILES = models.TextField(default=[]) EXPORTED_COUNT = models.TextField(default={}) diff --git a/mobsf/StaticAnalyzer/views/android/binary_analysis.py b/mobsf/StaticAnalyzer/views/android/binary_analysis.py index 6b5bcd12..148bc12f 100755 --- a/mobsf/StaticAnalyzer/views/android/binary_analysis.py +++ b/mobsf/StaticAnalyzer/views/android/binary_analysis.py @@ -3,10 +3,12 @@ import logging from pathlib import Path -from django.conf import settings - import lief +from mobsf.MobSF.utils import ( + settings_enabled, +) + logger = logging.getLogger(__name__) @@ -192,14 +194,16 @@ def elf_analysis(app_dir: str) -> dict: """Perform elf analysis on shared object.""" elf = {'elf_analysis': [], 'elf_strings': []} try: - if not getattr(settings, 'SO_ANALYSIS_ENABLED', True): + if not settings_enabled('SO_ANALYSIS_ENABLED'): return elf logger.info('Binary Analysis Started') - # Supports APK, AAR and JAR + sdir = Path(app_dir) + # Supports APK, SO, AAR and JAR libs = [ - Path(app_dir) / 'lib', - Path(app_dir) / 'libs', - Path(app_dir) / 'jni'] + sdir, + sdir / 'lib', + sdir / 'libs', + sdir / 'jni'] for lib_dir in libs: if not lib_dir.is_dir(): continue diff --git a/mobsf/StaticAnalyzer/views/android/code_analysis.py b/mobsf/StaticAnalyzer/views/android/code_analysis.py index 7ca3e7a5..9d088c88 100755 --- a/mobsf/StaticAnalyzer/views/android/code_analysis.py +++ b/mobsf/StaticAnalyzer/views/android/code_analysis.py @@ -6,7 +6,10 @@ from pathlib import Path from django.conf import settings -from mobsf.MobSF.utils import filename_from_path +from mobsf.MobSF.utils import ( + filename_from_path, + get_android_src_dir, +) from mobsf.StaticAnalyzer.views.common.shared_func import ( url_n_email_extract, ) @@ -31,16 +34,7 @@ def code_analysis(app_dir, typ, manifest_file): url_n_file = [] url_list = [] app_dir = Path(app_dir) - if typ == 'apk': - src = app_dir / 'java_source' - elif typ == 'studio': - src = app_dir / 'app' / 'src' / 'main' / 'java' - kt = app_dir / 'app' / 'src' / 'main' / 'kotlin' - if not src.exists() and kt.exists(): - src = kt - elif typ == 'eclipse': - src = app_dir / 'src' - src = src.as_posix() + '/' + src = get_android_src_dir(app_dir, typ).as_posix() + '/' skp = settings.SKIP_CLASS_PATH logger.info('Code Analysis Started on - %s', filename_from_path(src)) diff --git a/mobsf/StaticAnalyzer/views/android/converter.py b/mobsf/StaticAnalyzer/views/android/converter.py index 82093d5f..423d0e49 100755 --- a/mobsf/StaticAnalyzer/views/android/converter.py +++ b/mobsf/StaticAnalyzer/views/android/converter.py @@ -16,6 +16,7 @@ from mobsf.MobSF.utils import ( filename_from_path, find_java_binary, is_file_exists, + settings_enabled, ) @@ -31,7 +32,7 @@ def get_dex_files(app_dir): def dex_2_smali(app_dir, tools_dir): """Run dex2smali.""" try: - if not getattr(settings, 'DEX2SMALI_ENABLED', True): + if not settings_enabled('DEX2SMALI_ENABLED'): return logger.info('DEX -> SMALI') dexes = get_dex_files(app_dir) diff --git a/mobsf/StaticAnalyzer/views/android/db_interaction.py b/mobsf/StaticAnalyzer/views/android/db_interaction.py index 2a63f9a4..49a5572c 100755 --- a/mobsf/StaticAnalyzer/views/android/db_interaction.py +++ b/mobsf/StaticAnalyzer/views/android/db_interaction.py @@ -69,7 +69,7 @@ def get_context_from_db_entry(db_entry: QuerySet) -> dict: 'urls': python_list(db_entry[0].URLS), 'domains': python_dict(db_entry[0].DOMAINS), 'emails': python_list(db_entry[0].EMAILS), - 'strings': python_list(db_entry[0].STRINGS), + 'strings': python_dict(db_entry[0].STRINGS), 'firebase_urls': python_list(db_entry[0].FIREBASE_URLS), 'files': python_list(db_entry[0].FILES), 'exported_count': python_dict(db_entry[0].EXPORTED_COUNT), @@ -140,7 +140,7 @@ def get_context_from_analysis(app_dic, 'urls': code_an_dic['urls'], 'domains': code_an_dic['domains'], 'emails': code_an_dic['emails'], - 'strings': app_dic['strings'], + 'strings': code_an_dic['strings'], 'firebase_urls': code_an_dic['firebase'], 'files': app_dic['files'], 'exported_count': man_an_dic['exported_cnt'], @@ -148,7 +148,7 @@ def get_context_from_analysis(app_dic, 'quark': quark_report, 'trackers': trackers, 'playstore_details': app_dic['playstore'], - 'secrets': app_dic['secrets'], + 'secrets': code_an_dic['secrets'], } return context except Exception: @@ -202,7 +202,7 @@ def save_or_update(update_type, 'URLS': code_an_dic['urls'], 'DOMAINS': code_an_dic['domains'], 'EMAILS': code_an_dic['emails'], - 'STRINGS': app_dic['strings'], + 'STRINGS': code_an_dic['strings'], 'FIREBASE_URLS': code_an_dic['firebase'], 'FILES': app_dic['files'], 'EXPORTED_COUNT': man_an_dic['exported_cnt'], @@ -211,7 +211,7 @@ def save_or_update(update_type, 'TRACKERS': trackers, 'PLAYSTORE_DETAILS': app_dic['playstore'], 'NETWORK_SECURITY': man_an_dic['network_security'], - 'SECRETS': app_dic['secrets'], + 'SECRETS': code_an_dic['secrets'], } if update_type == 'save': db_entry = StaticAnalyzerAndroid.objects.filter( diff --git a/mobsf/StaticAnalyzer/views/android/generate_downloads.py b/mobsf/StaticAnalyzer/views/android/generate_downloads.py index d95af42f..a02cfe07 100644 --- a/mobsf/StaticAnalyzer/views/android/generate_downloads.py +++ b/mobsf/StaticAnalyzer/views/android/generate_downloads.py @@ -15,9 +15,9 @@ logger = logging.getLogger(__name__) def run(request): - """Generate downloads for apk, jar, aar, java zip and smali zip.""" + """Generate downloads for apk, jar, aar, so, java zip and smali zip.""" try: - allowed = ('apk', 'smali', 'java', 'jar', 'aar') + allowed = ('apk', 'smali', 'java', 'jar', 'aar', 'so') logger.info('Generating Downloads') md5 = request.GET['hash'] file_type = request.GET['file_type'] @@ -44,7 +44,7 @@ def run(request): shutil.make_archive( dwd_file.as_posix(), 'zip', directory.as_posix()) file_name = f'{md5}-smali.zip' - elif file_type in ('apk', 'ipa', 'jar', 'aar'): + elif file_type in ('apk', 'ipa', 'jar', 'aar', 'so'): # Binaries file_name = f'{md5}.{file_type}' src = app_dir / file_name diff --git a/mobsf/StaticAnalyzer/views/android/jar_aar.py b/mobsf/StaticAnalyzer/views/android/jar_aar.py index b25981e1..fe13ac96 100644 --- a/mobsf/StaticAnalyzer/views/android/jar_aar.py +++ b/mobsf/StaticAnalyzer/views/android/jar_aar.py @@ -25,7 +25,9 @@ from mobsf.StaticAnalyzer.views.android.manifest_analysis import ( manifest_analysis, manifest_data, ) -from mobsf.StaticAnalyzer.views.android.strings import strings_from_apk +from mobsf.StaticAnalyzer.views.android.strings import ( + get_strings_metadata, +) from mobsf.StaticAnalyzer.views.android.binary_analysis import elf_analysis from mobsf.StaticAnalyzer.views.android.cert_analysis import ( cert_info, @@ -151,29 +153,24 @@ def common_analysis(request, app_dic, rescan, api, analysis_type): quark_results = [] - # Get the strings from android resource and shared objects - string_res = strings_from_apk( + # Get the strings and metadata + get_strings_metadata( app_dic['app_file'], app_dic['app_dir'], - elf_dict['elf_strings']) - if string_res: - app_dic['strings'] = string_res['strings'] - app_dic['secrets'] = string_res['secrets'] - code_an_dic['urls_list'].extend( - string_res['urls_list']) - code_an_dic['urls'].extend(string_res['url_nf']) - code_an_dic['emails'].extend(string_res['emails_nf']) - else: - app_dic['strings'] = [] - app_dic['secrets'] = [] + elf_dict['elf_strings'], + 'apk', + ['.java'], + code_an_dic) + # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) # Domain Extraction and Malware Check logger.info( 'Performing Malware Check on extracted Domains') code_an_dic['domains'] = MalwareDomainCheck().scan( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) + app_dic['zipped'] = analysis_type app_dic['icon_hidden'] = True app_dic['icon_found'] = False diff --git a/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml b/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml index d6cef08e..17b9d1d9 100644 --- a/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml +++ b/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml @@ -561,10 +561,11 @@ masvs: crypto-3 ref: https://github.com/MobSF/owasp-mstg/blob/master/Document/0x04g-Testing-Cryptography.md#mobile-app-cryptography - id: android_world_writable - message: The file is World Writable. Any App can write to the file + message: The file or SharedPreference is World Writable. Any App can write to the file type: RegexOr pattern: - MODE_WORLD_WRITABLE + - \.getSharedPreferences\(.{0,50}2\) - 'openFileOutput\(\s*".{1,48}"\s*,\s*2\s*\)' severity: high input_case: exact @@ -575,10 +576,11 @@ masvs: storage-2 ref: https://github.com/MobSF/owasp-mstg/blob/master/Document/0x05d-Testing-Data-Storage.md#testing-local-storage-for-sensitive-data-mstg-storage-1-and-mstg-storage-2 - id: android_world_readable - message: The file is World Readable. Any App can read from the file + message: The file or SharedPreference is World Readable. Any App can read from the file type: RegexOr pattern: - MODE_WORLD_READABLE + - \.getSharedPreferences\(.{0,50}1\) - 'openFileOutput\(\s*".{1,48}"\s*,\s*1\s*\)' severity: high input_case: exact diff --git a/mobsf/StaticAnalyzer/views/android/so.py b/mobsf/StaticAnalyzer/views/android/so.py new file mode 100644 index 00000000..945ba94f --- /dev/null +++ b/mobsf/StaticAnalyzer/views/android/so.py @@ -0,0 +1,184 @@ +"""Handle (.so) Shared Object file.""" +import logging + +from django.conf import settings +from django.shortcuts import render + +import mobsf.MalwareAnalyzer.views.VirusTotal as VirusTotal +from mobsf.MobSF.utils import ( + file_size, +) +from mobsf.StaticAnalyzer.views.common.shared_func import ( + firebase_analysis, + hash_gen, + update_scan_timestamp, +) +from mobsf.StaticAnalyzer.views.android.binary_analysis import ( + elf_analysis, +) +from mobsf.StaticAnalyzer.views.android.strings import ( + get_strings_metadata, +) +from mobsf.StaticAnalyzer.models import ( + StaticAnalyzerAndroid, +) +from mobsf.StaticAnalyzer.views.android.db_interaction import ( + get_context_from_analysis, + get_context_from_db_entry, + save_or_update, +) +from mobsf.MalwareAnalyzer.views.MalwareDomainCheck import MalwareDomainCheck + + +logger = logging.getLogger(__name__) + + +def so_analysis(request, app_dic, rescan, api): + app_dic['app_file'] = f'{app_dic["md5"]}.so' # NEW FILENAME + app_dic['app_path'] = (app_dic['app_dir'] / app_dic['app_file']).as_posix() + app_dic['app_dir'] = app_dic['app_dir'].as_posix() + '/' + db_entry = StaticAnalyzerAndroid.objects.filter(MD5=app_dic['md5']) + if db_entry.exists() and not rescan: + context = get_context_from_db_entry(db_entry) + else: + app_dic['size'] = f'{str(file_size(app_dic["app_path"]))}MB' + app_dic['sha1'], app_dic['sha256'] = hash_gen(app_dic['app_path']) + app_dic['files'] = [] + app_dic['certz'] = [] + app_dic['playstore'] = {'error': True} + app_dic['manifest_file'] = None + app_dic['parsed_xml'] = '' + app_dic['mani'] = '' + man_data_dic = { + 'services': [], + 'activities': [], + 'receivers': [], + 'providers': [], + 'libraries': [], + 'categories': [], + 'perm': {}, + 'packagename': app_dic['app_name'], + 'mainactivity': '', + 'min_sdk': '', + 'max_sdk': '', + 'target_sdk': '', + 'androver': '', + 'androvername': '', + 'icons': [], + } + man_an_dic = { + 'manifest_anal': [], + 'exported_act': [], + 'exported_cnt': { + 'exported_activities': 0, + 'exported_services': 0, + 'exported_receivers': 0, + 'exported_providers': 0, + }, + 'browsable_activities': {}, + 'permissions': {}, + 'icon_hidden': True, + 'network_security': { + 'network_findings': [], + 'network_summary': {}, + }, + } + cert_dic = { + 'certificate_info': '', + 'certificate_findings': [], + 'certificate_summary': {}, + } + app_dic['real_name'] = '' + elf_dict = elf_analysis(app_dic['app_dir']) + apkid_results = {} + tracker_res = {} + code_an_dic = { + 'api': {}, + 'findings': {}, + 'niap': {}, + 'urls_list': [], + 'urls': [], + 'emails': [], + } + quark_results = [] + + # Get the strings and metadata from shared object + get_strings_metadata( + None, + None, + elf_dict['elf_strings'], + None, + None, + code_an_dic) + + # Firebase DB Check + code_an_dic['firebase'] = firebase_analysis( + code_an_dic['urls_list']) + # Domain Extraction and Malware Check + logger.info( + 'Performing Malware Check on extracted Domains') + code_an_dic['domains'] = MalwareDomainCheck().scan( + code_an_dic['urls_list']) + + app_dic['zipped'] = 'so' + app_dic['icon_hidden'] = True + app_dic['icon_found'] = False + logger.info('Connecting to Database') + try: + # SAVE TO DB + if rescan: + logger.info('Updating Database...') + save_or_update( + 'update', + app_dic, + man_data_dic, + man_an_dic, + code_an_dic, + cert_dic, + elf_dict['elf_analysis'], + apkid_results, + quark_results, + tracker_res, + ) + update_scan_timestamp(app_dic['md5']) + else: + logger.info('Saving to Database') + save_or_update( + 'save', + app_dic, + man_data_dic, + man_an_dic, + code_an_dic, + cert_dic, + elf_dict['elf_analysis'], + apkid_results, + quark_results, + tracker_res, + ) + except Exception: + logger.exception('Saving to Database Failed') + context = get_context_from_analysis( + app_dic, + man_data_dic, + man_an_dic, + code_an_dic, + cert_dic, + elf_dict['elf_analysis'], + apkid_results, + quark_results, + tracker_res, + ) + context['appsec'] = {} + context['average_cvss'] = None + context['dynamic_analysis_done'] = False + context['virus_total'] = None + if settings.VT_ENABLED: + vt = VirusTotal.VirusTotal() + context['virus_total'] = vt.get_result( + app_dic['app_path'], + app_dic['md5']) + template = 'static_analysis/android_binary_analysis.html' + if api: + return context + else: + return render(request, template, context) diff --git a/mobsf/StaticAnalyzer/views/android/static_analyzer.py b/mobsf/StaticAnalyzer/views/android/static_analyzer.py index 35a32e8c..c81cd176 100755 --- a/mobsf/StaticAnalyzer/views/android/static_analyzer.py +++ b/mobsf/StaticAnalyzer/views/android/static_analyzer.py @@ -55,7 +55,9 @@ from mobsf.StaticAnalyzer.views.android.manifest_analysis import ( manifest_data, ) from mobsf.StaticAnalyzer.views.android.playstore import get_app_details -from mobsf.StaticAnalyzer.views.android.strings import strings_from_apk +from mobsf.StaticAnalyzer.views.android.strings import ( + get_strings_metadata, +) from mobsf.StaticAnalyzer.views.android.xapk import ( handle_split_apk, handle_xapk, @@ -64,6 +66,9 @@ from mobsf.StaticAnalyzer.views.android.jar_aar import ( aar_analysis, jar_analysis, ) +from mobsf.StaticAnalyzer.views.android.so import ( + so_analysis, +) from mobsf.StaticAnalyzer.views.common.shared_func import ( firebase_analysis, get_avg_cvss, @@ -105,8 +110,8 @@ def static_analyzer(request, api=False): match = re.match('^[0-9a-f]{32}$', checksum) if (match and filename.lower().endswith( - ('.apk', '.xapk', '.zip', '.apks', '.jar', '.aar')) - and typ in ['zip', 'apk', 'xapk', 'apks', 'jar', 'aar']): + ('.apk', '.xapk', '.zip', '.apks', '.jar', '.aar', '.so')) + and typ in ['zip', 'apk', 'xapk', 'apks', 'jar', 'aar', 'so']): app_dic['dir'] = Path(settings.BASE_DIR) # BASE DIR app_dic['app_name'] = filename # APP ORIGINAL NAME app_dic['md5'] = checksum # MD5 @@ -213,29 +218,24 @@ def static_analyzer(request, api=False): app_dic['app_dir'], app_dic['app_path']) - # Get the strings from android resource and shared objects - string_res = strings_from_apk( + # Get the strings and metadata + get_strings_metadata( app_dic['app_file'], app_dic['app_dir'], - elf_dict['elf_strings']) - if string_res: - app_dic['strings'] = string_res['strings'] - app_dic['secrets'] = string_res['secrets'] - code_an_dic['urls_list'].extend( - string_res['urls_list']) - code_an_dic['urls'].extend(string_res['url_nf']) - code_an_dic['emails'].extend(string_res['emails_nf']) - else: - app_dic['strings'] = [] - app_dic['secrets'] = [] + elf_dict['elf_strings'], + 'apk', + ['.java'], + code_an_dic) + # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) # Domain Extraction and Malware Check logger.info( 'Performing Malware Check on extracted Domains') code_an_dic['domains'] = MalwareDomainCheck().scan( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) + app_dic['zipped'] = 'apk' logger.info('Connecting to Database') @@ -304,6 +304,8 @@ def static_analyzer(request, api=False): return jar_analysis(request, app_dic, rescan, api) elif typ == 'aar': return aar_analysis(request, app_dic, rescan, api) + elif typ == 'so': + return so_analysis(request, app_dic, rescan, api) elif typ == 'zip': ret = ( '/static_analyzer_ios/?name=' @@ -424,14 +426,25 @@ def static_analyzer(request, api=False): app_dic['app_dir'], pro_type, app_dic['manifest_file']) + + # Get the strings and metadata + get_strings_metadata( + None, + app_dic['app_dir'], + None, + pro_type, + ['.java', '.kt'], + code_an_dic) + # Firebase DB Check code_an_dic['firebase'] = firebase_analysis( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) # Domain Extraction and Malware Check logger.info( 'Performing Malware Check on extracted Domains') code_an_dic['domains'] = MalwareDomainCheck().scan( - list(set(code_an_dic['urls_list']))) + code_an_dic['urls_list']) + # Extract Trackers from Domains trk = Trackers.Trackers( None, app_dic['tools_dir']) @@ -506,7 +519,7 @@ def static_analyzer(request, api=False): else: return render(request, template, context) else: - err = ('Only APK, JAR, AAR, IPA and Zipped ' + err = ('Only APK, JAR, AAR, SO, IPA and Zipped ' 'Android/iOS Source code supported now!') logger.error(err) else: diff --git a/mobsf/StaticAnalyzer/views/android/strings.py b/mobsf/StaticAnalyzer/views/android/strings.py index 826ddc1a..257cbeed 100755 --- a/mobsf/StaticAnalyzer/views/android/strings.py +++ b/mobsf/StaticAnalyzer/views/android/strings.py @@ -2,27 +2,59 @@ """Module for strings-method for java.""" import logging import os +from pathlib import Path from androguard.core.bytecodes import apk from mobsf.StaticAnalyzer.views.common.shared_func import ( - is_secret, + is_secret_key, + strings_and_entropies, url_n_email_extract, ) +from mobsf.StaticAnalyzer.views.common.entropy import ( + get_entropies, +) +from mobsf.MobSF.utils import ( + get_android_src_dir, +) logger = logging.getLogger(__name__) logging.getLogger('androguard').setLevel(logging.ERROR) -def strings_from_apk(app_file, app_dir, elf_strings): - """Extract the strings from an app.""" +def strings_from_so(elf_strings): + """Extract Strings from so file.""" + sos = [] try: - logger.info('Extracting Strings from APK') - dat = [] - secrets = [] - urls = [] - urls_nf = [] - emails_nf = [] + for solib in elf_strings: + for so, str_list in solib.items(): + if not str_list: + continue + # extract url, email + so_str = ' '.join(str_list) + so_urls, so_urls_nf, so_emails_nf = url_n_email_extract( + so_str, so) + sos.append({so: { + 'secrets': list(get_entropies(so_str)), + 'strings': list(set(str_list)), + 'urls_list': so_urls, + 'urls_nf': so_urls_nf, + 'emails_nf': so_emails_nf, + }}) + except Exception: + logger.exception('Extracting Data from SO') + return sos + + +def strings_from_apk(app_file, app_dir): + """Extract Strings from an APK.""" + dat = [] + secrets = [] + urls = [] + urls_nf = [] + emails_nf = [] + try: + logger.info('Extracting Data from APK') apk_file = os.path.join(app_dir, app_file) and_a = apk.APK(apk_file) rsrc = and_a.get_android_resources() @@ -31,35 +63,85 @@ def strings_from_apk(app_file, app_dir, elf_strings): rsrc.get_strings_resources() for i in rsrc.values[pkg].keys(): res_string = rsrc.values[pkg][i].get('string') - if res_string: - for duo in res_string: - cap_str = '"' + duo[0] + '" : "' + duo[1] + '"' - if is_secret(duo[0] + '"') and ' ' not in duo[1]: - secrets.append(cap_str) - dat.append(cap_str) - data_string = ''.join(dat) + if not res_string: + continue + for duo in res_string: + cap_str = '"' + duo[0] + '" : "' + duo[1] + '"' + # Extract possible secret holding keys + if is_secret_key(duo[0] + '"') and ' ' not in duo[1]: + secrets.append(cap_str) + dat.append(cap_str) + # Extract URLs and Emails from Android String Resources urls, urls_nf, emails_nf = url_n_email_extract( - data_string, 'Android String Resource') - if elf_strings: - for solib in elf_strings: - for so, str_list in solib.items(): - # add to strings from jar - dat.extend(str_list) - # extract url, email - so_str = ' '.join(str_list) - su, suf, sem = url_n_email_extract( - so_str, so) - urls.extend(su) - urls_nf.extend(suf) - emails_nf.extend(sem) - strings_dat = list(set(dat)) - return { - 'strings': strings_dat, - 'urls_list': urls, - 'url_nf': urls_nf, - 'emails_nf': emails_nf, - 'secrets': secrets, - } + ''.join(dat), 'Android String Resource') except Exception: - logger.exception('Extracting Strings from APK') - return {} + logger.exception('Extracting Data from APK') + return { + 'strings': list(set(dat)), + 'urls_list': urls, + 'urls_nf': urls_nf, + 'emails_nf': emails_nf, + 'secrets': secrets, + } + + +def strings_from_code(src_dir, typ, exts): + """Extract Strings and Secrets from Java/Kotlin code.""" + data = { + 'strings': set(), + 'secrets': set(), + } + try: + src_dir = get_android_src_dir(Path(src_dir), typ) + data = strings_and_entropies(src_dir, exts) + except Exception: + logger.exception('Extracting Data from Code') + return data + + +def get_strings_metadata(app_file, app_dir, elf_strings, typ, exts, code_dic): + """Get Strings, secrets, entropies, URLs, emails.""" + strings = { + 'strings_apk_res': {}, + 'strings_so': [], + 'strings_code': {}, + } + urls_list = [] + urls_n_files = [] + emails_n_files = [] + secrets = [] + if app_file: + # APK + apk_res = strings_from_apk(app_file, app_dir) + strings['strings_apk_res'] = apk_res['strings'] + urls_list.extend(apk_res['urls_list']) + urls_n_files.extend(apk_res['urls_nf']) + emails_n_files.extend(apk_res['emails_nf']) + secrets.extend(apk_res['secrets']) + if elf_strings: + # ELF (.so) by file + sos = strings_from_so(elf_strings) + so_strings = [] + for so in sos: + for so_file, s in so.items(): + so_strings.append({so_file: s['strings']}) + urls_list.extend(s['urls_list']) + urls_n_files.extend(s['urls_nf']) + emails_n_files.extend(s['emails_nf']) + secrets.extend(s['secrets']) + secrets = list(set(secrets)) + strings['strings_so'] = so_strings + + if exts: + # Source Code + code_res = strings_from_code(app_dir, typ, exts) + strings['strings_code'] = list(code_res['strings']) + secrets.extend(code_res['secrets']) + + code_dic['strings'] = strings + code_dic['secrets'] = list(secrets) + # Code Analysis has urls, urlsnfiles and emailsnfiles + code_dic['urls'].extend(urls_n_files) + code_dic['emails'].extend(emails_n_files) + code_dic['urls_list'].extend(urls_list) + code_dic['urls_list'] = list(set(code_dic['urls_list'])) diff --git a/mobsf/StaticAnalyzer/views/android/view_source.py b/mobsf/StaticAnalyzer/views/android/view_source.py index 1c4d37fb..80c6dffe 100644 --- a/mobsf/StaticAnalyzer/views/android/view_source.py +++ b/mobsf/StaticAnalyzer/views/android/view_source.py @@ -52,7 +52,7 @@ def run(request, api=False): try: src, syntax, _ = find_java_source_folder(base) except StopIteration: - msg = 'Invalid Directory Structure' + msg = 'Invalid directory or file extension' return print_n_send_error_response(request, msg, api) sfile = src / fil diff --git a/mobsf/StaticAnalyzer/views/common/appsec.py b/mobsf/StaticAnalyzer/views/common/appsec.py index 632e4bb1..22c0e2f2 100644 --- a/mobsf/StaticAnalyzer/views/common/appsec.py +++ b/mobsf/StaticAnalyzer/views/common/appsec.py @@ -176,8 +176,10 @@ def common_fields(findings, data): warn = len(findings.get('warning')) sec = len(findings.get('secure')) total = high + warn + sec - score = int(100 - ( - ((high * 1) + (warn * .5) - (sec * .2)) / total) * 100) + score = 0 + if total > 0: + score = int(100 - ( + ((high * 1) + (warn * .5) - (sec * .2)) / total) * 100) if score > 100: score = 100 findings['security_score'] = score diff --git a/mobsf/StaticAnalyzer/views/common/entropy.py b/mobsf/StaticAnalyzer/views/common/entropy.py new file mode 100644 index 00000000..f20c4690 --- /dev/null +++ b/mobsf/StaticAnalyzer/views/common/entropy.py @@ -0,0 +1,63 @@ + +# -*- coding: utf_8 -*- +"""Entropy Scanner.""" +import re +import math + +MAX_LENGTH = 21 +ENTROPY_PATTERNS = [ + { + # Base64 + 'pattern': re.compile(r'(?m)[a-zA-Z\d+/=]{20,}'), + 'charset': ('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef' + 'ghijklmnopqrstuvwxyz0123456789+/='), + 'score': 4.5, + }, + { + # Hex + 'pattern': re.compile(r'(?m)[a-fA-F\d-]{20,}'), + 'charset': '1234567890abcdefABCDEF', + 'score': 3.0, + }, +] + + +def entropy(data, charset): + """Shannon Entropy score calculations.""" + # Based on + # http://blog.dkbza.org/2007/05/ + # scanning-data-for-entropy-anomalies.html + entropy = 0 + for char in charset: + p_x = float(data.count(char)) / len(data) + if p_x > 0: + entropy += -p_x * math.log2(p_x) + return entropy + + +def exclude(secret): + """Exclude entropies.""" + excludes = ('abcdefghi', 'kotlin/') + if secret.startswith('L') and '/' in secret: + return True + if any(i in secret.lower() for i in excludes): + return True + if secret.count('/') > 1: + # URLs/paths, ignore them + return True + if secret.isalpha(): + return True + return False + + +def get_entropies(data): + patterns = set() + for ascii_string in data: + if len(ascii_string) < MAX_LENGTH: + continue + for p in ENTROPY_PATTERNS: + for i in re.findall(p['pattern'], ascii_string): + score = p['score'] + if entropy(i, p['charset']) > score and not exclude(i): + patterns.add(i) + return patterns diff --git a/mobsf/StaticAnalyzer/views/common/shared_func.py b/mobsf/StaticAnalyzer/views/common/shared_func.py index 19425f5f..fd794c6d 100755 --- a/mobsf/StaticAnalyzer/views/common/shared_func.py +++ b/mobsf/StaticAnalyzer/views/common/shared_func.py @@ -27,9 +27,26 @@ from mobsf.MobSF.utils import ( upstream_proxy, ) from mobsf.StaticAnalyzer.models import RecentScansDB -from mobsf.StaticAnalyzer.views.comparer import generic_compare +from mobsf.StaticAnalyzer.views.comparer import ( + generic_compare, +) +from mobsf.StaticAnalyzer.views.common.entropy import ( + get_entropies, +) + logger = logging.getLogger(__name__) +# Regex to capture strings between quotes or tag +STRINGS_REGEX = re.compile(r'(?<=\")(.+?)(?=\")|(?<=\)(.+?)(?=\<)') +# MobSF Custom regex to catch maximum URI like strings +URL_REGEX = re.compile( + ( + r'((?:https?://|s?ftps?://|' + r'file://|javascript:|data:|www\d{0,3}[.])' + r'[\w().=/;,#:@?&~*+!$%\'{}-]+)' + ), + re.UNICODE) +EMAIL_REGEX = re.compile(r'[\w.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}') def hash_gen(app_path) -> tuple: @@ -91,15 +108,8 @@ def url_n_email_extract(dat, relative_path): urllist = [] url_n_file = [] email_n_file = [] - # URLs Extraction My Custom regex - pattern = re.compile( - ( - r'((?:https?://|s?ftps?://|' - r'file://|javascript:|data:|www\d{0,3}[.])' - r'[\w().=/;,#:@?&~*+!$%\'{}-]+)' - ), - re.UNICODE) - urllist = re.findall(pattern, dat) + # URL Extraction + urllist = URL_REGEX.findall(dat.lower()) uflag = 0 for url in urllist: if url not in urls: @@ -109,10 +119,9 @@ def url_n_email_extract(dat, relative_path): url_n_file.append( {'urls': urls, 'path': escape(relative_path)}) - # Email Extraction Regex - regex = re.compile(r'[\w.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}') + # Email Extraction eflag = 0 - for email in regex.findall(dat.lower()): + for email in EMAIL_REGEX.findall(dat.lower()): if (email not in emails) and (not email.startswith('//')): emails.append(email) eflag = 1 @@ -207,8 +216,8 @@ def find_java_source_folder(base_folder: Path): if p[0].exists()) -def is_secret(inp): - """Check if captures string is a possible secret.""" +def is_secret_key(inp): + """Check if the key in the key/value pair is interesting.""" inp = inp.lower() iden = ( 'api"', 'key"', 'api_', 'key_', 'secret"', @@ -231,3 +240,30 @@ def is_secret(inp): ) not_str = any(i in inp for i in not_string) return any(i in inp for i in iden) and not not_str + + +def strings_and_entropies(src, exts): + """Get Strings and Entropies.""" + logger.info('Extracting Data from Source Code') + data = { + 'strings': set(), + 'secrets': set(), + } + try: + if not src.exists(): + return data + for p in src.rglob('*'): + if p.suffix not in exts or not p.exists(): + continue + matches = STRINGS_REGEX.finditer( + p.read_text(encoding='utf-8', errors='ignore'), + re.MULTILINE) + for match in matches: + if len(match.group()) < 4: + continue + data['strings'].add(match.group()) + if data['strings']: + data['secrets'] = get_entropies(data['strings']) + except Exception: + logger.exception('Extracting Data from Code') + return data diff --git a/mobsf/StaticAnalyzer/views/ios/plist_analysis.py b/mobsf/StaticAnalyzer/views/ios/plist_analysis.py index 476d9ebc..4c90a08d 100755 --- a/mobsf/StaticAnalyzer/views/ios/plist_analysis.py +++ b/mobsf/StaticAnalyzer/views/ios/plist_analysis.py @@ -30,7 +30,7 @@ from mobsf.StaticAnalyzer.views.ios.app_transport_security import ( check_transport_security, ) from mobsf.StaticAnalyzer.views.common.shared_func import ( - is_secret, + is_secret_key, ) logger = logging.getLogger(__name__) @@ -234,11 +234,11 @@ def get_plist_secrets(xml_string): xml_list = xml_string.split('\n') for index, line in enumerate(xml_list): - if '' in line and is_secret(_remove_tags(line)): + if '' in line and is_secret_key(_remove_tags(line)): nxt = index + 1 value = ( _remove_tags(xml_list[nxt])if nxt < len(xml_list) else False) if value and ' ' not in value: result_list.append( - f'{_remove_tags(line)} : {_remove_tags(value)}') + f'{_remove_tags(line)} : {_remove_tags(value)}') return result_list diff --git a/mobsf/StaticAnalyzer/views/ios/static_analyzer.py b/mobsf/StaticAnalyzer/views/ios/static_analyzer.py index 87827b00..350cbf12 100755 --- a/mobsf/StaticAnalyzer/views/ios/static_analyzer.py +++ b/mobsf/StaticAnalyzer/views/ios/static_analyzer.py @@ -36,6 +36,7 @@ from mobsf.StaticAnalyzer.views.common.shared_func import ( firebase_analysis, get_avg_cvss, hash_gen, + strings_and_entropies, unzip, update_scan_timestamp, ) @@ -224,6 +225,11 @@ def static_analyzer_ios(request, api=False): infoplist_dict['plist_xml']) code_analysis_dic = ios_source_analysis( app_dict['app_dir']) + ios_strs = strings_and_entropies( + Path(app_dict['app_dir']), + ['.swift', '.m', '.h', '.plist']) + if ios_strs['secrets']: + app_dict['secrets'].extend(list(ios_strs['secrets'])) # Get App Icon app_dict['icon_found'] = get_icon_source( app_dict['md5_hash'], @@ -241,7 +247,7 @@ def static_analyzer_ios(request, api=False): 'checksec': {}, 'libraries': [], 'bin_code_analysis': {}, - 'strings': [], + 'strings': list(ios_strs['strings']), 'bin_info': {}, 'bin_type': code_analysis_dic['source_type'], } diff --git a/mobsf/StaticAnalyzer/views/sast_engine.py b/mobsf/StaticAnalyzer/views/sast_engine.py index 543d5dd0..75e288ec 100644 --- a/mobsf/StaticAnalyzer/views/sast_engine.py +++ b/mobsf/StaticAnalyzer/views/sast_engine.py @@ -2,10 +2,12 @@ """SAST engine.""" import logging -from django.conf import settings - from libsast import Scanner +from mobsf.MobSF.utils import ( + settings_enabled, +) + logger = logging.getLogger(__name__) @@ -28,7 +30,7 @@ def scan(rule, extensions, paths, ignore_paths=None): def niap_scan(rule, extensions, paths, apath, ignore_paths=None): """NIAP scan.""" - if not getattr(settings, 'NIAP_ENABLED', True): + if not settings_enabled('NIAP_ENABLED'): return {} try: logger.info('Running NIAP Analyzer') diff --git a/mobsf/templates/general/home.html b/mobsf/templates/general/home.html index 72f5304c..dc0e3474 100644 --- a/mobsf/templates/general/home.html +++ b/mobsf/templates/general/home.html @@ -225,10 +225,10 @@ // Is valid file extensions function isValidExt(file_name){ var val = file_name.toLowerCase(); - var regex = new RegExp("^(.{1,300}?)\.(ipa|apk|apks|xapk|jar|aar|zip|appx)$"); + var regex = new RegExp("^(.{1,300}?)\.(ipa|apk|apks|xapk|jar|aar|so|zip|appx)$"); val = val.replace(/^.*[\\\/]/, ''); if (!(regex.test(val))) { - _('status').innerText = "Please upload APK/APKS/XAPK/JAR/AAR/IPA/ZIP/APPX only!"; + _('status').innerText = "Please upload APK/APKS/XAPK/JAR/AAR/SO/IPA/ZIP/APPX only!"; return false; } return true; diff --git a/mobsf/templates/general/recent.html b/mobsf/templates/general/recent.html index 153cdc57..7a0cfb09 100644 --- a/mobsf/templates/general/recent.html +++ b/mobsf/templates/general/recent.html @@ -55,10 +55,12 @@ - {% if e.PACKAGE_NAME %} + {% if e.PACKAGE_NAME or e.APP_NAME %}
{{ e.APP_NAME }} {% if e.VERSION_NAME %} - {{ e.VERSION_NAME }} {% endif %}
{{ e.PACKAGE_NAME }} + {% if '.so' != e.FILE_NAME|slice:"-3:"%}

MobSF Scorecard

+ {% endif %}

Static Report {% if '.apk' == e.FILE_NAME|slice:"-4:" or '.xapk' == e.FILE_NAME|slice:"-5:" or '.apks' == e.FILE_NAME|slice:"-5:"%} Dynamic Report @@ -76,6 +78,7 @@ {% elif '.apks' == e.FILE_NAME|slice:"-5:"%} {% elif '.jar' == e.FILE_NAME|slice:"-4:"%} {% elif '.aar' == e.FILE_NAME|slice:"-4:"%} + {% elif '.so' == e.FILE_NAME|slice:"-3:"%} {% elif '.ipa' == e.FILE_NAME|slice:"-4:"%} {% elif '.zip' == e.FILE_NAME|slice:"-4:"%} {% elif '.appx' == e.FILE_NAME|slice:"-5:"%} diff --git a/mobsf/templates/pdf/android_report.html b/mobsf/templates/pdf/android_report.html index d7c7d2af..220307e5 100755 --- a/mobsf/templates/pdf/android_report.html +++ b/mobsf/templates/pdf/android_report.html @@ -64,6 +64,7 @@

Scan Date:

{{ timestamp }}

+ {% if app_type not in 'so' %} {% if average_cvss %}

Average CVSS Score:

@@ -122,6 +123,7 @@

0 %} class="warning" {% endif %}>{{ trackers.detected_trackers }}/{{ trackers.total_trackers }}

{% endif %} + {% endif %} {% if virus_total and virus_total.items|length > 9 and 'request successfully queued' not in virus_total.verbose_msg %}

VirusTotal Detection:

@@ -130,6 +132,7 @@ {% endif %} + {% if app_type not in 'so' %}





@@ -159,7 +162,7 @@ - + {% endif %}
@@ -171,7 +174,7 @@
SHA1:
{{ sha1 }}
SHA256:
{{ sha256 }}
- +{% if app_type not in 'so' %}

APP INFORMATION

App Name:
{{ app_name }}
@@ -182,8 +185,9 @@
Max SDK:
{{ max_sdk }}
Android Version Name:
{{ version_name }}
Android Version Code:
{{ version_code }}
- -{% if app_type not in 'jar,aar' %} +{% endif %} + +{% if app_type not in 'jar,aar,so' %}

APP COMPONENTS

Activities:
{{ activities | length }}
@@ -195,6 +199,7 @@
Exported Receivers:
0 %} class="danger" {% endif %}> {{ exported_count.exported_receivers }}
Exported Providers:
0 %} class="danger" {% endif %}> {{ exported_count.exported_providers }}
{% endif %} + {% if app_type not in 'so' %}

CERTIFICATE INFORMATION

{% if certificate_analysis.certificate_info %}
@@ -203,6 +208,7 @@ {% else %} Failed to read Code Signing Certificate or none available. {% endif %} + {% endif %} {% if permissions %}

APPLICATION PERMISSIONS

@@ -400,7 +406,7 @@ {% endif %} - + {% if app_type not in 'so' %}

NETWORK SECURITY

{% if network_security and 'network_summary' in network_security and network_security.network_summary|length > 0 %} HIGH: @@ -617,7 +623,7 @@ {% endif %} - + {% endif %} {% if binary_analysis %}

SHARED LIBRARY BINARY ANALYSIS

@@ -672,7 +678,7 @@ {% endif %} - + {% if app_type not in 'so' %}

NIAP ANALYSIS v1.3

@@ -704,7 +710,7 @@ {% endfor %}
- + {% endif %} +{% if app_type in 'so' %} {% if urls %}

URLS

@@ -828,7 +835,7 @@
{% endif %} ---> +{% endif %} {% if firebase_urls %}

FIREBASE DATABASES

diff --git a/mobsf/templates/static_analysis/android_binary_analysis.html b/mobsf/templates/static_analysis/android_binary_analysis.html index 3d6f818b..18b4122f 100755 --- a/mobsf/templates/static_analysis/android_binary_analysis.html +++ b/mobsf/templates/static_analysis/android_binary_analysis.html @@ -75,6 +75,7 @@

+ {% if app_type not in 'so' %}
  • @@ -107,7 +108,7 @@

    • - + {% endif %}
  • @@ -117,6 +118,7 @@

  • @@ -174,6 +179,7 @@

    • + {% if app_type not in 'so' %}
  • @@ -302,6 +312,7 @@
    • + {% endif %}
  • @@ -328,7 +339,7 @@
    • {% else %} - {% if app_type not in 'jar,aar' %} + {% if app_type not in 'jar,aar,so' %}
  • @@ -372,6 +383,7 @@
    + {% if app_type not in 'so' %}

    APP SCORES

    @@ -399,6 +411,7 @@

    MobSF Scorecard

    + {% endif %}

    FILE INFORMATION

    File Name @@ -412,6 +425,7 @@ SHA256 {{ sha256 }}
    + {% if app_type not in 'jar,aar,so' %}

    APP INFORMATION

    App Name @@ -431,6 +445,7 @@ Android Version Code {{ version_code }}
    + {% endif %}
    @@ -486,7 +501,7 @@ - {% if app_type not in 'jar,aar' %} + {% if app_type not in 'jar,aar,so' %}
    @@ -636,9 +651,14 @@ SCAN OPTIONS

    Rescan + {% if app_type in 'so' %} + Download {{ app_type | upper}} + {% endif %} + {% if app_type not in 'so' %} Manage Suppressions + {% endif %}

    - {% if app_type not in 'jar,aar' %} + {% if app_type not in 'jar,aar,so' %}

    Start Dynamic Analysis

    @@ -646,6 +666,7 @@
    + {% if app_type not in 'so' %}
    @@ -671,12 +692,14 @@
    + {% endif %} + {% if app_type not in 'so' %}
    @@ -1304,6 +1327,7 @@
    + {% endif %}
    @@ -1318,8 +1342,10 @@
    • + {% if app_type not in 'so' %} + {% endif %} @@ -1334,8 +1360,10 @@ {% endif %} {% for so in binary_analysis %} + {% if app_type not in 'so' %} + {% endif %} {% endfor %} @@ -1150,6 +1156,31 @@ + +
    +
    +
    +
    +
    +
    +

    + STRINGS +

    +
    +

    + {% for val in strings %} + {{ val }}
    + {% endfor %} +

    +
    +
    +
    +
    +
    + +
    +
    +
    diff --git a/tox.ini b/tox.ini index 580c4d4f..8f3ce758 100644 --- a/tox.ini +++ b/tox.ini @@ -91,6 +91,8 @@ ignore = # Use python sort imports R701, # TODO fix this later too complex + C901, + # TODO function too complex N400, # For ASCII art W605,
    NO SHARED OBJECTNX STACK CANARY RPATH
    {{ forloop.counter }} {{so.name}}{{so.nx.is_nx}}
    {{so.nx.severity}} @@ -1374,6 +1402,7 @@ + {% if app_type not in 'so' %}
    @@ -1598,6 +1627,7 @@
    +{% endif %} {% if virus_total %}
    @@ -1905,6 +1935,7 @@
    +{% if app_type not in 'so' %}
    @@ -1951,6 +1982,32 @@
    + {% endif %} + +
    +
    +
    +
    +
    +
    +

    + POSSIBLE HARDCODED SECRETS +

    +
    +

    + {% for val in secrets %} + {{ val }}
    + {% endfor %} +

    +
    +
    +
    +
    +
    + +
    +
    +
    @@ -1962,9 +2019,41 @@ STRINGS

    + {% if app_type not in 'so' %} +

    From APK Resource

    - {% for val in strings %} - {{ val }}
    + {% for key, val in strings.items %} + {% if key == 'strings_apk_res' %} + {% for v in val %} + {{ v }}
    + {% endfor %} + {% endif %} + {% endfor %} +

    +

    From Code

    +

    + {% for key, val in strings.items %} + {% if key == 'strings_code' %} + {% for v in val %} + {{ v }}
    + {% endfor %} + {% endif %} + {% endfor %} +

    + {% endif %} +

    From Shared Objects

    +

    + {% for key, val in strings.items %} + {% if key == 'strings_so' %} + {% for ls in val %} + {% for k,v in ls.items %} +


    {{ k }}

    + {% for i in v %} + {{ i }}
    + {% endfor %} + {% endfor %} + {% endfor %} + {% endif %} {% endfor %}

    @@ -1976,32 +2065,7 @@
    - -
    -
    -
    -
    -
    -
    -

    - POSSIBLE HARDCODED SECRETS -

    -
    -

    - {% for val in secrets %} - {{ val }}
    - {% endfor %} -

    -
    -
    -
    -
    -
    - -
    -
    - - + {% if app_type not in 'so' %}
    @@ -2153,6 +2217,7 @@
    + {% endif %} diff --git a/mobsf/templates/static_analysis/android_source_analysis.html b/mobsf/templates/static_analysis/android_source_analysis.html index 0d121691..e9045a7f 100755 --- a/mobsf/templates/static_analysis/android_source_analysis.html +++ b/mobsf/templates/static_analysis/android_source_analysis.html @@ -177,6 +177,12 @@

    Trackers

    +
  • + + +

    Hardcoded Secrets

    +     +
  • @@ -1465,31 +1471,61 @@ - -
    -
    -
    -
    -
    -
    -

    - STRINGS -

    -
    -

    - {% for val in strings %} - {{ val }}
    - {% endfor %} -

    + +
    +
    +
    +
    +
    +
    +

    + POSSIBLE HARDCODED SECRETS +

    +
    +

    + {% for val in secrets %} + {{ val }}
    + {% endfor %} +

    +
    +
    -
    -
    - -
    -
    - + + + + + +
    +
    +
    +
    +
    +
    +

    + STRINGS +

    +
    +

    From Code

    +

    + {% for key, val in strings.items %} + {% if key == 'strings_code' %} + {% for v in val %} + {{ v }}
    + {% endfor %} + {% endif %} + {% endfor %} +

    +
    +
    +
    +
    +
    + +
    +
    +
    diff --git a/mobsf/templates/static_analysis/ios_source_analysis.html b/mobsf/templates/static_analysis/ios_source_analysis.html index 6e659dec..1a2243b6 100755 --- a/mobsf/templates/static_analysis/ios_source_analysis.html +++ b/mobsf/templates/static_analysis/ios_source_analysis.html @@ -178,6 +178,12 @@

    Hardcoded Secrets

    • +
  • + + +

    Strings

    +     +
  • @@ -1063,7 +1069,7 @@ {% endfor %}
    		• - {{email_dict|key:"path"}} + {{email_dict|key:"path"}}