From 6c68150676a1dd2dd6e3817acfcaba6fd197e589 Mon Sep 17 00:00:00 2001 From: Ajin Abraham Date: Thu, 6 May 2021 20:20:04 -0700 Subject: [PATCH] Multiple Features Static Analyzer & Dynamic Analyzer (#1730) * Improved Split APK support now supports .apks file. * Ability to look up and download APK from apktada/apkpure/apkplz. * Dynamic Analyzer: Get Runtime Application Third party dependencies. * Static Analysis Rule QA and Typo Fix. * Persist Frida Code change in session storage. * Show Base64 strings decoded at runtime and the called class. * Detect Trackers from Runtime Dependencies and Network Traffic * Windows Binskim version pinning --- .../auxiliary/get_loaded_classes.js | 1 - .../views/android/dynamic_analyzer.py | 2 +- .../views/android/environment.py | 5 +- .../views/android/frida_core.py | 9 + .../views/android/frida_scripts.py | 2 +- .../views/android/operations.py | 20 --- mobsf/DynamicAnalyzer/views/android/report.py | 17 +- .../views/android/tests_frida.py | 69 +++++++- mobsf/MalwareAnalyzer/views/Trackers.py | 67 ++++++- mobsf/MobSF/settings.py | 3 + mobsf/MobSF/urls.py | 3 + mobsf/MobSF/utils.py | 27 +++ mobsf/MobSF/views/api/api_dynamic_analysis.py | 13 ++ mobsf/MobSF/views/apk_downloader.py | 163 ++++++++++++++++++ mobsf/MobSF/views/helpers.py | 15 +- mobsf/MobSF/views/home.py | 22 +++ mobsf/MobSF/views/scanning.py | 39 ++++- .../views/android/rules/android_rules.yaml | 8 +- .../views/android/static_analyzer.py | 20 ++- mobsf/StaticAnalyzer/views/android/strings.py | 43 +++-- mobsf/StaticAnalyzer/views/android/xapk.py | 29 +++- .../views/ios/rules/objective_c_rules.yaml | 4 +- .../views/ios/rules/swift_rules.yaml | 10 +- mobsf/install/windows/setup.py | 1 + .../android/dynamic_analyzer.html | 33 +++- .../android/dynamic_report.html | 127 ++++++++++++++ .../dynamic_analysis/dynamic_analysis.html | 14 +- mobsf/templates/general/apidocs.html | 75 ++++++++ mobsf/templates/general/home.html | 106 +++++++++--- mobsf/templates/general/recent.html | 3 +- mobsf/templates/pdf/android_report.html | 27 +-- .../android_binary_analysis.html | 10 +- 32 files changed, 851 insertions(+), 136 deletions(-) create mode 100644 mobsf/MobSF/views/apk_downloader.py diff --git a/mobsf/DynamicAnalyzer/tools/frida_scripts/auxiliary/get_loaded_classes.js b/mobsf/DynamicAnalyzer/tools/frida_scripts/auxiliary/get_loaded_classes.js index 7a61fe05..3b1baecc 100644 --- a/mobsf/DynamicAnalyzer/tools/frida_scripts/auxiliary/get_loaded_classes.js +++ b/mobsf/DynamicAnalyzer/tools/frida_scripts/auxiliary/get_loaded_classes.js @@ -5,7 +5,6 @@ */ Java.perform(function() { var classes = Java.enumerateLoadedClassesSync(); - send('[AUXILIARY] Loaded Classes'); classes.forEach(function(aClass) { try{ var className = aClass.match(/[L](.*);/)[1].replace(/\//g, "."); diff --git a/mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py b/mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py index ad75980d..2d2dd222 100755 --- a/mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py +++ b/mobsf/DynamicAnalyzer/views/android/dynamic_analyzer.py @@ -17,7 +17,6 @@ from django.shortcuts import render from mobsf.DynamicAnalyzer.views.android.environment import Environment from mobsf.DynamicAnalyzer.views.android.operations import ( get_package_name, - strict_package_check, ) from mobsf.DynamicAnalyzer.tools.webproxy import ( get_http_tools_url, @@ -30,6 +29,7 @@ from mobsf.MobSF.utils import ( get_proxy_ip, is_md5, print_n_send_error_response, + strict_package_check, ) from mobsf.MobSF.views.scanning import add_to_recent_scan from mobsf.StaticAnalyzer.models import StaticAnalyzerAndroid diff --git a/mobsf/DynamicAnalyzer/views/android/environment.py b/mobsf/DynamicAnalyzer/views/android/environment.py index e7d7eee3..9178c03b 100644 --- a/mobsf/DynamicAnalyzer/views/android/environment.py +++ b/mobsf/DynamicAnalyzer/views/android/environment.py @@ -378,11 +378,14 @@ class Environment: or any(char.isdigit() for char in ver)): logger.info('Found Genymotion x86 Android VM') return 'genymotion' + elif b'corellium' in out: + logger.info('Found Corellium ARM Android VM') + return 'corellium' else: logger.warning( 'Unable to identify Dynamic Analysis environment. ' 'Official support is available only for Android ' - 'Emulator and Genymotion VM') + 'Emulator, Corellium, and Genymotion') return '' def get_android_version(self): diff --git a/mobsf/DynamicAnalyzer/views/android/frida_core.py b/mobsf/DynamicAnalyzer/views/android/frida_core.py index db2e81b2..8808216b 100644 --- a/mobsf/DynamicAnalyzer/views/android/frida_core.py +++ b/mobsf/DynamicAnalyzer/views/android/frida_core.py @@ -40,6 +40,7 @@ class Frida: self.apk_dir = os.path.join(settings.UPLD_DIR, self.hash + '/') self.api_mon = os.path.join(self.apk_dir, 'mobsf_api_monitor.txt') self.frida_log = os.path.join(self.apk_dir, 'mobsf_frida_out.txt') + self.deps = os.path.join(self.apk_dir, 'mobsf_app_deps.txt') def get_default_scripts(self): """Get default Frida Scripts.""" @@ -65,6 +66,9 @@ class Frida: for itm in self.auxiliary: if itm == 'enum_class': scripts.append(get_loaded_classes()) + elif itm == 'get_dependencies': + scripts.append(get_loaded_classes().replace( + '[AUXILIARY] ', '[RUNTIME-DEPS] ')) elif itm == 'string_catch': scripts.append(string_catch()) elif itm == 'string_compare': @@ -95,10 +99,15 @@ class Frida: msg = message['payload'] api_mon = 'MobSF-API-Monitor: ' aux = '[AUXILIARY] ' + deps = '[RUNTIME-DEPS] ' if not isinstance(msg, str): msg = str(msg) if msg.startswith(api_mon): self.write_log(self.api_mon, msg.replace(api_mon, '')) + elif msg.startswith(deps): + info = msg.replace(deps, '') + '\n' + self.write_log(self.deps, info) + self.write_log(self.frida_log, info) elif msg.startswith(aux): self.write_log(self.frida_log, msg.replace(aux, '[*] ') + '\n') diff --git a/mobsf/DynamicAnalyzer/views/android/frida_scripts.py b/mobsf/DynamicAnalyzer/views/android/frida_scripts.py index c633f145..59e369ac 100644 --- a/mobsf/DynamicAnalyzer/views/android/frida_scripts.py +++ b/mobsf/DynamicAnalyzer/views/android/frida_scripts.py @@ -2,7 +2,7 @@ import os from django.conf import settings -from mobsf.DynamicAnalyzer.views.android.operations import strict_package_check +from mobsf.MobSF.utils import strict_package_check def get_content(file_name): diff --git a/mobsf/DynamicAnalyzer/views/android/operations.py b/mobsf/DynamicAnalyzer/views/android/operations.py index e2b2491b..98b6facc 100644 --- a/mobsf/DynamicAnalyzer/views/android/operations.py +++ b/mobsf/DynamicAnalyzer/views/android/operations.py @@ -65,26 +65,6 @@ def is_attack_pattern(user_input): return stat -def strict_package_check(user_input): - """Strict package name check.""" - pat = re.compile(r'^\w+\.*[\w\.\$]+$') - resp = re.match(pat, user_input) - if not resp: - logger.error('Invalid package/class name') - return resp - - -def is_path_traversal(user_input): - """Check for path traversal.""" - if (('../' in user_input) - or ('%2e%2e' in user_input) - or ('..' in user_input) - or ('%252e' in user_input)): - logger.error('Path traversal attack detected') - return True - return False - - def invalid_params(api=False): """Standard response for invalid params.""" msg = 'Invalid Parameters' diff --git a/mobsf/DynamicAnalyzer/views/android/report.py b/mobsf/DynamicAnalyzer/views/android/report.py index 87f00382..1d785dd6 100644 --- a/mobsf/DynamicAnalyzer/views/android/report.py +++ b/mobsf/DynamicAnalyzer/views/android/report.py @@ -10,6 +10,7 @@ from django.shortcuts import render from django.template.defaulttags import register from django.utils.html import escape +import mobsf.MalwareAnalyzer.views.Trackers as Trackers from mobsf.DynamicAnalyzer.views.android.analysis import ( generate_download, get_screenshots, @@ -17,17 +18,18 @@ from mobsf.DynamicAnalyzer.views.android.analysis import ( ) from mobsf.DynamicAnalyzer.views.android.operations import ( get_package_name, - is_path_traversal, ) from mobsf.DynamicAnalyzer.views.android.tests_xposed import ( droidmon_api_analysis, ) from mobsf.DynamicAnalyzer.views.android.tests_frida import ( apimon_analysis, + dependency_analysis, ) from mobsf.MobSF.utils import ( is_file_exists, is_md5, + is_path_traversal, is_safe_path, print_n_send_error_response, read_sqlite, @@ -49,6 +51,7 @@ def view_report(request, checksum, api=False): try: droidmon = {} apimon = {} + b64_strings = [] if not is_md5(checksum): # We need this check since checksum is not validated # in REST API @@ -64,6 +67,7 @@ def view_report(request, checksum, api=False): api) app_dir = os.path.join(settings.UPLD_DIR, checksum + '/') download_dir = settings.DWD_DIR + tools_dir = settings.TOOLS_DIR if not is_file_exists(os.path.join(app_dir, 'logcat.txt')): msg = ('Dynamic Analysis report is not available ' 'for this app. Perform Dynamic Analysis ' @@ -71,14 +75,18 @@ def view_report(request, checksum, api=False): return print_n_send_error_response(request, msg, api) fd_log = os.path.join(app_dir, 'mobsf_frida_out.txt') droidmon = droidmon_api_analysis(app_dir, package) - apimon = apimon_analysis(app_dir) + apimon, b64_strings = apimon_analysis(app_dir) + deps = dependency_analysis(package, app_dir) analysis_result = run_analysis(app_dir, checksum, package) + domains = analysis_result['domains'] + trk = Trackers.Trackers(app_dir, tools_dir) + trackers = trk.get_runtime_trackers(domains, deps) generate_download(app_dir, checksum, download_dir, package) images = get_screenshots(checksum, download_dir) context = {'hash': checksum, 'emails': analysis_result['emails'], 'urls': analysis_result['urls'], - 'domains': analysis_result['domains'], + 'domains': domains, 'clipboard': analysis_result['clipboard'], 'xml': analysis_result['xml'], 'sqlite': analysis_result['sqlite'], @@ -89,7 +97,10 @@ def view_report(request, checksum, api=False): 'exported_activity_tester': images['exported_activities'], 'droidmon': droidmon, 'apimon': apimon, + 'base64_strings': b64_strings, + 'trackers': trackers, 'frida_logs': is_file_exists(fd_log), + 'runtime_dependencies': deps, 'package': package, 'version': settings.MOBSF_VER, 'title': 'Dynamic Analysis'} diff --git a/mobsf/DynamicAnalyzer/views/android/tests_frida.py b/mobsf/DynamicAnalyzer/views/android/tests_frida.py index 67f28588..5762f77f 100644 --- a/mobsf/DynamicAnalyzer/views/android/tests_frida.py +++ b/mobsf/DynamicAnalyzer/views/android/tests_frida.py @@ -48,6 +48,29 @@ def list_frida_scripts(request, api=False): # AJAX +@require_http_methods(['POST']) +def get_runtime_dependencies(request, api=False): + """Get App runtime dependencies.""" + data = { + 'status': 'failed', + 'message': 'Failed to get runtime dependencies'} + try: + checksum = request.POST['hash'] + if not is_md5(checksum): + return invalid_params(api) + package = get_package_name(checksum) + if not package: + return invalid_params(api) + get_dependencies(package, checksum) + return send_response( + {'status': 'ok'}, + api) + except Exception: + pass + return send_response(data, api) +# AJAX + + @require_http_methods(['POST']) def get_script(request, api=False): """Get frida scripts from others.""" @@ -240,9 +263,10 @@ def apimon_analysis(app_dir): """API Analysis.""" api_details = {} try: + strings = [] location = os.path.join(app_dir, 'mobsf_api_monitor.txt') if not is_file_exists(location): - return {} + return api_details, strings logger.info('Frida API Monitor Analysis') with open(location, 'r', encoding='utf8', @@ -262,6 +286,7 @@ def apimon_analysis(app_dir): if to_decode: api['decoded'] = decode_base64( to_decode).decode('utf-8', 'ignore') + strings.append((api['calledFrom'], api['decoded'])) except Exception: pass api['icon'] = get_icon_map(api['name']) @@ -271,4 +296,44 @@ def apimon_analysis(app_dir): api_details[api['name']] = [api] except Exception: logger.exception('API Monitor Analysis') - return api_details + return api_details, strings + + +def get_dependencies(package, checksum): + """Get 3rd party dependencies at runtime.""" + frd = Frida( + checksum, + package, + ['ssl_pinning_bypass', 'debugger_check_bypass', 'root_bypass'], + ['get_dependencies'], + None, + None, + ) + location = Path(frd.deps) + if location.exists(): + location.write_text('') + trd = threading.Thread(target=frd.connect) + trd.daemon = True + trd.start() + + +def dependency_analysis(package, app_dir): + deps = set() + msg = 'Collecting Runtime Dependency Analysis data' + try: + ignore = ( + package, + 'android.', 'androidx.', 'kotlin.', 'kotlinx.', 'java.', 'javax.', + 'sun.', 'com.android.', 'j$', 'dalvik.system.', 'libcore.', + 'com.google.', 'org.kxml2.', 'org.apache.', 'org.json.') + location = Path(app_dir) / 'mobsf_app_deps.txt' + if not location.exists(): + return deps + logger.info(msg) + data = location.read_text('utf-8', 'ignore').splitlines() + for dep in data: + if not dep.startswith(ignore): + deps.add(dep.rsplit('.', 1)[0]) + except Exception: + logger.exception(msg) + return deps diff --git a/mobsf/MalwareAnalyzer/views/Trackers.py b/mobsf/MalwareAnalyzer/views/Trackers.py index 6739b9c4..29dad6ad 100644 --- a/mobsf/MalwareAnalyzer/views/Trackers.py +++ b/mobsf/MalwareAnalyzer/views/Trackers.py @@ -30,6 +30,7 @@ class Trackers: self.signatures = None self.nb_trackers_signature = 0 self.compiled_tracker_signature = None + self.compiled_network_tracker_sig = None self.classes = None self.tools_dir = tools_dir self._update_tracker_db() @@ -78,9 +79,14 @@ class Trackers: :return: A compiled list of signatures. """ self.compiled_tracker_signature = [] + self.compiled_network_tracker_sig = [] try: - self.compiled_tracker_signature = [re.compile(track.code_signature) - for track in self.signatures] + self.compiled_tracker_signature = [ + re.compile(track.code_signature) + for track in self.signatures] + self.compiled_network_tracker_sig = [ + re.compile(track.network_signature) + for track in self.signatures] except TypeError: logger.exception('compiling tracker signature failed') @@ -128,9 +134,9 @@ class Trackers: self.classes = classes return self.classes - def detect_trackers_in_list(self, class_list): + def detect_trackers_in_list(self, class_list, network=False): """ - Detect embedded trackers in the provided classes list. + Detect embedded trackers in the provided classes list/urls. :return: list of embedded trackers """ @@ -144,7 +150,17 @@ class Trackers: return None results = [] - args = [(self.compiled_tracker_signature[index], tracker, class_list) + if network: + compiled = self.compiled_network_tracker_sig + args = [ + (compiled[index], tracker, class_list) + for (index, tracker) in enumerate(self.signatures) if + len(tracker.network_signature) > 3] + + else: + compiled = self.compiled_tracker_signature + args = [ + (compiled[index], tracker, class_list) for (index, tracker) in enumerate(self.signatures) if len(tracker.code_signature) > 3] @@ -169,6 +185,22 @@ class Trackers: return self.detect_trackers_in_list(eclasses) return [] + def detect_runtime_trackers(self, items, deps=False): + """ + Detect runtime trackers. + + :return: list of embedded trackers + """ + if self.signatures is None: + self.load_trackers_signatures() + if items and not deps: + # Domains + return self.detect_trackers_in_list(items.keys(), True) + elif items and deps: + # Runtime Dependencies + return self.detect_trackers_in_list(items) + return [] + def get_trackers(self): """Get Trackers.""" logger.info('Detecting Trackers') @@ -178,5 +210,28 @@ class Trackers: 'trackers': []} for trk in trackers: trk_url = '{}/trackers/{}'.format(settings.EXODUS_URL, trk.id) - tracker_dict['trackers'].append({trk.name: trk_url}) + tracker_dict['trackers'].append({ + 'name': trk.name, + 'categories': ', '.join(trk.categories), + 'url': trk_url, + }) return tracker_dict + + def get_runtime_trackers(self, domains, deps): + """Get Trackers from Runtime.""" + trackers = [] + logger.info('Detecting Trackers from HTTP(s) traffic') + network = self.detect_runtime_trackers(domains) + logger.info('Detecting Trackers from Runtime dependencies') + runtime = self.detect_runtime_trackers(deps, True) + for i in runtime: + if i not in network: + network.append(i) + for trk in network: + trk_url = '{}/trackers/{}'.format(settings.EXODUS_URL, trk.id) + trackers.append({ + 'name': trk.name, + 'categories': ', '.join(trk.categories), + 'url': trk_url, + }) + return trackers diff --git a/mobsf/MobSF/settings.py b/mobsf/MobSF/settings.py index b2223a9a..129351ac 100644 --- a/mobsf/MobSF/settings.py +++ b/mobsf/MobSF/settings.py @@ -114,6 +114,9 @@ GITHUB_URL = ('https://raw.githubusercontent.com/' FRIDA_SERVER = 'https://api.github.com/repos/frida/frida/releases/tags/' GOOGLE = 'https://www.google.com' BAIDU = 'https://www.baidu.com/' +APKPURE = 'https://m.apkpure.com/android/{}/download?from=details' +APKTADA = 'https://apktada.com/download-apk/' +APKPLZ = 'https://apkplz.net/download-app/' # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # ============DJANGO SETTINGS ================= diff --git a/mobsf/MobSF/urls.py b/mobsf/MobSF/urls.py index ecaed312..04b9b1f9 100755 --- a/mobsf/MobSF/urls.py +++ b/mobsf/MobSF/urls.py @@ -60,6 +60,7 @@ urlpatterns = [ url(r'^api/v1/frida/logs$', api_dz.api_frida_logs), url(r'^api/v1/frida/list_scripts$', api_dz.api_list_frida_scripts), url(r'^api/v1/frida/get_script$', api_dz.api_get_script), + url(r'^api/v1/frida/get_dependencies$', api_dz.api_get_dependencies), ] if settings.API_ONLY == '0': urlpatterns.extend([ @@ -67,6 +68,7 @@ if settings.API_ONLY == '0': url(r'^$', home.index, name='home'), url(r'^upload/$', home.Upload.as_view), url(r'^download/', home.download), + url(r'^download_scan/', home.download_apk), url(r'^about$', home.about, name='about'), url(r'^api_docs$', home.api_docs, name='api_docs'), url(r'^recent_scans/$', home.recent_scans, name='recent'), @@ -131,6 +133,7 @@ if settings.API_ONLY == '0': url(r'^frida_logs/$', tests_frida.frida_logs), url(r'^list_frida_scripts/$', tests_frida.list_frida_scripts), url(r'^get_script/$', tests_frida.get_script), + url(r'^get_dependencies/$', tests_frida.get_runtime_dependencies), # Report url(r'^dynamic_report/(?P[0-9a-f]{32})$', report.view_report), diff --git a/mobsf/MobSF/utils.py b/mobsf/MobSF/utils.py index 4dbf86d1..761a0f4f 100755 --- a/mobsf/MobSF/utils.py +++ b/mobsf/MobSF/utils.py @@ -569,3 +569,30 @@ def cmd_injection_check(data): '%7C%7C', '%0a', '%0d%0a', ] return any(i in data for i in breakers) + + +def strict_package_check(user_input): + """Strict package name check.""" + pat = re.compile(r'^\w+\.*[\w\.\$]+$') + resp = re.match(pat, user_input) + if not resp: + logger.error('Invalid package/class name') + return resp + + +def is_path_traversal(user_input): + """Check for path traversal.""" + if (('../' in user_input) + or ('%2e%2e' in user_input) + or ('..' in user_input) + or ('%252e' in user_input)): + logger.error('Path traversal attack detected') + return True + return False + + +def is_zip_magic(file_obj): + magic = file_obj.read(4) + file_obj.seek(0, 0) + # ZIP magic PK.. no support for spanned and empty arch + return bool(magic == b'\x50\x4B\x03\x04') diff --git a/mobsf/MobSF/views/api/api_dynamic_analysis.py b/mobsf/MobSF/views/api/api_dynamic_analysis.py index 5e890e77..9b4083ba 100644 --- a/mobsf/MobSF/views/api/api_dynamic_analysis.py +++ b/mobsf/MobSF/views/api/api_dynamic_analysis.py @@ -234,6 +234,19 @@ def api_get_script(request): return make_api_response(resp, 500) +@request_method(['POST']) +@csrf_exempt +def api_get_dependencies(request): + """POST - Frida Get Runtime Dependencies.""" + if 'hash' not in request.POST: + return make_api_response( + {'error': 'Missing Parameters'}, 422) + resp = tests_frida.get_runtime_dependencies(request, True) + if resp['status'] == 'ok': + return make_api_response(resp, 200) + return make_api_response(resp, 500) + + # Report APIs @request_method(['POST']) @csrf_exempt diff --git a/mobsf/MobSF/views/apk_downloader.py b/mobsf/MobSF/views/apk_downloader.py new file mode 100644 index 00000000..0dcc7160 --- /dev/null +++ b/mobsf/MobSF/views/apk_downloader.py @@ -0,0 +1,163 @@ +# -*- coding: utf_8 -*- +"""Android APK Downloader.""" +import logging +from tempfile import gettempdir +from pathlib import Path +from zipfile import ZipFile + +import requests + +from bs4 import BeautifulSoup + +from django.conf import settings + +from mobsf.MobSF.views.scanning import ( + add_to_recent_scan, + handle_uploaded_file, +) +from mobsf.MobSF.utils import ( + is_path_traversal, + is_zip_magic, + strict_package_check, + upstream_proxy, +) + + +logger = logging.getLogger(__name__) + + +def fetch_html(url): + """Get Result HTML.""" + headers = { + 'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/39.0.2171.95 Safari/537.36'), + 'Accept-Encoding': 'deflate, gzip'} + try: + proxies, verify = upstream_proxy('https') + res = requests.get(url, + headers=headers, + proxies=proxies, + verify=verify, + stream=True) + if res.status_code == 200: + return BeautifulSoup(res.text, features='lxml') + except Exception: + pass + return None + + +def download_file(url, outfile): + try: + logger.info('Downloading APK...') + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(outfile, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + return outfile + except Exception: + pass + return None + + +def get_scan_type(location): + """Get APK type.""" + with ZipFile(location, 'r') as zf: + for fil in zf.namelist(): + if fil.endswith('.apk'): + return 'apks' + return 'apk' + + +def add_apk(dwd_file, filename): + """Add APK to MobSF.""" + with dwd_file.open('rb') as flip: + if not is_zip_magic(flip): + logger.warning('Downloaded file is not an APK/Split APK') + return None + md5 = handle_uploaded_file(flip, '.apk') + apk = Path(settings.UPLD_DIR) / md5 / f'{md5}.apk' + scan_type = get_scan_type(apk) + data = { + 'analyzer': 'static_analyzer', + 'status': 'success', + 'hash': md5, + 'scan_type': scan_type, + 'file_name': filename, + } + add_to_recent_scan(data) + return data + return None + + +def find_apk_link(url, domain): + """Find APK download link.""" + try: + logger.info('Looking for download link form %s', domain) + bsp = fetch_html(url) + if not bsp: + return None + link = bsp.find('a', href=True, string='click here') + if link: + logger.info('Download link found from %s', domain) + return link['href'] + logger.warning('Download link not found in %s', domain) + except Exception: + logger.warning('Failed to obtain download link from %s', domain) + return None + + +def try_provider(package, provider, domain): + """Try using a provider.""" + downloaded_file = None + data = None + apk_name = f'{package}.apk' + temp_file = Path(gettempdir()) / apk_name + link = find_apk_link(provider, domain) + if link: + downloaded_file = download_file(link, temp_file) + if downloaded_file: + data = add_apk(downloaded_file, apk_name) + if data: + return data + return None + + +def apk_download(package): + """Download APK.""" + downloaded_file = None + data = None + try: + if not strict_package_check(package) or is_path_traversal(package): + return None + logger.info('Attempting to download: %s', package) + # APKTADA + data = try_provider( + package, + f'{settings.APKTADA}{package}', + 'apktada.com') + if data: + return data + # APKPURE + data = try_provider( + package, + settings.APKPURE.format(package), + 'apkpure.com') + if data: + return data + # APKPLZ + data = try_provider( + package, + f'{settings.APKPLZ}{package}', + 'apkplz.net') + if data: + return data + logger.warning('Unable to find download link for %s', package) + return None + except Exception: + logger.exception('Failed to download the apk') + return None + finally: + if downloaded_file: + downloaded_file.unlink() diff --git a/mobsf/MobSF/views/helpers.py b/mobsf/MobSF/views/helpers.py index 4135c56d..29688641 100644 --- a/mobsf/MobSF/views/helpers.py +++ b/mobsf/MobSF/views/helpers.py @@ -1,6 +1,8 @@ """Helpers.""" import functools +from mobsf.MobSF.utils import is_zip_magic + from django.conf import settings from django.http import HttpRequest, HttpResponseNotAllowed @@ -13,7 +15,7 @@ class FileType(object): def __init__(self, file_obj): self.file_type = file_obj.content_type self.file_name_lower = file_obj.name.lower() - self.zip = self.is_zip_magic(file_obj) + self.zip = is_zip_magic(file_obj) def is_allow_file(self): """ @@ -26,15 +28,14 @@ class FileType(object): or self.is_xapk() or self.is_zip() or self.is_ipa() - or self.is_appx()): + or self.is_appx() + or self.is_apks()): return True return False - def is_zip_magic(self, file_obj): - magic = file_obj.read(4) - file_obj.seek(0, 0) - # ZIP magic PK.. no support for spanned and empty arch - return bool(magic == b'\x50\x4B\x03\x04') + def is_apks(self): + return (self.file_type in settings.APK_MIME + and self.file_name_lower.endswith('.apks')) def is_xapk(self): return (self.file_type in settings.APK_MIME diff --git a/mobsf/MobSF/views/home.py b/mobsf/MobSF/views/home.py index f03eb42f..877a824f 100755 --- a/mobsf/MobSF/views/home.py +++ b/mobsf/MobSF/views/home.py @@ -24,6 +24,7 @@ from mobsf.MobSF.utils import ( ) from mobsf.MobSF.views.helpers import FileType from mobsf.MobSF.views.scanning import Scanning +from mobsf.MobSF.views.apk_downloader import apk_download from mobsf.StaticAnalyzer.models import ( RecentScansDB, StaticAnalyzerAndroid, @@ -137,6 +138,8 @@ class Upload(object): return scanning.scan_apk() elif self.file_type.is_xapk(): return scanning.scan_xapk() + elif self.file_type.is_apks(): + return scanning.scan_apks() elif self.file_type.is_zip(): return scanning.scan_zip() elif self.file_type.is_ipa(): @@ -219,6 +222,25 @@ def recent_scans(request): return render(request, template, context) +def download_apk(request): + """Download and APK by package name.""" + package = request.POST['package'] + # Package validated in apk_download() + context = { + 'status': 'failed', + 'description': 'Unable to download APK', + } + res = apk_download(package) + if res: + context = res + context['status'] = 'ok' + context['package'] = package + resp = HttpResponse( + json.dumps(context), + content_type='application/json; charset=utf-8') + return resp + + def search(request): """Search Scan by MD5 Route.""" md5 = request.GET['md5'] diff --git a/mobsf/MobSF/views/scanning.py b/mobsf/MobSF/views/scanning.py index bef8b100..2e0d36f7 100644 --- a/mobsf/MobSF/views/scanning.py +++ b/mobsf/MobSF/views/scanning.py @@ -1,6 +1,7 @@ # -*- coding: utf_8 -*- import hashlib import logging +import io import os from django.conf import settings @@ -30,18 +31,30 @@ def add_to_recent_scan(data): logger.exception('Adding Scan URL to Database') -def handle_uploaded_file(filecnt, typ): +def handle_uploaded_file(content, typ): """Write Uploaded File.""" - md5 = hashlib.md5() # modify if crash for large - for chunk in filecnt.chunks(): - md5.update(chunk) + md5 = hashlib.md5() + bfr = isinstance(content, io.BufferedReader) + if bfr: + # Not File upload + while chunk := content.read(8192): + md5.update(chunk) + else: + # File upload + for chunk in content.chunks(): + md5.update(chunk) md5sum = md5.hexdigest() anal_dir = os.path.join(settings.UPLD_DIR, md5sum + '/') if not os.path.exists(anal_dir): os.makedirs(anal_dir) with open(anal_dir + md5sum + typ, 'wb+') as destination: - for chunk in filecnt.chunks(): - destination.write(chunk) + if bfr: + content.seek(0, 0) + while chunk := content.read(8192): + destination.write(chunk) + else: + for chunk in content.chunks(): + destination.write(chunk) return md5sum @@ -79,6 +92,20 @@ class Scanning(object): logger.info('Performing Static Analysis of Android XAPK base APK') return data + def scan_apks(self): + """Android Split APK.""" + md5 = handle_uploaded_file(self.file, '.apk') + data = { + 'analyzer': 'static_analyzer', + 'status': 'success', + 'hash': md5, + 'scan_type': 'apks', + 'file_name': self.file_name, + } + add_to_recent_scan(data) + logger.info('Performing Static Analysis of Android Split APK') + return data + def scan_zip(self): """Android /iOS Zipped Source.""" md5 = handle_uploaded_file(self.file, '.zip') diff --git a/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml b/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml index 8b0e6657..f66080bd 100644 --- a/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml +++ b/mobsf/StaticAnalyzer/views/android/rules/android_rules.yaml @@ -202,7 +202,7 @@ ref: https://github.com/MobSF/owasp-mstg/blob/master/Document/0x04c-Tampering-and-Reverse-Engineering.md#debugging-and-tracing - id: android_clipboard_listen description: >- - This app listens to Clipboard changes. Some malwares also listen to + This app listens to Clipboard changes. Some malware also listen to Clipboard changes. type: RegexAnd pattern: @@ -449,8 +449,8 @@ ref: https://github.com/MobSF/owasp-mstg/blob/master/Document/0x05j-Testing-Resiliency-Against-Reverse-Engineering.md#testing-root-detection-mstg-resilience-1 - id: android_jackson_deserialize description: >- - The app uses jackson deserialization libraryDeserialization of untrusted - input can result inarbitary code execution + The app uses jackson deserialization library. Deserialization of untrusted + input can result in arbitrary code execution. type: RegexAnd pattern: - com\.fasterxml\.jackson\.databind\.ObjectMapper @@ -626,7 +626,7 @@ cvss: 7.4 cwe: cwe-312 description: >- - Files may contain hardcoded sensitive informations like usernames, + Files may contain hardcoded sensitive information like usernames, passwords, keys etc. input_case: lower masvs: storage-14 diff --git a/mobsf/StaticAnalyzer/views/android/static_analyzer.py b/mobsf/StaticAnalyzer/views/android/static_analyzer.py index ebc0bcb8..2328ecd4 100755 --- a/mobsf/StaticAnalyzer/views/android/static_analyzer.py +++ b/mobsf/StaticAnalyzer/views/android/static_analyzer.py @@ -53,7 +53,10 @@ from mobsf.StaticAnalyzer.views.android.manifest_analysis import ( ) from mobsf.StaticAnalyzer.views.android.playstore import get_app_details from mobsf.StaticAnalyzer.views.android.strings import strings_from_apk -from mobsf.StaticAnalyzer.views.android.xapk import handle_xapk +from mobsf.StaticAnalyzer.views.android.xapk import ( + handle_split_apk, + handle_xapk, +) from mobsf.StaticAnalyzer.views.shared_func import ( firebase_analysis, hash_gen, @@ -94,8 +97,9 @@ def static_analyzer(request, api=False): app_dic = {} match = re.match('^[0-9a-f]{32}$', checksum) if (match - and filename.lower().endswith(('.apk', '.xapk', '.zip')) - and typ in ['zip', 'apk', 'xapk']): + and filename.lower().endswith( + ('.apk', '.xapk', '.zip', '.apks')) + and typ in ['zip', 'apk', 'xapk', 'apks']): app_dic['dir'] = Path(settings.BASE_DIR) # BASE DIR app_dic['app_name'] = filename # APP ORGINAL NAME app_dic['md5'] = checksum # MD5 @@ -103,14 +107,18 @@ def static_analyzer(request, api=False): app_dic['app_dir'] = Path(settings.UPLD_DIR) / checksum app_dic['tools_dir'] = app_dic['dir'] / 'StaticAnalyzer' / 'tools' app_dic['tools_dir'] = app_dic['tools_dir'].as_posix() - logger.info('Starting Analysis on : %s', app_dic['app_name']) + logger.info('Starting Analysis on: %s', app_dic['app_name']) if typ == 'xapk': # Handle XAPK # Base APK will have the MD5 of XAPK - res = handle_xapk(app_dic) - if not res: + if not handle_xapk(app_dic): raise Exception('Invalid XAPK File') typ = 'apk' + elif typ == 'apks': + # Handle Split APK + if not handle_split_apk(app_dic): + raise Exception('Invalid Split APK File') + typ = 'apk' if typ == 'apk': app_dic['app_file'] = app_dic['md5'] + '.apk' # NEW FILENAME app_dic['app_path'] = ( diff --git a/mobsf/StaticAnalyzer/views/android/strings.py b/mobsf/StaticAnalyzer/views/android/strings.py index a587b6a2..ec112c04 100755 --- a/mobsf/StaticAnalyzer/views/android/strings.py +++ b/mobsf/StaticAnalyzer/views/android/strings.py @@ -42,22 +42,26 @@ def strings_from_apk(app_file, app_dir, elf_strings): logger.info('Extracting Strings from APK') dat = [] secrets = [] + urls = [] + urls_nf = [] + emails_nf = [] apk_file = os.path.join(app_dir, app_file) and_a = apk.APK(apk_file) rsrc = and_a.get_android_resources() - pkg = rsrc.get_packages_names()[0] - rsrc.get_strings_resources() - for i in rsrc.values[pkg].keys(): - res_string = rsrc.values[pkg][i].get('string') - if res_string: - for duo in res_string: - cap_str = '"' + duo[0] + '" : "' + duo[1] + '"' - if is_secret(duo[0] + '"'): - secrets.append(cap_str) - dat.append(cap_str) - data_string = ''.join(dat) - urls, urls_nf, emails_nf = url_n_email_extract( - data_string, 'Android String Resource') + if rsrc: + pkg = rsrc.get_packages_names()[0] + rsrc.get_strings_resources() + for i in rsrc.values[pkg].keys(): + res_string = rsrc.values[pkg][i].get('string') + if res_string: + for duo in res_string: + cap_str = '"' + duo[0] + '" : "' + duo[1] + '"' + if is_secret(duo[0] + '"'): + secrets.append(cap_str) + dat.append(cap_str) + data_string = ''.join(dat) + urls, urls_nf, emails_nf = url_n_email_extract( + data_string, 'Android String Resource') if elf_strings: for solib in elf_strings: for so, str_list in solib.items(): @@ -71,12 +75,13 @@ def strings_from_apk(app_file, app_dir, elf_strings): urls_nf.extend(suf) emails_nf.extend(sem) strings_dat = list(set(dat)) - return {'strings': strings_dat, - 'urls_list': urls, - 'url_nf': urls_nf, - 'emails_nf': emails_nf, - 'secrets': secrets, - } + return { + 'strings': strings_dat, + 'urls_list': urls, + 'url_nf': urls_nf, + 'emails_nf': emails_nf, + 'secrets': secrets, + } except Exception: logger.exception('Extracting Strings from APK') return {} diff --git a/mobsf/StaticAnalyzer/views/android/xapk.py b/mobsf/StaticAnalyzer/views/android/xapk.py index ce6d8f99..6f17e35c 100644 --- a/mobsf/StaticAnalyzer/views/android/xapk.py +++ b/mobsf/StaticAnalyzer/views/android/xapk.py @@ -5,6 +5,7 @@ from json import load from shutil import move from mobsf.StaticAnalyzer.views.shared_func import unzip +from mobsf.MobSF.utils import is_safe_path logger = logging.getLogger(__name__) @@ -12,8 +13,9 @@ logger = logging.getLogger(__name__) def handle_xapk(app_dic): """Unzip and Extract APK.""" data = None - xapk = app_dic['app_dir'] / (app_dic['md5'] + '.xapk') - apk = app_dic['app_dir'] / (app_dic['md5'] + '.apk') + checksum = app_dic['md5'] + xapk = app_dic['app_dir'] / f'{checksum}.xapk' + apk = app_dic['app_dir'] / f'{checksum}.apk' files = unzip(xapk.as_posix(), app_dic['app_dir']) if 'manifest.json' not in files: logger.error('Manifest file not found in XAPK') @@ -31,6 +33,25 @@ def handle_xapk(app_dic): for a in apks: if a['id'] == 'base': base_apk = app_dic['app_dir'] / a['file'] - move(base_apk, apk) - return True + if is_safe_path(app_dic['app_dir'], base_apk): + move(base_apk, apk) + return True + return None + + +def handle_split_apk(app_dic): + """Unzip and Extract Split APKs.""" + checksum = app_dic['md5'] + apks = app_dic['app_dir'] / f'{checksum}.apk' + # Check if previously extracted + manifest = app_dic['app_dir'] / 'AndroidManifest.xml' + if manifest.exists(): + return True + files = unzip(apks.as_posix(), app_dic['app_dir']) + for apk in files: + if not apk.startswith('config.') and apk.endswith('.apk'): + full_path = app_dic['app_dir'] / apk + if is_safe_path(app_dic['app_dir'], full_path): + move(full_path, apks) + return True return None diff --git a/mobsf/StaticAnalyzer/views/ios/rules/objective_c_rules.yaml b/mobsf/StaticAnalyzer/views/ios/rules/objective_c_rules.yaml index 7f150624..5018788e 100644 --- a/mobsf/StaticAnalyzer/views/ios/rules/objective_c_rules.yaml +++ b/mobsf/StaticAnalyzer/views/ios/rules/objective_c_rules.yaml @@ -11,7 +11,7 @@ - id: ios_hardcoded_info cvss: 7.4 cwe: cwe-312 - description: Files may contain hardcoded sensitive informations like usernames, + description: Files may contain hardcoded sensitive information like usernames, passwords, keys etc. input_case: lower masvs: storage-14 @@ -160,7 +160,7 @@ cvss: 0 cwe: '' description: >- - This app listens to Clipboard changes. Some malwares also listen to + This app listens to Clipboard changes. Some malware also listen to Clipboard changes. input_case: exact masvs: platform-4 diff --git a/mobsf/StaticAnalyzer/views/ios/rules/swift_rules.yaml b/mobsf/StaticAnalyzer/views/ios/rules/swift_rules.yaml index c0aee7b0..787206f4 100644 --- a/mobsf/StaticAnalyzer/views/ios/rules/swift_rules.yaml +++ b/mobsf/StaticAnalyzer/views/ios/rules/swift_rules.yaml @@ -11,7 +11,7 @@ - id: ios_hardcoded_info cvss: 7.4 cwe: cwe-312 - description: Files may contain hardcoded sensitive informations like usernames, + description: Files may contain hardcoded sensitive information like usernames, passwords, keys etc. input_case: lower masvs: storage-14 @@ -136,7 +136,6 @@ masvs: crypto-4 owasp-mobile: m5 pattern: - - (?i)MD2\( - CC_MD2\( severity: high type: RegexOr @@ -149,7 +148,6 @@ masvs: crypto-4 owasp-mobile: m5 pattern: - - (?i)MD4\( - CC_MD4\( severity: high type: RegexOr @@ -194,7 +192,7 @@ - id: ios_keyboard_cache cvss: 0 cwe: '' - description: Keyboard cache should be disabled for all sensitve data inputs. + description: Keyboard cache should be disabled for all sensitive data inputs. input_case: exact masvs: storage-5 owasp-mobile: '' @@ -218,7 +216,7 @@ - id: ios_detect_reversing cvss: 0 cwe: '' - description: This App may have Reverse enginering detection capabilities. + description: This App may have Reverse engineering detection capabilities. input_case: exact masvs: resilience-4 owasp-mobile: '' @@ -243,7 +241,7 @@ - id: ios_biometric_bool cvss: 0 cwe: '' - description: 'Biometric authentication should be based on Keychain, not based on bool.' + description: 'Biometric authentication should be based on iOS Keychain, not based on bool.' input_case: exact masvs: auth-8 owasp-mobile: m1 diff --git a/mobsf/install/windows/setup.py b/mobsf/install/windows/setup.py index c1f3f921..3a8f707f 100755 --- a/mobsf/install/windows/setup.py +++ b/mobsf/install/windows/setup.py @@ -171,6 +171,7 @@ def tools_binskim(): [ nuget, 'install', binskim_nuget, '-Pre', + '-Version', '1.7.2', '-o', mobsf_subdir_tools]) # Some code to determine the version on the fly so we don't have to fix the diff --git a/mobsf/templates/dynamic_analysis/android/dynamic_analyzer.html b/mobsf/templates/dynamic_analysis/android/dynamic_analyzer.html index c084b084..445cc6ba 100644 --- a/mobsf/templates/dynamic_analysis/android/dynamic_analyzer.html +++ b/mobsf/templates/dynamic_analysis/android/dynamic_analyzer.html @@ -84,6 +84,9 @@ {% endif %} Exported Activity Tester Activity Tester + {% if android_version >= 5 %} + Get Dependencies + {% endif %} Take a Screenshot Logcat Stream @@ -469,8 +472,17 @@ var editor = CodeMirror.fromTextArea(document.getElementById("code-js"), { gutters: ["CodeMirror-lint-markers"], lint: true, lineWrapping: true, + onLoad: restore(), }); +//Restore Code +function restore(){ + var custom_code = window.sessionStorage.getItem("code"); + if (custom_code !== null) { + setTimeout(function(){ editor.getDoc().setValue(custom_code); }, 2000); + } +} + // Terminal var util = util || {}; util.toArray = function (list) { @@ -955,9 +967,13 @@ $("#loadscript").click(function() { return false; }); + // Frida Spawn $("#frida_spawn").click(function() { print_status("Instrumenting the application with Frida."); + // Save Frida Code to Session Storage + var code = editor.getDoc().getValue(); + window.sessionStorage.setItem('code', code); $('#fd_log').removeAttr('disabled'); $('#frida-tab').removeClass('disabled'); var default_hooks = []; @@ -991,6 +1007,20 @@ $("#frida_spawn").click(function() { return false; }); +//Frida Get Runtime Dependencies +$("#deps").click(function() { + print_status("Getting application runtime dependencies."); + action(document.location.origin + '/get_dependencies/', {hash: '{{ hash }}'}, function(json) { + if (json.status==="ok"){ + $('#fd_log').removeAttr('disabled'); + $('#frida-tab').removeClass('disabled'); + print_status("Dependencies collected!"); + } else { + print_status(json.message); + } + }); + return false; +}); //Dump App data function dump(){ @@ -1026,8 +1056,7 @@ $("#stop").one( "click", function() { dump(); }); return false; -}); - +}); //end document.ready }); diff --git a/mobsf/templates/dynamic_analysis/android/dynamic_report.html b/mobsf/templates/dynamic_analysis/android/dynamic_report.html index 52e4d642..da2d7211 100644 --- a/mobsf/templates/dynamic_analysis/android/dynamic_report.html +++ b/mobsf/templates/dynamic_analysis/android/dynamic_report.html @@ -114,6 +114,14 @@

+
  • + + +

    + Runtime Dependencies +

    +     +
  • @@ -164,6 +172,18 @@

    Emails
    • +
  • + + +

    Trackers

    +     +
    • +
  • + + +

    Base64 Strings

    +     +
  • @@ -519,6 +539,31 @@ + + +
    +
    +
    +
    +
    +
    +

    + RUNTIME DEPENDENCIES +

    + {% if runtime_dependencies %} +
    + {% for f in runtime_dependencies %} + {{ f }}
    + {% endfor %} + {% endif %} +
    +
    +
    +
    + +
    +
    +