Reduce Repo Size, download frida-server dynamically (#1648)

Reduce Repo Size, download frida-server dynamically (#1648)

.gitattributes

@@ -1 +0,0 @@
-mobsf/DynamicAnalyzer/tools/onDevice/frida/ filter=lfs diff=lfs merge=lfs -text

mobsf/DynamicAnalyzer/views/android/environment.py

@@ -18,6 +18,9 @@ from mobsf.DynamicAnalyzer.tools.webproxy import (
     start_proxy,
     stop_httptools,
 )
+from mobsf.DynamicAnalyzer.views.android import (
+    frida_server_download as fserver,
+)
 from mobsf.MobSF.utils import (
     get_adb,
     get_device,
@@ -564,7 +567,6 @@ class Environment:
     def frida_setup(self):
         """Setup Frida."""
         frida_arch = None
-        frida_dir = 'onDevice/frida/'
         arch = self.get_android_arch()
         logger.info('Android OS architecture identified as %s', arch)
         if arch in ['armeabi-v7a', 'armeabi']:
@@ -580,12 +582,15 @@ class Environment:
                          ' or Android Studio Emulator'
                          ' instance is running')
             return
-        frida_bin = 'frida-server-{}-android-{}'.format(
-            FRIDA_VERSION,
-            frida_arch)
-        frida_path = os.path.join(self.tools_dir,
-                                  frida_dir,
-                                  frida_bin)
+        frida_bin = f'frida-server-{FRIDA_VERSION}-android-{frida_arch}'
+        stat = fserver.update_frida_server(frida_arch, FRIDA_VERSION)
+        if not stat:
+            msg = ('Cannot download frida-server binary. You will need'
+                   f' {frida_bin} in {settings.DWD_DIR} for '
+                   'Dynamic Analysis to work')
+            logger.error(msg)
+            return
+        frida_path = os.path.join(settings.DWD_DIR, frida_bin)
         logger.info('Copying frida server for %s', frida_arch)
         self.adb_command(['push', frida_path, '/system/fd_server'])
         self.adb_command(['chmod', '755', '/system/fd_server'], True)

mobsf/DynamicAnalyzer/views/android/frida_server_download.py

@@ -0,0 +1,75 @@
+# -*- coding: utf_8 -*-
+"""Download Frida Server."""
+import logging
+from pathlib import Path
+from lzma import LZMAFile
+from shutil import copyfileobj
+
+import requests
+
+from django.conf import settings
+
+from mobsf.MobSF.utils import (
+    is_internet_available,
+    upstream_proxy,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def clean_up_old_binaries(dirc, version):
+    """Delete Old Binaries."""
+    for f in Path(dirc).iterdir():
+        if f.is_file() and f.name.startswith('frida-server'):
+            if version in f.name:
+                continue
+            try:
+                f.unlink()
+            except Exception:
+                pass
+
+
+def download_frida_server(url, version, fname):
+    """Download frida-server-binary."""
+    try:
+        download_dir = Path(settings.DWD_DIR)
+        logger.info('Downloading binary %s', fname)
+        dwd_loc = download_dir / fname
+        with requests.get(url, stream=True) as r:
+            with LZMAFile(r.raw) as f:
+                with open(dwd_loc, 'wb') as flip:
+                    copyfileobj(f, flip)
+        clean_up_old_binaries(download_dir, version)
+        return True
+    except Exception:
+        logger.exception('[ERROR] Downloading Frida Server Binary')
+    return False
+
+
+def update_frida_server(arch, version):
+    """Download Assets of a given version."""
+    download_dir = Path(settings.DWD_DIR)
+    fserver = f'frida-server-{version}-android-{arch}'
+    frida_bin = download_dir / fserver
+    if frida_bin.is_file():
+        return True
+    if not is_internet_available():
+        return False
+    try:
+        proxies, verify = upstream_proxy('https')
+    except Exception:
+        logger.exception('[ERROR] Setting upstream proxy')
+    try:
+        response = requests.get(f'{settings.FRIDA_SERVER}{version}',
+                                timeout=3,
+                                proxies=proxies,
+                                verify=verify)
+        for item in response.json()['assets']:
+            if item['name'] == f'{fserver}.xz':
+                url = item['browser_download_url']
+                return download_frida_server(url, version, fserver)
+        return False
+    except Exception:
+        logger.exception('[ERROR] Fetching Frida Server Release')
+    return False

mobsf/MobSF/settings.py

@@ -20,8 +20,8 @@ logger = logging.getLogger(__name__)
 #       MOBSF CONFIGURATIONS
 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-VERSION = '3.2.6'
-MOBSF_VER = 'v3.2.6 Beta'
+VERSION = '3.2.7'
+MOBSF_VER = 'v3.2.7 Beta'
 # Remove this later with f string
 BANNER = """
   __  __       _    ____  _____   _____  ____  
@@ -211,6 +211,20 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 268435456
 # In this mode, web UI related urls are disabled.
 API_ONLY = os.getenv('MOBSF_API_ONLY', '0')
 
+# -----External URLS--------------------------
+MALWARE_DB_URL = 'https://www.malwaredomainlist.com/mdlcsv.php'
+MALTRAIL_DB_URL = ('https://raw.githubusercontent.com/stamparm/aux/'
+                   'master/maltrail-malware-domains.txt')
+VIRUS_TOTAL_BASE_URL = 'https://www.virustotal.com/vtapi/v2/file/'
+EXODUS_URL = 'https://reports.exodus-privacy.eu.org'
+APPMONSTA_URL = 'https://api.appmonsta.com/v1/stores/android/details/'
+ITUNES_URL = 'https://itunes.apple.com/lookup'
+GITHUB_URL = ('https://raw.githubusercontent.com/'
+              'MobSF/Mobile-Security-Framework-MobSF/'
+              'master/mobsf/MobSF/settings.py')
+FRIDA_SERVER = 'https://api.github.com/repos/frida/frida/releases/tags/'
+GOOGLE = 'https://www.google.com'
+BAIDU = 'https://www.baidu.com/'
 # ===================
 # USER CONFIGURATION
 # ===================
@@ -223,7 +237,7 @@ else:
     If 'USE_HOME' is set to True,
     then below user configuration settings are not considered.
     The user configuration will be loaded from
-    config.py in MobSF Home directory.
+    .MobSF/config.py in user's home directory.
     """
     # ^CONFIG-START^: Do not edit this line
     # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -324,26 +338,10 @@ else:
     APKID_ENABLED = True
     # ==============================================
 
-    # -----External URLS--------------------------
-    MALWARE_DB_URL = 'https://www.malwaredomainlist.com/mdlcsv.php'
-    MALTRAIL_DB_URL = ('https://raw.githubusercontent.com/stamparm/aux/'
-                       'master/maltrail-malware-domains.txt')
-    VIRUS_TOTAL_BASE_URL = 'https://www.virustotal.com/vtapi/v2/file/'
-    EXODUS_URL = 'https://reports.exodus-privacy.eu.org'
-    APPMONSTA_URL = 'https://api.appmonsta.com/v1/stores/android/details/'
-    ITUNES_URL = 'https://itunes.apple.com/lookup'
-    GITHUB_URL = ('https://raw.githubusercontent.com/'
-                  'MobSF/Mobile-Security-Framework-MobSF/'
-                  'master/mobsf/MobSF/settings.py')
-    GOOGLE = 'https://www.google.com'
-    BAIDU = 'https://www.baidu.com/'
-
-    # -------External -----------------------------
+    # ========DISABLED COMPONENTS===================
+    # -------External API Keys----------------------
     # Get AppMonsta API from https://appmonsta.com/dashboard/get_api_key/
     APPMONSTA_API = ''
-
-    # ========DISABLED COMPONENTS===================
-
     # ----------VirusTotal--------------------------
     VT_ENABLED = False
     VT_API_KEY = ''

mobsf/MobSF/utils.py

@@ -415,7 +415,7 @@ def update_local_db(db_name, url, local_file):
     update = None
     inmemoryfile = None
     try:
-        proxies, verify = upstream_proxy('http')
+        proxies, verify = upstream_proxy('https')
     except Exception:
         logger.exception('[ERROR] Setting upstream proxy')
     try:

mobsf/templates/dynamic_analysis/dynamic_analysis.html

@@ -254,7 +254,7 @@ div.loading{
                   <div class="row">
                      <div class="col-md-9">
                         <h4>Dynamic Analyzer Supports</h4>
-                        <h5><strong>• Genymotion Android VM</strong> version 4.1 to 10.0 (x86, upto API 29)<br/><strong>• Android Studio Emulator AVD</strong> version 5.0 to 9.0 (arm, arm64, and x86 upto API 28)<br/>We recommend using Android 7.0 and above.</h5>
+                        <h5><strong>• Genymotion Android VM</strong> version 4.1 to 10.0 (x86, upto API 29)<br/><strong>• Android Studio Emulator AVD</strong> version 5.0 to 9.0 (arm, arm64, x86, and x86_64 upto API 28)<br/>We recommend using Android 7.0 and above.</h5>
                         <p>For Android versions less than <strong>5</strong>, we use Xposed Framework and therefore, you must MobSFy the Android Runtime prior to Dynamic Analysis.<br /> Other versions use Frida and are automatically MobSFyed on the go.</p>
                         <p><h5> HTTPS Traffic Interception</h5>
                         <p>For Android version <strong>4.1 - 4.3</strong>, set Android VM proxy as <strong>{% if proxy_ip %} {{ proxy_ip }}{% else %} <i>&lt;Host IP&gt;</i>{% endif %}:{{proxy_port}}</strong><br/ >For Android versions <strong>4.4 - 10</strong>, global proxy settings are automatically applied.<br/>For more information, please refer Documentation.

scripts/update_frida.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-import requests
-
-import lzma
-import shutil
-
-
-def get_latest_frida_binaries():
-    r = requests.get('https://api.github.com/'
-                     'repos/frida/frida/releases/latest')
-    for item in r.json()['assets']:
-        url = item['browser_download_url']
-        if ('frida-server' in url and 'android' in url):
-            download_file(url)
-
-
-def download_file(url):
-    fname = url.split('/')[-1]
-    print(f'Downloading & Extracting - {fname}')
-    base = '../mobsf/DynamicAnalyzer/tools/onDevice/frida/'
-    dwd_file = fname.replace('.xz', '')
-    dwd_loc = f'{base}{dwd_file}'
-    with requests.get(url, stream=True) as r:
-        with lzma.LZMAFile(r.raw) as f:
-            with open(dwd_loc, 'wb') as flip:
-                shutil.copyfileobj(f, flip)
-
-
-if __name__ == '__main__':
-    get_latest_frida_binaries()