4.2.2 (#2452)

* QA
* Verbose

mobsf/DynamicAnalyzer/views/android/environment.py

@@ -535,6 +535,7 @@ class Environment:
                                  'MobSF documentation!')
                 return False
         except Exception:
+            logger.exception('System check failed')
             logger.error(err_msg)
             return False
         return True

mobsf/MobSF/settings.py

@@ -351,6 +351,7 @@ Q_CLUSTER = {
     'label': 'scan_queue',
     'orm': 'default',
     'max_attempts': 2,
+    'save_limit': -1,
 }
 QUEUE_MAX_SIZE = 100
 ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')

mobsf/MobSF/urls.py

@@ -204,6 +204,7 @@ if settings.API_ONLY == '0':
         re_path(r'^status/$', home.scan_status, name='status'),
         re_path(r'^error/$', home.error, name='error'),
         re_path(r'^zip_format/$', home.zip_format),
+        re_path(r'^robots.txt$', home.robots_txt),
         re_path(r'^dynamic_analysis/$', home.dynamic_analysis, name='dynamic'),
         re_path(r'^tasks$',
                 async_task.list_tasks,

mobsf/MobSF/views/home.py

@@ -249,6 +249,11 @@ def zip_format(request):
     return render(request, template, context)
 
 
+def robots_txt(request):
+    content = 'User-agent: *\nDisallow: /*/\nAllow: /*\n'
+    return HttpResponse(content, content_type='text/plain')
+
+
 @login_required
 def dynamic_analysis(request):
     """Dynamic Analysis Landing."""
@@ -305,6 +310,7 @@ def recent_scans(request, page_size=10, page_number=1):
         'entries': entries,
         'version': settings.MOBSF_VER,
         'page_obj': page_obj,
+        'async_scans': settings.ASYNC_ANALYSIS,
     }
     template = 'general/recent.html'
     return render(request, template, context)

mobsf/StaticAnalyzer/models.py

@@ -175,7 +175,7 @@ class EnqueuedTask(models.Model):
     checksum = models.CharField(max_length=255)
     file_name = models.CharField(max_length=255)
     created_at = models.DateTimeField(default=timezone.now)
-    status = models.CharField(max_length=50, default='Enqueued')
+    status = models.CharField(max_length=255, default='Enqueued')
     completed_at = models.DateTimeField(null=True)
     app_name = models.CharField(max_length=255, default='')
 

mobsf/StaticAnalyzer/views/android/apk.py

@@ -83,13 +83,17 @@ from mobsf.MobSF.views.authorization import (
 logger = logging.getLogger(__name__)
 
 
-def initialize_app_dic(checksum, app_dic, file_ext):
+def initialize_app_dic(app_dic, file_ext):
+    checksum = app_dic['md5']
     app_dic['app_file'] = f'{checksum}.{file_ext}'
     app_dic['app_path'] = (app_dic['app_dir'] / app_dic['app_file']).as_posix()
     app_dic['app_dir'] = app_dic['app_dir'].as_posix() + '/'
+    return checksum
+
+
+def get_size_and_hashes(app_dic):
     app_dic['size'] = str(file_size(app_dic['app_path'])) + 'MB'
-    app_dic['sha1'], app_dic['sha256'] = hash_gen(checksum, app_dic['app_path'])
-    return app_dic
+    app_dic['sha1'], app_dic['sha256'] = hash_gen(app_dic['md5'], app_dic['app_path'])
 
 
 def get_manifest_data(checksum, app_dic, andro_apk=None):
@@ -147,7 +151,7 @@ def apk_analysis_task(checksum, app_dic, rescan, queue=False):
         if queue:
             settings.ASYNC_ANALYSIS = True
         append_scan_status(checksum, 'init')
-        initialize_app_dic(checksum, app_dic, 'apk')
+        get_size_and_hashes(app_dic)
         msg = 'Extracting APK'
         logger.info(msg)
         append_scan_status(checksum, msg)
@@ -281,7 +285,7 @@ def generate_dynamic_context(request, app_dic, checksum, context, api):
 
 def apk_analysis(request, app_dic, rescan, api):
     """APK Analysis."""
-    checksum = app_dic['md5']
+    checksum = initialize_app_dic(app_dic, 'apk')
     db_entry = StaticAnalyzerAndroid.objects.filter(MD5=checksum)
     if db_entry.exists() and not rescan:
         context = get_context_from_db_entry(db_entry)
@@ -402,7 +406,7 @@ def generate_dynamic_src_context(request, context, api):
 
 def src_analysis(request, app_dic, rescan, api):
     """Source Code Analysis."""
-    checksum = app_dic['md5']
+    checksum = initialize_app_dic(app_dic, 'zip')
     ret = f'/static_analyzer_ios/{checksum}/'
     db_entry = StaticAnalyzerAndroid.objects.filter(
         MD5=checksum)
@@ -416,7 +420,7 @@ def src_analysis(request, app_dic, rescan, api):
     else:
         # Initialize for both Android and iOS Source Analysis
         append_scan_status(checksum, 'init')
-        initialize_app_dic(checksum, app_dic, 'zip')
+        get_size_and_hashes(app_dic)
         msg = 'Extracting ZIP'
         logger.info(msg)
         append_scan_status(checksum, msg)

mobsf/StaticAnalyzer/views/common/async_task.py

@@ -84,9 +84,9 @@ def async_analysis(checksum, app_name, func, *args):
 def update_enqueued_task(checksum, app_name, status):
     """Update the Enqueued Task and others that matches the checksum."""
     EnqueuedTask.objects.filter(checksum=checksum).update(
-        app_name=app_name,
+        app_name=app_name[:254],
         completed_at=timezone.now(),
-        status=status,
+        status=status[:254],
     )
     return True
 

mobsf/StaticAnalyzer/views/ios/ipa.py

@@ -68,12 +68,18 @@ from mobsf.MobSF.views.authorization import (
 logger = logging.getLogger(__name__)
 
 
-def initialize_app_dic(app_dic, checksum, scan_type):
+def initialize_app_dic(app_dic, file_ext):
     """Initialize App Dictionary."""
-    app_dic['app_file'] = f'{checksum}.{scan_type}'
+    checksum = app_dic['md5_hash']
+    app_dic['app_file'] = f'{checksum}.{file_ext}'
     app_dic['app_path'] = (app_dic['app_dirp'] / app_dic['app_file']).as_posix()
+    return checksum
+
+
+def get_size_and_hashes(app_dic):
     app_dic['size'] = str(file_size(app_dic['app_path'])) + 'MB'
-    app_dic['sha1'], app_dic['sha256'] = hash_gen(checksum, app_dic['app_path'])
+    app_dic['sha1'], app_dic['sha256'] = hash_gen(
+        app_dic['md5_hash'], app_dic['app_path'])
 
 
 def extract_and_check_ipa(checksum, app_dic):
@@ -163,12 +169,11 @@ def ipa_analysis_task(checksum, app_dic, rescan, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-        scan_type = 'ipa'
         append_scan_status(checksum, 'init')
         msg = 'iOS Binary (IPA) Analysis Started'
         logger.info(msg)
         append_scan_status(checksum, msg)
-        initialize_app_dic(app_dic, checksum, scan_type)
+        get_size_and_hashes(app_dic)
 
         if not extract_and_check_ipa(checksum, app_dic):
             msg = ('IPA is malformed! MobSF cannot find Payload directory')
@@ -177,7 +182,7 @@ def ipa_analysis_task(checksum, app_dic, rescan, queue=False):
                 return update_enqueued_task(
                     checksum, 'Failed', msg)
             return context, msg
-        common_analysis(scan_type, app_dic, checksum)
+        common_analysis('ipa', app_dic, checksum)
 
         # IPA Binary Analysis
         bin_dict = binary_analysis(
@@ -241,7 +246,7 @@ def generate_dynamic_context(request, app_dic, context, checksum, api):
 
 def ipa_analysis(request, app_dic, rescan, api):
     """IPA Analysis."""
-    checksum = app_dic['md5_hash']
+    checksum = initialize_app_dic(app_dic, 'ipa')
     ipa_db = StaticAnalyzerIOS.objects.filter(MD5=checksum)
     if ipa_db.exists() and not rescan:
         context = get_context_from_db_entry(ipa_db)
@@ -267,13 +272,12 @@ def ios_analysis_task(checksum, app_dic, rescan, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-        scan_type = 'zip'
         logger.info('iOS Source Code Analysis Started')
-        initialize_app_dic(app_dic, checksum, scan_type)
+        get_size_and_hashes(app_dic)
 
         # ANALYSIS BEGINS - Already Unzipped
         # append_scan_status init done in android static analyzer
-        common_analysis(scan_type, app_dic, checksum)
+        common_analysis('zip', app_dic, checksum)
 
         # IOS Source Code Analysis
         code_dict = ios_source_analysis(
@@ -327,7 +331,7 @@ def generate_dynamic_ios_context(request, context, api):
 
 def ios_analysis(request, app_dic, rescan, api):
     """IOS Source Code Analysis."""
-    checksum = app_dic['md5_hash']
+    checksum = initialize_app_dic(app_dic, 'zip')
     ios_zip_db = StaticAnalyzerIOS.objects.filter(MD5=checksum)
     if ios_zip_db.exists() and not rescan:
         context = get_context_from_db_entry(ios_zip_db)

mobsf/templates/general/recent.html

@@ -35,6 +35,9 @@
                  <div class="box">
         <div class="box-header with-border">
             <h3 class="box-title"><i class="fa fa-rocket"></i> Recent Scans</h3>
+            {% if async_scans %}
+            <p class="lead">View scan progress in the <strong><a href="{% url 'list_tasks' %}">Scan Queue</a></strong></p>
+            {% endif %}
         </div>
 
         <div class="box-body">

mobsf/templates/general/tasks.html

@@ -41,7 +41,7 @@
                    <div class="box">
           <div class="box-header with-border">
               <h3 class="box-title"><i class="fa fa-rocket"></i> Scan Queue</h3>
-              <p class="lead">The scan results will be available in <strong><a href="{% url 'recent' %}">Recent Scans</a></strong> upon completion.</p>
+              <p class="lead">The scan results will appear in <strong><a href="{% url 'recent' %}">Recent Scans</a></strong> once completed.</p>
           </div>
           <div class="box-body">
             <div class="table-responsive">