diff --git a/mobsf/MobSF/init.py b/mobsf/MobSF/init.py index 91c38a38..b4aec6c1 100644 --- a/mobsf/MobSF/init.py +++ b/mobsf/MobSF/init.py @@ -18,7 +18,7 @@ from mobsf.install.windows.setup import windows_config_local logger = logging.getLogger(__name__) -VERSION = '4.2.4' +VERSION = '4.2.5' BANNER = r""" __ __ _ ____ _____ _ _ ____ | \/ | ___ | |__/ ___|| ___|_ _| || | |___ \ diff --git a/mobsf/MobSF/settings.py b/mobsf/MobSF/settings.py index ef40f776..e2df6380 100644 --- a/mobsf/MobSF/settings.py +++ b/mobsf/MobSF/settings.py @@ -341,12 +341,14 @@ LOGGING = { }, }, } +ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1') +ASYNC_ANALYSIS_TIMEOUT = int(os.getenv('MOBSF_ASYNC_ANALYSIS_TIMEOUT', '60')) Q_CLUSTER = { 'name': 'scan_queue', 'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', 3)), 'recycle': 5, - 'timeout': 3600, - 'retry': 3700, + 'timeout': ASYNC_ANALYSIS_TIMEOUT * 60, + 'retry': (ASYNC_ANALYSIS_TIMEOUT * 60) + 100, 'compress': True, 'label': 'scan_queue', 'orm': 'default', @@ -355,7 +357,6 @@ Q_CLUSTER = { 'ack_failures': True, } QUEUE_MAX_SIZE = 100 -ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1') MULTIPROCESSING = os.getenv('MOBSF_MULTIPROCESSING') JADX_TIMEOUT = int(os.getenv('MOBSF_JADX_TIMEOUT', 1000)) SAST_TIMEOUT = int(os.getenv('MOBSF_SAST_TIMEOUT', 1000)) diff --git a/mobsf/MobSF/utils.py b/mobsf/MobSF/utils.py index e6edf739..1cc2122d 100755 --- a/mobsf/MobSF/utils.py +++ b/mobsf/MobSF/utils.py @@ -61,6 +61,8 @@ EMAIL_REGEX = re.compile(r'[\w+.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}') USERNAME_REGEX = re.compile(r'^\w[\w\-\@\.]{1,35}$') GOOGLE_API_KEY_REGEX = re.compile(r'AIza[0-9A-Za-z-_]{35}$') GOOGLE_APP_ID_REGEX = re.compile(r'\d{1,2}:\d{1,50}:android:[a-f0-9]{1,50}') +PKG_REGEX = re.compile( + r'package\s+([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*);') class Color(object): diff --git a/mobsf/MobSF/views/api/api_static_analysis.py b/mobsf/MobSF/views/api/api_static_analysis.py index ede9aa43..436ce029 100755 --- a/mobsf/MobSF/views/api/api_static_analysis.py +++ b/mobsf/MobSF/views/api/api_static_analysis.py @@ -113,7 +113,6 @@ def api_scan_logs(request): return make_api_response({'logs': resp}, 200) - @request_method(['POST']) @csrf_exempt def api_tasks(request): diff --git a/mobsf/MobSF/views/home.py b/mobsf/MobSF/views/home.py index b6bbdf62..cf37275c 100755 --- a/mobsf/MobSF/views/home.py +++ b/mobsf/MobSF/views/home.py @@ -7,9 +7,11 @@ import platform import re import shutil from pathlib import Path +from datetime import timedelta from wsgiref.util import FileWrapper from django.conf import settings +from django.utils.timezone import now from django.core.paginator import Paginator from django.http import HttpResponse, HttpResponseRedirect from django.views.decorators.http import require_http_methods @@ -541,10 +543,14 @@ def delete_scan(request, api=False): if settings.ASYNC_ANALYSIS: # Handle Async Tasks et = EnqueuedTask.objects.filter(checksum=md5_hash).first() - if et and not et.completed_at: - # Queue is in progress, cannot delete the task - return send_response({ - 'deleted': 'A scan can only be deleted after it is completed'}, api) + if et: + max_time_passed = now() - et.created_at > timedelta( + minutes=settings.ASYNC_ANALYSIS_TIMEOUT) + if not (et.completed_at or max_time_passed): + # Queue is in progress, cannot delete the task + return send_response( + {'deleted': 'A scan can only be deleted after it is completed'}, + api) # Delete all related DB entries EnqueuedTask.objects.filter(checksum=md5_hash).all().delete() RecentScansDB.objects.filter(MD5=md5_hash).delete() @@ -565,7 +571,7 @@ def delete_scan(request, api=False): os.remove(item_path) # Delete related directories if is_dir_exists(item_path) and valid_item: - shutil.rmtree(item_path) + shutil.rmtree(item_path, ignore_errors=True) return send_response({'deleted': 'yes'}, api) except Exception as exp: msg = str(exp) diff --git a/mobsf/StaticAnalyzer/models.py b/mobsf/StaticAnalyzer/models.py index 3749b995..bc7a22f5 100755 --- a/mobsf/StaticAnalyzer/models.py +++ b/mobsf/StaticAnalyzer/models.py @@ -82,6 +82,7 @@ class StaticAnalyzerAndroid(models.Model): PLAYSTORE_DETAILS = models.TextField(default={}) NETWORK_SECURITY = models.TextField(default=[]) SECRETS = models.TextField(default=[]) + SBOM = models.TextField(default={}) class StaticAnalyzerIOS(models.Model): diff --git a/mobsf/StaticAnalyzer/views/android/code_analysis.py b/mobsf/StaticAnalyzer/views/android/code_analysis.py index f5d57a40..c2cc7f43 100755 --- a/mobsf/StaticAnalyzer/views/android/code_analysis.py +++ b/mobsf/StaticAnalyzer/views/android/code_analysis.py @@ -26,6 +26,9 @@ from mobsf.StaticAnalyzer.views.sast_engine import ( from mobsf.MalwareAnalyzer.views.android import ( behaviour_analysis, ) +from mobsf.StaticAnalyzer.views.android import ( + sbom_analysis, +) logger = logging.getLogger(__name__) @@ -83,6 +86,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions): email_n_file = [] url_n_file = [] url_list = [] + sbom = {} app_dir = Path(app_dir) src = get_android_src_dir(app_dir, typ).as_posix() + '/' skp = settings.SKIP_CLASS_PATH @@ -95,10 +99,17 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions): 'match_extensions': {'.java', '.kt'}, 'ignore_paths': skp, } - # Code Analysis sast = SastEngine(options, src) # Read data once and pass it to all the analysis file_data = sast.read_files() + + # SBOM Analysis + sbom = sbom_analysis.sbom(app_dir, file_data) + msg = 'Android SBOM Analysis Completed' + logger.info(msg) + append_scan_status(checksum, msg) + + # Code Analysis code_findings = sast.run_rules(file_data, code_rules.as_posix()) msg = 'Android SAST Completed' logger.info(msg) @@ -186,6 +197,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions): 'urls_list': url_list, 'urls': url_n_file, 'emails': email_n_file, + 'sbom': sbom, } return code_an_dic except Exception as exp: diff --git a/mobsf/StaticAnalyzer/views/android/db_interaction.py b/mobsf/StaticAnalyzer/views/android/db_interaction.py index e95419ac..a6d03673 100755 --- a/mobsf/StaticAnalyzer/views/android/db_interaction.py +++ b/mobsf/StaticAnalyzer/views/android/db_interaction.py @@ -88,6 +88,7 @@ def get_context_from_db_entry(db_entry: QuerySet) -> dict: 'playstore_details': python_dict(db_entry[0].PLAYSTORE_DETAILS), 'secrets': python_list(db_entry[0].SECRETS), 'logs': get_scan_logs(db_entry[0].MD5), + 'sbom': python_dict(db_entry[0].SBOM), } return context except Exception: @@ -161,6 +162,7 @@ def get_context_from_analysis(app_dic, 'playstore_details': app_dic['playstore'], 'secrets': code_an_dic['secrets'], 'logs': get_scan_logs(app_dic['md5']), + 'sbom': code_an_dic['sbom'], } return context except Exception as exp: @@ -226,6 +228,7 @@ def save_or_update(update_type, 'PLAYSTORE_DETAILS': app_dic['playstore'], 'NETWORK_SECURITY': man_an_dic['network_security'], 'SECRETS': code_an_dic['secrets'], + 'SBOM': code_an_dic['sbom'], } if update_type == 'save': db_entry = StaticAnalyzerAndroid.objects.filter( diff --git a/mobsf/StaticAnalyzer/views/android/sbom_analysis.py b/mobsf/StaticAnalyzer/views/android/sbom_analysis.py new file mode 100644 index 00000000..3f41e15a --- /dev/null +++ b/mobsf/StaticAnalyzer/views/android/sbom_analysis.py @@ -0,0 +1,78 @@ +# -*- coding: utf_8 -*- +"""Extract packages from APK.""" +import logging + +from mobsf.MobSF.utils import ( + PKG_REGEX, +) + + +logger = logging.getLogger(__name__) + + +def merge_common_packages(items): + """Merge common packages.""" + items = list(items) + items.sort() # Sort items lexicographically + merged = [] + for item in items: + if not merged or not item.startswith(merged[-1] + '.'): + merged.append(item) + return merged + + +def extract_packages(file_data): + """Extract package names from file data.""" + packages = set() + try: + for item in file_data: + # tuple has file path and file content + # we are interested in file content's first line + pkg = item[1].split('\n')[0] + match = PKG_REGEX.search(pkg) + if match and match.group(1) != '_COROUTINE': + packages.add(match.group(1)) + packages = merge_common_packages(packages) + except Exception: + logger.exception('Extracting packages from file data') + return sorted(packages) + + +def get_group_name(file_name, group): + """Get group and name from file name.""" + parts = file_name.split('_') + + if parts and len(parts) == 2: + group, name = parts[0], parts[1] + else: + name = file_name.replace('_', '-') + if name.startswith('kotlinx-'): + group = 'org.jetbrains.kotlinx' + + return group, name + + +def android_sbom(app_dir): + """Extract SBOM from files.""" + sbom = set() + for vfile in app_dir.rglob('*.version'): + try: + dependency = vfile.stem + group, name = '', '' + version = vfile.read_text().strip() or '' + version = 'dynamic' if version.startswith('task') else version + + if '_' in dependency: + group, name = get_group_name(dependency, group) + sbom.add(f'{group}:{name}@{version}') + except Exception: + pass + return sorted(sbom) + + +def sbom(app_dir, file_data): + """Extract SBOM from version files and decompiled source code.""" + return { + 'sbom_versioned': android_sbom(app_dir), + 'sbom_packages': extract_packages(file_data), + } diff --git a/mobsf/StaticAnalyzer/views/android/so.py b/mobsf/StaticAnalyzer/views/android/so.py index b6916939..4066b5c8 100644 --- a/mobsf/StaticAnalyzer/views/android/so.py +++ b/mobsf/StaticAnalyzer/views/android/so.py @@ -123,6 +123,7 @@ def so_analysis(request, app_dic, rescan, api): 'urls_list': [], 'urls': [], 'emails': [], + 'sbom': {}, } # Get the strings and metadata from shared object get_strings_metadata( diff --git a/mobsf/StaticAnalyzer/views/common/async_task.py b/mobsf/StaticAnalyzer/views/common/async_task.py index 0e85ce62..053cc7c0 100644 --- a/mobsf/StaticAnalyzer/views/common/async_task.py +++ b/mobsf/StaticAnalyzer/views/common/async_task.py @@ -52,11 +52,13 @@ def async_analysis(checksum, api, file_name, func, *args, **kwargs): recent = RecentScansDB.objects.filter(MD5=checksum) scan_completed = recent[0].APP_NAME or recent[0].PACKAGE_NAME # Check if the task is updated within the last 60 minutes - active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(minutes=60) + active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta( + minutes=settings.ASYNC_ANALYSIS_TIMEOUT) # Check if the task is already enqueued within the last 60 minutes queued_recently = EnqueuedTask.objects.filter( checksum=checksum, - created_at__gte=timezone.now() - timedelta(minutes=60), + created_at__gte=timezone.now() - timedelta( + minutes=settings.ASYNC_ANALYSIS_TIMEOUT), ).exists() # Additional checks on recent queue diff --git a/mobsf/templates/static_analysis/android_binary_analysis.html b/mobsf/templates/static_analysis/android_binary_analysis.html index c5133f86..aab82d7b 100755 --- a/mobsf/templates/static_analysis/android_binary_analysis.html +++ b/mobsf/templates/static_analysis/android_binary_analysis.html @@ -324,6 +324,12 @@

Libraries

+ +
  • + + +

    SBOM

    +
  • @@ -2337,6 +2343,28 @@ + +
    +
    +
    +
    +
    +
    +

    + SBOM +

    +
    + {% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %} + {% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %} +
    +
    +
    +
    +
    + +
    +
    +
    diff --git a/mobsf/templates/static_analysis/android_source_analysis.html b/mobsf/templates/static_analysis/android_source_analysis.html index fcbf4869..93519c2c 100755 --- a/mobsf/templates/static_analysis/android_source_analysis.html +++ b/mobsf/templates/static_analysis/android_source_analysis.html @@ -241,6 +241,12 @@

    Libraries

    +
    • +
  • + + +

    SBOM

    +
  • @@ -1744,6 +1750,28 @@ + +
    +
    +
    +
    +
    +
    +

    + SBOM +

    +
    + {% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %} + {% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %} +
    +
    +
    +
    +
    + +
    +
    +
    diff --git a/mobsf/templates/static_analysis/ios_source_analysis.html b/mobsf/templates/static_analysis/ios_source_analysis.html index 0d95ea76..dc2204f8 100755 --- a/mobsf/templates/static_analysis/ios_source_analysis.html +++ b/mobsf/templates/static_analysis/ios_source_analysis.html @@ -7,7 +7,6 @@ -