MobSF/Mobile-Security-Framework-MobSF
1
0
Fork 0
mirror of https://github.com/MobSF/Mobile-Security-Framework-MobSF.git synced 2026-10-11 22:50:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

4.2.5 (#2457)

Browse Source 
* Unified async scan timeout
* Allow incomplete scan delete after async scan timeout duration
* Added support for Android SBOM analysis
* Make dependencies unpinned (Address #2458)
This commit is contained in:
Ajin Abraham
2024-11-23 13:48:12 -08:00
committed by GitHub
parent 4b394a2b8a
commit b4cf9b70ac
16 changed files with 204 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobsf/MobSF/init.py
+1 -1
View File
@@ -18,7 +18,7 @@ from mobsf.install.windows.setup import windows_config_local
logger = logging.getLogger(__name__)
VERSION = '4.2.4'
VERSION = '4.2.5'
BANNER = r"""
__ __ _ ____ _____ _ _ ____
| \/ | ___ | |__/ ___|| ___|_ _| || | |___ \
mobsf/MobSF/settings.py
+4 -3
View File
@@ -341,12 +341,14 @@ LOGGING = {
},
},
}
ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
ASYNC_ANALYSIS_TIMEOUT = int(os.getenv('MOBSF_ASYNC_ANALYSIS_TIMEOUT', '60'))
Q_CLUSTER = {
'name': 'scan_queue',
'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', 3)),
'recycle': 5,
'timeout': 3600,
'retry': 3700,
'timeout': ASYNC_ANALYSIS_TIMEOUT * 60,
'retry': (ASYNC_ANALYSIS_TIMEOUT * 60) + 100,
'compress': True,
'label': 'scan_queue',
'orm': 'default',
@@ -355,7 +357,6 @@ Q_CLUSTER = {
'ack_failures': True,
}
QUEUE_MAX_SIZE = 100
ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
MULTIPROCESSING = os.getenv('MOBSF_MULTIPROCESSING')
JADX_TIMEOUT = int(os.getenv('MOBSF_JADX_TIMEOUT', 1000))
SAST_TIMEOUT = int(os.getenv('MOBSF_SAST_TIMEOUT', 1000))
mobsf/MobSF/utils.py
+2
View File
@@ -61,6 +61,8 @@ EMAIL_REGEX = re.compile(r'[\w+.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}')
USERNAME_REGEX = re.compile(r'^\w[\w\-\@\.]{1,35}$')
GOOGLE_API_KEY_REGEX = re.compile(r'AIza[0-9A-Za-z-_]{35}$')
GOOGLE_APP_ID_REGEX = re.compile(r'\d{1,2}:\d{1,50}:android:[a-f0-9]{1,50}')
PKG_REGEX = re.compile(
r'package\s+([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*);')
class Color(object):
mobsf/MobSF/views/api/api_static_analysis.py
-1
View File
@@ -113,7 +113,6 @@ def api_scan_logs(request):
return make_api_response({'logs': resp}, 200)
@request_method(['POST'])
@csrf_exempt
def api_tasks(request):
mobsf/MobSF/views/home.py
+11 -5
View File
@@ -7,9 +7,11 @@ import platform
import re
import shutil
from pathlib import Path
from datetime import timedelta
from wsgiref.util import FileWrapper
from django.conf import settings
from django.utils.timezone import now
from django.core.paginator import Paginator
from django.http import HttpResponse, HttpResponseRedirect
from django.views.decorators.http import require_http_methods
@@ -541,10 +543,14 @@ def delete_scan(request, api=False):
if settings.ASYNC_ANALYSIS:
# Handle Async Tasks
et = EnqueuedTask.objects.filter(checksum=md5_hash).first()
if et and not et.completed_at:
# Queue is in progress, cannot delete the task
return send_response({
'deleted': 'A scan can only be deleted after it is completed'}, api)
if et:
max_time_passed = now() - et.created_at > timedelta(
minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
if not (et.completed_at or max_time_passed):
# Queue is in progress, cannot delete the task
return send_response(
{'deleted': 'A scan can only be deleted after it is completed'},
api)
# Delete all related DB entries
EnqueuedTask.objects.filter(checksum=md5_hash).all().delete()
RecentScansDB.objects.filter(MD5=md5_hash).delete()
@@ -565,7 +571,7 @@ def delete_scan(request, api=False):
os.remove(item_path)
# Delete related directories
if is_dir_exists(item_path) and valid_item:
shutil.rmtree(item_path)
shutil.rmtree(item_path, ignore_errors=True)
return send_response({'deleted': 'yes'}, api)
except Exception as exp:
msg = str(exp)
mobsf/StaticAnalyzer/models.py
+1
View File
@@ -82,6 +82,7 @@ class StaticAnalyzerAndroid(models.Model):
PLAYSTORE_DETAILS = models.TextField(default={})
NETWORK_SECURITY = models.TextField(default=[])
SECRETS = models.TextField(default=[])
SBOM = models.TextField(default={})
class StaticAnalyzerIOS(models.Model):
mobsf/StaticAnalyzer/views/android/code_analysis.py
+13 -1
View File
@@ -26,6 +26,9 @@ from mobsf.StaticAnalyzer.views.sast_engine import (
from mobsf.MalwareAnalyzer.views.android import (
behaviour_analysis,
)
from mobsf.StaticAnalyzer.views.android import (
sbom_analysis,
)
logger = logging.getLogger(__name__)
@@ -83,6 +86,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
email_n_file = []
url_n_file = []
url_list = []
sbom = {}
app_dir = Path(app_dir)
src = get_android_src_dir(app_dir, typ).as_posix() + '/'
skp = settings.SKIP_CLASS_PATH
@@ -95,10 +99,17 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
'match_extensions': {'.java', '.kt'},
'ignore_paths': skp,
}
# Code Analysis
sast = SastEngine(options, src)
# Read data once and pass it to all the analysis
file_data = sast.read_files()
# SBOM Analysis
sbom = sbom_analysis.sbom(app_dir, file_data)
msg = 'Android SBOM Analysis Completed'
logger.info(msg)
append_scan_status(checksum, msg)
# Code Analysis
code_findings = sast.run_rules(file_data, code_rules.as_posix())
msg = 'Android SAST Completed'
logger.info(msg)
@@ -186,6 +197,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
'urls_list': url_list,
'urls': url_n_file,
'emails': email_n_file,
'sbom': sbom,
}
return code_an_dic
except Exception as exp:
mobsf/StaticAnalyzer/views/android/db_interaction.py
+3
View File
@@ -88,6 +88,7 @@ def get_context_from_db_entry(db_entry: QuerySet) -> dict:
'playstore_details': python_dict(db_entry[0].PLAYSTORE_DETAILS),
'secrets': python_list(db_entry[0].SECRETS),
'logs': get_scan_logs(db_entry[0].MD5),
'sbom': python_dict(db_entry[0].SBOM),
}
return context
except Exception:
@@ -161,6 +162,7 @@ def get_context_from_analysis(app_dic,
'playstore_details': app_dic['playstore'],
'secrets': code_an_dic['secrets'],
'logs': get_scan_logs(app_dic['md5']),
'sbom': code_an_dic['sbom'],
}
return context
except Exception as exp:
@@ -226,6 +228,7 @@ def save_or_update(update_type,
'PLAYSTORE_DETAILS': app_dic['playstore'],
'NETWORK_SECURITY': man_an_dic['network_security'],
'SECRETS': code_an_dic['secrets'],
'SBOM': code_an_dic['sbom'],
}
if update_type == 'save':
db_entry = StaticAnalyzerAndroid.objects.filter(
mobsf/StaticAnalyzer/views/android/sbom_analysis.py
+78
View File
@@ -0,0 +1,78 @@
# -*- coding: utf_8 -*-
"""Extract packages from APK."""
import logging
from mobsf.MobSF.utils import (
PKG_REGEX,
)
logger = logging.getLogger(__name__)
def merge_common_packages(items):
"""Merge common packages."""
items = list(items)
items.sort() # Sort items lexicographically
merged = []
for item in items:
if not merged or not item.startswith(merged[-1] + '.'):
merged.append(item)
return merged
def extract_packages(file_data):
"""Extract package names from file data."""
packages = set()
try:
for item in file_data:
# tuple has file path and file content
# we are interested in file content's first line
pkg = item[1].split('\n')[0]
match = PKG_REGEX.search(pkg)
if match and match.group(1) != '_COROUTINE':
packages.add(match.group(1))
packages = merge_common_packages(packages)
except Exception:
logger.exception('Extracting packages from file data')
return sorted(packages)
def get_group_name(file_name, group):
"""Get group and name from file name."""
parts = file_name.split('_')
if parts and len(parts) == 2:
group, name = parts[0], parts[1]
else:
name = file_name.replace('_', '-')
if name.startswith('kotlinx-'):
group = 'org.jetbrains.kotlinx'
return group, name
def android_sbom(app_dir):
"""Extract SBOM from files."""
sbom = set()
for vfile in app_dir.rglob('*.version'):
try:
dependency = vfile.stem
group, name = '', ''
version = vfile.read_text().strip() or ''
version = 'dynamic' if version.startswith('task') else version
if '_' in dependency:
group, name = get_group_name(dependency, group)
sbom.add(f'{group}:{name}@{version}')
except Exception:
pass
return sorted(sbom)
def sbom(app_dir, file_data):
"""Extract SBOM from version files and decompiled source code."""
return {
'sbom_versioned': android_sbom(app_dir),
'sbom_packages': extract_packages(file_data),
}
mobsf/StaticAnalyzer/views/android/so.py
+1
View File
@@ -123,6 +123,7 @@ def so_analysis(request, app_dic, rescan, api):
'urls_list': [],
'urls': [],
'emails': [],
'sbom': {},
}
# Get the strings and metadata from shared object
get_strings_metadata(
mobsf/StaticAnalyzer/views/common/async_task.py
+4 -2
View File
@@ -52,11 +52,13 @@ def async_analysis(checksum, api, file_name, func, *args, **kwargs):
recent = RecentScansDB.objects.filter(MD5=checksum)
scan_completed = recent[0].APP_NAME or recent[0].PACKAGE_NAME
# Check if the task is updated within the last 60 minutes
active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(minutes=60)
active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(
minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
# Check if the task is already enqueued within the last 60 minutes
queued_recently = EnqueuedTask.objects.filter(
checksum=checksum,
created_at__gte=timezone.now() - timedelta(minutes=60),
created_at__gte=timezone.now() - timedelta(
minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
).exists()
# Additional checks on recent queue
mobsf/templates/static_analysis/android_binary_analysis.html
+28
View File
@@ -324,6 +324,12 @@
<i class="fab fa-buffer nav-icon"></i>
<p>Libraries</p>
</a>
</li>
<li class="nav-item">
<a href="#sbom" class="nav-link">
<i class="fa fa-archive nav-icon"></i>
<p>SBOM</p>
</a>
</li>
<li class="nav-item">
<a href="#files" class="nav-link">
@@ -2337,6 +2343,28 @@
</div>
</section>
<!-- ===========================end libraries ================================== -->
<a id="sbom" class="anchor"></a>
<section class="content">
<div class="container-fluid">
<div class="row">
<div class="col-lg-12">
<div class="card">
<div class="card-body">
<p>
<strong><i class="fa fa-archive"></i> SBOM</strong>
</p>
<div class="list-group">
{% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
{% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
</div>
</div>
</div>
</div><!-- /.card -->
</div>
<!-- end row -->
</div>
</section>
<!-- ===========================end sbom ================================== -->
<a id="files" class="anchor"></a>
<section class="content">
<div class="container-fluid">
mobsf/templates/static_analysis/android_source_analysis.html
+28
View File
@@ -241,6 +241,12 @@
<i class="fab fa-buffer nav-icon"></i>
<p>Libraries</p>
</a>
</li>
<li class="nav-item">
<a href="#sbom" class="nav-link">
<i class="fa fa-archive nav-icon"></i>
<p>SBOM</p>
</a>
</li>
<li class="nav-item">
<a href="#files" class="nav-link">
@@ -1744,6 +1750,28 @@
</div>
</section>
<!-- ===========================end libraries ================================== -->
<a id="sbom" class="anchor"></a>
<section class="content">
<div class="container-fluid">
<div class="row">
<div class="col-lg-12">
<div class="card">
<div class="card-body">
<p>
<strong><i class="fa fa-archive"></i> SBOM</strong>
</p>
<div class="list-group">
{% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
{% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
</div>
</div>
</div>
</div><!-- /.card -->
</div>
<!-- end row -->
</div>
</section>
<!-- ===========================end sbom ================================== -->
<a id="files" class="anchor"></a>
<section class="content">
<div class="container-fluid">
mobsf/templates/static_analysis/ios_source_analysis.html
-1
View File
@@ -7,7 +7,6 @@
<!-- DataTables -->
<link href="{% static "datatables/css/datatables.combined.min.css" %}" rel="stylesheet">
<link href="{% static "adminlte/plugins/sweetalert2/sweetalert2.min.css" %}" rel="stylesheet">
<style type="text/css" media="print">
@page { size: landscape; }
@media print {
poetry.lock
Generated
+20 -19
View File
@@ -903,13 +903,13 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2
[[package]]
name = "ip2location"
version = "8.10.0"
version = "8.10.4"
description = "This is an IP geolocation library that enables the user to find the country, region, city, latitude and longitude, ZIP code, time zone, ISP, domain name, area code, weather info, mobile info, elevation, usage type, address type and IAB category from an IP address. It supports both IPv4 and IPv6 lookup."
optional = false
python-versions = "*"
files = [
{file = "IP2Location-8.10.0-py3-none-any.whl", hash = "sha256:b6573b422569581b2f40425e1e8dd5fdba1f8a75d3a1be366c1f0886fede3ba4"},
{file = "IP2Location-8.10.0.tar.gz", hash = "sha256:f0eea26c655f701406fa9379a38a0b56cc0dc7d0c6a221d44bda5280978a75ab"},
{file = "IP2Location-8.10.4-py3-none-any.whl", hash = "sha256:4323b6a25c6afb29c2f2b0b8aca3c3d1868a0199b705afaded1174cdbd8dacef"},
{file = "ip2location-8.10.4.tar.gz", hash = "sha256:395b3db260c0fa1de6b003808fdea144b0aa76ceed26c7a49de929b454e903c5"},
]
[[package]]
@@ -1442,29 +1442,26 @@ files = [
[[package]]
name = "openstep-parser"
version = "1.5.4"
version = "2.0.1"
description = "OpenStep plist reader into python objects"
optional = false
python-versions = "*"
python-versions = ">=3.8"
files = [
{file = "openstep_parser-1.5.4-py3-none-any.whl", hash = "sha256:56d7d25f6bfb1b17ce6c522f2467cf114f70b0c910693a07d4513109e19e9b18"},
{file = "openstep_parser-1.5.4.tar.gz", hash = "sha256:f03c7fd9c78ac718c470cb0f667c1f09fa7f58786314f77fef8767302abe1658"},
{file = "openstep_parser-2.0.1-py3-none-any.whl", hash = "sha256:5921efe694fb48137ad5eece58eac49a5a4e0745da5868eb3bd98bc1e41b35b2"},
{file = "openstep_parser-2.0.1.tar.gz", hash = "sha256:f37ed387ede1aa6fa2610f9db88f2528d7218b2f42fd34271b4a20b578c68c0e"},
]
[[package]]
name = "packaging"
version = "21.3"
version = "24.2"
description = "Core utilities for Python packages"
optional = false
python-versions = ">=3.6"
python-versions = ">=3.8"
files = [
{file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
{file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
{file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},
{file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
]
[package.dependencies]
pyparsing = ">=2.0.2,<3.0.5 || >3.0.5"
[[package]]
name = "paramiko"
version = "3.5.0"
@@ -2214,13 +2211,13 @@ lxml = "*"
[[package]]
name = "tldextract"
version = "3.4.4"
version = "5.1.3"
description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well."
optional = false
python-versions = ">=3.7"
python-versions = ">=3.9"
files = [
{file = "tldextract-3.4.4-py3-none-any.whl", hash = "sha256:581e7dbefc90e7bb857bb6f768d25c811a3c5f0892ed56a9a2999ddb7b1b70c2"},
{file = "tldextract-3.4.4.tar.gz", hash = "sha256:5fe3210c577463545191d45ad522d3d5e78d55218ce97215e82004dcae1e1234"},
{file = "tldextract-5.1.3-py3-none-any.whl", hash = "sha256:78de310cc2ca018692de5ddf320f9d6bd7c5cf857d0fd4f2175f0cdf4440ea75"},
{file = "tldextract-5.1.3.tar.gz", hash = "sha256:d43c7284c23f5dc8a42fd0fee2abede2ff74cc622674e4cb07f514ab3330c338"},
]
[package.dependencies]
@@ -2229,6 +2226,10 @@ idna = "*"
requests = ">=2.1.0"
requests-file = ">=1.4"
[package.extras]
release = ["build", "twine"]
testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "tox-uv", "types-filelock", "types-requests"]
[[package]]
name = "tornado"
version = "6.4.1"
@@ -2665,4 +2666,4 @@ cffi = ["cffi (>=1.11)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "4ce52e297085b7230f5456701fecc3913fabd942f2558f2cd85b67cbc8fedb97"
content-hash = "47edc939d8d6d36767bbc3210b4440106dbda32948de859d18a100e458815e0c"
pyproject.toml
+10 -10
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "mobsf"
version = "4.2.4"
version = "4.2.5"
description = "Mobile Security Framework (MobSF) is an automated, all-in-one mobile application (Android/iOS/Windows) pen-testing, malware analysis and security assessment framework capable of performing static and dynamic analysis."
keywords = ["mobsf", "mobile security framework", "mobile security", "security tool", "static analysis", "dynamic analysis", "malware analysis"]
authors = ["Ajin Abraham <[email protected]>"]
@@ -39,26 +39,26 @@ psutil = ">=5.8.0"
shelljob = ">=0.6.2"
asn1crypto = ">=1.4.0"
distro = ">=1.5.0"
ip2location = "8.10.0"
ip2location = "^8.10.4"
pdfkit = ">=0.6.1"
google-play-scraper = ">=0.1.2"
apkid = "2.1.5"
apkid = "^2.1.5"
frida = ">=16.1.4"
tldextract = "3.4.4"
openstep-parser = "1.5.4"
svgutils = "0.3.4"
arpy = "2.3.0"
apksigtool = "0.1.0"
tldextract = ">=5.1.3"
openstep-parser = ">=2.0.1"
svgutils = ">=0.3.4"
arpy = ">=2.3.0"
apksigtool = "^0.1.0"
tzdata = "^2023.3"
http-tools = ">=5.0.0"
libsast = "^3.1.6"
paramiko = "^3.3.1"
six = ">=1.16.0"
python3-saml = "^1.16.0"
bcrypt = "4.0.1"
bcrypt = "4.0.1" # Remove this in mitmproxy 12
psycopg2-binary = "^2.9.10"
lief = "^0.15.1"
packaging = ">=21.3,<22.0"
packaging = ">=21.3"
django-ratelimit = "^4.1.0"
django-q2 = "^1.7.4"