[HOTFIX] Migrate from setup.py to poetry, tox QA (#2249)

* Migrate from setup.py to use poetry build and publish
* Tox QA
* Version is now configured only at pyproject.toml
* Added poetry build test
* Updated mobsf PyPI publishing workflow 
* Update local DBs

.dockerignore

@@ -5,6 +5,7 @@
 .dockerignore
 .sonarcloud.properties
 .pyup.yml
+.python-version
 tox.ini
 mobsf/db.sqlite3
 Dockerfile
@@ -15,8 +16,6 @@ build
 docker-compose.yml
 *.md
 venv
-setup.py
-MANIFEST.in
 setup.bat
 run.bat
 setup.sh

.github/workflows/mobsf-test.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         python -m pip install pip==22.3.1 poetry==1.6.1
 
-    - name: Lint
+    - name: Lint on Ubuntu
       if: startsWith(matrix.os, 'ubuntu')
       run: |
         python -m pip install --upgrade tox
@@ -52,14 +52,22 @@ jobs:
 
     - name: Install Python dependencies
       run: |
-        poetry install --only main --no-interaction --no-ansi || poetry install --only main --no-interaction --no-ansi || poetry install --only main --no-interaction --no-ansi
+        poetry install --only main --no-root --no-interaction --no-ansi || poetry install --only main --no-root --no-interaction --no-ansi || poetry install --only main --no-root --no-interaction --no-ansi
 
     - name: Migrate Database
       run: |
         poetry run python manage.py makemigrations
         poetry run python manage.py makemigrations StaticAnalyzer
         poetry run python manage.py migrate
-    - name: Unit Tests
+
+    - name: Unit Tests on Ubuntu, macOS and Windows
       run: |
         git submodule update --init --recursive
         poetry run python manage.py test mobsf
+
+    - name: Python Package Test in Ubuntu and macOS
+      if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macOS')
+      run: |
+        poetry build
+        python -m pip install dist/*.whl
+        mobsf db

.github/workflows/python-publish.yml

@@ -18,11 +18,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools twine
+        pip install poetry==1.6.1
     - name: Build and publish
       env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist
+        poetry config pypi-token.pypi ${PYPI_TOKEN}
-        twine upload dist/*
+        poetry build
+        poetry publish

.gitignore

@@ -68,6 +68,7 @@ classes-error.zip
 */migrations/*
 .fuse_hidden*
 .tox
+.python-version
 
 #MobSF Files
 mobsf/debug.log

Dockerfile

@@ -64,7 +64,7 @@ RUN adduser $MOBSF_USER --shell /bin/false -u 9901 --ingroup $MOBSF_USER --gecos
 COPY poetry.lock pyproject.toml ./
 RUN python3 -m pip install --upgrade --no-cache-dir pip poetry==${POETRY_VERSION} && \
     poetry config virtualenvs.create false && \
-    poetry install --only main --no-interaction --no-ansi
+    poetry install --only main --no-root --no-interaction --no-ansi
 
 # Cleanup
 RUN \

MANIFEST.in

@@ -1,12 +0,0 @@
-
-recursive-include mobsf/StaticAnalyzer/tools *
-recursive-include mobsf/DynamicAnalyzer/tools *
-recursive-include mobsf/StaticAnalyzer/views/android/rules *
-recursive-include mobsf/StaticAnalyzer/views/ios/rules *
-recursive-include mobsf/signatures *
-recursive-include mobsf/static *
-recursive-include mobsf/templates *
-recursive-include mobsf/install *
-recursive-include LICENSES *
-include requirements.txt
-recursive-exclude *.pyc

mobsf/__main__.py

@@ -3,10 +3,13 @@ import sys
 import platform
 
 from django.core.management import execute_from_command_line
+from django.db import connection
+
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mobsf.MobSF.settings')
 
 
 def db():
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mobsf.MobSF.settings')
     execute_from_command_line([
         '',
         'makemigrations',
@@ -23,10 +26,20 @@ def db():
 
 
 def main():
-    if len(sys.argv) == 2:
+    try:
-        listen = sys.argv[1]
+        if not connection.introspection.table_names():
-    else:
+            db()
-        listen = '127.0.0.1:8000'
+    except Exception:
+        db()
+    listen = '127.0.0.1:8000'
+    if len(sys.argv) == 2 and sys.argv[1]:
+        if sys.argv[1] == 'db':
+            db()
+            listen = None
+        elif sys.argv[1]:
+            listen = sys.argv[1]
+    if not listen:
+        exit(0)
     if platform.system() != 'Windows':
         sys.argv = [
             '',
@@ -36,6 +49,11 @@ def main():
             '--workers=1',
             '--threads=10',
             '--timeout=3600',
+            '--log-level=citical',
+            '--log-file=-',
+            '--access-logfile=-',
+            '--error-logfile=-',
+            '--capture-output',
         ]
         from gunicorn.app.wsgiapp import run
         run()

mobsf/signatures/malwaredomainlist

@@ -510,7 +510,7 @@
 "2011/03/12_14:57","vkont.bos.ru","194.186.208.8","as3.centre.ru.","Trojan","-","3216","0","RU",
 "2011/03/13_21:47","download207.mediafire.com/2ezaemp68lyg/fe8lc6kqa1f06n1/Grand+Theft+Modification.zip","38.114.196.226","-","trojan","-","46179","0","US",
 "2011/03/18_19:29","-","69.50.192.250:8000","-","malware calls home","-","18866","0","US",
-"2014/01/06_12:28","dl.downf468.com/n/3.0.25/3299664/manual-básico-de-windows-movie-maker.exe","204.93.38.152","-","Win32/FirseriaInstaller.C","-","20940","0","US",
+"2014/01/06_12:28","dl.downf468.com/n/3.0.25/3299664/manual-b�sico-de-windows-movie-maker.exe","204.93.38.152","-","Win32/FirseriaInstaller.C","-","20940","0","US",
 "2011/03/20_17:23","-","70.38.124.97/distrmaincp/","-","SpyEye C&C","-","32613","0","CA",
 "2011/03/20_17:23","-","91.220.62.84/123/index.php","-","Eleonore Exploits pack version 1.4.4mod","-","51699","0","UA",
 "2011/03/20_17:23","-","91.220.62.84/123/stat.php","-","control panel of Eleonore Exploits pack version 1.4.4mod","-","51699","0","UA",
@@ -1167,7 +1167,7 @@
 "2014/02/07_09:44","www.t-gas.co.uk/1.txt","212.227.26.21","-","Leads to exploit at jolygoestobeinvester.ru","-","8560","0","DE",
 "2014/02/07_08:25","www.zatzy.com/allmaent/391854-volvos-nya-drive-e.html","81.201.217.115","-","compromised site leads to exploit kit","-","41175","0","SE",
 "2013/12/29_17:53","aippnetworks.com/plugins/authentication/joomla/aviancataca/aviancatacafreeticket.php","195.74.65.196","195-74-65-196.ip.aleto.nl.","Redirects to Trojan","-","25459","0","NL",
-"2014/01/06_12:28","dl.downf468.com/n/3.0.25/2629037/manual-de-configuración-del-putty.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
+"2014/01/06_12:28","dl.downf468.com/n/3.0.25/2629037/manual-de-configuraci�n-del-putty.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.2/11975718/ultrasurf+13.03.exe","96.17.161.145","a96-17-161-145.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/11999301/Silver+TV+3.0.exe","96.17.161.145","a96-17-161-145.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/7412352/BTV+Solo.exe","96.17.161.145","a96-17-161-145.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
@@ -1178,7 +1178,7 @@
 "2014/01/06_12:28","dl.downf468.com/n/3.0.24.1/12015430/NT+Key+Enterprise+Edition+3.80.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/12023961/Microsoft+Hesap+Makinesi++.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/4789416/LogiEscalierExpert.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
-"2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/11630753/iniciación-a-la-soldadura-con-estaño.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
+"2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/11630753/iniciaci�n-a-la-soldadura-con-esta�o.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/11812852/n5239.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/11358561/notepad+++.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/11707102/psx0139.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
@@ -1247,7 +1247,7 @@
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/11706811/psx0109.exe=","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/11784924/p1232.exeo:=","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.25/11560994/d3dx9_31.exe=","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
-"2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/3285684/lógica-y-teoría-de-conjuntos.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
+"2014/01/06_12:28","dl.downf468.com/n/3.0.26.1/3285684/l�gica-y-teor�a-de-conjuntos.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.3/4789537/setup.exe","96.17.161.137","a96-17-161-137.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.3/8311230/avast.exe","96.17.161.145","a96-17-161-145.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
 "2014/01/06_12:28","dl.downf468.com/n/3.0.26.3/4812448/pointagedheures.exe","96.17.161.145","a96-17-161-145.deploy.akamaitechnologies.com.","Win32/FirseriaInstaller.C","-","12989","0","US",
@@ -2253,4 +2253,3 @@
 "2017/06/02_08:38","sarahdaniella.com/swift/SWIFT%20$.pdf.ace","63.247.140.224","coriandertest.hmdnsgroup.com.","trojan","-","19271","0","US",
 "2017/12/04_18:50","textspeier.de","104.27.163.228","-","phishing/fraud","-","13335","0","US",
 "2017/10/26_13:48","photoscape.ch/Setup.exe","31.148.219.11","knigazdorovya.com.","trojan","-","14576","0","CZ",
-

pyproject.toml

@@ -1,10 +1,26 @@
 [tool.poetry]
-name = "mobile-security-framework-mobsf"
+name = "mobsf"
 version = "3.7.6"
 description = "Mobile Security Framework (MobSF) is an automated, all-in-one mobile application (Android/iOS/Windows) pen-testing, malware analysis and security assessment framework capable of performing static and dynamic analysis."
+keywords = ["mobsf", "mobile security framework", "mobile security", "security tool", "static analysis", "dynamic analysis", "malware analysis"]
 authors = ["Ajin Abraham <[email protected]>"]
-license = "GPLv3"
+license = "GPL-3.0-only"
 readme = "README.md"
+repository = "https://github.com/MobSF/Mobile-Security-Framework-MobSF"
+documentation = "https://mobsf.github.io/docs/"
+packages = [{ include = "mobsf", format = ["sdist", "wheel"]}]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Topic :: Security",
+    "Topic :: Software Development :: Quality Assurance",
+]
+
+[tool.poetry.urls]
+"Bug Tracker" = "https://github.com/MobSF/Mobile-Security-Framework-MobSF/issues"
+
+[tool.poetry.scripts]
+mobsf = "mobsf.__main__:main"
 
 [tool.poetry.dependencies]
 python = "^3.9"

requirements.txt

@@ -1,33 +0,0 @@
-Django>=3.1.5
-tzdata>2023.3
-lxml>=4.6.2
-rsa>=4.7
-biplist>=1.0.3
-requests>=2.25.1
-bs4>=0.0.1
-colorlog>=4.7.2
-macholib>=1.14
-whitenoise>=5.2.0
-waitress>=1.4.4;platform_system=='Windows'
-gunicorn>=20.0.4;platform_system!='Windows'
-psutil>=5.8.0
-shelljob>=0.6.2
-asn1crypto>=1.4.0
-oscrypto>=1.2.1
-distro>=1.5.0
-lief>=0.12.3 # pyup: ignore
-http-tools>=3.0.0
-libsast>=1.5.1
-pdfkit>=0.6.1
-google-play-scraper>=0.1.2
-# pinned versions
-IP2Location==8.10.0
-androguard==3.4.0a1
-apkid==2.1.5
-quark-engine==23.7.1
-frida==16.1.3
-tldextract==3.4.4
-openstep-parser==1.5.4
-svgutils==0.3.4
-arpy==2.3.0
-apksigtool==0.1.0 # test before updating

setup.bat

@@ -58,8 +58,8 @@ where python >nul 2>&1 && (
   echo [INSTALL] Installing Requirements
   python -m pip install --no-cache-dir wheel poetry==1.6.1
   python -m poetry lock
-  python -m poetry install --only main --no-interaction --no-ansi || python -m poetry install --only main --no-interaction --no-ansi || python -m poetry install --only main --no-interaction --no-ansi
+  python -m poetry install --only main --no-root --no-interaction --no-ansi || python -m poetry install --only main --no-root --no-interaction --no-ansi || python -m poetry install --only main --no-root --no-interaction --no-ansi
-  
+ 
   echo [INSTALL] Clean Up
   call scripts/clean.bat y
 

setup.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python3
-"""Setup for MobSF."""
-
-
-from setuptools import (
-    find_packages,
-    setup,
-)
-
-from pathlib import Path
-
-
-def read(rel_path):
-    init = Path(__file__).resolve().parent / rel_path
-    return init.read_text('utf-8', 'ignore')
-
-
-def get_version():
-    ver_path = 'mobsf/MobSF/init.py'
-    for line in read(ver_path).splitlines():
-        if line.startswith('VERSION'):
-            return line.split('\'')[1]
-    raise RuntimeError('Unable to find version string.')
-
-
-description = (
-    'Mobile Security Framework (MobSF) is an automated,'
-    ' all-in-one mobile application (Android/iOS/Windows) pen-testing,'
-    ' malware analysis and security assessment framework capable of '
-    'performing static and dynamic analysis.')
-
-setup(
-    name='mobsf',
-    version=get_version(),
-    description=description,
-    author='Ajin Abraham',
-    author_email='[email protected]',
-    classifiers=[
-        'Development Status :: 4 - Beta',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-        'Programming Language :: Python :: 3.9',
-        'Topic :: Security',
-        'Topic :: Software Development :: Quality Assurance',
-    ],
-    packages=find_packages(include=[
-        'mobsf', 'mobsf.*',
-    ]),
-    include_package_data=True,
-    python_requires='>=3.9',
-    entry_points={
-        'console_scripts': [
-            'mobsf = mobsf.__main__:main',
-            'mobsfdb = mobsf.__main__:db',
-        ],
-    },
-    url='https://github.com/MobSF/Mobile-Security-Framework-MobSF',
-    long_description=read('README.md'),
-    long_description_content_type='text/markdown',
-    install_requires=Path('requirements.txt').read_text().splitlines(),
-)

setup.sh

@@ -47,7 +47,7 @@ fi
 
 echo '[INSTALL] Installing Requirements'
 python3 -m pip install --no-cache-dir wheel poetry==1.6.1
-python3 -m poetry install --only main --no-interaction --no-ansi
+python3 -m poetry install --no-root --only main --no-interaction --no-ansi
 
 echo '[INSTALL] Clean Up'
 bash scripts/clean.sh y

tox.ini

@@ -1,20 +1,18 @@
 [tox]
-envlist = py37, py38, py39
+envlist = py39, py310, py311
 skipsdist = True
-
+isolated_build = true
-[testenv]
-deps =
-    -rrequirements.txt
 
 [testenv:test]
 skip_install = true
 deps =
-whitelist_externals =
+allowlist_externals =
     git
-    bash
+    poetry
 commands =
     git submodule update --init --recursive
-    bash -c '. venv/bin/activate && python manage.py test mobsf'
+    poetry install --no-root --only main --no-interaction --no-ansi
+    poetry run python manage.py test mobsf
 
 [testenv:lint]
 skip_install = true
@@ -39,12 +37,12 @@ deps =
 commands =
     flake8 {posargs}
     codespell --ignore-words-list="doubleclick,dout,ne,upto,lief,afile" \
-        --skip="./mobsf/StaticAnalyzer/tools/*,./mobsf/signatures/*,*.map,*.js,*.svg,./.tox/*,./venv/*,./.git/*,./poetry.lock"
+        --skip="./mobsf/StaticAnalyzer/tools/*,./mobsf/signatures/*,*.map,*.js,*.svg,./build/*,./.tox/*,./venv/*,./.git/*,./poetry.lock"
 
 [testenv:clean]
 deps =
 skip_install = true
-whitelist_externals =
+allowlist_externals =
     bash
     find
     rm
@@ -52,7 +50,7 @@ commands =
     find . -name "*.py[co]" -delete
     bash -c 'find . -name "__pycache__" -exec rm -fr \{\} \; -prune'
     bash -c 'find . -depth -name "*.egg-info" -exec rm -fr \{\} \;'
-    rm -rf .tox
+    rm -rf build dist
     bash -c './scripts/clean.sh y'