feat: Update URL extraction to preserve case sensitivity (#2550)

Bug 2491, which stated that the URLs were being converted to lowercase, which potentially caused issues for URLs with uppercase characters.

Co-authored-by: Sai Prathik R <[email protected]>

mobsf/DynamicAnalyzer/views/common/shared.py

@@ -27,17 +27,18 @@ logger = logging.getLogger(__name__)
 def extract_urls_domains_emails(checksum, data):
     """Extract URLs, Domains and Emails."""
     # URL Extraction
-    urls = re.findall(URL_REGEX, data.lower())
+    urls = re.findall(URL_REGEX, data)
     if urls:
         urls = list(set(urls))
     else:
         urls = []
     # Domain Extraction and Malware Check
     logger.info('Performing Malware check on extracted domains')
+    # For domain extraction, use lowercased URLs
     domains = MalwareDomainCheck().scan(
         checksum,
-        urls)
-    # Email Etraction Regex
+        [u.lower() for u in urls if isinstance(u, str)])
+    # Email Extraction Regex
     emails = set()
     for email in EMAIL_REGEX.findall(data.lower()):
         if email.startswith('//'):

mobsf/MobSF/utils.py

@@ -55,7 +55,7 @@ URL_REGEX = re.compile(
         r'file://|javascript:|data:|www\d{0,3}[.])'
         r'[\w().=/;,#:@?&~*+!$%\'{}-]+)'
     ),
-    re.UNICODE)
+    re.UNICODE | re.IGNORECASE)
 EMAIL_REGEX = re.compile(r'[\w+.-]{1,20}@[\w-]{1,20}\.[\w]{2,10}')
 USERNAME_REGEX = re.compile(r'^\w[\w\-\@\.]{1,35}$')
 GOOGLE_API_KEY_REGEX = re.compile(r'AIza[0-9A-Za-z-_]{35}$')

mobsf/StaticAnalyzer/views/common/shared_func.py

@@ -358,7 +358,7 @@ def url_n_email_extract(dat, relative_path):
     url_n_file = []
     email_n_file = []
     # URL Extraction
-    urllist = URL_REGEX.findall(dat.lower())
+    urllist = URL_REGEX.findall(dat)
     for url in urllist:
         urls.add(url)
     if urls: