diff --git a/setup.py b/setup.py index 5929672..bfb15e7 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,18 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import sys +import os +import platform +import re +import zipfile +import shutil +import struct import codecs +import subprocess +import hashlib +from collections import namedtuple +from functools import partial try: from io import BytesIO except: @@ -9,26 +20,29 @@ except: from cStringIO import StringIO as BytesIO except: from StringIO import StringIO as BytesIO -import os -import platform -import re -from setuptools import setup -from setuptools.command.build_ext import build_ext -from setuptools.extension import Extension -import shutil -import struct -import subprocess -import sys try: from urllib.request import urlopen, Request except: from urllib2 import urlopen, Request try: - import xmlrpclib -except ImportError: - import xmlrpc.client as xmlrpclib -import zipfile + from urllib.parse import urljoin, urlparse +except: + from urlparse import urljoin, urlparse +try: + from html.parser import HTMLParser +except: + from HTMLParser import HTMLParser +from setuptools import setup +from setuptools.command.build_ext import build_ext +from setuptools.extension import Extension + + +DEFAULT_INDEX_URL = "https://pypi.org/simple/" + +python_version = sys.version_info[0:2] +python_major_version = python_version[0] +PY2 = python_major_version == 2 package_dir = os.path.dirname(os.path.realpath(__file__)) pkg_info = os.path.join(package_dir, "PKG-INFO") @@ -44,18 +58,11 @@ else: frida_extension = os.environ['FRIDA_EXTENSION'] frida_major_version = int(frida_version.split(".")[0]) +index_url_pip_configs = ("global.index-url", "global.extra-index-url") -class UrllibTransport(xmlrpclib.Transport): - def __init__(self, *args, **kwargs): - xmlrpclib.Transport.__init__(self, *args, **kwargs) - - def request(self, host, handler, request_body, verbose=0): - self.verbose = verbose - scheme = "https" - url = "%(scheme)s://%(host)s%(handler)s" % locals() - req = Request(url, data=request_body, headers={'Content-Type': 'text/xml'}) - fp = urlopen(req) - return self.parse_response(fp) +Tag = namedtuple("Tag", ["tagname", "attrs"]) +ParsedUrlInfo = namedtuple("ParsedUrlInfo", + ["url", "filename", "major", "minor", "micro"]) class FridaPrebuiltExt(build_ext): @@ -69,8 +76,6 @@ class FridaPrebuiltExt(build_ext): pass if in_source_package: - python_version = sys.version_info[0:2] - python_major_version = python_version[0] system = platform.system() arch = struct.calcsize('P') * 8 if system == 'Windows': @@ -106,37 +111,46 @@ class FridaPrebuiltExt(build_ext): print("prebuilt extension not found in home directory, will try downloading it") print("querying pypi for available prebuilds") - client = xmlrpclib.ServerProxy("https://pypi.python.org/pypi", transport=UrllibTransport()) - urls = client.release_urls("frida", frida_version) + # index_url is a url compatible with PEP 503 + index_url = get_index_url().strip() + frida_url = urljoin(index_url, "frida/") # slash is necessary here + timeout = 20 + errmsg = ( + "unable to download it within {} seconds; " + "please download it manually to {}" + ).format("{}", egg_path) - urls = [url for url in urls if url['python_version'] != 'source'] + print("downloading package list from", frida_url) + try: + links_html = urlopen(frida_url, timeout=timeout).read().decode("utf-8") + except Exception: + print(errmsg.format(timeout)) + raise - if python_major_version >= 3: - urls = [url for url in urls if parse_version(url['python_version'])[0] == python_major_version] - else: - urls = [url for url in urls if parse_version(url['python_version']) == python_version] - - os_suffix = "-{}.egg".format(os_version) - urls = [url for url in urls if url['filename'].endswith(os_suffix)] + parser = PEP503PageParser("frida", frida_version, os_version) + parser.feed(links_html) + urls = [url for url in parser.urls if url.major == python_major_version] if len(urls) == 0: raise NotImplementedError("could not find prebuilt Frida extension; " "prebuilds only provided for Python 2.7 and 3.4+") url = urls[0] - egg_url = url['url'] + egg_url = urljoin(frida_url, url.url) try: print("downloading prebuilt extension from", egg_url) timeout = 120 # We'll assume the user has at least 200 kB/s transfer speed. egg_data = urlopen(egg_url, timeout=timeout).read() - except Exception as e: - message = "unable to download it within 120 seconds; please download it manually to {}" - print(message.format(egg_path)) - raise e + except Exception: + print(errmsg.format(timeout)) + raise egg_file = BytesIO(egg_data) + print("checking hash") + check_pep503_hash(egg_file, egg_url) + print("extracting prebuilt extension") egg_zip = zipfile.ZipFile(egg_file) extension_member = [info for info in egg_zip.infolist() if info.filename.endswith(target_extension)][0] @@ -150,51 +164,148 @@ class FridaPrebuiltExt(build_ext): shutil.copyfile(frida_extension, target) -def parse_version(version): - return tuple(map(int, version.split("."))) +def get_index_url(): + """get `index-url` from environment or pip + Use FRIDA_INDEX_URL environment variable to customize index-url compatible + with PEP 503. + """ + index_url = os.environ.get("FRIDA_INDEX_URL", None) + if index_url is not None: + return index_url + + for config_name in index_url_pip_configs: + try: + index_url = get_index_url_from_pip(config_name) + except (subprocess.CalledProcessError, OSError): + pass + else: + return index_url + + print("using default index URL: {}".format(DEFAULT_INDEX_URL)) + return DEFAULT_INDEX_URL -setup( - name="frida", - version=frida_version, - description="Dynamic instrumentation toolkit for developers, reverse-engineers, and security researchers", - long_description=long_description, - long_description_content_type="text/markdown", - author="Frida Developers", - author_email="oleavr@frida.re", - url="https://frida.re", - install_requires=["setuptools"], - license="wxWindows Library Licence, Version 3.1", - keywords="frida debugger dynamic instrumentation inject javascript windows macos linux ios iphone ipad android qnx", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Environment :: MacOS X", - "Environment :: Win32 (MS Windows)", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved", - "Natural Language :: English", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: JavaScript", - "Topic :: Software Development :: Debuggers", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - packages=['frida'], - ext_modules=[Extension('_frida', [])], - cmdclass={ - 'build_ext': FridaPrebuiltExt - }, - zip_safe=False -) +def get_index_url_from_pip(config_name): + assert config_name in index_url_pip_configs + + return subprocess.check_output([sys.executable, "-m", "pip", "config", "get", config_name], + stderr=subprocess.PIPE) + + +class PEP503PageParser(HTMLParser): + def __init__(self, name, version, os_version): + HTMLParser.__init__(self) + filename_pattern = ( + r"^{}\-{}\-py(?P\d+)\.(?P\d+)(\.(?P\d+))?-{}.egg$" + ).format(*map(re.escape, [name, version, os_version])) + if PY2: + filename_pattern = filename_pattern.decode("utf-8") + self._filename_pattern = re.compile(filename_pattern) + + def reset(self): + HTMLParser.reset(self) + self._path = [] + self.urls = [] + + def handle_starttag(self, tag, attrs): + self._path.append(Tag(tag, dict(attrs))) + + def handle_endtag(self, tag): + if tag == u"a": + while True: + if self._path.pop().tagname == tag: + break + else: + if len(self._path) > 0 and self._path[-1].tagname == tag: + self._path.pop() + + def handle_data(self, data): + if not (len(self._path) > 0 + and self._path[-1].tagname == u"a" + and self._path[-1].attrs.get("href")): + return + + match = self._filename_pattern.match(data) + if match is not None: + self.urls.append(ParsedUrlInfo( + self._path[-1].attrs["href"], + data, + *map( + lambda g: int(g) if g else None, + map(match.group, ["major", "minor", "micro"]) + ) + )) + + +def check_pep503_hash(bytes_io, url): + parse_result = urlparse(url) + fragment = parse_result.fragment + if fragment == "": + return + + hashname, hashvalue = fragment.split("=") + if hashname not in {"md5", "sha1", "sha224", "sha256", "sha348", "sha512"}: + raise ValueError("Unsupported hash algorithm: {}, hashvalue={}".format( + hashname, hashvalue, + )) + + h = hashlib.new(hashname) + for block in iter(partial(bytes_io.read, 4096), b""): # iterate until EOF + h.update(block) + digest = h.hexdigest() + bytes_io.seek(0) # reset offset + + if digest == hashvalue: + return + else: + raise ValueError( + "`{}` hash checking failed! Expected: {}, but got: {}".format( + hashname, hashvalue, digest) + ) + + +if __name__ == "__main__": + setup( + name="frida", + version=frida_version, + description="Dynamic instrumentation toolkit for developers, reverse-engineers, and security researchers", + long_description=long_description, + long_description_content_type="text/markdown", + author="Frida Developers", + author_email="oleavr@frida.re", + url="https://frida.re", + install_requires=["setuptools"], + license="wxWindows Library Licence, Version 3.1", + keywords="frida debugger dynamic instrumentation inject javascript windows macos linux ios iphone ipad android qnx", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: MacOS X", + "Environment :: Win32 (MS Windows)", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved", + "Natural Language :: English", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: JavaScript", + "Topic :: Software Development :: Debuggers", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + packages=['frida'], + ext_modules=[Extension('_frida', [])], + cmdclass={ + 'build_ext': FridaPrebuiltExt + }, + zip_safe=False + ) diff --git a/tests/test_pep503_page_parser.py b/tests/test_pep503_page_parser.py new file mode 100644 index 0000000..b19b65a --- /dev/null +++ b/tests/test_pep503_page_parser.py @@ -0,0 +1,211 @@ +# coding: utf-8 + +import sys +try: + import unittest2 as unittest +except: + import unittest + +import setup + + +py_major_version = sys.version_info[0] +htmls = [] +cases = [] + + +html = ( + '\n\n \n Links for frida\n ' + ' \n \n

Links for frida

\n frida-1.4.1-py2.6-macosx-10.9-' + 'intel.egg
\nfrida-9.1.9.t' + 'ar.gz
\n \n\n' +) +htmls.append(html) +cases.extend([ + ( + setup.PEP503PageParser("frida", "15.1.1", "win-amd64"), + html, + [] + ), + ( + setup.PEP503PageParser("frida", "1.4.1", "macosx-10.9-intel"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/5d/80/3b140c5998df9d81e40169f188a2347b6c705156a2b556ff308e2f8b7e0a/frida-1.4.1-py2.6-macosx-10.9-intel.egg#sha256=eef92210084ef083b34f8972078550c6ef45255e444905f95495792c7f709546', + filename='frida-1.4.1-py2.6-macosx-10.9-intel.egg', + major=2, minor=6, micro=None + ) + ] + ), + ( + setup.PEP503PageParser("frida", "1.4.1", "macosx-11.0-arm64"), + html, + [] + ), +]) + +html = ( + 'frida-15.0.7-py2.7-linux-i686.egg
' + 'frida-15.0.7-py2.7-linux-x86_64.egg
' + 'frida-15.0.7-py2.7-macosx-10.9-x86_64.egg
' + 'frida-15.0.7-py2.7-macosx-11.0-fat64.egg
' + 'frida-15.0.7-py2.7-win-amd64.egg
' + 'frida-15.0.7-py2.7-win32.egg
' + 'frida-15.0.7-py3.8-android-aarch64.egg
' + 'frida-15.0.7-py3.8-linux-i686.egg
' + 'frida-15.0.7-py3.8-linux-x86_64.egg
' + 'frida-15.0.7-py3.8-macosx-10.9-x86_64.egg
' + 'frida-15.0.7-py3.8-macosx-11.0-arm64.egg
' + 'frida-15.0.7-py3.8-win-amd64.egg
' + 'frida-15.0.7-py3.8-win32.egg
' + 'frida-15.0.7.tar.gz
' + 'frida-15.0.8-py2.7-linux-aarch64.egg
' + 'frida-15.0.8-py2.7-linux-armv7l.egg
' + 'frida-15.0.8-py2.7-linux-i686.egg
' + 'frida-15.0.8-py2.7-linux-x86_64.egg
' + 'frida-15.0.8-py2.7-macosx-10.9-x86_64.egg
' + 'frida-15.0.8-py2.7-macosx-11.0-fat64.egg
' + 'frida-15.0.8-py2.7-win-amd64.egg
' + 'frida-15.0.8-py2.7-win32.egg
' + 'frida-15.0.8-py3.6-linux-aarch64.egg
' + 'frida-15.0.8-py3.6-linux-armv7l.egg
' + 'frida-15.0.8-py3.8-android-aarch64.egg
' + 'frida-15.0.8-py3.8-linux-i686.egg
' + 'frida-15.0.8-py3.8-linux-x86_64.egg
' + 'frida-15.0.8-py3.8-macosx-10.9-x86_64.egg
' + 'frida-15.0.8-py3.8-macosx-11.0-arm64.egg
' + 'frida-15.0.8-py3.8-win-amd64.egg
' + 'frida-15.0.8-py3.8-win32.egg
' + 'frida-15.0.8.tar.gz
' +) +htmls.append(html) +cases.extend([ + ( + setup.PEP503PageParser("frida", "15.1.0", "win-amd64"), + html, + [] + ), + ( + setup.PEP503PageParser("frida", "15.0.7", "win-amd64"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/e0/5c/b45c8f27482d81179eb640726b703f95c624cc4f32ae3ed3f8bc858ae5d9/frida-15.0.7-py2.7-win-amd64.egg#sha256=eb696528b9c19f1895123e731b094363a87a4412d7ea4fcb54ef71841f7b3c1e', + filename='frida-15.0.7-py2.7-win-amd64.egg', + major=2, minor=7, micro=None + ), + setup.ParsedUrlInfo( + url='../../packages/77/34/6ebaea697f3df72818e60c6494a716c51f7f13b3da323598c1711d21779c/frida-15.0.7-py3.8-win-amd64.egg#sha256=a9964cc6dd4e3ea71c42b1800c79571c670905dc82cd769302b066499fff7bf4', + filename='frida-15.0.7-py3.8-win-amd64.egg', + major=3, minor=8, micro=None + ), + ] + ), + ( + setup.PEP503PageParser("frida", "15.0.7", "linux-i686"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/e3/21/da75f6207f76750799d68938707a74d46512e666293eb550247bf5314613/frida-15.0.7-py2.7-linux-i686.egg#sha256=444246bad3b2222efec301e96c2d6ac5da039d41acd655f6d5b6e548637cae09', + filename='frida-15.0.7-py2.7-linux-i686.egg', + major=2, minor=7, micro=None + ), + setup.ParsedUrlInfo( + url='../../packages/19/d3/a4a1980005e232399575aeb2ae973d2087a94ec7dbaf6d7a481612979fc7/frida-15.0.7-py3.8-linux-i686.egg#sha256=ed922ec0258e95f39b4004066b72fb48546041d28602e55d44ca12effa80e8bf', + filename='frida-15.0.7-py3.8-linux-i686.egg', + major=3, minor=8, micro=None + ), + ] + ), + ( + setup.PEP503PageParser("frida", "15.0.8", "linux-x86_64"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/64/4a/1e1735a8c2f606c953cccfb9d7086c15d19b5151ebd6e0cbcab2e817d6e2/frida-15.0.8-py2.7-linux-x86_64.egg#sha256=e5b29da8394ef5643fc42877856859d544cd2aba0a874a4a700f2ce4521d9780', + filename='frida-15.0.8-py2.7-linux-x86_64.egg', + major=2, minor=7, micro=None + ), + setup.ParsedUrlInfo( + url='../../packages/0b/20/11101c2cc053bbe3695c8778ffb239e49c0bc24066257bc3246ef67770d9/frida-15.0.8-py3.8-linux-x86_64.egg#sha256=6b3f42225c22a1f149107f963abe9f7b5f32eb4915fe8fa8286e5657a7b6c789', + filename='frida-15.0.8-py3.8-linux-x86_64.egg', + major=3, minor=8, micro=None + ), + ] + ), + ( + setup.PEP503PageParser("frida", "15.0.7", "linux-amd64"), + html, + [] + ), + ( + setup.PEP503PageParser("frida", "15.0.8", "macosx-11.0-fat64"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/d1/20/a65170d6a898541839acb03a16d1dd26499928c937350078765fe1e4beb3/frida-15.0.8-py2.7-macosx-11.0-fat64.egg#sha256=f9e58ff7f6d53640a991d3e77711b0095927103d7bdfef55268b58091938f72e', + filename='frida-15.0.8-py2.7-macosx-11.0-fat64.egg', + major=2, minor=7, micro=None + ), + ] + ), +]) + +html = ( + '

frida-15.1.1-py3.8-linux-x86_64.egg

' + 'frida-15.1.1-py3.8-' + 'linux-x86_64.egg
' +) +htmls.append(html) +cases.extend([ + ( + setup.PEP503PageParser("frida", "15.1.1", "linux-x86_64"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/e4/c1/82e361bbaa535b334f5b1b432b4573a7871fa973edeb3aab9dbb6b3b4cdc/frida-15.1.1-py3.8-linux-x86_64.egg#sha256=505f4ffa34cc7d68664fcd00d469f5d832e6778800d112aadb8a13692f984b40', + filename='frida-15.1.1-py3.8-linux-x86_64.egg', + major=3, minor=8, micro=None + ), + ] + ), +]) + +html = ( + 'frida-15.0.1-py3.8-android-aarch64.egg' + 'frida-15.0.1-py3' + '.8-android-aarch64.egg
' +) +htmls.append(html) +cases.extend([ + ( + setup.PEP503PageParser("frida", "15.0.1", "android-aarch64"), + html, + [ + setup.ParsedUrlInfo( + url='../../packages/3e/80/78fa3ed5fd636b606dc06157069b37eb677652cd985739cde35a86d7a362/frida-15.0.1-py3.8-android-aarch64.egg#sha256=d44bc341590dd8cf2623089b54aa16d697536fb016fde0ecd6df5262723c652b', + filename='frida-15.0.1-py3.8-android-aarch64.egg', + major=3, minor=8, micro=None + ), + ] + ), +]) + + +class TestPEP503PageParser(unittest.TestCase): + def test_parse_html(self): + for parser, html, result in cases: + for _ in range(2): + parser.reset() + parser.feed(html) + assert parser.urls == result, (parser.urls, result)