From 2c805431bd9a53fe33dbe2f3b5cd7fb0fa597969 Mon Sep 17 00:00:00 2001 From: Orip Date: Sat, 3 Sep 2022 00:32:38 +0300 Subject: [PATCH] Remove PEP503 parser and egg downloader from setup.py --- setup.py | 250 +------------------------------ tests/test_pep503_page_parser.py | 217 --------------------------- 2 files changed, 5 insertions(+), 462 deletions(-) diff --git a/setup.py b/setup.py index ede543b..2449dde 100755 --- a/setup.py +++ b/setup.py @@ -1,265 +1,25 @@ import codecs -import hashlib import os -import platform -import re import shutil -import struct -import subprocess -import sys -import zipfile -from collections import namedtuple -from functools import partial -from html.parser import HTMLParser -from io import BytesIO -from urllib.parse import urljoin, urlparse, urlunparse -from urllib.request import urlopen from setuptools import setup from setuptools.command.build_ext import build_ext from setuptools.extension import Extension -DEFAULT_INDEX_URL = "https://pypi.org/simple/" - -python_version = sys.version_info[0:2] - package_dir = os.path.dirname(os.path.realpath(__file__)) -pkg_info = os.path.join(package_dir, "PKG-INFO") -in_source_package = os.path.isfile(pkg_info) -if in_source_package: - with codecs.open(pkg_info, "r", "utf-8") as f: - version_line = [line.rstrip("\r") for line in f.read().split("\n") if line.startswith("Version: ")][0] - frida_version = version_line[9:] - long_description = None -else: - frida_version = os.environ.get("FRIDA_VERSION", None) - long_description = codecs.open(os.path.join(package_dir, "README.md"), "r", "utf-8").read() - frida_extension = os.environ.get("FRIDA_EXTENSION", None) -index_url_pip_configs = ("global.index-url", "global.extra-index-url") - -Tag = namedtuple("Tag", ["tagname", "attrs"]) -ParsedUrlInfo = namedtuple("ParsedUrlInfo", ["url", "filename", "major", "minor", "micro"]) +frida_version = os.environ.get("FRIDA_VERSION", None) +long_description = codecs.open(os.path.join(package_dir, "README.md"), "r", "utf-8").read() +frida_extension = os.environ.get("FRIDA_EXTENSION", None) class FridaPrebuiltExt(build_ext): def build_extension(self, ext): target = self.get_ext_fullpath(ext.name) - target_extension = os.path.splitext(target)[1] target_dir = os.path.dirname(target) - try: - os.makedirs(target_dir) - except: - pass + os.makedirs(target_dir, exist_ok=True) - if in_source_package: - system = platform.system() - arch = struct.calcsize("P") * 8 - if system == "Windows": - os_version = "win-amd64" if arch == 64 else "win32" - elif system == "Darwin": - if platform.machine() == "x86_64": - os_version = "macosx-10.9-x86_64" - else: - os_version = "macosx-11.0-arm64" - elif system == "Linux": - os_name = ( - "android" - if subprocess.check_output(["uname", "-o"]).decode("utf-8").rstrip() == "Android" - else "linux" - ) - machine = platform.machine() - if machine == "" or "86" in machine: - arch_name = "x86_64" if arch == 64 else "i686" - elif os_name == "android" and machine.startswith("armv"): - arch_name = "armv7l" - else: - arch_name = machine - os_version = f"{os_name}-{arch_name}" - elif system == "FreeBSD": - os_version = "freebsd-" + platform.machine() - else: - raise NotImplementedError("unsupported OS") - - egg_path = os.path.expanduser( - f"~{os.sep}frida-{frida_version}-py{python_version[0]}.{python_version[1]}-{os_version}.egg" - ) - print("looking for prebuilt extension in home directory, i.e.", egg_path) - - try: - with open(egg_path, "rb") as cache: - egg_data = cache.read() - except: - egg_data = None - - if egg_data is None: - print("prebuilt extension not found in home directory, will try downloading it") - - print("querying pypi for available prebuilds") - # index_url is a url compatible with PEP 503 - index_url = get_index_url().strip() - index_url = normalize_url(index_url) - frida_url = urljoin(index_url, "frida/") # slash is necessary here - timeout = 20 - errmsg = "unable to download it within {} seconds; " f"please download it manually to {egg_path}" - - print("downloading package list from", frida_url) - try: - links_html = urlopen(frida_url, timeout=timeout).read().decode("utf-8") - except Exception: - print(errmsg.format(timeout)) - raise - - parser = PEP503PageParser("frida", frida_version, os_version) - parser.feed(links_html) - - if len(parser.urls) == 0: - raise NotImplementedError( - "could not find prebuilt Frida extension; " "prebuilds only provided for Python 3.4+" - ) - - url = parser.urls[0] - egg_url = urljoin(frida_url, url.url) - - try: - print("downloading prebuilt extension from", egg_url) - timeout = 120 # We'll assume the user has at least 200 kB/s transfer speed. - egg_data = urlopen(egg_url, timeout=timeout).read() - except Exception: - print(errmsg.format(timeout)) - raise - else: - egg_url = None - - egg_file = BytesIO(egg_data) - - if egg_url is not None: - print("checking hash") - check_pep503_hash(egg_file, egg_url) - - print("extracting prebuilt extension") - egg_zip = zipfile.ZipFile(egg_file) - extension_member = [info for info in egg_zip.infolist() if info.filename.endswith(target_extension)][0] - extension_data = egg_zip.read(extension_member) - if system == "Windows": - trailer = b"\x00" if python_version[1] >= 10 else b"\x00\x00" - extension_data = re.sub( - b"python[3-9][0-9][0-9]\\.dll\x00", - "python{}{}.dll".format(*python_version).encode("utf-8") + trailer, - extension_data, - ) - with open(target, "wb") as f: - f.write(extension_data) - else: - shutil.copyfile(frida_extension, target) - - -def get_index_url(): - """get `index-url` from environment or pip - Use FRIDA_INDEX_URL environment variable to customize index-url compatible - with PEP 503. - """ - index_url = os.environ.get("FRIDA_INDEX_URL", None) - if index_url is not None: - return index_url - - for config_name in index_url_pip_configs: - try: - index_url = get_index_url_from_pip(config_name) - except (subprocess.CalledProcessError, OSError): - pass - else: - return index_url - - print(f"using default index URL: {DEFAULT_INDEX_URL}") - return DEFAULT_INDEX_URL - - -def get_index_url_from_pip(config_name): - assert config_name in index_url_pip_configs - - return subprocess.check_output( - [sys.executable, "-m", "pip", "config", "get", config_name], stderr=subprocess.PIPE - ).decode("utf-8") - - -def normalize_url(url): - parse_result = urlparse(url) - path = parse_result.path - if not path.endswith("/"): - path += "/" - return urlunparse( - ( - parse_result.scheme, - parse_result.netloc, - path, - parse_result.params, - parse_result.query, - parse_result.fragment, - ) - ) - - -class PEP503PageParser(HTMLParser): - def __init__(self, name, version, os_version): - HTMLParser.__init__(self) - filename_pattern = (r"^{}\-{}\-py(?P\d+)\.(?P\d+)(\.(?P\d+))?-{}.egg$").format( - *map(re.escape, [name, version, os_version]) - ) - self._filename_pattern = re.compile(filename_pattern) - - def reset(self): - HTMLParser.reset(self) - self._path = [] - self.urls = [] - - def handle_starttag(self, tag, attrs): - self._path.append(Tag(tag, dict(attrs))) - - def handle_endtag(self, tag): - if tag == "a": - while True: - if self._path.pop().tagname == tag: - break - else: - if len(self._path) > 0 and self._path[-1].tagname == tag: - self._path.pop() - - def handle_data(self, data): - if not (len(self._path) > 0 and self._path[-1].tagname == "a" and self._path[-1].attrs.get("href")): - return - - match = self._filename_pattern.match(data) - if match is not None: - self.urls.append( - ParsedUrlInfo( - self._path[-1].attrs["href"], - data, - *map(lambda g: int(g) if g else None, map(match.group, ["major", "minor", "micro"])), - ) - ) - - -def check_pep503_hash(bytes_io, url): - parse_result = urlparse(url) - fragment = parse_result.fragment - if fragment == "": - return - - hashname, hashvalue = fragment.split("=") - if hashname not in {"md5", "sha1", "sha224", "sha256", "sha348", "sha512"}: - raise ValueError(f"Unsupported hash algorithm: {hashname}, hashvalue={hashvalue}") - - h = hashlib.new(hashname) - for block in iter(partial(bytes_io.read, 4096), b""): # iterate until EOF - h.update(block) - digest = h.hexdigest() - bytes_io.seek(0) # reset offset - - if digest == hashvalue: - return - else: - raise ValueError(f"`{hashname}` hash checking failed! Expected: {hashvalue}, but got: {digest}") + shutil.copyfile(frida_extension, target) if __name__ == "__main__": diff --git a/tests/test_pep503_page_parser.py b/tests/test_pep503_page_parser.py index c85d692..e69de29 100644 --- a/tests/test_pep503_page_parser.py +++ b/tests/test_pep503_page_parser.py @@ -1,217 +0,0 @@ -import sys -import unittest - -import setup - -py_major_version = sys.version_info[0] -htmls = [] -cases = [] - - -html = ( - "\n\n \n Links for frida\n " - ' \n \n

Links for frida

\n frida-1.4.1-py2.6-macosx-10.9-' - 'intel.egg
\nfrida-9.1.9.t' - "ar.gz
\n \n\n" -) -htmls.append(html) -cases.extend( - [ - (setup.PEP503PageParser("frida", "15.1.1", "win-amd64"), html, []), - ( - setup.PEP503PageParser("frida", "1.4.1", "macosx-10.9-intel"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/5d/80/3b140c5998df9d81e40169f188a2347b6c705156a2b556ff308e2f8b7e0a/frida-1.4.1-py2.6-macosx-10.9-intel.egg#sha256=eef92210084ef083b34f8972078550c6ef45255e444905f95495792c7f709546", - filename="frida-1.4.1-py2.6-macosx-10.9-intel.egg", - major=2, - minor=6, - micro=None, - ) - ], - ), - (setup.PEP503PageParser("frida", "1.4.1", "macosx-11.0-arm64"), html, []), - ] -) - -html = ( - 'frida-15.0.7-py2.7-linux-i686.egg
' - 'frida-15.0.7-py2.7-linux-x86_64.egg
' - 'frida-15.0.7-py2.7-macosx-10.9-x86_64.egg
' - 'frida-15.0.7-py2.7-macosx-11.0-fat64.egg
' - 'frida-15.0.7-py2.7-win-amd64.egg
' - 'frida-15.0.7-py2.7-win32.egg
' - 'frida-15.0.7-py3.8-android-aarch64.egg
' - 'frida-15.0.7-py3.8-linux-i686.egg
' - 'frida-15.0.7-py3.8-linux-x86_64.egg
' - 'frida-15.0.7-py3.8-macosx-10.9-x86_64.egg
' - 'frida-15.0.7-py3.8-macosx-11.0-arm64.egg
' - 'frida-15.0.7-py3.8-win-amd64.egg
' - 'frida-15.0.7-py3.8-win32.egg
' - 'frida-15.0.7.tar.gz
' - 'frida-15.0.8-py2.7-linux-aarch64.egg
' - 'frida-15.0.8-py2.7-linux-armv7l.egg
' - 'frida-15.0.8-py2.7-linux-i686.egg
' - 'frida-15.0.8-py2.7-linux-x86_64.egg
' - 'frida-15.0.8-py2.7-macosx-10.9-x86_64.egg
' - 'frida-15.0.8-py2.7-macosx-11.0-fat64.egg
' - 'frida-15.0.8-py2.7-win-amd64.egg
' - 'frida-15.0.8-py2.7-win32.egg
' - 'frida-15.0.8-py3.6-linux-aarch64.egg
' - 'frida-15.0.8-py3.6-linux-armv7l.egg
' - 'frida-15.0.8-py3.8-android-aarch64.egg
' - 'frida-15.0.8-py3.8-linux-i686.egg
' - 'frida-15.0.8-py3.8-linux-x86_64.egg
' - 'frida-15.0.8-py3.8-macosx-10.9-x86_64.egg
' - 'frida-15.0.8-py3.8-macosx-11.0-arm64.egg
' - 'frida-15.0.8-py3.8-win-amd64.egg
' - 'frida-15.0.8-py3.8-win32.egg
' - 'frida-15.0.8.tar.gz
' -) -htmls.append(html) -cases.extend( - [ - (setup.PEP503PageParser("frida", "15.1.0", "win-amd64"), html, []), - ( - setup.PEP503PageParser("frida", "15.0.7", "win-amd64"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/e0/5c/b45c8f27482d81179eb640726b703f95c624cc4f32ae3ed3f8bc858ae5d9/frida-15.0.7-py2.7-win-amd64.egg#sha256=eb696528b9c19f1895123e731b094363a87a4412d7ea4fcb54ef71841f7b3c1e", - filename="frida-15.0.7-py2.7-win-amd64.egg", - major=2, - minor=7, - micro=None, - ), - setup.ParsedUrlInfo( - url="../../packages/77/34/6ebaea697f3df72818e60c6494a716c51f7f13b3da323598c1711d21779c/frida-15.0.7-py3.8-win-amd64.egg#sha256=a9964cc6dd4e3ea71c42b1800c79571c670905dc82cd769302b066499fff7bf4", - filename="frida-15.0.7-py3.8-win-amd64.egg", - major=3, - minor=8, - micro=None, - ), - ], - ), - ( - setup.PEP503PageParser("frida", "15.0.7", "linux-i686"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/e3/21/da75f6207f76750799d68938707a74d46512e666293eb550247bf5314613/frida-15.0.7-py2.7-linux-i686.egg#sha256=444246bad3b2222efec301e96c2d6ac5da039d41acd655f6d5b6e548637cae09", - filename="frida-15.0.7-py2.7-linux-i686.egg", - major=2, - minor=7, - micro=None, - ), - setup.ParsedUrlInfo( - url="../../packages/19/d3/a4a1980005e232399575aeb2ae973d2087a94ec7dbaf6d7a481612979fc7/frida-15.0.7-py3.8-linux-i686.egg#sha256=ed922ec0258e95f39b4004066b72fb48546041d28602e55d44ca12effa80e8bf", - filename="frida-15.0.7-py3.8-linux-i686.egg", - major=3, - minor=8, - micro=None, - ), - ], - ), - ( - setup.PEP503PageParser("frida", "15.0.8", "linux-x86_64"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/64/4a/1e1735a8c2f606c953cccfb9d7086c15d19b5151ebd6e0cbcab2e817d6e2/frida-15.0.8-py2.7-linux-x86_64.egg#sha256=e5b29da8394ef5643fc42877856859d544cd2aba0a874a4a700f2ce4521d9780", - filename="frida-15.0.8-py2.7-linux-x86_64.egg", - major=2, - minor=7, - micro=None, - ), - setup.ParsedUrlInfo( - url="../../packages/0b/20/11101c2cc053bbe3695c8778ffb239e49c0bc24066257bc3246ef67770d9/frida-15.0.8-py3.8-linux-x86_64.egg#sha256=6b3f42225c22a1f149107f963abe9f7b5f32eb4915fe8fa8286e5657a7b6c789", - filename="frida-15.0.8-py3.8-linux-x86_64.egg", - major=3, - minor=8, - micro=None, - ), - ], - ), - (setup.PEP503PageParser("frida", "15.0.7", "linux-amd64"), html, []), - ( - setup.PEP503PageParser("frida", "15.0.8", "macosx-11.0-fat64"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/d1/20/a65170d6a898541839acb03a16d1dd26499928c937350078765fe1e4beb3/frida-15.0.8-py2.7-macosx-11.0-fat64.egg#sha256=f9e58ff7f6d53640a991d3e77711b0095927103d7bdfef55268b58091938f72e", - filename="frida-15.0.8-py2.7-macosx-11.0-fat64.egg", - major=2, - minor=7, - micro=None, - ), - ], - ), - ] -) - -html = ( - "

frida-15.1.1-py3.8-linux-x86_64.egg

" - 'frida-15.1.1-py3.8-' - "linux-x86_64.egg
" -) -htmls.append(html) -cases.extend( - [ - ( - setup.PEP503PageParser("frida", "15.1.1", "linux-x86_64"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/e4/c1/82e361bbaa535b334f5b1b432b4573a7871fa973edeb3aab9dbb6b3b4cdc/frida-15.1.1-py3.8-linux-x86_64.egg#sha256=505f4ffa34cc7d68664fcd00d469f5d832e6778800d112aadb8a13692f984b40", - filename="frida-15.1.1-py3.8-linux-x86_64.egg", - major=3, - minor=8, - micro=None, - ), - ], - ), - ] -) - -html = ( - 'frida-15.0.1-py3.8-android-aarch64.egg' - 'frida-15.0.1-py3' - ".8-android-aarch64.egg
" -) -htmls.append(html) -cases.extend( - [ - ( - setup.PEP503PageParser("frida", "15.0.1", "android-aarch64"), - html, - [ - setup.ParsedUrlInfo( - url="../../packages/3e/80/78fa3ed5fd636b606dc06157069b37eb677652cd985739cde35a86d7a362/frida-15.0.1-py3.8-android-aarch64.egg#sha256=d44bc341590dd8cf2623089b54aa16d697536fb016fde0ecd6df5262723c652b", - filename="frida-15.0.1-py3.8-android-aarch64.egg", - major=3, - minor=8, - micro=None, - ), - ], - ), - ] -) - - -class TestPEP503PageParser(unittest.TestCase): - def test_parse_html(self): - for parser, html, result in cases: - for _ in range(2): - parser.reset() - parser.feed(html) - assert parser.urls == result, (parser.urls, result)