diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..fb824d0 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,65 @@ +name: PyPI 📦 Distribution + +on: [push] + +jobs: + run: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + platform: [x32, x64] + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Set up MSVC x86 + if: matrix.os == 'windows-latest' && matrix.platform == 'x32' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + - name: Set up MSVC x64 + if: matrix.os == 'windows-latest' && matrix.platform == 'x64' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Install dependencies + run: | + pip install setuptools wheel + + - name: Build distribution 📦 + shell: bash + run: | + if [ ${{ matrix.platform }} == 'x32' ] && [ ${{ matrix.os }} == 'windows-latest' ]; then + cd bindings/python && python setup.py build -p win32 bdist_wheel -p win32 + elif [ ${{ matrix.platform }} == 'x32' ] && [ ${{ matrix.os }} == 'ubuntu-latest' ]; then + docker run --rm -v `pwd`/:/work dockcross/manylinux1-x86 > ./dockcross + chmod +x ./dockcross + ./dockcross bindings/python/build_wheel.sh + elif [ ${{ matrix.platform }} == 'x64' ] && [ ${{ matrix.os }} == 'ubuntu-latest' ]; then + docker run --rm -v `pwd`/:/work dockcross/manylinux1-x64 > ./dockcross + chmod +x ./dockcross + ./dockcross bindings/python/build_wheel.sh + elif [ ${{ matrix.platform }} == 'x32' ] && [ ${{ matrix.os }} == 'macos-latest' ]; then + cd bindings/python && python setup.py sdist + else + cd bindings/python && python setup.py bdist_wheel + fi + + - uses: actions/upload-artifact@v2 + with: + name: keystone-wheel-${{ matrix.os }}-${{ matrix.platform }} + path: ${{ github.workspace }}/bindings/python/dist/* + + - name: Publish distribution 📦 to PyPI + if: startsWith(github.event.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_pass }} + packages_dir: ${{ github.workspace }}/bindings/python/dist/ diff --git a/CREDITS.TXT b/CREDITS.TXT index 46f0a96..b3b25cb 100644 --- a/CREDITS.TXT +++ b/CREDITS.TXT @@ -51,3 +51,4 @@ David Zimmer: VB6 binding. Michael Mohr: Debian packaging. Jämes Ménétrey (ZenLulz): Java binding. Philippe Antoine (Catena cyber): fuzzing. +Kevin Foo (chfl4gs): PyPI packaging, Travis-CI, fixes for Python binding. diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index 0197892..c71bfc2 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -2,3 +2,5 @@ recursive-include src * recursive-include prebuilt * include LICENSE.TXT include README +global-exclude *.pyc +global-exclude __pycache__ diff --git a/bindings/python/build_wheel.sh b/bindings/python/build_wheel.sh new file mode 100755 index 0000000..d842a7b --- /dev/null +++ b/bindings/python/build_wheel.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e -x + +cd /work/bindings/python + +sudo rm /usr/bin/python && sudo ln -s /opt/python/cp36-cp36m/bin/python /usr/bin/python; python -V + +function repair_wheel { + wheel="$1" + if ! auditwheel show "$wheel"; then + echo "Skipping non-platform wheel $wheel" + else + auditwheel repair "$wheel" -w /work/bindings/python/dist/ + fi +} + + +# Compile wheels +/opt/python/cp36-cp36m/bin/python setup.py bdist_wheel -d wheelhouse + +# Bundle external shared libraries into the wheels +for whl in wheelhouse/*.whl; do + repair_wheel "$whl" +done diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 3ceb327..3b262f2 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -7,33 +7,39 @@ import glob import os import shutil +import subprocess import stat import sys -from distutils import dir_util, file_util +import platform from distutils import log -from distutils.command.build_clib import build_clib -from distutils.command.install_lib import install_lib -from distutils.command.sdist import sdist -from distutils.core import setup - -# prebuilt libraries for Windows - for sdist -PATH_LIB64 = "prebuilt/win64/keystone.dll" -PATH_LIB32 = "prebuilt/win32/keystone.dll" - -# package name can be 'keystone-engine' or 'keystone-engine-windows' -PKG_NAME = 'keystone-engine' -if os.path.exists(PATH_LIB64) and os.path.exists(PATH_LIB32): - PKG_NAME = 'keystone-engine-windows' +from setuptools import setup +from distutils.util import get_platform +from distutils.command.build import build as _build +from distutils.command.sdist import sdist as _sdist +from setuptools.command.bdist_egg import bdist_egg as _bdist_egg +from setuptools.command.develop import develop as _develop VERSION = '0.9.1-3' SYSTEM = sys.platform +IS_64BITS = platform.architecture()[0] == '64bit' -SETUP_DATA_FILES = [] +# paths +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +LIBS_DIR = os.path.join(ROOT_DIR, 'keystone') +SRC_DIR = os.path.join(ROOT_DIR, 'src') +BUILD_DIR = os.path.join(SRC_DIR, 'build') -# adapted from commit e504b81 of Nguyen Tan Cong -# Reference: https://docs.python.org/2/library/platform.html#cross-platform -is_64bits = sys.maxsize > 2 ** 32 +if SYSTEM == 'darwin': + LIBRARY_FILE = "libkeystone.dylib" + MAC_LIBRARY_FILE = "libkeystone*.dylib" +elif SYSTEM in ('win32', 'cygwin'): + LIBRARY_FILE = "keystone.dll" +else: + LIBRARY_FILE = "libkeystone.so" +# prebuilt libraries for Windows - for sdist +PATH_LIB64 = os.path.join(ROOT_DIR, 'prebuilt', 'win64') +PATH_LIB32 = os.path.join(ROOT_DIR, 'prebuilt', 'win32') def copy_sources(): """Copy the C sources into the source directory. @@ -42,158 +48,154 @@ def copy_sources(): """ src = [] - try: - dir_util.remove_tree("src/") - except (IOError, OSError): - pass + os.system('make clean') + shutil.rmtree(SRC_DIR, ignore_errors=True) + os.mkdir(SRC_DIR) - dir_util.copy_tree("../../llvm", "src/llvm/") - dir_util.copy_tree("../../include", "src/include/") - dir_util.copy_tree("../../suite", "src/suite") + shutil.copytree(os.path.join(ROOT_DIR, '../../llvm'), os.path.join(SRC_DIR, 'llvm/')) + shutil.copytree(os.path.join(ROOT_DIR, '../../include'), os.path.join(SRC_DIR, 'include/')) + shutil.copytree(os.path.join(ROOT_DIR, '../../suite'), os.path.join(SRC_DIR, 'suite/')) - src.extend(glob.glob("../../*.h")) - src.extend(glob.glob("../../*.cpp")) - src.extend(glob.glob("../../*.inc")) - src.extend(glob.glob("../../*.def")) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.h"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.cpp"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.inc"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.def"))) - src.extend(glob.glob("../../CMakeLists.txt")) - src.extend(glob.glob("../../CMakeUninstall.in")) - src.extend(glob.glob("../../*.txt")) - src.extend(glob.glob("../../*.TXT")) - src.extend(glob.glob("../../COPYING")) - src.extend(glob.glob("../../LICENSE*")) - src.extend(glob.glob("../../EXCEPTIONS-CLIENT")) - src.extend(glob.glob("../../README.md")) - src.extend(glob.glob("../../RELEASE_NOTES")) - src.extend(glob.glob("../../ChangeLog")) - src.extend(glob.glob("../../SPONSORS.TXT")) - src.extend(glob.glob("../../*.cmake")) - src.extend(glob.glob("../../*.sh")) - src.extend(glob.glob("../../*.bat")) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../CMakeLists.txt"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../CMakeUninstall.in"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.txt"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.TXT"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../COPYING"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../LICENSE*"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../EXCEPTIONS-CLIENT"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../README.md"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../RELEASE_NOTES"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../ChangeLog"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../SPONSORS.TXT"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.cmake"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.sh"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.bat"))) for filename in src: - outpath = os.path.join("./src/", os.path.basename(filename)) + outpath = os.path.join(SRC_DIR, os.path.basename(filename)) log.info("%s -> %s" % (filename, outpath)) shutil.copy(filename, outpath) +def build_libraries(): + cur_dir = os.getcwd() -class custom_sdist(sdist): - """Reshuffle files for distribution.""" - - def run(self): - # if prebuilt libraries are existent, then do not copy source - if os.path.exists(PATH_LIB64) and os.path.exists(PATH_LIB32): - return sdist.run(self) + if SYSTEM in ("win32", "cygwin"): + # if Windows prebuilt library is available, then include it + if IS_64BITS and os.path.exists(os.path.join(PATH_LIB64, LIBRARY_FILE)): + shutil.copy(os.path.join(PATH_LIB64, LIBRARY_FILE), LIBS_DIR) + return + elif os.path.exists(os.path.join(PATH_LIB32, LIBRARY_FILE)): + shutil.copy(os.path.join(PATH_LIB32, LIBRARY_FILE), LIBS_DIR) + return + # cd src/build + if not os.path.isdir(SRC_DIR): copy_sources() - return sdist.run(self) + os.chdir(SRC_DIR) + if not os.path.isdir(BUILD_DIR): + os.mkdir(BUILD_DIR) + os.chdir(BUILD_DIR) + if SYSTEM == "win32": + if IS_64BITS: + subprocess.call([r'..\nmake-dll.bat']) + else: + subprocess.call([r'..\nmake-dll.bat', 'X86']) + winobj_dir = os.path.join(BUILD_DIR, 'llvm', 'bin') + shutil.copy(os.path.join(winobj_dir, LIBRARY_FILE), LIBS_DIR) + else: + cmd = ['sh', '../make-share.sh', 'lib_only'] + subprocess.call(cmd) + obj_dir = os.path.join(BUILD_DIR, 'llvm', 'lib') + obj64_dir = os.path.join(BUILD_DIR, 'llvm', 'lib64') + if SYSTEM == 'darwin': + for file in glob.glob(os.path.join(obj_dir, MAC_LIBRARY_FILE)): + try: + shutil.copy(file, LIBS_DIR, follow_symlinks=False) + except: + shutil.copy(file, LIBS_DIR) + else: + try: + shutil.copy(os.path.join(obj_dir, LIBRARY_FILE), LIBS_DIR) + except: + shutil.copy(os.path.join(obj64_dir, LIBRARY_FILE), LIBS_DIR) + # back to root dir + os.chdir(cur_dir) -class custom_build_clib(build_clib): - """Customized build_clib command.""" - +class sdist(_sdist): def run(self): - log.info('running custom_build_clib') - build_clib.run(self) + copy_sources() + return _sdist.run(self) - def finalize_options(self): - # We want build-clib to default to build-lib as defined by the "build" - # command. This is so the compiled library will be put in the right - # place along side the python code. - self.set_undefined_options('build', - ('build_lib', 'build_clib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) - - build_clib.finalize_options(self) - - def build_libraries(self, libraries): - - cur_dir = os.path.realpath(os.curdir) - - if SYSTEM in ("win32", "cygwin"): - # if Windows prebuilt library is available, then include it - if is_64bits and os.path.exists(PATH_LIB64): - SETUP_DATA_FILES.append(PATH_LIB64) - return - elif os.path.exists(PATH_LIB32): - SETUP_DATA_FILES.append(PATH_LIB32) - return - - try: - for (lib_name, build_info) in libraries: - log.info("building '%s' library", lib_name) - - # cd src/build - os.chdir("../..") - if not os.path.isdir('build'): - os.mkdir('build') - os.chdir("build") - - # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform - if SYSTEM == "cygwin": - os.chmod("make.sh", stat.S_IREAD | stat.S_IEXEC) - if is_64bits: - os.system("KEYSTONE_BUILD_CORE_ONLY=yes ./make.sh cygwin-mingw64") - else: - os.system("KEYSTONE_BUILD_CORE_ONLY=yes ./make.sh cygwin-mingw32") - SETUP_DATA_FILES.append("src/build/keystone.dll") - else: # Unix - os.chmod("../make-share.sh", stat.S_IREAD | stat.S_IEXEC) - os.system("../make-share.sh lib_only") - base_dir = os.path.realpath(os.curdir) - if SYSTEM == "darwin": - SETUP_DATA_FILES.append(base_dir + "/llvm/lib/libkeystone.dylib") - else: # Non-OSX - SETUP_DATA_FILES.append(base_dir + "/llvm/lib/libkeystone.so") - - # back to root dir - os.chdir(cur_dir) - - except Exception as e: - log.error(e) - finally: - os.chdir(cur_dir) - - -class custom_install(install_lib): - def install(self): - install_lib.install(self) - ks_install_dir = os.path.join(self.install_dir, 'keystone') - for lib_file in SETUP_DATA_FILES: - file_util.copy_file(lib_file, ks_install_dir) +class build(_build): + def run(self): + log.info("Building C++ extensions") + build_libraries() + return _build.run(self) +class develop(_develop): + def run(self): + log.info("Building C++ extensions") + build_libraries() + return _develop.run(self) +class bdist_egg(_bdist_egg): + def run(self): + self.run_command('build') + return _bdist_egg.run(self) + def dummy_src(): return [] +if 'bdist_wheel' in sys.argv and '--plat-name' not in sys.argv: + idx = sys.argv.index('bdist_wheel') + 1 + sys.argv.insert(idx, '--plat-name') + name = get_platform() + if 'linux' in name: + # linux_* platform tags are disallowed because the python ecosystem is fubar + # linux builds should be built in the centos 5 vm for maximum compatibility + # see https://github.com/pypa/manylinux + # see also https://github.com/angr/angr-dev/blob/master/bdist.sh + sys.argv.insert(idx + 1, 'manylinux1_' + platform.machine()) + elif 'mingw' in name: + if IS_64BITS: + sys.argv.insert(idx + 1, 'win_amd64') + else: + sys.argv.insert(idx + 1, 'win32') + else: + # https://www.python.org/dev/peps/pep-0425/ + sys.argv.insert(idx + 1, name.replace('.', '_').replace('-', '_')) + + +with open("README.pypi-src", "r") as ld: + long_desc = ld.read() setup( provides=['keystone'], packages=['keystone'], - name=PKG_NAME, + name='keystone-engine', version=VERSION, author='Nguyen Anh Quynh', author_email='aquynh@gmail.com', description='Keystone assembler engine', - url='http://www.keystone-engine.org', + long_description=long_desc, + long_description_content_type="text/markdown", + url='https://www.keystone-engine.org', classifiers=[ 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', ], requires=['ctypes'], - cmdclass=dict( - build_clib=custom_build_clib, - sdist=custom_sdist, - install_lib=custom_install, - ), - - libraries=[( - 'keystone', dict( - package='keystone', - sources=dummy_src() - ), - )], + cmdclass={'build': build, 'develop': develop, 'sdist': sdist, 'bdist_egg': bdist_egg}, + zip_safe=True, + include_package_data=True, + is_pure=False, + package_data={ + 'keystone': ['*'] + } )