diff --git a/.gitignore b/.gitignore index 4a5682aa..9faec4cd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dist/ MANIFEST .coverage delocate*info/ -.cache \ No newline at end of file +.cache +delocate/tests/data_platform_tag/*.cc \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 9cad2ae5..f43eedb2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include LICENSE include README.rst include AUTHORS recursive-include delocate/tests/data * +recursive-include delocate/tests/data_platform_tag * diff --git a/delocate/cmd/delocate_wheel.py b/delocate/cmd/delocate_wheel.py index caa38792..92062fc5 100644 --- a/delocate/cmd/delocate_wheel.py +++ b/delocate/cmd/delocate_wheel.py @@ -40,7 +40,12 @@ def main(): Option("--require-archs", action="store", type='string', help="Architectures that all wheel libraries should " - "have (from 'intel', 'i386', 'x86_64', 'i386,x86_64')")]) + "have (from 'intel', 'i386', 'x86_64', 'i386,x86_64')"), + Option("--verify-name", action="store_true", + help="Verify if platform tag in wheel name is proper"), + Option("--fix-name", action="store_true", + help="calculate proper platform tag base on libraries requirements") + ]) (opts, wheels) = parser.parse_args() if len(wheels) < 1: parser.print_help() @@ -69,7 +74,10 @@ def main(): copied = delocate_wheel(wheel, out_wheel, lib_filt_func=lib_filt_func, lib_sdir=opts.lib_sdir, require_archs=require_archs, - check_verbose=opts.verbose) + check_verbose=opts.verbose, + check_wheel_name=opts.verify_name, + fix_wheel_name=opts.fix_name + ) if opts.verbose and len(copied): print("Copied to package {0} directory:".format(opts.lib_sdir)) copy_lines = [' ' + name for name in sorted(copied)] diff --git a/delocate/delocating.py b/delocate/delocating.py index 8bc98012..78ad9ee0 100644 --- a/delocate/delocating.py +++ b/delocate/delocating.py @@ -3,7 +3,9 @@ from __future__ import division, print_function +import re import os +import sys from os.path import (join as pjoin, dirname, basename, exists, abspath, relpath, realpath) import shutil @@ -13,7 +15,7 @@ from .pycompat import string_types from .libsana import tree_libs, stripped_lib_dict, get_rp_stripper from .tools import (set_install_name, zip2dir, dir2zip, validate_signature, - find_package_dirs, set_install_id, get_archs) + find_package_dirs, set_install_id, get_archs, back_tick) from .tmpdirs import TemporaryDirectory from .wheeltools import rewrite_record, InWheel @@ -308,6 +310,8 @@ def delocate_wheel(in_wheel, copy_filt_func=filter_system_libs, require_archs=None, check_verbose=False, + check_wheel_name=False, + fix_wheel_name=False ): """ Update wheel by copying required libraries to `lib_sdir` in wheel @@ -347,6 +351,10 @@ def delocate_wheel(in_wheel, (e.g "i386" or "x86_64"). check_verbose : bool, optional If True, print warning messages about missing required architectures + check_wheel_name: bool, optional + If true check if wheel name correctly inform about minimal dependencies + fix_wheel_name: bool, optional + If true then change wheel name to satisfy minimal system requirements Returns ------- @@ -391,6 +399,7 @@ def delocate_wheel(in_wheel, print(bads_report(bads, pjoin(tmpdir, 'wheel'))) raise DelocationError( "Some missing architectures in wheel") + # Change install ids to be unique within Python space install_id_root = (DLC_PREFIX + relpath(package_path, wheel_dir) + @@ -401,6 +410,14 @@ def delocate_wheel(in_wheel, set_install_id(copied_path, install_id_root + lib_base) validate_signature(copied_path) _merge_lib_dict(all_copied, copied_libs) + if check_wheel_name or fix_wheel_name: + wheel_name = os.path.basename(in_wheel) + new_name = update_wheel_name(wheel_name, wheel_dir) + if check_wheel_name and new_name != wheel_name: + raise DelocationError("Wheel name do not satisfy minimal package requirements") + if new_name != wheel_name: + in_place = False # maybe something better + out_wheel = os.path.join(os.path.basename(out_wheel), new_name) if len(all_copied): rewrite_record(wheel_dir) if len(all_copied) or not in_place: @@ -408,6 +425,111 @@ def delocate_wheel(in_wheel, return stripped_lib_dict(all_copied, wheel_dir + os.path.sep) +def analise_lib_file(file_path): + """ + file_path: str + path to lib file to analise + + Returns + ------- + system_requirements per architecture + """ + arches = get_archs(file_path) + needed_os_version = {} + for arch in arches: + lines = back_tick(["otool", "-arch", arch, "-l", file_path]).split("\n") + for i, line in enumerate(lines): + if "LC_BUILD_VERSION" in line: + for line in lines[i+1: i+5]: + cmd, val = line.split() + if cmd == "platform" and val != "macos": + if arch in needed_os_version: + del needed_os_version[arch] + break + if cmd == "minos": + version = [int(x) for x in val.split(".")] + if len(version) == 3: + version = version[:2] + needed_os_version[arch] = tuple(version) + break + if "LC_VERSION_MIN_MACOSX" in line: + for line in lines[i+1: i+4]: + cmd, val = line.split() + if cmd == "version": + version = [int(x) for x in val.split(".")] + if len(version) == 3: + version = version[:2] + needed_os_version[arch] = tuple(version) + break + + return needed_os_version + + +def version_to_str(version, sep="."): + return sep.join([str(x) for x in version]) + + +def update_wheel_name(wheel_name, root_dir): + """ + wheel_name: str + current name of wheel. + copied_libs: + + Returns + ------- + new_wheel_name: str + new wheel name which proper inform about minimal dependencies. + """ + # from pep 427 + # The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl. + tag_reg = re.compile(r"(.*)-macosx_(\d+)_(\d+)_(.+).whl") + parsed_wheel_name = tag_reg.match(wheel_name) + if parsed_wheel_name is None: + raise DelocationError("Cannot parse wheel name. Do not use check_wheel_name or fix_wheel_name options") + + package_name, major, minor, arch_list = parsed_wheel_name.groups() + arch_list = ['i386', 'x86_64'] if arch_list == "intel" else [arch_list] + versions_dict = {} + current_version = int(major), int(minor) + final_version = {arch: current_version for arch in arch_list} + for (dir_path, dir_names, filenames) in os.walk(root_dir): + for filename in filenames: + if filename.endswith('.dylib') or filename.endswith('.so'): + lib_path = os.path.join(dir_path, filename) + versions_dict[os.path.relpath(lib_path, root_dir)] = analise_lib_file(lib_path) + for library_path, arch_version_dict in versions_dict.items(): + if not arch_version_dict: + continue + for arch_name in arch_list: + if arch_name not in arch_version_dict: + if arch_name in final_version: + del final_version[arch_name] + print("Library {} force to remove {} from list of supported architectures".format( + library_path, arch_name), file=sys.stderr) + if not final_version: + raise DelocationError("Empty list of architectures") + + for library_path, arch_version_dict in versions_dict.items(): + for arch_name, value in final_version.items(): + if current_version < arch_version_dict[arch_name]: + print("Library {} force tu bump macos version to {}".format( + library_path, ".".join(version_to_str(arch_version_dict[arch_name]))), file=sys.stderr) + if value < arch_version_dict[arch_name]: + final_version[arch_name] = arch_version_dict[arch_name] + + if "i386" in final_version and "x86_64" in final_version: + if final_version["x86_64"] >= final_version["i386"]: + final_version["intel"] = final_version["x86_64"] + if final_version["x86_64"] < final_version["i386"]: + final_version["intel"] = final_version["i386"] + del final_version["x86_64"] + del final_version["i386"] + + assert len(final_version) == 1 + arch, version = next(iter(final_version.items())) + return package_name + "-macosx_" + version_to_str(version, "_") + "_" + arch + ".whl" + + def patch_wheel(in_wheel, patch_fname, out_wheel=None): """ Apply ``-p1`` style patch in `patch_fname` to contents of `in_wheel` diff --git a/delocate/tests/data_platform_tag/a.o b/delocate/tests/data_platform_tag/a.o new file mode 100644 index 00000000..fb21903d Binary files /dev/null and b/delocate/tests/data_platform_tag/a.o differ diff --git a/delocate/tests/data_platform_tag/liba.14.dylib b/delocate/tests/data_platform_tag/liba.14.dylib new file mode 100755 index 00000000..47cb53b6 Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.14.dylib differ diff --git a/delocate/tests/data_platform_tag/liba.14.so b/delocate/tests/data_platform_tag/liba.14.so new file mode 100755 index 00000000..9f13ea63 Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.14.so differ diff --git a/delocate/tests/data_platform_tag/liba.6.dylib b/delocate/tests/data_platform_tag/liba.6.dylib new file mode 100755 index 00000000..5cca1b5d Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.6.dylib differ diff --git a/delocate/tests/data_platform_tag/liba.6.so b/delocate/tests/data_platform_tag/liba.6.so new file mode 100755 index 00000000..753de30f Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.6.so differ diff --git a/delocate/tests/data_platform_tag/liba.9.dylib b/delocate/tests/data_platform_tag/liba.9.dylib new file mode 100755 index 00000000..653d32cd Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.9.dylib differ diff --git a/delocate/tests/data_platform_tag/liba.9.so b/delocate/tests/data_platform_tag/liba.9.so new file mode 100755 index 00000000..f14581aa Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.9.so differ diff --git a/delocate/tests/data_platform_tag/liba.a b/delocate/tests/data_platform_tag/liba.a new file mode 100644 index 00000000..35414602 Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.a differ diff --git a/delocate/tests/data_platform_tag/liba.dylib b/delocate/tests/data_platform_tag/liba.dylib new file mode 100755 index 00000000..9e83585d Binary files /dev/null and b/delocate/tests/data_platform_tag/liba.dylib differ diff --git a/delocate/tests/data_platform_tag/liba32.14.dylib b/delocate/tests/data_platform_tag/liba32.14.dylib new file mode 100755 index 00000000..8ca6ad5a Binary files /dev/null and b/delocate/tests/data_platform_tag/liba32.14.dylib differ diff --git a/delocate/tests/data_platform_tag/liba32.6.dylib b/delocate/tests/data_platform_tag/liba32.6.dylib new file mode 100755 index 00000000..f067705f Binary files /dev/null and b/delocate/tests/data_platform_tag/liba32.6.dylib differ diff --git a/delocate/tests/data_platform_tag/liba32.9.dylib b/delocate/tests/data_platform_tag/liba32.9.dylib new file mode 100755 index 00000000..eb0e4989 Binary files /dev/null and b/delocate/tests/data_platform_tag/liba32.9.dylib differ diff --git a/delocate/tests/data_platform_tag/liba_both.dylib b/delocate/tests/data_platform_tag/liba_both.dylib new file mode 100755 index 00000000..de9ee915 Binary files /dev/null and b/delocate/tests/data_platform_tag/liba_both.dylib differ diff --git a/delocate/tests/data_platform_tag/libb.14.dylib b/delocate/tests/data_platform_tag/libb.14.dylib new file mode 100755 index 00000000..2b2af845 Binary files /dev/null and b/delocate/tests/data_platform_tag/libb.14.dylib differ diff --git a/delocate/tests/data_platform_tag/libb.6.dylib b/delocate/tests/data_platform_tag/libb.6.dylib new file mode 100755 index 00000000..391ba0b7 Binary files /dev/null and b/delocate/tests/data_platform_tag/libb.6.dylib differ diff --git a/delocate/tests/data_platform_tag/libb.9.dylib b/delocate/tests/data_platform_tag/libb.9.dylib new file mode 100755 index 00000000..fd90f72e Binary files /dev/null and b/delocate/tests/data_platform_tag/libb.9.dylib differ diff --git a/delocate/tests/data_platform_tag/libb.dylib b/delocate/tests/data_platform_tag/libb.dylib new file mode 100755 index 00000000..0576a8b1 Binary files /dev/null and b/delocate/tests/data_platform_tag/libb.dylib differ diff --git a/delocate/tests/data_platform_tag/libc.14.dylib b/delocate/tests/data_platform_tag/libc.14.dylib new file mode 100755 index 00000000..c27775e6 Binary files /dev/null and b/delocate/tests/data_platform_tag/libc.14.dylib differ diff --git a/delocate/tests/data_platform_tag/libc.6.dylib b/delocate/tests/data_platform_tag/libc.6.dylib new file mode 100755 index 00000000..c7765374 Binary files /dev/null and b/delocate/tests/data_platform_tag/libc.6.dylib differ diff --git a/delocate/tests/data_platform_tag/libc.9.1.dylib b/delocate/tests/data_platform_tag/libc.9.1.dylib new file mode 100755 index 00000000..6dab09fb Binary files /dev/null and b/delocate/tests/data_platform_tag/libc.9.1.dylib differ diff --git a/delocate/tests/data_platform_tag/libc.9.dylib b/delocate/tests/data_platform_tag/libc.9.dylib new file mode 100755 index 00000000..35facd5a Binary files /dev/null and b/delocate/tests/data_platform_tag/libc.9.dylib differ diff --git a/delocate/tests/data_platform_tag/libc.dylib b/delocate/tests/data_platform_tag/libc.dylib new file mode 100755 index 00000000..9f904e8f Binary files /dev/null and b/delocate/tests/data_platform_tag/libc.dylib differ diff --git a/delocate/tests/data_platform_tag/make_libs.sh b/delocate/tests/data_platform_tag/make_libs.sh new file mode 100644 index 00000000..a05871d2 --- /dev/null +++ b/delocate/tests/data_platform_tag/make_libs.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Create libs used for testing +# Run in directory containing this file +# With thanks to https://dev.lsstcorp.org/trac/wiki/LinkingDarwin +# I ran this on a Snow Leopard machine with CXX=g++ ./make_libs.sh + +if [ "$CXX" = "" ]; then + CXX=c++ +fi + +cat << EOF > a.cc +#include +void a() { printf("a()\n"); } +EOF + +cat > b.cc << EOF +#include +void a(); +void b() { printf("b()\n"); a(); } +EOF + +cat > c.cc << EOF +#include +void b(); +void c() {printf("c()\n"); b(); } +EOF + +cat > d.cc << EOF +void c(); +int main(int, char**) { c(); return 0; } +EOF + +CXX_64="$CXX -arch x86_64" +CXX_32="$CXX -arch i386" + +rm ./*.dylib + +# 10.6 section + +$CXX_64 -o liba.6.dylib -mmacosx-version-min=10.6 -dynamiclib a.cc +$CXX_64 -o libb.6.dylib -mmacosx-version-min=10.6 -dynamiclib b.cc -L. -la.6 +$CXX_64 -o libc.6.dylib -mmacosx-version-min=10.6 -dynamiclib c.cc -L. -la.6 -lb.6 + +# 10.9 section +$CXX_64 -o liba.9.dylib -mmacosx-version-min=10.9 -dynamiclib a.cc +$CXX_64 -o libb.9.dylib -mmacosx-version-min=10.9 -dynamiclib b.cc -L. -la.9 +$CXX_64 -o libc.9.dylib -mmacosx-version-min=10.9 -dynamiclib c.cc -L. -la.9 -lb.9 + +# 10.9 base on 10.6 +$CXX_64 -o libc.9.1.dylib -mmacosx-version-min=10.9 -dynamiclib c.cc -L. -la.6 -lb.6 + +# 10.14 section +$CXX_64 -o liba.14.dylib -mmacosx-version-min=10.14 -dynamiclib a.cc +$CXX_64 -o libb.14.dylib -mmacosx-version-min=10.14 -dynamiclib b.cc -L. -la.14 +$CXX_64 -o libc.14.dylib -mmacosx-version-min=10.14 -dynamiclib c.cc -L. -la.14 -lb.14 + +$CXX_64 -o liba.dylib -dynamiclib a.cc +$CXX_32 -o liba32.6.dylib -mmacosx-version-min=10.6 -dynamiclib a.cc +$CXX_32 -o liba32.9.dylib -mmacosx-version-min=10.9 -dynamiclib a.cc +$CXX_32 -o liba32.14.dylib -mmacosx-version-min=10.14 -dynamiclib a.cc + +$CXX_64 -o a.o -c a.cc +ar rcs liba.a a.o +$CXX_64 -o libb.dylib -dynamiclib b.cc -L. -la +$CXX_64 -o libc.dylib -dynamiclib c.cc -L. -la -lb +$CXX_64 -o test-lib d.cc -L. -lc + +# Make a dual-arch library +lipo -create liba.9.dylib liba32.14.dylib -output liba_both.dylib + +$CXX_64 -o liba.6.so -mmacosx-version-min=10.6 -bundle a.cc +$CXX_64 -o liba.9.so -mmacosx-version-min=10.9 -bundle a.cc +$CXX_64 -o liba.14.so -mmacosx-version-min=10.14 -bundle a.cc \ No newline at end of file diff --git a/delocate/tests/data_platform_tag/test-lib b/delocate/tests/data_platform_tag/test-lib new file mode 100755 index 00000000..6520444b Binary files /dev/null and b/delocate/tests/data_platform_tag/test-lib differ diff --git a/delocate/tests/test_delocating.py b/delocate/tests/test_delocating.py index 2e068421..c64855bf 100644 --- a/delocate/tests/test_delocating.py +++ b/delocate/tests/test_delocating.py @@ -2,13 +2,17 @@ from __future__ import division, print_function +import itertools import os from os.path import (join as pjoin, dirname, basename, relpath, realpath, splitext) +import platform import shutil +import pytest + from ..delocating import (DelocationError, delocate_tree_libs, copy_recurse, - delocate_path, check_archs, bads_report) + delocate_path, check_archs, bads_report, update_wheel_name, analise_lib_file) from ..libsana import (tree_libs, search_environment_for_lib) from ..tools import (get_install_names, set_install_name, back_tick) @@ -571,3 +575,66 @@ def test_dyld_fallback_library_path_loses_to_basename(): # tmpdir can end up in /var, and that can be symlinked to # /private/var, so we'll use realpath to resolve the two assert_equal(predicted_lib_location, os.path.realpath(libb)) + + +def walk_mock(file_list, base_path): + def _walk_mock(top, topdown=True, onerror=None, followlinks=False): + yield (base_path, [], file_list) + + return _walk_mock + +@pytest.fixture +def tag_data(): + return os.path.join(os.path.dirname(__file__), "data_platform_tag") + + +@pytest.mark.parametrize( + "file_name,expected_result", + [("lib{}.{}.dylib".format(l, v), + {"x86_64": (10, v)}) for l, v in itertools.product(["a", "b", "c"], [6, 9, 14])] + + [("libc.9.1.dylib", {"x86_64": (10, 9)}), ("liba_both.dylib", {"x86_64": (10, 9), "i386": (10, 14)})] + + [("liba32.{}.dylib".format(v), {"i386": (10, v)}) for v in [6, 9, 14]] + + [("liba.{}.so".format(v), {"x86_64": (10, v)}) for v in [6, 9, 14]] +) +def test_analise_lib_file(file_name, expected_result, tag_data): + if (".14." in file_name or "liba_both.dylib" == file_name) and platform.mac_ver()[0]: + ver = tuple([int(x) for x in platform.mac_ver()[0].split(".")[:2]]) + if ver < (10, 13): + pytest.xfail("To low version of otool to support LC_BUILD_VERSION") + assert analise_lib_file(os.path.join(tag_data, file_name)) == expected_result + + +@pytest.mark.parametrize("start_version,expected_version,files", [ + ("10_10_x86_64", "10_10_x86_64", ["liba.6.dylib"]), + ("10_6_x86_64", "10_9_x86_64", ["liba.9.dylib"]), + ("10_10_x86_64", "10_10_x86_64", ["liba.9.dylib"]), + ("10_10_intel", "10_10_x86_64", ["liba.9.dylib"]), + ("10_10_intel", "10_10_i386", ["liba32.9.dylib"]), + ("10_10_intel", "10_14_intel", ["liba_both.dylib"]), + ("10_9_x86_64", "10_9_x86_64", ["liba_both.dylib"]), # ignore i386 requirements + ("10_10_x86_64", "10_14_x86_64", ["liba.6.dylib", "libb.14.dylib"]), + ("10_10_x86_64", "10_14_x86_64", ["liba.6.dylib", "liba.14.so"]), + ("10_6_x86_64", "10_9_x86_64", ["liba.6.dylib", "liba.9.so", "test-lib"]) # test-lib is compiled against 10.14 +]) +def test_update_wheel_name(start_version, expected_version, files, monkeypatch, tag_data): + if expected_version.startswith("10_14") and platform.mac_ver()[0]: + ver = tuple([int(x) for x in platform.mac_ver()[0].split(".")[:2]]) + if ver < (10, 13): + pytest.xfail("To low version of otool to support LC_BUILD_VERSION") + wheel_name_template = "test-0.1-cp36-cp36m-macosx_{}.whl" + monkeypatch.setattr(os, "walk", walk_mock(files, tag_data)) + assert update_wheel_name(wheel_name_template.format(start_version), tag_data) ==\ + wheel_name_template.format(expected_version) + +@pytest.mark.parametrize("start_version,files", [ + ("stupid_text", ["liba.6.dylib"]), + ("10_6_x86_64", ["liba32.9.dylib"]), + ("10_6_i386", ["liba.9.dylib"]), + ("10_6_intel", ["liba.9.dylib", "liba32.9.dylib"]), + ("10_6_stupid_arch", ["liba32.9.dylib"]), +]) +def test_update_wheel_name_fail(start_version, files, monkeypatch, tag_data): + wheel_name_template = "test-0.1-cp36-cp36m-macosx_{}.whl" + monkeypatch.setattr(os, "walk", walk_mock(files, tag_data)) + with pytest.raises(DelocationError): + update_wheel_name(wheel_name_template.format(start_version), tag_data) diff --git a/setup.py b/setup.py index 39f2264e..4fa149db 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,8 @@ #!/usr/bin/env python """ setup script for delocate package """ -import sys from os.path import join as pjoin from setuptools import setup, find_packages -# For some commands, use setuptools. -if len(set(('develop', 'bdist_egg', 'bdist_rpm', 'bdist', 'bdist_dumb', - 'install_egg_info', 'egg_info', 'easy_install', 'bdist_wheel', - 'bdist_mpkg')).intersection(sys.argv)) > 0: - import setuptools - import versioneer versioneer.VCS = 'git' @@ -43,7 +36,11 @@ pjoin('data', 'test-lib'), pjoin('data', '*patch'), pjoin('data', 'make_libs.sh'), - pjoin('data', 'icon.ico')]}, + pjoin('data', 'icon.ico'), + pjoin('data_platform_tag', '*.dylib'), + pjoin('data_platform_tag', '*.so'), + pjoin('data_platform_tag', 'test-lib'), + ]}, entry_points={ 'console_scripts': [ 'delocate-{} = delocate.cmd.delocate_{}:main'.format(name, name)