From 263bc44d21fd622f9e6bed7f2f6be38e2c0d752c Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Mon, 6 Jan 2025 14:46:51 +0100 Subject: [PATCH] Enhance Cython integration: Add support for `c++` mode, use hostpython3-provided Cython, allow pinning of specific Cython version in recipe (#957) * Add support to build cpp files from Cython, use Cython from hostpython3, and allow configuring Cython version to be used in recipe * PEP8 (noqa as ATM we do not care why it failed to Cythonize) * Add cymunk to broken recipes * Remove cpplink, now it uses the same liblink * Remove unused import * Avoid DRY violation --- .ci/constants.py | 2 +- kivy_ios/recipes/audiostream/__init__.py | 1 + kivy_ios/recipes/curly/__init__.py | 1 + kivy_ios/recipes/cymunk/__init__.py | 2 +- kivy_ios/recipes/ffpyplayer/__init__.py | 1 + kivy_ios/recipes/hostpython3/__init__.py | 13 +- .../hostpython3/fix-ldshared-override.patch | 31 ++++ kivy_ios/recipes/ios/__init__.py | 1 + kivy_ios/recipes/kivent_core/__init__.py | 1 + kivy_ios/recipes/kiwisolver/__init__.py | 18 +- kivy_ios/recipes/matplotlib/__init__.py | 10 +- kivy_ios/recipes/netifaces/__init__.py | 1 + kivy_ios/recipes/numpy/__init__.py | 2 +- kivy_ios/recipes/photolibrary/__init__.py | 1 + kivy_ios/recipes/pillow/__init__.py | 1 + kivy_ios/recipes/pycrypto/__init__.py | 1 + kivy_ios/toolchain.py | 18 +- kivy_ios/tools/cpplink | 159 ------------------ kivy_ios/tools/cythonize.py | 33 ++-- kivy_ios/tools/liblink | 3 + requirements.txt | 1 - setup.cfg | 1 - 22 files changed, 92 insertions(+), 210 deletions(-) create mode 100644 kivy_ios/recipes/hostpython3/fix-ldshared-override.patch delete mode 100755 kivy_ios/tools/cpplink diff --git a/.ci/constants.py b/.ci/constants.py index 20403d3b5..3ae849751 100644 --- a/.ci/constants.py +++ b/.ci/constants.py @@ -1,4 +1,4 @@ -BROKEN_RECIPES = set(["netifaces", "kivent_core"]) +BROKEN_RECIPES = set(["netifaces", "kivent_core", "cymunk"]) # recipes that were already built will be skipped CORE_RECIPES = set(["kivy", "hostpython3", "python3"]) diff --git a/kivy_ios/recipes/audiostream/__init__.py b/kivy_ios/recipes/audiostream/__init__.py index f5207ec80..ba7ac805a 100644 --- a/kivy_ios/recipes/audiostream/__init__.py +++ b/kivy_ios/recipes/audiostream/__init__.py @@ -7,6 +7,7 @@ class AudiostreamRecipe(CythonRecipe): library = "libaudiostream.a" depends = ["python", "sdl2", "sdl2_mixer"] pre_build_ext = True + hostpython_prerequisites = ["Cython==0.29.37"] recipe = AudiostreamRecipe() diff --git a/kivy_ios/recipes/curly/__init__.py b/kivy_ios/recipes/curly/__init__.py index 6c8b5ff7e..1adf0bde6 100644 --- a/kivy_ios/recipes/curly/__init__.py +++ b/kivy_ios/recipes/curly/__init__.py @@ -7,6 +7,7 @@ class CurlyRecipe(CythonRecipe): library = "libcurly.a" depends = ["python", "libcurl", "sdl2", "sdl2_image"] pre_build_ext = True + hostpython_prerequisites = ["Cython==0.29.37"] recipe = CurlyRecipe() diff --git a/kivy_ios/recipes/cymunk/__init__.py b/kivy_ios/recipes/cymunk/__init__.py index 7571bbdd9..d99e27131 100644 --- a/kivy_ios/recipes/cymunk/__init__.py +++ b/kivy_ios/recipes/cymunk/__init__.py @@ -12,7 +12,7 @@ class CymunkRecipe(CythonRecipe): name = 'cymunk' pre_build_ext = True library = 'libcymunk.a' - + hostpython_prerequisites = ["Cython==0.29.37"] depends = ['python'] def get_recipe_env(self, arch): diff --git a/kivy_ios/recipes/ffpyplayer/__init__.py b/kivy_ios/recipes/ffpyplayer/__init__.py index 8d76e3ca1..49e0e66ee 100644 --- a/kivy_ios/recipes/ffpyplayer/__init__.py +++ b/kivy_ios/recipes/ffpyplayer/__init__.py @@ -12,6 +12,7 @@ class FFPyplayerRecipe(CythonRecipe): "CoreMotion"] pbx_libraries = ["libiconv"] pre_build_ext = True + hostpython_prerequisites = ["Cython==0.29.37"] def get_recipe_env(self, plat): env = super(FFPyplayerRecipe, self).get_recipe_env(plat) diff --git a/kivy_ios/recipes/hostpython3/__init__.py b/kivy_ios/recipes/hostpython3/__init__.py index 9ffa195f7..9771a6bac 100644 --- a/kivy_ios/recipes/hostpython3/__init__.py +++ b/kivy_ios/recipes/hostpython3/__init__.py @@ -100,11 +100,22 @@ def install(self): self.ctx.dist_dir, "hostpython3", "lib", - "python3.11", + f"python{self.ctx.hostpython_ver}", "site-packages", "setuptools", ), ) + self.apply_patch( + "fix-ldshared-override.patch", + join( + self.ctx.dist_dir, + "hostpython3", + "lib", + f"python{self.ctx.hostpython_ver}", + "site-packages", + "setuptools", + ), + ) recipe = Hostpython3Recipe() diff --git a/kivy_ios/recipes/hostpython3/fix-ldshared-override.patch b/kivy_ios/recipes/hostpython3/fix-ldshared-override.patch new file mode 100644 index 000000000..a2a7a145f --- /dev/null +++ b/kivy_ios/recipes/hostpython3/fix-ldshared-override.patch @@ -0,0 +1,31 @@ +diff -Naur setuptools.orig/_distutils/unixccompiler.py setuptools/_distutils/unixccompiler.py +--- setuptools.orig/_distutils/unixccompiler.py 2024-02-11 18:42:58 ++++ setuptools/_distutils/unixccompiler.py 2024-02-11 18:45:30 +@@ -253,14 +253,20 @@ + building_exe = target_desc == CCompiler.EXECUTABLE + linker = (self.linker_exe if building_exe else self.linker_so)[:] + +- if target_lang == "c++" and self.compiler_cxx: +- env, linker_ne = _split_env(linker) +- aix, linker_na = _split_aix(linker_ne) +- _, compiler_cxx_ne = _split_env(self.compiler_cxx) +- _, linker_exe_ne = _split_env(self.linker_exe) ++ # Mirko: We need our LDSHARED also for c++ things, ++ # otherwise our hack to have static libs does not work ++ # properly. ++ # We will likely remove all these caveats once PEP 730 ++ # is implemented (and we will conform to it). + +- params = _linker_params(linker_na, linker_exe_ne) +- linker = env + aix + compiler_cxx_ne + params ++ #if target_lang == "c++" and self.compiler_cxx: ++ # env, linker_ne = _split_env(linker) ++ # aix, linker_na = _split_aix(linker_ne) ++ # _, compiler_cxx_ne = _split_env(self.compiler_cxx) ++ # _, linker_exe_ne = _split_env(self.linker_exe) ++ ++ # params = _linker_params(linker_na, linker_exe_ne) ++ # linker = env + aix + compiler_cxx_ne + params + + linker = compiler_fixup(linker, ld_args) + diff --git a/kivy_ios/recipes/ios/__init__.py b/kivy_ios/recipes/ios/__init__.py index cd6dab11e..b187af4e4 100644 --- a/kivy_ios/recipes/ios/__init__.py +++ b/kivy_ios/recipes/ios/__init__.py @@ -7,6 +7,7 @@ class IosRecipe(CythonRecipe): library = "libios.a" depends = ["python"] pbx_frameworks = ["MessageUI", "CoreMotion", "UIKit", "WebKit", "Photos"] + hostpython_prerequisites = ["Cython==0.29.37"] def install(self): self.install_python_package( diff --git a/kivy_ios/recipes/kivent_core/__init__.py b/kivy_ios/recipes/kivent_core/__init__.py index 2e0116d57..fdb4214d1 100644 --- a/kivy_ios/recipes/kivent_core/__init__.py +++ b/kivy_ios/recipes/kivent_core/__init__.py @@ -21,6 +21,7 @@ class KiventCoreRecipe(CythonRecipe): subbuilddir = False cythonize = True pbx_frameworks = ["OpenGLES"] # note: This line may be unnecessary + hostpython_prerequisites = ["Cython==0.29.37"] def get_recipe_env(self, plat): env = super(KiventCoreRecipe, self).get_recipe_env(plat) diff --git a/kivy_ios/recipes/kiwisolver/__init__.py b/kivy_ios/recipes/kiwisolver/__init__.py index ac10aec81..45b6789cf 100644 --- a/kivy_ios/recipes/kiwisolver/__init__.py +++ b/kivy_ios/recipes/kiwisolver/__init__.py @@ -2,14 +2,10 @@ This file is derived from the p4a recipe for kiwisolver. It is a dependency of matplotlib. -It is a C++ library, and it utilizes the cpplink script to handle -creating the library files needed for inclusion in an iOS project. - It also depends on the headers from the cppy package. ''' from kivy_ios.toolchain import CythonRecipe -from os.path import join class KiwiSolverRecipe(CythonRecipe): @@ -18,21 +14,9 @@ class KiwiSolverRecipe(CythonRecipe): version = '1.3.2' url = 'https://github.com/nucleic/kiwi/archive/{version}.zip' depends = ["python"] - hostpython_prerequisites = ["cppy"] + hostpython_prerequisites = ["cppy", "Cython==0.29.37"] cythonize = False library = "libkiwisolver.a" - def get_recipe_env(self, plat): - env = super().get_recipe_env(plat) - - # cpplink setup - env['CXX_ORIG'] = env['CXX'] - env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink") - - # setuptools uses CC for compiling and CXX for linking - env['CC'] = env['CXX'] - env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT']) - return env - recipe = KiwiSolverRecipe() diff --git a/kivy_ios/recipes/matplotlib/__init__.py b/kivy_ios/recipes/matplotlib/__init__.py index b2d26acab..2e4914f22 100644 --- a/kivy_ios/recipes/matplotlib/__init__.py +++ b/kivy_ios/recipes/matplotlib/__init__.py @@ -2,9 +2,6 @@ This file is derived from the p4a recipe for matplotlib. It is a dependency of matplotlib. -It is a C++ library, and it utilizes the cpplink script to handle -creating the library files needed for inclusion in an iOS project. - In addition to the original patch files for p4a, additional patch files are necessary to prevent duplicate symbols from appearing in the final link of a kivy-ios application. @@ -24,7 +21,7 @@ class MatplotlibRecipe(CythonRecipe): pre_build_ext = True python_depends = ['cycler', 'fonttools', 'packaging', 'pyparsing', 'python-dateutil', 'six'] - hostpython_prerequisites = ['pybind11', 'certifi'] + hostpython_prerequisites = ['pybind11', 'certifi', "Cython==0.29.37"] cythonize = False def generate_libraries_pc_files(self, plat): @@ -113,11 +110,6 @@ def get_recipe_env(self, plat): numpy_inc_dir = dirname(sh.glob(numpytype.get_build_dir(plat) + '/**/_numpyconfig.h', recursive=True)[0]) env['CFLAGS'] += f' -I{free_inc_dir} -I{numpy_inc_dir}' - env['CXX_ORIG'] = env['CXX'] - env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink") - - # setuptools uses CC for compiling and CXX for linking - env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT']) return env diff --git a/kivy_ios/recipes/netifaces/__init__.py b/kivy_ios/recipes/netifaces/__init__.py index 3276e0ac4..c31b7baff 100644 --- a/kivy_ios/recipes/netifaces/__init__.py +++ b/kivy_ios/recipes/netifaces/__init__.py @@ -11,6 +11,7 @@ class NetifacesRecipe(CythonRecipe): url = "https://pypi.io/packages/source/n/netifaces/netifaces-{version}.tar.gz" depends = ["python3"] python_depends = ["setuptools"] + hostpython_prerequisites = ["Cython==0.29.37"] library = "libnetifaces.a" cythonize = False diff --git a/kivy_ios/recipes/numpy/__init__.py b/kivy_ios/recipes/numpy/__init__.py index fe4528f34..5ea852bf3 100644 --- a/kivy_ios/recipes/numpy/__init__.py +++ b/kivy_ios/recipes/numpy/__init__.py @@ -11,7 +11,7 @@ class NumpyRecipe(CythonRecipe): libraries = ["libnpymath.a", "libnpyrandom.a"] include_dir = "numpy/core/include" depends = ["python"] - hostpython_prerequisites = ["Cython==0.29.36"] + hostpython_prerequisites = ["Cython==0.29.37"] cythonize = False def prebuild_platform(self, plat): diff --git a/kivy_ios/recipes/photolibrary/__init__.py b/kivy_ios/recipes/photolibrary/__init__.py index 81e5c500f..27c9989f5 100644 --- a/kivy_ios/recipes/photolibrary/__init__.py +++ b/kivy_ios/recipes/photolibrary/__init__.py @@ -7,6 +7,7 @@ class PhotoRecipe(CythonRecipe): library = "libphotolibrary.a" depends = ["python"] pbx_frameworks = ["UIKit", "Photos", "MobileCoreServices"] + hostpython_prerequisites = ["Cython==0.29.37"] def install(self): self.install_python_package(name="photolibrary.so", is_dir=False) diff --git a/kivy_ios/recipes/pillow/__init__.py b/kivy_ios/recipes/pillow/__init__.py index b91e0f3d6..37946de31 100644 --- a/kivy_ios/recipes/pillow/__init__.py +++ b/kivy_ios/recipes/pillow/__init__.py @@ -18,6 +18,7 @@ class PillowRecipe(CythonRecipe): python_depends = ["setuptools"] pbx_libraries = ["libz", "libbz2"] include_per_platform = True + hostpython_prerequisites = ["Cython==0.29.37"] cythonize = False def prebuild_platform(self, plat): diff --git a/kivy_ios/recipes/pycrypto/__init__.py b/kivy_ios/recipes/pycrypto/__init__.py index b155a4b87..b26d31ea5 100644 --- a/kivy_ios/recipes/pycrypto/__init__.py +++ b/kivy_ios/recipes/pycrypto/__init__.py @@ -12,6 +12,7 @@ class PycryptoRecipe(CythonRecipe): depends = ["python", "openssl"] include_per_platform = True library = "libpycrypto.a" + hostpython_prerequisites = ["Cython==0.29.37"] def build_platform(self, plat): build_env = plat.get_env() diff --git a/kivy_ios/toolchain.py b/kivy_ios/toolchain.py index 87aa66dd0..0cbdb06da 100755 --- a/kivy_ios/toolchain.py +++ b/kivy_ios/toolchain.py @@ -176,6 +176,19 @@ def get_env(self): include_dirs += ["-I{}".format( join(self.ctx.dist_dir, "include", self.name))] + # Add Python include directories + include_dirs += [ + "-I{}".format( + join( + self.ctx.dist_dir, + "root", + "python3", + "include", + f"python{self.ctx.hostpython_ver}", + ) + ) + ] + env = {} cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip() cxx = sh.xcrun("-find", "-sdk", self.sdk, "clang++").strip() @@ -249,6 +262,7 @@ def noicctempfile(): "-O3", self.version_min, ] + include_dirs) + env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] = " ".join([ "-arch", self.arch, # "--sysroot", self.sysroot, @@ -1134,6 +1148,7 @@ def reduce_python_package(self): class CythonRecipe(PythonRecipe): pre_build_ext = False cythonize = True + hostpython_prerequisites = ["Cython==3.0.11"] def cythonize_file(self, filename): if filename.startswith(self.build_dir): @@ -1143,7 +1158,8 @@ def cythonize_file(self, filename): # doesn't (yet) have the executable bit hence we explicitly call it # with the Python interpreter cythonize_script = join(self.ctx.root_dir, "tools", "cythonize.py") - shprint(sh.Command(sys.executable), cythonize_script, filename) + + shprint(sh.Command(self.ctx.hostpython), cythonize_script, filename) def cythonize_build(self): if not self.cythonize: diff --git a/kivy_ios/tools/cpplink b/kivy_ios/tools/cpplink deleted file mode 100755 index e9328e15c..000000000 --- a/kivy_ios/tools/cpplink +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -''' -C++ libraries are linked using environ['CXX'] compiler. This is different -from C libraries, which are linked using environ['LDSHARED']. - -This script behaves like a C++ compiler or a linker, depending on -whether its output file ends with '.so'. - -If for compiling, this script forwards all arguments in a call to the -compiler specified as environ['CXX_ORIG']. - -If for linking, this script collects the same object and dependent -library data. It then generates the same files as liblink would for an -ld target. The linker called is specified in environ['ARM_LD']. -''' - -import sys -import subprocess -from os import environ, remove - - -# this section is to quickly bypass the full script for cases of c++ compiling - -def get_output(args): - ''' - gets output file - ''' - output = None - get_next = False - - for arg in args: - if get_next: - output = arg - break - elif arg.startswith('-o'): - if arg == '-o': - get_next = True - else: - output = arg[2:] - break - - return output - - -def call_cpp(args): - ''' - call the c++ compiler and return error code - throws a RuntimeError if there is an exception in processing - ''' - result = subprocess.run([environ['CXX_ORIG'], *args]) - if result.returncode != 0: - raise RuntimeError("Compiling C++ failed") - - -def parse_linker_args(args): - ''' - parse arguments to the linker - ''' - libs = [] - objects = [] - i = 0 - while i < len(args): - opt = args[i] - i += 1 - - if opt == "-o": - i += 1 - continue - elif opt.startswith("-l") or opt.startswith("-L"): - libs.append(opt) - continue - elif opt in ("-r", "-pipe", "-no-cpp-precomp"): - continue - elif opt in ( - "--sysroot", "-isysroot", "-framework", "-undefined", - "-macosx_version_min" - ): - i += 1 - continue - elif opt.startswith(("-I", "-m", "-f", "-O", "-g", "-D", "-arch", "-Wl", "-W", "-stdlib=")): - continue - elif opt.startswith("-"): - raise RuntimeError(str(args) + "\nUnknown option: " + opt) - elif not opt.endswith('.o'): - continue - - objects.append(opt) - - if not objects: - raise RuntimeError('C++ Linker arguments contain no object files') - - return libs, objects - - -def call_linker(objects, output): - ''' - calls linker (environ['ARM_LD']) and returns error code - throws a RuntimeError if there is an exception in processing - ''' - print('cpplink redirect linking with', objects) - ld = environ.get('ARM_LD') - arch = environ.get('ARCH', 'arm64') - sdk = environ.get('PLATFORM_SDK', 'iphoneos') - if sdk == 'iphoneos': - min_version_flag = '-ios_version_min' - elif sdk == 'iphonesimulator': - min_version_flag = '-ios_simulator_version_min' - else: - raise ValueError("Unsupported SDK: {}".format(sdk)) - - call = [ld, '-r', '-o', output + '.o', min_version_flag, '9.0', '-arch', arch] - call += objects - print("Linking: {}".format(" ".join(call))) - result = subprocess.run(call) - - if result.returncode != 0: - raise RuntimeError("C++ Linking failed") - - -def delete_so_files(output): - ''' - delete shared object files needed for proper module loading - ''' - try: - remove(output) - except FileNotFoundError: - pass - - try: - remove(output + '.libs') - except FileNotFoundError: - pass - - -def write_so_files(output, libs): - ''' - Writes empty .so and .so.libs file which is needed for proper module loading - ''' - with open(output, "w") as f: - f.write('') - - with open(output + ".libs", "w") as f: - f.write(" ".join(libs)) - - -# command line arguments to the C++ Compiler/Linker -args = sys.argv[1:] -# get the output files -output = get_output(args) - -if not output.endswith('.so'): - # C++ Compiling - subprocess.run([environ['CXX_ORIG'], *args]) -else: - # C++ Linking - libs, objects = parse_linker_args(args) - delete_so_files(output) - call_linker(objects, output) - write_so_files(output, libs) diff --git a/kivy_ios/tools/cythonize.py b/kivy_ios/tools/cythonize.py index a1a011fcf..00c06587a 100755 --- a/kivy_ios/tools/cythonize.py +++ b/kivy_ios/tools/cythonize.py @@ -1,22 +1,16 @@ #!/usr/bin/env python3 -import os import sys -import subprocess +from Cython.Build.Cythonize import main as cythonize_main -# resolve cython executable -cython = None - -def resolve_cython(): - global cython - for executable in ('cython', 'cython-2.7'): - for path in os.environ['PATH'].split(':'): - if not os.path.exists(path): - continue - if executable in os.listdir(path): - cython = os.path.join(path, executable) - return +def is_cplus(fn): + # Check if there's the directive to compile as C++ + with open(fn) as fd: + for line in fd: + if line.startswith('# distutils: language = c++'): + return True + return False def do(fn): @@ -29,13 +23,13 @@ def do(fn): package = '_'.join(parts[:-1]) # cythonize - subprocess.Popen([cython, fn], env=os.environ).communicate() + cythonize_main([fn]) if not package: print('no need to rewrite', fn) else: # get the .c, and change the initXXX - fn_c = fn[:-3] + 'c' + fn_c = fn[:-3] + ('c' if not is_cplus(fn) else 'cpp') with open(fn_c) as fd: data = fd.read() modname = modname.split('.')[-1] @@ -53,6 +47,9 @@ def do(fn): if __name__ == '__main__': print('-- cythonize', sys.argv) - resolve_cython() for fn in sys.argv[1:]: - do(fn) + try: + do(fn) + except: # noqa: E722 + print("Failed to cythonize, this is not necessarily a problem") + pass diff --git a/kivy_ios/tools/liblink b/kivy_ios/tools/liblink index c48a14561..40f966fa9 100755 --- a/kivy_ios/tools/liblink +++ b/kivy_ios/tools/liblink @@ -58,6 +58,9 @@ while i < len(sys.argv): if opt.startswith("-W"): continue + if opt.startswith("-stdlib="): + continue + if opt.startswith("-"): print(sys.argv) print("Unknown option: ", opt) diff --git a/requirements.txt b/requirements.txt index 2933f3aae..c030afb37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ pbxproj==3.5.0 Pillow>=6.1.0 cookiecutter==2.1.1 sh==1.12.14 -Cython==0.29.36 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 710990f7a..491eea40a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,6 @@ project_urls = [options] python_requires >= "3.8.0" install_requires = - Cython==0.29.36 cookiecutter pbxproj Pillow