From a9549149baad821db73135c4c2030c85f646ffc1 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 16 Aug 2018 21:18:00 -0700 Subject: [PATCH 1/5] Add Julia._unbox_as --- julia/core.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/julia/core.py b/julia/core.py index 48d1d32d..fb0e0d38 100644 --- a/julia/core.py +++ b/julia/core.py @@ -346,6 +346,22 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _julia_runtime = [False] + +UNBOXABLE_TYPES = ( + 'bool', + 'int8', + 'uint8', + 'int16', + 'uint16', + 'int32', + 'uint32', + 'int64', + 'uint64', + 'float32', + 'float64', +) + + class Julia(object): """ Implements a bridge to the Julia interpreter or library. @@ -460,6 +476,17 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, self.api.jl_unbox_voidpointer.argtypes = [void_p] self.api.jl_unbox_voidpointer.restype = py_object + for c_type in UNBOXABLE_TYPES: + jl_unbox = getattr(self.api, "jl_unbox_{}".format(c_type)) + jl_unbox.argtypes = [void_p] + jl_unbox.restype = getattr(ctypes, "c_{}".format({ + "float32": "float", + "float64": "double", + }.get(c_type, c_type))) + + self.api.jl_typeof.argtypes = [void_p] + self.api.jl_typeof.restype = void_p + self.api.jl_exception_clear.restype = None self.api.jl_stderr_obj.argtypes = [] self.api.jl_stderr_obj.restype = void_p @@ -557,6 +584,29 @@ def _call(self, src): return ans + @staticmethod + def _check_unboxable(c_type): + if c_type not in UNBOXABLE_TYPES: + raise ValueError("Julia value cannot be unboxed as c_type={!r}.\n" + "c_type supported by PyJulia are:\n" + "{}".format(c_type, "\n".join(UNBOXABLE_TYPES))) + + def _is_unboxable_as(self, pointer, c_type): + self._check_unboxable(c_type) + jl_type = getattr(self.api, 'jl_{}_type'.format(c_type)) + desired = ctypes.cast(jl_type, ctypes.POINTER(ctypes.c_void_p))[0] + actual = self.api.jl_typeof(pointer) + return actual == desired + + def _unbox_as(self, pointer, c_type): + self._check_unboxable(c_type) + jl_unbox = getattr(self.api, 'jl_unbox_{}'.format(c_type)) + if self._is_unboxable_as(pointer, c_type): + return jl_unbox(pointer) + else: + raise TypeError("Cannot unbox pointer {} as {}" + .format(pointer, c_type)) + def check_exception(self, src=""): exoc = self.api.jl_exception_occurred() self._debug("exception occured? " + str(exoc)) From 87ca784b70a2a435f9b772a7d400afb47ca30699 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 16 Aug 2018 22:22:16 -0700 Subject: [PATCH 2/5] Fail with a helpful message if separate cache is not supported --- julia/core.py | 50 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/julia/core.py b/julia/core.py index fb0e0d38..d2f1d411 100644 --- a/julia/core.py +++ b/julia/core.py @@ -344,6 +344,34 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): return py_libpython == jl_libpython +def raise_separate_cache_error(runtime, pyprogramname): + message = """\ +It seems your Julia and PyJulia setup are not supported. + +Julia interpreter: + {runtime} +Python interpreter used by PyCall.jl: + {pyprogramname} +Python interpreter used to import PyJulia. + {sys.executable} + +In Julia >= 0.7, above two paths to the Python interpreters have to match +exactly in order for PyJulia to work. To configure PyCall.jl to use Python +interpreter "{sys.executable}", +run the following commands in the Julia interpreter: + + ENV["PYTHON"] = "{sys.executable}" + using Pkg + Pkg.build("PyCall") + +For more information, see: + https://github.com/JuliaPy/pyjulia + https://github.com/JuliaPy/PyCall.jl + """.format(runtime=runtime, pyprogramname=pyprogramname, + sys=sys) + raise RuntimeError(message) + + _julia_runtime = [False] @@ -498,14 +526,20 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, if init_julia: if use_separate_cache: # First check that this is supported - self._call(""" - if VERSION < v"0.5-" - error(\"""Using pyjulia with a statically-compiled version - of python or with a version of python that - differs from that used by PyCall.jl is not - supported on julia 0.4""\") - end - """) + version_range = self._unbox_as(self._call(""" + Int64(if VERSION < v"0.6-" + 2 + elseif VERSION >= v"0.7-" + 1 + else + 0 + end) + """), "int64") + if version_range == 2: + raise RuntimeError( + "PyJulia does not support Julia < 0.6 anymore") + elif version_range == 1: + raise_separate_cache_error(runtime, depsjlexe) # Intercept precompilation os.environ["PYCALL_PYTHON_EXE"] = sys.executable os.environ["PYCALL_JULIA_HOME"] = PYCALL_JULIA_HOME From 6712660b9b0e8dfb377d085e2f2bce7ceb3d7e32 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 5 Sep 2018 19:38:41 -0700 Subject: [PATCH 3/5] Emit different message when statically linked --- julia/core.py | 55 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/julia/core.py b/julia/core.py index d2f1d411..eed7122a 100644 --- a/julia/core.py +++ b/julia/core.py @@ -344,31 +344,60 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): return py_libpython == jl_libpython -def raise_separate_cache_error(runtime, pyprogramname): - message = """\ +_separate_cache_error_common_header = """\ It seems your Julia and PyJulia setup are not supported. Julia interpreter: {runtime} -Python interpreter used by PyCall.jl: - {pyprogramname} -Python interpreter used to import PyJulia. +Python interpreter and libpython used by PyCall.jl: + {jlinfo.pyprogramname} + {jl_libpython} +Python interpreter used to import PyJulia and its libpython. {sys.executable} + {py_libpython} +""" + -In Julia >= 0.7, above two paths to the Python interpreters have to match -exactly in order for PyJulia to work. To configure PyCall.jl to use Python +_separate_cache_error_common_footer = """ +For more information, see: + https://github.com/JuliaPy/pyjulia + https://github.com/JuliaPy/PyCall.jl +""" + + +_separate_cache_error_statically_linked = """ +Your Python interpreter "{sys.executable}" +is statically linked to libpython. Currently, PyJulia does not support +such Python interpreter. For available workarounds, see: + https://github.com/JuliaPy/pyjulia/issues/185 +""" + + +_separate_cache_error_incompatible_libpython = """ +In Julia >= 0.7, above two paths to `libpython` have to match exactly +in order for PyJulia to work. To configure PyCall.jl to use Python interpreter "{sys.executable}", run the following commands in the Julia interpreter: ENV["PYTHON"] = "{sys.executable}" using Pkg Pkg.build("PyCall") +""" -For more information, see: - https://github.com/JuliaPy/pyjulia - https://github.com/JuliaPy/PyCall.jl - """.format(runtime=runtime, pyprogramname=pyprogramname, - sys=sys) + +def raise_separate_cache_error(runtime, jlinfo): + template = _separate_cache_error_common_header + if determine_if_statically_linked(): + template += _separate_cache_error_statically_linked + else: + template += _separate_cache_error_incompatible_libpython + template += _separate_cache_error_common_footer + message = template.format( + runtime=runtime, + jlinfo=jlinfo, + py_libpython=find_libpython(), + jl_libpython=normalize_path(jlinfo.libpython), + sys=sys) raise RuntimeError(message) @@ -539,7 +568,7 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, raise RuntimeError( "PyJulia does not support Julia < 0.6 anymore") elif version_range == 1: - raise_separate_cache_error(runtime, depsjlexe) + raise_separate_cache_error(runtime, jlinfo) # Intercept precompilation os.environ["PYCALL_PYTHON_EXE"] = sys.executable os.environ["PYCALL_JULIA_HOME"] = PYCALL_JULIA_HOME From 8d747958b4d263551e9c5706e7191aa8d25accd0 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 5 Sep 2018 19:47:08 -0700 Subject: [PATCH 4/5] Rename: test_utils.py -> test_find_libpython.py --- test/{test_utils.py => test_find_libpython.py} | 4 ---- 1 file changed, 4 deletions(-) rename test/{test_utils.py => test_find_libpython.py} (91%) diff --git a/test/test_utils.py b/test/test_find_libpython.py similarity index 91% rename from test/test_utils.py rename to test/test_find_libpython.py index 0b6dc5c3..cf67b7ae 100644 --- a/test/test_utils.py +++ b/test/test_find_libpython.py @@ -1,7 +1,3 @@ -""" -Unit tests which can be done without loading `libjulia`. -""" - import platform import pytest From caeef4f6f47ff151c4b5885f038837bc02b3a414 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 5 Sep 2018 19:55:45 -0700 Subject: [PATCH 5/5] Test raise_separate_cache_error --- julia/core.py | 7 +++++-- test/test_utils.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 test/test_utils.py diff --git a/julia/core.py b/julia/core.py index eed7122a..47b6855c 100644 --- a/julia/core.py +++ b/julia/core.py @@ -385,9 +385,12 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): """ -def raise_separate_cache_error(runtime, jlinfo): +def raise_separate_cache_error( + runtime, jlinfo, + # For test: + _determine_if_statically_linked=determine_if_statically_linked): template = _separate_cache_error_common_header - if determine_if_statically_linked(): + if _determine_if_statically_linked(): template += _separate_cache_error_statically_linked else: template += _separate_cache_error_incompatible_libpython diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 00000000..98f9e872 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,42 @@ +""" +Unit tests which can be done without loading `libjulia`. +""" + +import os + +import pytest + +from julia.core import raise_separate_cache_error + +try: + from types import SimpleNamespace +except ImportError: + from argparse import Namespace as SimpleNamespace # Python 2 + + +def dummy_juliainfo(): + somepath = os.devnull # some random path + return SimpleNamespace( + pyprogramname=somepath, + libpython=somepath, + ) + + +def test_raise_separate_cache_error_statically_linked(): + runtime = "julia" + jlinfo = dummy_juliainfo() + with pytest.raises(RuntimeError) as excinfo: + raise_separate_cache_error( + runtime, jlinfo, + _determine_if_statically_linked=lambda: True) + assert "is statically linked" in str(excinfo.value) + + +def test_raise_separate_cache_error_dynamically_linked(): + runtime = "julia" + jlinfo = dummy_juliainfo() + with pytest.raises(RuntimeError) as excinfo: + raise_separate_cache_error( + runtime, jlinfo, + _determine_if_statically_linked=lambda: False) + assert "have to match exactly" in str(excinfo.value)