From c331f32500a78f018eb929291b6c339320af397c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 10:26:16 +0100 Subject: [PATCH 001/110] tmp --- external-deps/spyder-kernels-server/README.md | 1 + external-deps/spyder-kernels-server/setup.py | 73 +++++ .../spyder_kernels_server/__init__.py | 2 + .../spyder_kernels_server/__main__.py | 56 ++++ .../spyder_kernels_server/_version.py | 12 + .../spyder_kernels_server/kernel_client.py | 0 .../spyder_kernels_server/kernel_comm.py | 1 - .../spyder_kernels_server/kernel_manager.py | 2 +- .../spyder_kernels_server/kernel_server.py | 174 ++++++++++++ .../ipythonconsole/utils/kernel_handler.py | 265 ++++-------------- .../plugins/ipythonconsole/widgets/client.py | 42 --- .../ipythonconsole/widgets/main_widget.py | 15 +- .../plugins/ipythonconsole/widgets/mixins.py | 39 ++- .../plugins/ipythonconsole/widgets/shell.py | 9 +- spyder/plugins/maininterpreter/confpage.py | 92 +++++- 15 files changed, 511 insertions(+), 272 deletions(-) create mode 100644 external-deps/spyder-kernels-server/README.md create mode 100644 external-deps/spyder-kernels-server/setup.py create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/_version.py rename spyder/plugins/ipythonconsole/utils/client.py => external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py (100%) rename spyder/plugins/ipythonconsole/comms/kernelcomm.py => external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py (99%) rename spyder/plugins/ipythonconsole/utils/manager.py => external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py (98%) create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py diff --git a/external-deps/spyder-kernels-server/README.md b/external-deps/spyder-kernels-server/README.md new file mode 100644 index 00000000000..30404ce4c54 --- /dev/null +++ b/external-deps/spyder-kernels-server/README.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/external-deps/spyder-kernels-server/setup.py b/external-deps/spyder-kernels-server/setup.py new file mode 100644 index 00000000000..916e66fba76 --- /dev/null +++ b/external-deps/spyder-kernels-server/setup.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- + +"""Jupyter Kernels for the Spyder consoles.""" + +# Standard library imports +import ast +import io +import os + +# Third party imports +from setuptools import find_packages, setup + +HERE = os.path.abspath(os.path.dirname(__file__)) + +with io.open('README.md', encoding='utf-8') as f: + LONG_DESCRIPTION = f.read() + + +def get_version(module='spyder_kernels_server'): + """Get version.""" + with open(os.path.join(HERE, module, '_version.py'), 'r') as f: + data = f.read() + lines = data.split('\n') + for line in lines: + if line.startswith('VERSION_INFO'): + version_tuple = ast.literal_eval(line.split('=')[-1].strip()) + version = '.'.join(map(str, version_tuple)) + break + return version + + +REQUIREMENTS = [ + 'spyder-kernels', +] + +setup( + name='spyder-kernels-server', + version=get_version(), + keywords='spyder jupyter kernel ipython console', + url='https://github.com/spyder-ide/spyder-kernels', + download_url="https://www.spyder-ide.org/#fh5co-download", + license='MIT', + author='Spyder Development Team', + author_email="spyderlib@googlegroups.com", + description="Jupyter kernels for Spyder's console", + long_description=LONG_DESCRIPTION, + long_description_content_type='text/markdown', + packages=find_packages(exclude=['docs', '*tests']), + install_requires=REQUIREMENTS, + # extras_require={'test': TEST_REQUIREMENTS}, + include_package_data=True, + python_requires='>=3.7', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Framework :: Jupyter', + 'Framework :: IPython', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: Software Development :: Interpreters', + ] +) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py new file mode 100644 index 00000000000..633f866158a --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- + diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py new file mode 100644 index 00000000000..0d69363c8c8 --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- +import sys +import zmq +import json +from spyder_kernels_server.kernel_server import KernelServer + + +def main(port): + if len(sys.argv) > 1: + port = sys.argv[1] + + context = zmq.Context() + socket = context.socket(zmq.REP) + socket.bind("tcp://*:%s" % port) + print(f"Server running on port {port}") + kernel_server = KernelServer() + shutdown = False + + while not shutdown: + # Wait for next request from client + message = socket.recv_pyobj() + print(message) + cmd = message[0] + if cmd == "shutdown": + socket.send_pyobj(["shutting_down"]) + shutdown = True + kernel_server.shutdown() + + elif cmd == "open_kernel": + try: + cf = kernel_server.open_kernel(message[1]) + print(cf) + with open(cf, "br") as f: + cf = (cf, json.load(f)) + + except Exception as e: + cf = ("error", e) + socket.send_pyobj(["new_kernel", *cf]) + + elif cmd == "close_kernel": + socket.send_pyobj(["closing_kernel"]) + try: + kernel_server.close_kernel(message[1]) + except Exception: + pass + + +if __name__ == "__main__": + port = "5556" + main(port) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py b/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py new file mode 100644 index 00000000000..b9e5bdfe787 --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py @@ -0,0 +1,12 @@ +# +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- + +"""Version File.""" + +VERSION_INFO = (1, 0, 0, 'dev0') +__version__ = '.'.join(map(str, VERSION_INFO)) diff --git a/spyder/plugins/ipythonconsole/utils/client.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py similarity index 100% rename from spyder/plugins/ipythonconsole/utils/client.py rename to external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py diff --git a/spyder/plugins/ipythonconsole/comms/kernelcomm.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py similarity index 99% rename from spyder/plugins/ipythonconsole/comms/kernelcomm.py rename to external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py index c8636ec1f47..cc9f1fd6dce 100644 --- a/spyder/plugins/ipythonconsole/comms/kernelcomm.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py @@ -15,7 +15,6 @@ from qtpy.QtCore import QEventLoop, QObject, QTimer, Signal from spyder_kernels.comms.commbase import CommBase -from spyder.py3compat import TimeoutError logger = logging.getLogger(__name__) TIMEOUT_KERNEL_START = 30 diff --git a/spyder/plugins/ipythonconsole/utils/manager.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py similarity index 98% rename from spyder/plugins/ipythonconsole/utils/manager.py rename to external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py index 0beabef9531..a24c9c4c614 100644 --- a/spyder/plugins/ipythonconsole/utils/manager.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py @@ -30,7 +30,7 @@ class SpyderKernelManager(QtKernelManager): """ client_class = DottedObjectName( - 'spyder.plugins.ipythonconsole.utils.client.SpyderKernelClient') + 'spyder_kernels_server.kernel_client.SpyderKernelClient') def __init__(self, *args, **kwargs): self.shutting_down = False diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py new file mode 100644 index 00000000000..b2db62395e1 --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- + +"""Jupyter Kernels for the Spyder consoles.""" + +# Standard library imports +import os +import os.path as osp +from subprocess import PIPE +import uuid + +from threading import Thread + + +# Third-party imports +from jupyter_core.paths import jupyter_runtime_dir + +from spyder_kernels_server.kernel_manager import SpyderKernelManager +from spyder_kernels_server.kernel_comm import KernelComm + + +PERMISSION_ERROR_MSG = ( + "The directory {} is not writable and it is required to create IPython " + "consoles. Please make it writable." +) + +# kernel_comm needs a qthread +class StdThread(Thread): + """Poll for changes in std buffers.""" + def __init__(self, std_buffer, buffer_key, kernel_comm): + self._std_buffer = std_buffer + + self.buffer_key = buffer_key + self.kernel_comm = kernel_comm + super().__init__() + + def run(self): + txt = True + while txt: + txt = self._std_buffer.read1() + if txt: + # Needs to be on control so the message is sent to currently + # executing shell + self.kernel_comm.remote_call( + interrupt=True + ).print_remote( + txt.decode(), + self.buffer_key + ) + + +class ShutdownThread(Thread): + def __init__(self, kernel_dict): + self.kernel_dict = kernel_dict + super().__init__() + + def run(self): + """Shutdown kernel.""" + kernel_manager = self.kernel_dict["kernel"] + + if not kernel_manager.shutting_down: + kernel_manager.shutting_down = True + try: + kernel_manager.shutdown_kernel() + except Exception: + # kernel was externally killed + pass + if "stdout" in self.kernel_dict: + self.kernel_dict["stdout" ].join() + if "stderr" in self.kernel_dict: + self.kernel_dict["stderr" ].join() + + +class KernelServer: + + def __init__(self): + self._kernel_list = {} + + @staticmethod + def new_connection_file(): + """ + Generate a new connection file + + Taken from jupyter_client/console_app.py + Licensed under the BSD license + """ + # Check if jupyter_runtime_dir exists (Spyder addition) + if not osp.isdir(jupyter_runtime_dir()): + try: + os.makedirs(jupyter_runtime_dir()) + except (IOError, OSError): + return None + cf = "" + while not cf: + ident = str(uuid.uuid4()).split("-")[-1] + cf = os.path.join(jupyter_runtime_dir(), "kernel-%s.json" % ident) + cf = cf if not os.path.exists(cf) else "" + return cf + + + def open_kernel(self, kernel_spec): + """ + Create a new kernel. + + Might raise all kinds of exceptions + """ + connection_file = self.new_connection_file() + if connection_file is None: + raise RuntimeError( + PERMISSION_ERROR_MSG.format(jupyter_runtime_dir()) + ) + + # Kernel manager + kernel_manager = SpyderKernelManager( + connection_file=connection_file, + config=None, + autorestart=True, + ) + + kernel_manager._kernel_spec = kernel_spec + + kernel_manager.start_kernel( + stderr=PIPE, + stdout=PIPE, + env=kernel_spec.env, + ) + + kernel_key = connection_file + self._kernel_list[kernel_key] = { + "kernel": kernel_manager + } + + kernel_client = kernel_manager.client() + kernel_client.start_channels() + kernel_comm = KernelComm() + kernel_comm.open_comm(kernel_client) + self.connect_std_pipes(kernel_key, kernel_comm) + + + return connection_file + + def connect_std_pipes(self, kernel_key, kernel_comm): + """Connect to std pipes.""" + + kernel_manager = self._kernel_list[kernel_key]["kernel"] + stdout = kernel_manager.provisioner.process.stdout + stderr = kernel_manager.provisioner.process.stderr + + if stdout: + stdout_thread = StdThread( + stdout, "stdout", kernel_comm) + stdout_thread.start() + self._kernel_list[kernel_key]["stdout"] = stdout_thread + if stderr: + stderr_thread = StdThread( + stderr, "stderr", kernel_comm) + stderr_thread.start() + self._kernel_list[kernel_key]["stderr"] = stderr_thread + + def close_kernel(self, kernel_key): + """Close kernel""" + kernel_manager = self._kernel_list[kernel_key]["kernel"] + kernel_manager.stop_restarter() + shutdown_thread = ShutdownThread(self._kernel_list.pop(kernel_key)) + shutdown_thread.start() + + def shutdown(self): + for kernel_key in self._kernel_list: + self.close_kernel(kernel_key) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 26935b795f4..488d244f4ea 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -9,14 +9,10 @@ # Standard library imports import ast import os -import os.path as osp -from subprocess import PIPE -from threading import Lock import uuid # Third-party imports -from jupyter_core.paths import jupyter_runtime_dir -from qtpy.QtCore import QObject, QThread, Signal, Slot +from qtpy.QtCore import QObject, Signal from zmq.ssh import tunnel as zmqtunnel # Local imports @@ -24,9 +20,8 @@ from spyder.plugins.ipythonconsole import ( SPYDER_KERNELS_MIN_VERSION, SPYDER_KERNELS_MAX_VERSION, SPYDER_KERNELS_VERSION, SPYDER_KERNELS_CONDA, SPYDER_KERNELS_PIP) -from spyder.plugins.ipythonconsole.comms.kernelcomm import KernelComm -from spyder.plugins.ipythonconsole.utils.manager import SpyderKernelManager -from spyder.plugins.ipythonconsole.utils.client import SpyderKernelClient +from spyder_kernels_server.kernel_comm import KernelComm +from spyder_kernels_server.kernel_client import SpyderKernelClient from spyder.plugins.ipythonconsole.utils.ssh import openssh_tunnel from spyder.utils.programs import check_version_range @@ -89,39 +84,12 @@ class KernelConnectionState: Closed = 'closed' -class StdThread(QThread): - """Poll for changes in std buffers.""" - sig_out = Signal(str) - - def __init__(self, parent, std_buffer): - super().__init__(parent) - self._std_buffer = std_buffer - self._closing = False - - def run(self): - txt = True - while txt: - txt = self._std_buffer.read1() - if txt: - self.sig_out.emit(txt.decode()) - - class KernelHandler(QObject): """ A class to handle the kernel in several ways and store kernel connection information. """ - sig_stdout = Signal(str) - """ - A stdout message was received on the process stdout. - """ - - sig_stderr = Signal(str) - """ - A stderr message was received on the process stderr. - """ - sig_fault = Signal(str) """ A fault message was received. @@ -137,10 +105,15 @@ class KernelHandler(QObject): The kernel raised an error while connecting. """ + sig_request_close = Signal(str) + """ + This kernel would like to be closed + """ + def __init__( self, connection_file, - kernel_manager=None, + kernel_spec=None, kernel_client=None, known_spyder_kernel=False, hostname=None, @@ -150,7 +123,7 @@ def __init__( super().__init__() # Connection Informations self.connection_file = connection_file - self.kernel_manager = kernel_manager + self.kernel_spec = kernel_spec self.kernel_client = kernel_client self.known_spyder_kernel = known_spyder_kernel self.hostname = hostname @@ -165,18 +138,12 @@ def __init__( self.handle_comm_ready) # Internal - self._shutdown_thread = None - self._shutdown_lock = Lock() - self._stdout_thread = None - self._stderr_thread = None self._fault_args = None - self._init_stderr = "" - self._init_stdout = "" self._spyder_kernel_info_uuid = None self._shellwidget_connected = False # Start kernel - self.connect_std_pipes() + # self.connect_std_pipes() self.kernel_client.start_channels() self.check_kernel_info() @@ -191,13 +158,13 @@ def connect(self): elif self.connection_state == KernelConnectionState.Error: self.sig_kernel_connection_error.emit() - # Show initial io - if self._init_stderr: - self.sig_stderr.emit(self._init_stderr) - self._init_stderr = None - if self._init_stdout: - self.sig_stdout.emit(self._init_stdout) - self._init_stdout = None + # # Show initial io + # if self._init_stderr: + # self.sig_stderr.emit(self._init_stderr) + # self._init_stderr = None + # if self._init_stdout: + # self.sig_stdout.emit(self._init_stdout) + # self._init_stdout = None def check_kernel_info(self): """Send request to check kernel info.""" @@ -282,81 +249,6 @@ def handle_comm_ready(self): self.connection_state = KernelConnectionState.SpyderKernelReady self.sig_kernel_is_ready.emit() - def connect_std_pipes(self): - """Connect to std pipes.""" - self.close_std_threads() - - # Connect new threads - if self.kernel_manager is None: - return - - stdout = self.kernel_manager.provisioner.process.stdout - stderr = self.kernel_manager.provisioner.process.stderr - - if stdout: - self._stdout_thread = StdThread(self, stdout) - self._stdout_thread.sig_out.connect(self.handle_stdout) - self._stdout_thread.start() - if stderr: - self._stderr_thread = StdThread(self, stderr) - self._stderr_thread.sig_out.connect(self.handle_stderr) - self._stderr_thread.start() - - def disconnect_std_pipes(self): - """Disconnect old std pipes.""" - if self._stdout_thread and not self._stdout_thread._closing: - self._stdout_thread.sig_out.disconnect(self.handle_stdout) - self._stdout_thread._closing = True - if self._stderr_thread and not self._stderr_thread._closing: - self._stderr_thread.sig_out.disconnect(self.handle_stderr) - self._stderr_thread._closing = True - - def close_std_threads(self): - """Close std threads.""" - if self._stdout_thread is not None: - self._stdout_thread.wait() - self._stdout_thread = None - if self._stderr_thread is not None: - self._stderr_thread.wait() - self._stderr_thread = None - - @Slot(str) - def handle_stderr(self, err): - """Handle stderr""" - if self._shellwidget_connected: - self.sig_stderr.emit(err) - else: - self._init_stderr += err - - @Slot(str) - def handle_stdout(self, out): - """Handle stdout""" - if self._shellwidget_connected: - self.sig_stdout.emit(out) - else: - self._init_stdout += out - - @staticmethod - def new_connection_file(): - """ - Generate a new connection file - - Taken from jupyter_client/console_app.py - Licensed under the BSD license - """ - # Check if jupyter_runtime_dir exists (Spyder addition) - if not osp.isdir(jupyter_runtime_dir()): - try: - os.makedirs(jupyter_runtime_dir()) - except (IOError, OSError): - return None - cf = "" - while not cf: - ident = str(uuid.uuid4()).split("-")[-1] - cf = os.path.join(jupyter_runtime_dir(), "kernel-%s.json" % ident) - cf = cf if not os.path.exists(cf) else "" - return cf - @staticmethod def tunnel_to_kernel( connection_info, hostname, sshkey=None, password=None, timeout=10 @@ -380,45 +272,34 @@ def tunnel_to_kernel( return tuple(lports) @classmethod - def new_from_spec(cls, kernel_spec): + def new_from_spec( + cls, kernel_spec, connection_file, connection_info, + hostname=None, sshkey=None, password=None + ): """ Create a new kernel. - - Might raise all kinds of exceptions """ - connection_file = cls.new_connection_file() - if connection_file is None: - raise RuntimeError( - PERMISSION_ERROR_MSG.format(jupyter_runtime_dir()) + kernel_client = SpyderKernelClient() + kernel_client.load_connection_info(connection_info) + kernel_client = cls.tunnel_kernel_client( + kernel_client, + hostname, + sshkey, + password ) - # Kernel manager - kernel_manager = SpyderKernelManager( - connection_file=connection_file, - config=None, - autorestart=True, - ) - - kernel_manager._kernel_spec = kernel_spec - - kernel_manager.start_kernel( - stderr=PIPE, - stdout=PIPE, - env=kernel_spec.env, - ) - - # Kernel client - kernel_client = kernel_manager.client() - # Increase time (in seconds) to detect if a kernel is alive. # See spyder-ide/spyder#3444. kernel_client.hb_channel.time_to_dead = 25.0 return cls( connection_file=connection_file, - kernel_manager=kernel_manager, + kernel_spec=kernel_spec, kernel_client=kernel_client, known_spyder_kernel=True, + hostname=hostname, + sshkey=sshkey, + password=password, ) @classmethod @@ -426,22 +307,6 @@ def from_connection_file( cls, connection_file, hostname=None, sshkey=None, password=None ): """Create kernel for given connection file.""" - return cls( - connection_file, - hostname=hostname, - sshkey=sshkey, - password=password, - kernel_client=cls.init_kernel_client( - connection_file, - hostname, - sshkey, - password - ) - ) - - @classmethod - def init_kernel_client(cls, connection_file, hostname, sshkey, password): - """Create kernel client.""" kernel_client = SpyderKernelClient( connection_file=connection_file ) @@ -459,6 +324,24 @@ def init_kernel_client(cls, connection_file, hostname, sshkey, password): + str(e) ) + kernel_client = cls.tunnel_kernel_client( + kernel_client, + hostname, + sshkey, + password + ) + + return cls( + connection_file, + hostname=hostname, + sshkey=sshkey, + password=password, + kernel_client=kernel_client + ) + + @classmethod + def tunnel_kernel_client(cls, kernel_client, hostname, sshkey, password): + """Create kernel client.""" if hostname is not None: try: connection_info = dict( @@ -490,20 +373,8 @@ def close(self, shutdown_kernel=True, now=False): """Close kernel""" self.close_comm() - if shutdown_kernel and self.kernel_manager is not None: - km = self.kernel_manager - km.stop_restarter() - self.disconnect_std_pipes() - - if now: - km.shutdown_kernel(now=True) - self.after_shutdown() - else: - shutdown_thread = QThread(None) - shutdown_thread.run = self._thread_shutdown_kernel - shutdown_thread.start() - shutdown_thread.finished.connect(self.after_shutdown) - self._shutdown_thread = shutdown_thread + if shutdown_kernel and self.kernel_spec is not None: + self.sig_request_close.emit(self.connection_file) if ( self.kernel_client is not None @@ -515,34 +386,6 @@ def after_shutdown(self): """Cleanup after shutdown""" self.close_std_threads() self.kernel_comm.remove(only_closing=True) - self._shutdown_thread = None - - def _thread_shutdown_kernel(self): - """Shutdown kernel.""" - with self._shutdown_lock: - # Avoid calling shutdown_kernel on the same manager twice - # from different threads to avoid crash. - if self.kernel_manager.shutting_down: - return - self.kernel_manager.shutting_down = True - try: - self.kernel_manager.shutdown_kernel() - except Exception: - # kernel was externally killed - pass - - def wait_shutdown_thread(self): - """Wait shutdown thread.""" - thread = self._shutdown_thread - if thread is None: - return - if thread.isRunning(): - try: - thread.kernel_manager._kill_kernel() - except Exception: - pass - thread.quit() - thread.wait() def copy(self): """Copy kernel.""" @@ -558,7 +401,7 @@ def copy(self): return self.__class__( connection_file=self.connection_file, - kernel_manager=self.kernel_manager, + kernel_spec=self.kernel_spec, known_spyder_kernel=self.known_spyder_kernel, hostname=self.hostname, sshkey=self.sshkey, diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 58c906467ee..d4e18e2c741 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -320,8 +320,6 @@ def connect_kernel(self, kernel_handler, first_connect=True): self.kernel_handler = kernel_handler # Connect standard streams. - kernel_handler.sig_stderr.connect(self.print_stderr) - kernel_handler.sig_stdout.connect(self.print_stdout) kernel_handler.sig_fault.connect(self.print_fault) kernel_handler.sig_kernel_is_ready.connect( self._when_kernel_is_ready) @@ -336,51 +334,11 @@ def disconnect_kernel(self, shutdown_kernel): if not kernel_handler: return - kernel_handler.sig_stderr.disconnect(self.print_stderr) - kernel_handler.sig_stdout.disconnect(self.print_stdout) kernel_handler.sig_fault.disconnect(self.print_fault) self.shellwidget.disconnect_kernel(shutdown_kernel) self.kernel_handler = None - @Slot(str) - def print_stderr(self, stderr): - """Print stderr written in PIPE.""" - if not stderr: - return - - if self.is_benign_error(stderr): - return - - if self.shellwidget.isHidden(): - error_text = '%s' % stderr - # Avoid printing the same thing again - if self.error_text != error_text: - if self.error_text: - # Append to error text - error_text = self.error_text + error_text - self.show_kernel_error(error_text) - - if self.shellwidget._starting: - self.shellwidget.banner = ( - stderr + '\n' + self.shellwidget.banner) - else: - self.shellwidget._append_plain_text( - stderr, before_prompt=True) - - @Slot(str) - def print_stdout(self, stdout): - """Print stdout written in PIPE.""" - if not stdout: - return - - if self.shellwidget._starting: - self.shellwidget.banner = ( - stdout + '\n' + self.shellwidget.banner) - else: - self.shellwidget._append_plain_text( - stdout, before_prompt=True) - def connect_shellwidget_signals(self): """Configure shellwidget after kernel is connected.""" # Set exit callback diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 57a9a7efc68..3699d00e7f8 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -38,7 +38,8 @@ from spyder.plugins.ipythonconsole.widgets import ( ClientWidget, ConsoleRestartDialog, COMPLETION_WIDGET_TYPE, KernelConnectionDialog, PageControlWidget) -from spyder.plugins.ipythonconsole.widgets.mixins import CachedKernelMixin +from spyder.plugins.ipythonconsole.widgets.mixins import ( + CachedKernelMixin, KernelConnectorMixin) from spyder.py3compat import PY38_OR_MORE from spyder.utils import encoding, programs, sourcecode from spyder.utils.misc import get_error_match, remove_backslashes @@ -114,7 +115,9 @@ class IPythonConsoleWidgetTabsContextMenuSections: # --- Widgets # ---------------------------------------------------------------------------- -class IPythonConsoleWidget(PluginMainWidget, CachedKernelMixin): +class IPythonConsoleWidget( + PluginMainWidget, CachedKernelMixin, KernelConnectorMixin +): """ IPython Console plugin @@ -1402,7 +1405,7 @@ def create_client_for_kernel(self, connection_file, hostname, sshkey, related_clients = [] for cl in self.clients: - if connection_file in cl.connection_file: + if cl.connection_file and connection_file in cl.connection_file: if ( cl.kernel_handler is not None and hostname == cl.kernel_handler.hostname and @@ -1772,8 +1775,8 @@ def restart_kernel(self, client=None, ask_before_restart=True): if client is None: return - km = client.kernel_handler.kernel_manager - if km is None: + ks = client.kernel_handler.kernel_spec + if ks is None: client.shellwidget._append_plain_text( _('Cannot restart a kernel not started by Spyder\n'), before_prompt=True @@ -1798,7 +1801,7 @@ def restart_kernel(self, client=None, ask_before_restart=True): # Get new kernel try: - kernel_handler = self.get_cached_kernel(km._kernel_spec) + kernel_handler = self.get_cached_kernel(ks) except Exception as e: client.show_kernel_error(e) return diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 7c5b829d77c..792a42f51a4 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -7,14 +7,47 @@ """ IPython Console mixins. """ +import zmq # Local imports from spyder.plugins.ipythonconsole.utils.kernel_handler import KernelHandler +class KernelConnectorMixin: + """Needs https://github.com/jupyter/jupyter_client/pull/835""" + def __init__(self): + super().__init__() + self.port = "5556" + self.context = zmq.Context() + self.socket = self.context.socket(zmq.REQ) + self.socket.connect("tcp://localhost:%s" % self.port) + + def new_kernel(self, kernel_spec): + """Get a new kernel""" + self.socket.send_pyobj(["open_kernel", kernel_spec]) + cmd, connection_file, connection_info = self.socket.recv_pyobj() + if connection_file == "error": + raise connection_info + + hostname, sshkey, password = None, None, None + + kernel_handler = KernelHandler.new_from_spec( + kernel_spec, connection_file, connection_info, + hostname, sshkey, password + ) + + kernel_handler.sig_request_close.connect(self.close_kernel) + return kernel_handler + + def close_kernel(self, connection_file): + self.socket.send_pyobj(["close_kernel", connection_file]) + # Wait for confirmation + self.socket.recv_pyobj() + + + class CachedKernelMixin: """Cached kernel mixin.""" - def __init__(self): super().__init__() self._cached_kernel_properties = None @@ -56,7 +89,7 @@ def check_cached_kernel_spec(self, kernel_spec): def get_cached_kernel(self, kernel_spec, cache=True): """Get a new kernel, and cache one for next time.""" # Cache another kernel for next time. - new_kernel_handler = KernelHandler.new_from_spec(kernel_spec) + new_kernel_handler = self.new_kernel(kernel_spec) if not cache: # remove/don't use cache if requested @@ -81,6 +114,6 @@ def get_cached_kernel(self, kernel_spec, cache=True): ) if cached_kernel_handler is None: - return KernelHandler.new_from_spec(kernel_spec) + return self.new_kernel(kernel_spec) return cached_kernel_handler diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 757e55d6b1d..053c197df24 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -150,7 +150,6 @@ def __init__(self, ipyclient, additional_options, interpreter_versions, self.set_bracket_matcher_color_scheme(self.syntax_style) self.shutting_down = False - self.kernel_manager = None self.kernel_client = None self._init_kernel_setup = False handlers.update({ @@ -197,7 +196,6 @@ def connect_kernel(self, kernel_handler, first_connect=True): kernel_client.stopped_channels.connect(self.notify_deleted) self.kernel_client = kernel_client - self.kernel_manager = kernel_handler.kernel_manager self.kernel_handler = kernel_handler if first_connect: @@ -254,7 +252,6 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): self.reset_kernel_state() self.kernel_client = None - self.kernel_manager = None self.kernel_handler = None def handle_kernel_is_ready(self): @@ -332,7 +329,7 @@ def call_kernel(self, interrupt=False, blocking=False, callback=None, @property def is_external_kernel(self): """Check if this is an external kernel.""" - return self.kernel_manager is None + return self.kernel_handler.kernel_spec is None def setup_spyder_kernel(self): """Setup spyder kernel""" @@ -728,7 +725,7 @@ def _perform_reset(self, message): # kernels. # See spyder-ide/spyder#9505. try: - kernel_env = self.kernel_manager._kernel_spec.env + kernel_env = self.kernel_handler.kernel_spec.env except AttributeError: kernel_env = {} @@ -1098,7 +1095,7 @@ def _banner_default(self): def _kernel_restarted_message(self, died=True): msg = _("Kernel died, restarting") if died else _("Kernel restarting") - if died and self.kernel_manager is None: + if died and self.is_external_kernel: # The kernel might never restart, show position of fault file msg += ( "\n" + _("Its crash file is located at:") + " " diff --git a/spyder/plugins/maininterpreter/confpage.py b/spyder/plugins/maininterpreter/confpage.py index 0f4c3ff795d..4e5bbeda0ed 100644 --- a/spyder/plugins/maininterpreter/confpage.py +++ b/spyder/plugins/maininterpreter/confpage.py @@ -12,8 +12,11 @@ import sys # Third party imports -from qtpy.QtWidgets import (QButtonGroup, QGroupBox, QInputDialog, QLabel, - QLineEdit, QMessageBox, QPushButton, QVBoxLayout) +from qtpy.QtWidgets import ( + QButtonGroup, QGroupBox, QInputDialog, QLabel, + QLineEdit, QMessageBox, QPushButton, QVBoxLayout, QRadioButton, + QHBoxLayout, QGridLayout, QSpacerItem) +from qtpy.compat import getopenfilename # Local imports from spyder.api.translations import get_translation @@ -23,6 +26,7 @@ from spyder.utils.conda import get_list_conda_envs_cache from spyder.utils.misc import get_python_executable from spyder.utils.pyenv import get_list_pyenv_envs_cache +from spyder.config.base import get_home_dir # Localization _ = get_translation('spyder') @@ -66,6 +70,84 @@ def initialize(self): def setup_page(self): newcb = self.create_checkbox + + # Remote kernel groupbox + self.rm_group = QGroupBox(_("Use a remote kernel server (via SSH)")) + + # SSH connection + hn_label = QLabel(_('Hostname:')) + self.hn = QLineEdit() + pn_label = QLabel(_('Port:')) + self.pn = QLineEdit() + self.pn.setMaximumWidth(75) + + un_label = QLabel(_('Username:')) + self.un = QLineEdit() + + # SSH authentication + auth_group = QGroupBox(_("Authentication method:")) + self.pw_radio = QRadioButton() + pw_label = QLabel(_('Password:')) + self.kf_radio = QRadioButton() + kf_label = QLabel(_('SSH keyfile:')) + + self.pw = QLineEdit() + self.pw.setEchoMode(QLineEdit.Password) + self.pw_radio.toggled.connect(self.pw.setEnabled) + self.kf_radio.toggled.connect(self.pw.setDisabled) + + self.kf = QLineEdit() + kf_open_btn = QPushButton(_('Browse')) + kf_open_btn.clicked.connect(self.select_ssh_key) + kf_layout = QHBoxLayout() + kf_layout.addWidget(self.kf) + kf_layout.addWidget(kf_open_btn) + + kfp_label = QLabel(_('Passphase:')) + self.kfp = QLineEdit() + self.kfp.setPlaceholderText(_('Optional')) + self.kfp.setEchoMode(QLineEdit.Password) + + self.kf_radio.toggled.connect(self.kf.setEnabled) + self.kf_radio.toggled.connect(self.kfp.setEnabled) + self.kf_radio.toggled.connect(kf_open_btn.setEnabled) + self.kf_radio.toggled.connect(kfp_label.setEnabled) + self.pw_radio.toggled.connect(self.kf.setDisabled) + self.pw_radio.toggled.connect(self.kfp.setDisabled) + self.pw_radio.toggled.connect(kf_open_btn.setDisabled) + self.pw_radio.toggled.connect(kfp_label.setDisabled) + + # SSH layout + ssh_layout = QGridLayout() + ssh_layout.addWidget(hn_label, 0, 0, 1, 2) + ssh_layout.addWidget(self.hn, 0, 2) + ssh_layout.addWidget(pn_label, 0, 3) + ssh_layout.addWidget(self.pn, 0, 4) + ssh_layout.addWidget(un_label, 1, 0, 1, 2) + ssh_layout.addWidget(self.un, 1, 2, 1, 3) + + # SSH authentication layout + auth_layout = QGridLayout() + auth_layout.addWidget(self.pw_radio, 1, 0) + auth_layout.addWidget(pw_label, 1, 1) + auth_layout.addWidget(self.pw, 1, 2) + auth_layout.addWidget(self.kf_radio, 2, 0) + auth_layout.addWidget(kf_label, 2, 1) + auth_layout.addLayout(kf_layout, 2, 2) + auth_layout.addWidget(kfp_label, 3, 1) + auth_layout.addWidget(self.kfp, 3, 2) + auth_group.setLayout(auth_layout) + + # Remote kernel layout + rm_layout = QVBoxLayout() + rm_layout.addLayout(ssh_layout) + rm_layout.addSpacerItem(QSpacerItem(0, 8)) + rm_layout.addWidget(auth_group) + self.rm_group.setLayout(rm_layout) + self.rm_group.setCheckable(True) + self.rm_group.toggled.connect(self.pw_radio.setChecked) + + # Python executable Group pyexec_group = QGroupBox(_("Python interpreter")) pyexec_bg = QButtonGroup(pyexec_group) @@ -104,6 +186,7 @@ def setup_page(self): self.def_exec_radio.toggled.connect(self.cus_exec_combo.setDisabled) self.cus_exec_radio.toggled.connect(self.cus_exec_combo.setEnabled) pyexec_layout.addWidget(self.cus_exec_combo) + pyexec_layout.addWidget(self.rm_group) pyexec_group.setLayout(pyexec_layout) self.pyexec_edit = self.cus_exec_combo.combobox.lineEdit() @@ -159,6 +242,11 @@ def setup_page(self): vlayout.addStretch(1) self.setLayout(vlayout) + def select_ssh_key(self): + kf = getopenfilename(self, _('Select SSH keyfile'), + get_home_dir(), '*.pem;;*')[0] + self.kf.setText(kf) + def warn_python_compatibility(self, pyexec): if not osp.isfile(pyexec): return From 18d090a99c9509092f1152d820d72e21ea6cb05a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 10:23:05 +0100 Subject: [PATCH 002/110] run server locally --- RELEASE.md | 1 + external-deps/qtconsole/qtconsole/comms.py | 2 +- external-deps/spyder-kernels-server/.gitignore | 3 +++ installers/Windows/req-extras-pull-request.txt | 1 + spyder/plugins/ipythonconsole/widgets/mixins.py | 16 ++++++++++++---- 5 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 external-deps/spyder-kernels-server/.gitignore diff --git a/RELEASE.md b/RELEASE.md index dc9108d1251..01b89a9ef50 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -123,6 +123,7 @@ To release a new version of Spyder you need to follow these steps: * Update our subrepos with the following commands, but only if new versions are available! git subrepo pull external-deps/spyder-kernels + git subrepo pull external-deps/spyder-kernels-server git subrepo pull external-deps/python-lsp-server git subrepo pull external-deps/qtconsole diff --git a/external-deps/qtconsole/qtconsole/comms.py b/external-deps/qtconsole/qtconsole/comms.py index 74f8ce5c990..78cf5641cff 100644 --- a/external-deps/qtconsole/qtconsole/comms.py +++ b/external-deps/qtconsole/qtconsole/comms.py @@ -114,9 +114,9 @@ def get_comm(self, comm_id, closing=False): except KeyError: if closing: return - self.log.warning("No such comm: %s", comm_id) # don't create the list of keys if debug messages aren't enabled if self.log.isEnabledFor(logging.DEBUG): + self.log.debug("No such comm: %s", comm_id) self.log.debug("Current comms: %s", list(self.comms.keys())) # comm message handlers diff --git a/external-deps/spyder-kernels-server/.gitignore b/external-deps/spyder-kernels-server/.gitignore new file mode 100644 index 00000000000..570961b3a3a --- /dev/null +++ b/external-deps/spyder-kernels-server/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +*.egg-info +__pycache__ diff --git a/installers/Windows/req-extras-pull-request.txt b/installers/Windows/req-extras-pull-request.txt index df35dfafbff..766417e3f29 100644 --- a/installers/Windows/req-extras-pull-request.txt +++ b/installers/Windows/req-extras-pull-request.txt @@ -24,4 +24,5 @@ spyder-terminal>=1.2.2 # Spyder external dependencies (spyder-kernels and qdarkstyle) ./external-deps/spyder-kernels +./external-deps/spyder-kernels-server ./external-deps/qdarkstyle diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 792a42f51a4..c96adaebc85 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -8,6 +8,9 @@ IPython Console mixins. """ import zmq +import sys + +from qtpy.QtCore import QProcess # Local imports from spyder.plugins.ipythonconsole.utils.kernel_handler import KernelHandler @@ -17,11 +20,19 @@ class KernelConnectorMixin: """Needs https://github.com/jupyter/jupyter_client/pull/835""" def __init__(self): super().__init__() + self.hostname = None + self.sshkey = None + self.password = None + self.start_local_server() self.port = "5556" self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) self.socket.connect("tcp://localhost:%s" % self.port) + def start_local_server(self): + self.server = QProcess(self) + self.server.start(sys.executable, ["-m", "spyder_kernels_server"]) + def new_kernel(self, kernel_spec): """Get a new kernel""" self.socket.send_pyobj(["open_kernel", kernel_spec]) @@ -29,11 +40,9 @@ def new_kernel(self, kernel_spec): if connection_file == "error": raise connection_info - hostname, sshkey, password = None, None, None - kernel_handler = KernelHandler.new_from_spec( kernel_spec, connection_file, connection_info, - hostname, sshkey, password + self.hostname, self.sshkey, self.password ) kernel_handler.sig_request_close.connect(self.close_kernel) @@ -45,7 +54,6 @@ def close_kernel(self, connection_file): self.socket.recv_pyobj() - class CachedKernelMixin: """Cached kernel mixin.""" def __init__(self): From 7052977a0e1812f2d7d269e7e05b91079f2f9fb1 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 10:29:44 +0100 Subject: [PATCH 003/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "a1386a921" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "a1386a921" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 6 +++--- .../spyder-kernels/spyder_kernels/console/kernel.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 8289cc5780e..9436590b4fc 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -5,8 +5,8 @@ ; [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git - branch = master - commit = 34a6d634383f27c61cd92172be94c1b344bc5cf1 - parent = f80c897ac6aafb65c421da50a63e2092f32ff7e2 + branch = print_remote + commit = a1386a921df7ab5fdaa175ec9e6a754ec29bf607 + parent = 18d090a99c9509092f1152d820d72e21ea6cb05a method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 7949189469c..2e544186ba1 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -90,6 +90,7 @@ def __init__(self, *args, **kwargs): 'request_pdb_stop': self.shell.request_pdb_stop, 'raise_interrupt_signal': self.shell.raise_interrupt_signal, 'get_fault_text': self.get_fault_text, + 'print_remote': self.print_remote, } for call_id in handlers: self.frontend_comm.register_call_handler( @@ -1002,3 +1003,14 @@ def post_handler_hook(self): self.shell.register_debugger_sigint() # Reset tracing function so that pdb.set_trace works sys.settrace(None) + + def print_remote(self, text, file_name=None): + """Remote print""" + file = None + if file_name == "stdout": + file = sys.stdout + elif file_name == "stderr": + file = sys.stderr + print(text, file=file) + if file: + file.flush() From 69159cae8ce95bb02591c968a796911cf509f351 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 10:42:19 +0100 Subject: [PATCH 004/110] spyder-kernels-server --- installers/Windows/req-pull-request.txt | 1 + spyder/plugins/ipythonconsole/comms/tests/test_comms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/installers/Windows/req-pull-request.txt b/installers/Windows/req-pull-request.txt index 861f2e8f9cf..9e691fa7c57 100644 --- a/installers/Windows/req-pull-request.txt +++ b/installers/Windows/req-pull-request.txt @@ -3,4 +3,5 @@ spyder-terminal>=1.2.2 # Spyder external dependencies (spyder-kernels and qdarkstyle) ./external-deps/spyder-kernels +./external-deps/spyder-kernels-server ./external-deps/qdarkstyle diff --git a/spyder/plugins/ipythonconsole/comms/tests/test_comms.py b/spyder/plugins/ipythonconsole/comms/tests/test_comms.py index f8bf989e002..6248dc129a6 100644 --- a/spyder/plugins/ipythonconsole/comms/tests/test_comms.py +++ b/spyder/plugins/ipythonconsole/comms/tests/test_comms.py @@ -20,7 +20,7 @@ # Local imports from spyder_kernels.utils.test_utils import get_kernel from spyder_kernels.comms.frontendcomm import FrontendComm -from spyder.plugins.ipythonconsole.comms.kernelcomm import KernelComm +from spyder_kernels_server.kernel_comm import KernelComm # ============================================================================= From 5713f2bbd0a9142c9704c5078e73694b5b9e7e1b Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 12:42:25 +0100 Subject: [PATCH 005/110] kernel server preferences --- .../spyder_kernels_server/kernel_server.py | 2 +- spyder/config/main.py | 10 ++ spyder/plugins/maininterpreter/confpage.py | 156 ++++++++++-------- spyder/plugins/preferences/api.py | 25 +++ 4 files changed, 123 insertions(+), 70 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index b2db62395e1..bf944fb6f67 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -44,6 +44,7 @@ def run(self): while txt: txt = self._std_buffer.read1() if txt: + print(txt) # Needs to be on control so the message is sent to currently # executing shell self.kernel_comm.remote_call( @@ -141,7 +142,6 @@ def open_kernel(self, kernel_spec): kernel_comm.open_comm(kernel_client) self.connect_std_pipes(kernel_key, kernel_comm) - return connection_file def connect_std_pipes(self, kernel_key, kernel_comm): diff --git a/spyder/config/main.py b/spyder/config/main.py index 03350d028b5..0fb34a143d1 100644 --- a/spyder/config/main.py +++ b/spyder/config/main.py @@ -133,6 +133,16 @@ 'umr/namelist': [], 'custom_interpreters_list': [], 'custom_interpreter': '', + 'kernel_server/external_server': False, + 'kernel_server/use_ssh': False, + 'kernel_server/host': "127.0.0.1", + 'kernel_server/port': 5556, + 'kernel_server/username': "", + 'kernel_server/password_auth': True, + 'kernel_server/keyfile_auth': False, + 'kernel_server/password': "", + 'kernel_server/keyfile': "", + 'kernel_server/passphrase': "", }), ('ipython_console', { diff --git a/spyder/plugins/maininterpreter/confpage.py b/spyder/plugins/maininterpreter/confpage.py index 4e5bbeda0ed..0f2b0d74fb2 100644 --- a/spyder/plugins/maininterpreter/confpage.py +++ b/spyder/plugins/maininterpreter/confpage.py @@ -12,6 +12,7 @@ import sys # Third party imports +from qtpy.QtCore import Qt from qtpy.QtWidgets import ( QButtonGroup, QGroupBox, QInputDialog, QLabel, QLineEdit, QMessageBox, QPushButton, QVBoxLayout, QRadioButton, @@ -70,83 +71,105 @@ def initialize(self): def setup_page(self): newcb = self.create_checkbox - # Remote kernel groupbox - self.rm_group = QGroupBox(_("Use a remote kernel server (via SSH)")) + rm_group = self.create_checkable_groupbox( + _("Use an external kernel server"), + 'kernel_server/external_server', + ) - # SSH connection - hn_label = QLabel(_('Hostname:')) - self.hn = QLineEdit() - pn_label = QLabel(_('Port:')) - self.pn = QLineEdit() - self.pn.setMaximumWidth(75) + # Hostname Layout + hostname = self.create_lineedit( + _("Hostnane:"), + 'kernel_server/host', + alignment=Qt.Horizontal, + word_wrap=False + ) + port = self.create_spinbox( + ":", "", 'kernel_server/port', min_=1, max_=65535, step=1 + ) - un_label = QLabel(_('Username:')) - self.un = QLineEdit() + hostname_layout = QHBoxLayout() + hostname_layout.addWidget(hostname) + hostname_layout.addWidget(port) # SSH authentication - auth_group = QGroupBox(_("Authentication method:")) - self.pw_radio = QRadioButton() - pw_label = QLabel(_('Password:')) - self.kf_radio = QRadioButton() - kf_label = QLabel(_('SSH keyfile:')) - - self.pw = QLineEdit() - self.pw.setEchoMode(QLineEdit.Password) - self.pw_radio.toggled.connect(self.pw.setEnabled) - self.kf_radio.toggled.connect(self.pw.setDisabled) - - self.kf = QLineEdit() - kf_open_btn = QPushButton(_('Browse')) - kf_open_btn.clicked.connect(self.select_ssh_key) - kf_layout = QHBoxLayout() - kf_layout.addWidget(self.kf) - kf_layout.addWidget(kf_open_btn) - - kfp_label = QLabel(_('Passphase:')) - self.kfp = QLineEdit() - self.kfp.setPlaceholderText(_('Optional')) - self.kfp.setEchoMode(QLineEdit.Password) - - self.kf_radio.toggled.connect(self.kf.setEnabled) - self.kf_radio.toggled.connect(self.kfp.setEnabled) - self.kf_radio.toggled.connect(kf_open_btn.setEnabled) - self.kf_radio.toggled.connect(kfp_label.setEnabled) - self.pw_radio.toggled.connect(self.kf.setDisabled) - self.pw_radio.toggled.connect(self.kfp.setDisabled) - self.pw_radio.toggled.connect(kf_open_btn.setDisabled) - self.pw_radio.toggled.connect(kfp_label.setDisabled) - - # SSH layout - ssh_layout = QGridLayout() - ssh_layout.addWidget(hn_label, 0, 0, 1, 2) - ssh_layout.addWidget(self.hn, 0, 2) - ssh_layout.addWidget(pn_label, 0, 3) - ssh_layout.addWidget(self.pn, 0, 4) - ssh_layout.addWidget(un_label, 1, 0, 1, 2) - ssh_layout.addWidget(self.un, 1, 2, 1, 3) + auth_group = self.create_checkable_groupbox( + _("Authentication method (via SSH):"), + 'kernel_server/use_ssh', + ) + + username = self.create_lineedit( + _("Username:"), + 'kernel_server/username', + alignment=Qt.Horizontal, + word_wrap=False + ) + + auth_bg = QButtonGroup(auth_group) + password_radio = self.create_radiobutton( + _("Password:"), + 'kernel_server/password_auth', + button_group=auth_bg, + ) + keyfile_radio = self.create_radiobutton( + _('SSH keyfile:'), + 'kernel_server/keyfile_auth', + button_group=auth_bg, + ) + + password = self.create_lineedit( + "", + 'kernel_server/password', + alignment=Qt.Horizontal, + word_wrap=False + ) + password.textbox.setEchoMode(QLineEdit.Password) + password_radio.toggled.connect(password.setEnabled) + keyfile_radio.toggled.connect(password.setDisabled) + + keyfile = self.create_file_combobox( + _('SSH Keyfile'), + self.get_option('custom_interpreters_list'), + 'kernel_server/keyfile', + default_line_edit=True, + adjust_to_contents=True, + ) + passphrase = self.create_lineedit( + _('Passphase:'), + 'kernel_server/passphrase', + alignment=Qt.Horizontal, + word_wrap=False + ) + + passphrase.textbox.setPlaceholderText(_('Optional')) + passphrase.textbox.setEchoMode(QLineEdit.Password) + + keyfile_radio.toggled.connect(keyfile.setEnabled) + keyfile_radio.toggled.connect(passphrase.setEnabled) + password_radio.toggled.connect(keyfile.setDisabled) + password_radio.toggled.connect(passphrase.setDisabled) + + # SSH authentication layout auth_layout = QGridLayout() - auth_layout.addWidget(self.pw_radio, 1, 0) - auth_layout.addWidget(pw_label, 1, 1) - auth_layout.addWidget(self.pw, 1, 2) - auth_layout.addWidget(self.kf_radio, 2, 0) - auth_layout.addWidget(kf_label, 2, 1) - auth_layout.addLayout(kf_layout, 2, 2) - auth_layout.addWidget(kfp_label, 3, 1) - auth_layout.addWidget(self.kfp, 3, 2) + auth_layout.addWidget(username, 0, 0, 1, 2) + auth_layout.addWidget(password_radio, 1, 0) + auth_layout.addWidget(password, 1, 1) + auth_layout.addWidget(keyfile_radio, 2, 0) + auth_layout.addWidget(keyfile, 2, 1) + auth_layout.addWidget(passphrase, 3, 0, 1, 2) auth_group.setLayout(auth_layout) # Remote kernel layout rm_layout = QVBoxLayout() - rm_layout.addLayout(ssh_layout) + rm_layout.addLayout(hostname_layout) rm_layout.addSpacerItem(QSpacerItem(0, 8)) rm_layout.addWidget(auth_group) - self.rm_group.setLayout(rm_layout) - self.rm_group.setCheckable(True) - self.rm_group.toggled.connect(self.pw_radio.setChecked) - + rm_group.setLayout(rm_layout) + auth_group.setCheckable(True) + auth_group.toggled.connect(password_radio.setChecked) + rm_group.setCheckable(True) # Python executable Group pyexec_group = QGroupBox(_("Python interpreter")) @@ -186,7 +209,7 @@ def setup_page(self): self.def_exec_radio.toggled.connect(self.cus_exec_combo.setDisabled) self.cus_exec_radio.toggled.connect(self.cus_exec_combo.setEnabled) pyexec_layout.addWidget(self.cus_exec_combo) - pyexec_layout.addWidget(self.rm_group) + pyexec_layout.addWidget(rm_group) pyexec_group.setLayout(pyexec_layout) self.pyexec_edit = self.cus_exec_combo.combobox.lineEdit() @@ -242,11 +265,6 @@ def setup_page(self): vlayout.addStretch(1) self.setLayout(vlayout) - def select_ssh_key(self): - kf = getopenfilename(self, _('Select SSH keyfile'), - get_home_dir(), '*.pem;;*')[0] - self.kf.setText(kf) - def warn_python_compatibility(self, pyexec): if not osp.isfile(pyexec): return diff --git a/spyder/plugins/preferences/api.py b/spyder/plugins/preferences/api.py index 28a0aae3552..e1dad076f66 100644 --- a/spyder/plugins/preferences/api.py +++ b/spyder/plugins/preferences/api.py @@ -448,6 +448,31 @@ def show_message(is_checked=False): checkbox.restart_required = restart return checkbox + def create_checkable_groupbox( + self, text, option, default=NoDefault, + tip=None, msg_warning=None, msg_info=None, + msg_if_enabled=False, section=None, restart=False + ): + checkbox = QGroupBox(text) + checkbox.setCheckable(True) + self.checkboxes[checkbox] = (section, option, default) + if section is not None and section != self.CONF_SECTION: + self.cross_section_options[option] = section + if tip is not None: + checkbox.setToolTip(tip) + if msg_warning is not None or msg_info is not None: + def show_message(is_checked=False): + if is_checked or not msg_if_enabled: + if msg_warning is not None: + QMessageBox.warning(self, self.get_name(), + msg_warning, QMessageBox.Ok) + if msg_info is not None: + QMessageBox.information(self, self.get_name(), + msg_info, QMessageBox.Ok) + checkbox.clicked.connect(show_message) + checkbox.restart_required = restart + return checkbox + def create_radiobutton(self, text, option, default=NoDefault, tip=None, msg_warning=None, msg_info=None, msg_if_enabled=False, button_group=None, From 07bc477357fcfaaf22a1ce2dc421829f5fb4d6e7 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 16:50:34 +0100 Subject: [PATCH 006/110] implement custom server --- .../ipythonconsole/utils/kernel_handler.py | 15 ++- .../plugins/ipythonconsole/widgets/mixins.py | 127 +++++++++++++++--- 2 files changed, 120 insertions(+), 22 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 488d244f4ea..eeeda4f83b5 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -105,11 +105,6 @@ class KernelHandler(QObject): The kernel raised an error while connecting. """ - sig_request_close = Signal(str) - """ - This kernel would like to be closed - """ - def __init__( self, connection_file, @@ -119,6 +114,7 @@ def __init__( hostname=None, sshkey=None, password=None, + socket=None, ): super().__init__() # Connection Informations @@ -131,6 +127,7 @@ def __init__( self.password = password self.kernel_error_message = None self.connection_state = KernelConnectionState.Connecting + self.socket = socket # For closing # Comm self.kernel_comm = KernelComm() @@ -274,7 +271,8 @@ def tunnel_to_kernel( @classmethod def new_from_spec( cls, kernel_spec, connection_file, connection_info, - hostname=None, sshkey=None, password=None + hostname=None, sshkey=None, password=None, + socket=None ): """ Create a new kernel. @@ -300,6 +298,7 @@ def new_from_spec( hostname=hostname, sshkey=sshkey, password=password, + socket=socket, ) @classmethod @@ -374,7 +373,9 @@ def close(self, shutdown_kernel=True, now=False): self.close_comm() if shutdown_kernel and self.kernel_spec is not None: - self.sig_request_close.emit(self.connection_file) + self.socket.send_pyobj(["close_kernel", self.connection_file]) + # Wait for confirmation + self.socket.recv_pyobj() if ( self.kernel_client is not None diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index c96adaebc85..7f3a2589ec8 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -9,29 +9,127 @@ """ import zmq import sys +import os from qtpy.QtCore import QProcess # Local imports +from spyder.api.config.mixins import SpyderConfigurationObserver from spyder.plugins.ipythonconsole.utils.kernel_handler import KernelHandler +from spyder.api.config.decorators import on_conf_change +from zmq.ssh import tunnel as zmqtunnel +from spyder.plugins.ipythonconsole.utils.ssh import openssh_tunnel +if os.name == "nt": + ssh_tunnel = zmqtunnel.paramiko_tunnel +else: + ssh_tunnel = openssh_tunnel -class KernelConnectorMixin: +class KernelConnectorMixin(SpyderConfigurationObserver): """Needs https://github.com/jupyter/jupyter_client/pull/835""" def __init__(self): super().__init__() + self.context = zmq.Context() + self.on_kernel_server_conf_changed() + + @on_conf_change( + option=[ + 'kernel_server/external_server', + 'kernel_server/use_ssh', + 'kernel_server/host', + 'kernel_server/port', + 'kernel_server/username', + 'kernel_server/password_auth', + 'kernel_server/keyfile_auth', + 'kernel_server/password', + 'kernel_server/keyfile', + 'kernel_server/passphrase', + ], + section='main_interpreter' + ) + def on_kernel_server_conf_changed(self, option=None, value=None): + """Start server""" self.hostname = None self.sshkey = None self.password = None - self.start_local_server() - self.port = "5556" - self.context = zmq.Context() - self.socket = self.context.socket(zmq.REQ) - self.socket.connect("tcp://localhost:%s" % self.port) + + is_remote = self.get_conf( + option='kernel_server/external_server', + section='main_interpreter') + + if not is_remote: + self.start_local_server() + return + # Remote server + + remote_port = int(self.get_conf( + option='kernel_server/port', + section='main_interpreter')) + if not remote_port: + remote_port = 22 + remote_ip = self.get_conf( + option='kernel_server/host', + section='main_interpreter') + + + is_ssh = self.get_conf( + option='kernel_server/use_ssh', + section='main_interpreter') + + if not is_ssh: + self.connect_socket(f"{remote_ip}:{remote_port}") + return + + username = self.get_conf( + option='kernel_server/username', + section='main_interpreter') + + self.hostname = f"{username}@{remote_ip}:{remote_port}" + + # Now we deal with ssh + uses_password = self.get_conf( + option='kernel_server/password_auth', + section='main_interpreter') + uses_keyfile = self.get_conf( + option='kernel_server/keyfile_auth', + section='main_interpreter') + + if uses_password: + self.password = self.get_conf( + option='kernel_server/password', + section='main_interpreter') + self.sshkey = None + elif uses_keyfile: + self.password = self.get_conf( + option='kernel_server/passphrase', + section='main_interpreter') + self.sshkey = self.get_conf( + option='kernel_server/keyfile', + section='main_interpreter') + else: + raise NotImplementedError("This should not be possible.") + + local_port = zmqtunnel.select_random_ports(1) + local_ip = "localhost" + timeout = 10 + ssh_tunnel( + local_port, remote_port, + local_ip, remote_ip, + self.sshkey, self.password, timeout) + self.connect_socket(f"{local_ip}:{local_port}") def start_local_server(self): + """Start a server with the current interpreter.""" + port = str(zmqtunnel.select_random_ports(1)[0]) self.server = QProcess(self) - self.server.start(sys.executable, ["-m", "spyder_kernels_server"]) + self.server.start( + sys.executable, ["-m", "spyder_kernels_server", port] + ) + self.connect_socket(f"localhost:{port}") + + def connect_socket(self, hostname): + self.socket = self.context.socket(zmq.REQ) + self.socket.connect(f"tcp://{hostname}") def new_kernel(self, kernel_spec): """Get a new kernel""" @@ -41,18 +139,17 @@ def new_kernel(self, kernel_spec): raise connection_info kernel_handler = KernelHandler.new_from_spec( - kernel_spec, connection_file, connection_info, - self.hostname, self.sshkey, self.password + kernel_spec=kernel_spec, + connection_file=connection_file, + connection_info=connection_info, + hostname=self.hostname, + sshkey=self.sshkey, + password=self.password, + socket=self.socket, ) - kernel_handler.sig_request_close.connect(self.close_kernel) return kernel_handler - def close_kernel(self, connection_file): - self.socket.send_pyobj(["close_kernel", connection_file]) - # Wait for confirmation - self.socket.recv_pyobj() - class CachedKernelMixin: """Cached kernel mixin.""" From e4d2243fb885d29abcfca1bbef0f675e58b1cbcf Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 4 Dec 2022 16:55:23 +0100 Subject: [PATCH 007/110] select random port --- .../spyder_kernels_server/__main__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 0d69363c8c8..b885cc97e13 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -9,11 +9,14 @@ import zmq import json from spyder_kernels_server.kernel_server import KernelServer +from zmq.ssh import tunnel as zmqtunnel -def main(port): +def main(): if len(sys.argv) > 1: port = sys.argv[1] + else: + port = str(zmqtunnel.select_random_ports(1)[0]) context = zmq.Context() socket = context.socket(zmq.REP) @@ -48,9 +51,9 @@ def main(port): try: kernel_server.close_kernel(message[1]) except Exception: + print("Nope") pass if __name__ == "__main__": - port = "5556" - main(port) + main() From 9913876c76dab3f1063c545c70f7602554c29b91 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 5 Dec 2022 22:27:36 +0100 Subject: [PATCH 008/110] QApplication --- .../spyder_kernels_server/__main__.py | 57 ++++++++++++------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index b885cc97e13..5543f5ca4cf 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -10,50 +10,63 @@ import json from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel +from qtpy.QtWidgets import QApplication +from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication -def main(): - if len(sys.argv) > 1: - port = sys.argv[1] - else: - port = str(zmqtunnel.select_random_ports(1)[0]) +class Server(QObject): - context = zmq.Context() - socket = context.socket(zmq.REP) - socket.bind("tcp://*:%s" % port) - print(f"Server running on port {port}") - kernel_server = KernelServer() - shutdown = False + def __init__(self): + super().__init__() - while not shutdown: + if len(sys.argv) > 1: + port = sys.argv[1] + else: + port = str(zmqtunnel.select_random_ports(1)[0]) + + context = zmq.Context() + self.socket = context.socket(zmq.REP) + self.socket.bind("tcp://*:%s" % port) + print(f"Server running on port {port}") + self.kernel_server = KernelServer() + + self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), + QSocketNotifier.Read, self) + self._notifier.activated.connect(self._socket_activity) + + def _socket_activity(self): + self._notifier.setEnabled(False) # Wait for next request from client - message = socket.recv_pyobj() + message = self.socket.recv_pyobj() print(message) cmd = message[0] if cmd == "shutdown": - socket.send_pyobj(["shutting_down"]) - shutdown = True - kernel_server.shutdown() + self.socket.send_pyobj(["shutting_down"]) + self.kernel_server.shutdown() elif cmd == "open_kernel": try: - cf = kernel_server.open_kernel(message[1]) + cf = self.kernel_server.open_kernel(message[1]) print(cf) with open(cf, "br") as f: cf = (cf, json.load(f)) except Exception as e: cf = ("error", e) - socket.send_pyobj(["new_kernel", *cf]) + self.socket.send_pyobj(["new_kernel", *cf]) elif cmd == "close_kernel": - socket.send_pyobj(["closing_kernel"]) + self.socket.send_pyobj(["closing_kernel"]) try: - kernel_server.close_kernel(message[1]) + self.kernel_server.close_kernel(message[1]) except Exception: print("Nope") pass + self._notifier.setEnabled(True) + -if __name__ == "__main__": - main() +if __name__ == '__main__': + app = QApplication(sys.argv) + w = Server() + sys.exit(app.exec_()) \ No newline at end of file From bd3a25aeb429aa7cf2c7f1a21a2c2642f0d30bd0 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 06:07:27 +0200 Subject: [PATCH 009/110] git subrepo clone (merge) --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "c1e60e0e4" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "c1e60e0e4" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 8 ++++---- .../spyder-kernels/requirements/posix.txt | 2 +- .../spyder-kernels/requirements/windows.txt | 2 +- external-deps/spyder-kernels/setup.py | 2 +- .../spyder_kernels/comms/frontendcomm.py | 15 +++++++++------ 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 41bbff86bda..9b8a4d9c1a9 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -5,8 +5,8 @@ ; [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git - branch = master - commit = a40a207f3f739761fa6be980e001c9a9e9a75c04 - parent = 41eb51124a578de8677b1a8818ba5eeaea0e9c08 + branch = print_remote + commit = c1e60e0e4fa38ddf5e953efa4f551735997a34a1 + parent = e8200d860ee587efb4702faa2a7e780937e16bfd method = merge - cmdver = 0.4.3 + cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/requirements/posix.txt b/external-deps/spyder-kernels/requirements/posix.txt index 2503a2c29a4..005ccb31de1 100644 --- a/external-deps/spyder-kernels/requirements/posix.txt +++ b/external-deps/spyder-kernels/requirements/posix.txt @@ -1,5 +1,5 @@ cloudpickle -ipykernel>=6.16.1,<7 +ipykernel>=6.23.2,<7 ipython>=7.31.1,<9 jupyter_client>=7.4.9,<9 pyzmq>=22.1.0 diff --git a/external-deps/spyder-kernels/requirements/windows.txt b/external-deps/spyder-kernels/requirements/windows.txt index da1002d040f..d87d5415bee 100644 --- a/external-deps/spyder-kernels/requirements/windows.txt +++ b/external-deps/spyder-kernels/requirements/windows.txt @@ -1,5 +1,5 @@ cloudpickle -ipykernel>=6.16.1,<7 +ipykernel>=6.23.2,<7 ipython>=7.31.1,<9 jupyter_client>=7.4.9,<9 pyzmq>=22.1.0 diff --git a/external-deps/spyder-kernels/setup.py b/external-deps/spyder-kernels/setup.py index a0b7be2ab71..78460701b44 100644 --- a/external-deps/spyder-kernels/setup.py +++ b/external-deps/spyder-kernels/setup.py @@ -37,7 +37,7 @@ def get_version(module='spyder_kernels'): REQUIREMENTS = [ 'cloudpickle', - 'ipykernel>=6.16.1,<7', + 'ipykernel>=6.23.2,<7', 'ipython>=7.31.1,<9,!=8.8.0,!=8.9.0,!=8.10.0,!=8.11.0,!=8.12.0,!=8.12.1', 'jupyter-client>=7.4.9,<9', 'pyzmq>=22.1.0', diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index 1e7e86f49cd..abfeee82d1e 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -130,10 +130,11 @@ def _check_comm_reply(self): """ Send comm message to frontend to check if the iopub channel is ready """ - if len(self._pending_comms) == 0: - return - for comm in self._pending_comms.values(): - self._notify_comm_ready(comm) + with self.comm_lock: + if len(self._pending_comms) == 0: + return + for comm in self._pending_comms.values(): + self._notify_comm_ready(comm) self.kernel.io_loop.call_later(1, self._check_comm_reply) def _notify_comm_ready(self, comm): @@ -145,7 +146,8 @@ def _notify_comm_ready(self, comm): def _comm_ready_callback(self, ret): """A comm has replied, so process all cached messages related to it.""" - comm = self._pending_comms.pop(self.calling_comm_id, None) + with self.comm_lock: + comm = self._pending_comms.pop(self.calling_comm_id, None) if not comm: return # Cached messages for that comm @@ -179,7 +181,8 @@ def _comm_open(self, comm, msg): # IOPub might not be connected yet, keep sending messages until a # reply is received. - self._pending_comms[comm.comm_id] = comm + with self.comm_lock: + self._pending_comms[comm.comm_id] = comm self._notify_comm_ready(comm) self.kernel.io_loop.call_later(.3, self._check_comm_reply) From b0d21d34f410569cfa304797c5a7dea223d3617f Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 06:20:21 +0200 Subject: [PATCH 010/110] backport fixes --- .../spyder_kernels_server/kernel_comm.py | 15 +++++++++++++++ .../spyder_kernels_server/kernel_manager.py | 10 +++++++--- .../ipythonconsole/utils/kernel_handler.py | 8 -------- .../plugins/ipythonconsole/widgets/main_widget.py | 3 +-- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py index cc9f1fd6dce..43a5334eb5b 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py @@ -10,6 +10,7 @@ """ from contextlib import contextmanager import logging +import os import pickle from qtpy.QtCore import QEventLoop, QObject, QTimer, Signal @@ -20,6 +21,17 @@ TIMEOUT_KERNEL_START = 30 +def get_debug_level(): + debug_env = os.environ.get('SPYDER_DEBUG', '') + if not debug_env.isdigit(): + debug_env = bool(debug_env) + return int(debug_env) + + +def running_under_pytest(): + return bool(os.environ.get('SPYDER_PYTEST')) + + class KernelComm(CommBase, QObject): """ Class with the necessary attributes and methods to handle @@ -70,6 +82,9 @@ def comm_channel_manager(self, comm_id, queue_message=False): def _set_call_return_value(self, call_dict, data, is_error=False): """Override to use the comm_channel for all replies.""" with self.comm_channel_manager(self.calling_comm_id, False): + if is_error and (get_debug_level() or running_under_pytest()): + # Disable error muting when debugging or testing + call_dict['settings']['display_error'] = True super(KernelComm, self)._set_call_return_value( call_dict, data, is_error) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py index a24c9c4c614..ab68a79a833 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py @@ -64,12 +64,16 @@ async def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True, children.append(parent) for child_process in children: - # This is necessary to avoid an error when restarting the - # kernel that started a PyQt5 application in the background. + # This is necessary to avoid an error when restarting the kernel + # that started a PyQt5 application in the background. It also fixes + # a problem when some of the kernel children are not available + # anymore, probably because they were removed by the OS before this + # method is able to run. + # Fixes spyder-ide/spyder#21012 # Fixes spyder-ide/spyder#13999 try: child_process.send_signal(sig) - except psutil.AccessDenied: + except (psutil.AccessDenied, psutil.NoSuchProcess): return ([], []) gone, alive = psutil.wait_procs( diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index c6246120b95..129f29c03fa 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -101,14 +101,6 @@ class KernelHandler(QObject): The kernel raised an error while connecting. """ - _shutdown_thread_list = [] - """List of running shutdown threads""" - - _shutdown_thread_list_lock = Lock() - """ - Lock to add threads to _shutdown_thread_list or clear that list. - """ - def __init__( self, connection_file, diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 95abb23e2a1..d52b55217d7 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -38,10 +38,9 @@ from spyder.plugins.ipythonconsole.utils.style import create_qss_style from spyder.plugins.ipythonconsole.widgets import ( ClientWidget, ConsoleRestartDialog, COMPLETION_WIDGET_TYPE, - KernelConnectionDialog, PageControlWidget) + KernelConnectionDialog, PageControlWidget, MatplotlibStatus) from spyder.plugins.ipythonconsole.widgets.mixins import ( CachedKernelMixin, KernelConnectorMixin) -from spyder.py3compat import PY38_OR_MORE from spyder.utils import encoding, programs, sourcecode from spyder.utils.envs import get_list_envs from spyder.utils.misc import get_error_match, remove_backslashes From 1ec7f5e1aae0e5f21ca2d9d76ee276c317477940 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 11:37:46 +0200 Subject: [PATCH 011/110] wait for connection to connect --- .../ipythonconsole/utils/kernel_handler.py | 75 +++++++++++++------ .../plugins/ipythonconsole/widgets/client.py | 1 + .../plugins/ipythonconsole/widgets/mixins.py | 7 -- .../plugins/ipythonconsole/widgets/shell.py | 6 +- 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 129f29c03fa..5184fbcb4cc 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -12,8 +12,9 @@ import uuid # Third-party imports -from qtpy.QtCore import QObject, Signal +from qtpy.QtCore import QObject, Signal, QSocketNotifier from zmq.ssh import tunnel as zmqtunnel +import zmq # Local imports from spyder.api.translations import _ @@ -123,7 +124,15 @@ def __init__( self.password = password self.kernel_error_message = None self.connection_state = KernelConnectionState.Connecting + + # socket + self._socket_waiting = False self.socket = socket # For closing + self._notifier = None + if socket is not None: + self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), + QSocketNotifier.Read, self) + self._notifier.activated.connect(self._socket_activity) # Comm self.kernel_comm = KernelComm() @@ -134,14 +143,20 @@ def __init__( self._fault_args = None self._spyder_kernel_info_uuid = None self._shellwidget_connected = False - - # Start kernel - # self.connect_std_pipes() - self.kernel_client.start_channels() - self.check_kernel_info() + + if self.kernel_client: + # Start kernel + # self.connect_std_pipes() + self.kernel_client.start_channels() + self.check_kernel_info() def connect(self): """Connect to shellwidget.""" + if self.kernel_spec is not None: + print("Send open_kernel") + self._socket_waiting = True + self.socket.send_pyobj(["open_kernel", self.kernel_spec]) + self._shellwidget_connected = True # Emit signal in case the connection is already made if self.connection_state in [ @@ -158,6 +173,38 @@ def connect(self): # if self._init_stdout: # self.sig_stdout.emit(self._init_stdout) # self._init_stdout = None + + def _socket_activity(self): + if not self._socket_waiting: + return + self._socket_waiting = False + print("Got Notified") + # Wait for next request from client + message = self.socket.recv_pyobj() + print(message) + cmd = message[0] + if cmd == "new_kernel": + cmd, self.connection_file, self.connection_info = message + if self.connection_file == "error": + print(self.connection_info) + self.kernel_client = SpyderKernelClient() + self.kernel_client.load_connection_info(self.connection_info) + self.kernel_client = self.tunnel_kernel_client( + self.kernel_client, + self.hostname, + self.sshkey, + self.password + ) + + # Increase time (in seconds) to detect if a kernel is alive. + # See spyder-ide/spyder#3444. + self.kernel_client.hb_channel.time_to_dead = 25.0 + + # Start kernel + # self.connect_std_pipes() + self.kernel_client.start_channels() + self.check_kernel_info() + def check_kernel_info(self): """Send request to check kernel info.""" @@ -266,30 +313,16 @@ def tunnel_to_kernel( @classmethod def new_from_spec( - cls, kernel_spec, connection_file, connection_info, + cls, kernel_spec, connection_file=None, connection_info=None, hostname=None, sshkey=None, password=None, socket=None ): """ Create a new kernel. """ - kernel_client = SpyderKernelClient() - kernel_client.load_connection_info(connection_info) - kernel_client = cls.tunnel_kernel_client( - kernel_client, - hostname, - sshkey, - password - ) - - # Increase time (in seconds) to detect if a kernel is alive. - # See spyder-ide/spyder#3444. - kernel_client.hb_channel.time_to_dead = 25.0 - return cls( connection_file=connection_file, kernel_spec=kernel_spec, - kernel_client=kernel_client, known_spyder_kernel=True, hostname=hostname, sshkey=sshkey, diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 420236c32a8..296baa03eb2 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -540,6 +540,7 @@ def close_client(self, is_last_client): def shutdown(self, is_last_client): """Shutdown connection and kernel if needed.""" + return self.dialog_manager.close_all() shutdown_kernel = ( is_last_client and not self.shellwidget.is_external_kernel diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 7f3a2589ec8..ad21cf6802f 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -133,15 +133,8 @@ def connect_socket(self, hostname): def new_kernel(self, kernel_spec): """Get a new kernel""" - self.socket.send_pyobj(["open_kernel", kernel_spec]) - cmd, connection_file, connection_info = self.socket.recv_pyobj() - if connection_file == "error": - raise connection_info - kernel_handler = KernelHandler.new_from_spec( kernel_spec=kernel_spec, - connection_file=connection_file, - connection_info=connection_info, hostname=self.hostname, sshkey=self.sshkey, password=self.password, diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 4af3234517d..3a22a318129 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -203,10 +203,6 @@ def spyder_kernel_ready(self): def connect_kernel(self, kernel_handler, first_connect=True): """Connect to the kernel using our handler.""" # Kernel client - kernel_client = kernel_handler.kernel_client - kernel_client.stopped_channels.connect(self.notify_deleted) - self.kernel_client = kernel_client - self.kernel_handler = kernel_handler if first_connect: @@ -271,6 +267,8 @@ def handle_kernel_is_ready(self): self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady ): + self.kernel_client = self.kernel_handler.kernel_client + self.kernel_client.stopped_channels.connect(self.notify_deleted) self.setup_spyder_kernel() return From d02be3f014c1c7f4c45165fdf30f1c68cbd39625 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 5 Dec 2022 22:27:36 +0100 Subject: [PATCH 012/110] Revert "QApplication" This reverts commit 9913876c76dab3f1063c545c70f7602554c29b91. --- .../spyder_kernels_server/__main__.py | 57 +++++++------------ 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 5543f5ca4cf..b885cc97e13 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -10,63 +10,50 @@ import json from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel -from qtpy.QtWidgets import QApplication -from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication -class Server(QObject): +def main(): + if len(sys.argv) > 1: + port = sys.argv[1] + else: + port = str(zmqtunnel.select_random_ports(1)[0]) - def __init__(self): - super().__init__() + context = zmq.Context() + socket = context.socket(zmq.REP) + socket.bind("tcp://*:%s" % port) + print(f"Server running on port {port}") + kernel_server = KernelServer() + shutdown = False - if len(sys.argv) > 1: - port = sys.argv[1] - else: - port = str(zmqtunnel.select_random_ports(1)[0]) - - context = zmq.Context() - self.socket = context.socket(zmq.REP) - self.socket.bind("tcp://*:%s" % port) - print(f"Server running on port {port}") - self.kernel_server = KernelServer() - - self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), - QSocketNotifier.Read, self) - self._notifier.activated.connect(self._socket_activity) - - def _socket_activity(self): - self._notifier.setEnabled(False) + while not shutdown: # Wait for next request from client - message = self.socket.recv_pyobj() + message = socket.recv_pyobj() print(message) cmd = message[0] if cmd == "shutdown": - self.socket.send_pyobj(["shutting_down"]) - self.kernel_server.shutdown() + socket.send_pyobj(["shutting_down"]) + shutdown = True + kernel_server.shutdown() elif cmd == "open_kernel": try: - cf = self.kernel_server.open_kernel(message[1]) + cf = kernel_server.open_kernel(message[1]) print(cf) with open(cf, "br") as f: cf = (cf, json.load(f)) except Exception as e: cf = ("error", e) - self.socket.send_pyobj(["new_kernel", *cf]) + socket.send_pyobj(["new_kernel", *cf]) elif cmd == "close_kernel": - self.socket.send_pyobj(["closing_kernel"]) + socket.send_pyobj(["closing_kernel"]) try: - self.kernel_server.close_kernel(message[1]) + kernel_server.close_kernel(message[1]) except Exception: print("Nope") pass - self._notifier.setEnabled(True) - -if __name__ == '__main__': - app = QApplication(sys.argv) - w = Server() - sys.exit(app.exec_()) \ No newline at end of file +if __name__ == "__main__": + main() From 264df3a2eb8a0295ce11efb56441238f61adf740 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 14:05:33 +0200 Subject: [PATCH 013/110] QCoreApplicaiton --- .../spyder_kernels_server/__main__.py | 63 ++++++++++++------- .../ipythonconsole/utils/kernelspec.py | 15 ++++- spyder/utils/environ.py | 14 ----- 3 files changed, 55 insertions(+), 37 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index b885cc97e13..6b8cc4f6df1 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -5,55 +5,74 @@ # Licensed under the terms of the MIT License # (see spyder_kernels/__init__.py for details) # ----------------------------------------------------------------------------- +import os import sys import zmq import json from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel +# from qtpy.QtWidgets import QApplication +from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication -def main(): - if len(sys.argv) > 1: - port = sys.argv[1] - else: - port = str(zmqtunnel.select_random_ports(1)[0]) +class Server(QObject): - context = zmq.Context() - socket = context.socket(zmq.REP) - socket.bind("tcp://*:%s" % port) - print(f"Server running on port {port}") - kernel_server = KernelServer() - shutdown = False + def __init__(self): + super().__init__() - while not shutdown: + if len(sys.argv) > 1: + port = sys.argv[1] + else: + port = str(zmqtunnel.select_random_ports(1)[0]) + + context = zmq.Context() + self.socket = context.socket(zmq.REP) + self.socket.bind("tcp://*:%s" % port) + print(f"Server running on port {port}") + self.kernel_server = KernelServer() + + self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), + QSocketNotifier.Read, self) + self._notifier.activated.connect(self._socket_activity) + + def _socket_activity(self): + self._notifier.setEnabled(False) # Wait for next request from client - message = socket.recv_pyobj() + message = self.socket.recv_pyobj() print(message) cmd = message[0] if cmd == "shutdown": - socket.send_pyobj(["shutting_down"]) - shutdown = True - kernel_server.shutdown() + self.socket.send_pyobj(["shutting_down"]) + self.kernel_server.shutdown() elif cmd == "open_kernel": try: - cf = kernel_server.open_kernel(message[1]) + cf = self.kernel_server.open_kernel(message[1]) print(cf) with open(cf, "br") as f: cf = (cf, json.load(f)) except Exception as e: cf = ("error", e) - socket.send_pyobj(["new_kernel", *cf]) + self.socket.send_pyobj(["new_kernel", *cf]) elif cmd == "close_kernel": - socket.send_pyobj(["closing_kernel"]) + self.socket.send_pyobj(["closing_kernel"]) try: - kernel_server.close_kernel(message[1]) + self.kernel_server.close_kernel(message[1]) except Exception: print("Nope") pass + self._notifier.setEnabled(True) + + # This is necessary for some reason. + # Otherwise the socket only works twice ! + self.socket.getsockopt(zmq.EVENTS) + -if __name__ == "__main__": - main() +if __name__ == '__main__': + print(os.getpid()) + app = QCoreApplication(sys.argv) + w = Server() + sys.exit(app.exec_()) \ No newline at end of file diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 20bfaa7838b..aa2a6829ff8 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -27,7 +27,6 @@ SpyderKernelError) from spyder.utils.conda import (add_quotes, get_conda_env_path, is_conda_env, find_conda) -from spyder.utils.environ import clean_env from spyder.utils.misc import get_python_executable from spyder.utils.programs import is_python_interpreter, is_module_installed @@ -53,6 +52,20 @@ "") +def clean_env(env_vars): + """ + Remove non-ascii entries from a dictionary of environments variables. + + The values will be converted to strings or bytes (on Python 2). If an + exception is raised, an empty string will be used. + """ + new_env_vars = env_vars.copy() + for key, var in env_vars.items(): + new_env_vars[key] = str(var) + + return new_env_vars + + def is_different_interpreter(pyexec): """Check that pyexec is a different interpreter from sys.executable.""" # Paths may be symlinks diff --git a/spyder/utils/environ.py b/spyder/utils/environ.py index d100570359c..0f406b9f6de 100644 --- a/spyder/utils/environ.py +++ b/spyder/utils/environ.py @@ -159,20 +159,6 @@ def amend_user_shell_init(text="", restore=False): init_file.write_text(_script.rstrip() + "\n") -def clean_env(env_vars): - """ - Remove non-ascii entries from a dictionary of environments variables. - - The values will be converted to strings or bytes (on Python 2). If an - exception is raised, an empty string will be used. - """ - new_env_vars = env_vars.copy() - for key, var in env_vars.items(): - new_env_vars[key] = str(var) - - return new_env_vars - - class RemoteEnvDialog(CollectionsEditor): """Remote process environment variables dialog.""" From cefc022fcbe6fa5af6c6ed0848f57cc3f8106e0d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 14:06:07 +0200 Subject: [PATCH 014/110] better request --- .../ipythonconsole/utils/kernel_handler.py | 84 +++++++------------ .../plugins/ipythonconsole/widgets/mixins.py | 68 ++++++++++++++- 2 files changed, 96 insertions(+), 56 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 5184fbcb4cc..fc5fbaf5e4e 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -101,6 +101,8 @@ class KernelHandler(QObject): """ The kernel raised an error while connecting. """ + + sig_remote_close = Signal(dict) def __init__( self, @@ -111,7 +113,6 @@ def __init__( hostname=None, sshkey=None, password=None, - socket=None, ): super().__init__() # Connection Informations @@ -124,15 +125,6 @@ def __init__( self.password = password self.kernel_error_message = None self.connection_state = KernelConnectionState.Connecting - - # socket - self._socket_waiting = False - self.socket = socket # For closing - self._notifier = None - if socket is not None: - self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), - QSocketNotifier.Read, self) - self._notifier.activated.connect(self._socket_activity) # Comm self.kernel_comm = KernelComm() @@ -152,11 +144,6 @@ def __init__( def connect(self): """Connect to shellwidget.""" - if self.kernel_spec is not None: - print("Send open_kernel") - self._socket_waiting = True - self.socket.send_pyobj(["open_kernel", self.kernel_spec]) - self._shellwidget_connected = True # Emit signal in case the connection is already made if self.connection_state in [ @@ -173,37 +160,6 @@ def connect(self): # if self._init_stdout: # self.sig_stdout.emit(self._init_stdout) # self._init_stdout = None - - def _socket_activity(self): - if not self._socket_waiting: - return - self._socket_waiting = False - print("Got Notified") - # Wait for next request from client - message = self.socket.recv_pyobj() - print(message) - cmd = message[0] - if cmd == "new_kernel": - cmd, self.connection_file, self.connection_info = message - if self.connection_file == "error": - print(self.connection_info) - self.kernel_client = SpyderKernelClient() - self.kernel_client.load_connection_info(self.connection_info) - self.kernel_client = self.tunnel_kernel_client( - self.kernel_client, - self.hostname, - self.sshkey, - self.password - ) - - # Increase time (in seconds) to detect if a kernel is alive. - # See spyder-ide/spyder#3444. - self.kernel_client.hb_channel.time_to_dead = 25.0 - - # Start kernel - # self.connect_std_pipes() - self.kernel_client.start_channels() - self.check_kernel_info() def check_kernel_info(self): @@ -314,8 +270,7 @@ def tunnel_to_kernel( @classmethod def new_from_spec( cls, kernel_spec, connection_file=None, connection_info=None, - hostname=None, sshkey=None, password=None, - socket=None + hostname=None, sshkey=None, password=None ): """ Create a new kernel. @@ -326,8 +281,7 @@ def new_from_spec( known_spyder_kernel=True, hostname=hostname, sshkey=sshkey, - password=password, - socket=socket, + password=password ) @classmethod @@ -402,9 +356,7 @@ def close(self, shutdown_kernel=True, now=False): self.close_comm() if shutdown_kernel and self.kernel_spec is not None: - self.socket.send_pyobj(["close_kernel", self.connection_file]) - # Wait for confirmation - self.socket.recv_pyobj() + self.sig_remote_close.emit(self.connection_file) if ( self.kernel_client is not None @@ -481,3 +433,29 @@ def reopen_comm(self): self.kernel_comm.remove() self.connection_state = KernelConnectionState.Connecting self.kernel_comm.open_comm(self.kernel_client) + + def set_connection(self, connection_file, connection_info, + hostname, sshkey, password): + self.connection_file = connection_file + self.connection_info = connection_info + self.hostname = hostname + self.sshkey = sshkey + self.password = password + + self.kernel_client = SpyderKernelClient() + self.kernel_client.load_connection_info(self.connection_info) + self.kernel_client = self.tunnel_kernel_client( + self.kernel_client, + self.hostname, + self.sshkey, + self.password + ) + + # Increase time (in seconds) to detect if a kernel is alive. + # See spyder-ide/spyder#3444. + self.kernel_client.hb_channel.time_to_dead = 25.0 + + # Start kernel + # self.connect_std_pipes() + self.kernel_client.start_channels() + self.check_kernel_info() \ No newline at end of file diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index ad21cf6802f..5bd552bce38 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -10,8 +10,9 @@ import zmq import sys import os +import queue -from qtpy.QtCore import QProcess +from qtpy.QtCore import QProcess, QSocketNotifier # Local imports from spyder.api.config.mixins import SpyderConfigurationObserver @@ -31,6 +32,9 @@ def __init__(self): super().__init__() self.context = zmq.Context() self.on_kernel_server_conf_changed() + self.kernel_handler_waitlist = [] + self.request_queue = queue.Queue() + self._notifier = None @on_conf_change( option=[ @@ -130,6 +134,11 @@ def start_local_server(self): def connect_socket(self, hostname): self.socket = self.context.socket(zmq.REQ) self.socket.connect(f"tcp://{hostname}") + self._notifier = QSocketNotifier( + self.socket.getsockopt(zmq.FD), + QSocketNotifier.Read, self + ) + self._notifier.activated.connect(self._socket_activity) def new_kernel(self, kernel_spec): """Get a new kernel""" @@ -138,10 +147,63 @@ def new_kernel(self, kernel_spec): hostname=self.hostname, sshkey=self.sshkey, password=self.password, - socket=self.socket, ) - + + kernel_handler.sig_remote_close.connect(self.request_close) + self.kernel_handler_waitlist.append(kernel_handler) + + + self.send_request(["open_kernel", kernel_spec]) + return kernel_handler + + def request_close(self, connection_file): + self.send_request(["close_kernel", connection_file]) + + def send_request(self, request): + + if self.socket.getsockopt(zmq.EVENTS) & zmq.POLLOUT: + print("send", request) + self.socket.send_pyobj(request) + else: + print("queue", request) + self.request_queue.put(request) + + def _socket_activity(self): + if not self.socket.getsockopt(zmq.EVENTS) & zmq.POLLIN: + return + print("Got Activity") + self._notifier.setEnabled(False) + # Wait for next request from client + message = self.socket.recv_pyobj() + print(message) + cmd = message[0] + if cmd == "new_kernel": + cmd, connection_file, connection_info = message + + if connection_file == "error": + print(connection_info) + + if len(self.kernel_handler_waitlist) == 0: + print("WTF???") + + kernel_handler = self.kernel_handler_waitlist.pop(0) + kernel_handler.set_connection( + connection_file, connection_info, + self.hostname, + self.sshkey, + self.password) + + self._notifier.setEnabled(True) + # This is necessary for some reason. + # Otherwise the socket only works twice ! + self.socket.getsockopt(zmq.EVENTS) + + try: + request = self.request_queue.get_nowait() + self.send_request(request) + except queue.Empty: + pass class CachedKernelMixin: From 0a712a248db082454b56cd7fbb302bacf8511dd0 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 14:15:09 +0200 Subject: [PATCH 015/110] correct shutdown --- spyder/plugins/ipythonconsole/utils/kernel_handler.py | 6 ++---- spyder/plugins/ipythonconsole/widgets/client.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index fc5fbaf5e4e..f02facbac02 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -106,7 +106,7 @@ class KernelHandler(QObject): def __init__( self, - connection_file, + connection_file=None, kernel_spec=None, kernel_client=None, known_spyder_kernel=False, @@ -269,14 +269,12 @@ def tunnel_to_kernel( @classmethod def new_from_spec( - cls, kernel_spec, connection_file=None, connection_info=None, - hostname=None, sshkey=None, password=None + cls, kernel_spec, hostname=None, sshkey=None, password=None ): """ Create a new kernel. """ return cls( - connection_file=connection_file, kernel_spec=kernel_spec, known_spyder_kernel=True, hostname=hostname, diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 296baa03eb2..420236c32a8 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -540,7 +540,6 @@ def close_client(self, is_last_client): def shutdown(self, is_last_client): """Shutdown connection and kernel if needed.""" - return self.dialog_manager.close_all() shutdown_kernel = ( is_last_client and not self.shellwidget.is_external_kernel From 97792e5fb7d7f851e956ebcdb5a61409e217ab5e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 15:25:27 +0200 Subject: [PATCH 016/110] Avoid repeat --- .../ipythonconsole/utils/kernel_handler.py | 2 +- .../plugins/ipythonconsole/widgets/mixins.py | 83 +++++++++---------- 2 files changed, 39 insertions(+), 46 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index f02facbac02..ebd06722b28 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -102,7 +102,7 @@ class KernelHandler(QObject): The kernel raised an error while connecting. """ - sig_remote_close = Signal(dict) + sig_remote_close = Signal(str) def __init__( self, diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 5bd552bce38..50f0af82e92 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -26,90 +26,83 @@ ssh_tunnel = openssh_tunnel +KERNEL_SERVER_OPTIONS = [ + 'kernel_server/external_server', + 'kernel_server/use_ssh', + 'kernel_server/host', + 'kernel_server/port', + 'kernel_server/username', + 'kernel_server/password_auth', + 'kernel_server/keyfile_auth', + 'kernel_server/password', + 'kernel_server/keyfile', + 'kernel_server/passphrase', +] + class KernelConnectorMixin(SpyderConfigurationObserver): """Needs https://github.com/jupyter/jupyter_client/pull/835""" def __init__(self): super().__init__() - self.context = zmq.Context() - self.on_kernel_server_conf_changed() + self.options = None self.kernel_handler_waitlist = [] self.request_queue = queue.Queue() - self._notifier = None + self.context = zmq.Context() + + self.on_kernel_server_conf_changed() @on_conf_change( - option=[ - 'kernel_server/external_server', - 'kernel_server/use_ssh', - 'kernel_server/host', - 'kernel_server/port', - 'kernel_server/username', - 'kernel_server/password_auth', - 'kernel_server/keyfile_auth', - 'kernel_server/password', - 'kernel_server/keyfile', - 'kernel_server/passphrase', - ], + option=KERNEL_SERVER_OPTIONS, section='main_interpreter' ) def on_kernel_server_conf_changed(self, option=None, value=None): """Start server""" + options = { + option: self.get_conf( + option=option, + section='main_interpreter') + for option in KERNEL_SERVER_OPTIONS + } + if self.options == options: + return + + self.options = options self.hostname = None self.sshkey = None self.password = None - is_remote = self.get_conf( - option='kernel_server/external_server', - section='main_interpreter') + is_remote = options['kernel_server/external_server'] if not is_remote: self.start_local_server() return # Remote server - remote_port = int(self.get_conf( - option='kernel_server/port', - section='main_interpreter')) + remote_port = int(options['kernel_server/port']) if not remote_port: remote_port = 22 - remote_ip = self.get_conf( - option='kernel_server/host', - section='main_interpreter') + remote_ip = options['kernel_server/host'] - is_ssh = self.get_conf( - option='kernel_server/use_ssh', - section='main_interpreter') + is_ssh = options['kernel_server/use_ssh'] if not is_ssh: self.connect_socket(f"{remote_ip}:{remote_port}") return - username = self.get_conf( - option='kernel_server/username', - section='main_interpreter') + username = options['kernel_server/username'] self.hostname = f"{username}@{remote_ip}:{remote_port}" # Now we deal with ssh - uses_password = self.get_conf( - option='kernel_server/password_auth', - section='main_interpreter') - uses_keyfile = self.get_conf( - option='kernel_server/keyfile_auth', - section='main_interpreter') + uses_password = options['kernel_server/password_auth'] + uses_keyfile = options['kernel_server/keyfile_auth'] if uses_password: - self.password = self.get_conf( - option='kernel_server/password', - section='main_interpreter') + self.password = options['kernel_server/password'] self.sshkey = None elif uses_keyfile: - self.password = self.get_conf( - option='kernel_server/passphrase', - section='main_interpreter') - self.sshkey = self.get_conf( - option='kernel_server/keyfile', - section='main_interpreter') + self.password = options['kernel_server/passphrase'] + self.sshkey = options['kernel_server/keyfile'] else: raise NotImplementedError("This should not be possible.") From f4e15ac30ade387a5319601d83a9f59ea982c1e8 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 15:26:08 +0200 Subject: [PATCH 017/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "67dceeb9b" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "67dceeb9b" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- .../spyder_kernels/comms/frontendcomm.py | 15 ++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 9b8a4d9c1a9..022e35e537a 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = c1e60e0e4fa38ddf5e953efa4f551735997a34a1 - parent = e8200d860ee587efb4702faa2a7e780937e16bfd + commit = 67dceeb9bab5c3ad26d878d002df094f05b06f48 + parent = 97792e5fb7d7f851e956ebcdb5a61409e217ab5e method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index abfeee82d1e..1e7e86f49cd 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -130,11 +130,10 @@ def _check_comm_reply(self): """ Send comm message to frontend to check if the iopub channel is ready """ - with self.comm_lock: - if len(self._pending_comms) == 0: - return - for comm in self._pending_comms.values(): - self._notify_comm_ready(comm) + if len(self._pending_comms) == 0: + return + for comm in self._pending_comms.values(): + self._notify_comm_ready(comm) self.kernel.io_loop.call_later(1, self._check_comm_reply) def _notify_comm_ready(self, comm): @@ -146,8 +145,7 @@ def _notify_comm_ready(self, comm): def _comm_ready_callback(self, ret): """A comm has replied, so process all cached messages related to it.""" - with self.comm_lock: - comm = self._pending_comms.pop(self.calling_comm_id, None) + comm = self._pending_comms.pop(self.calling_comm_id, None) if not comm: return # Cached messages for that comm @@ -181,8 +179,7 @@ def _comm_open(self, comm, msg): # IOPub might not be connected yet, keep sending messages until a # reply is received. - with self.comm_lock: - self._pending_comms[comm.comm_id] = comm + self._pending_comms[comm.comm_id] = comm self._notify_comm_ready(comm) self.kernel.io_loop.call_later(.3, self._check_comm_reply) From dece62b726e6c612458467a6453ab0b541632115 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 15:32:08 +0200 Subject: [PATCH 018/110] print faults --- .../plugins/ipythonconsole/widgets/mixins.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 50f0af82e92..33738818e37 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -156,16 +156,13 @@ def request_close(self, connection_file): def send_request(self, request): if self.socket.getsockopt(zmq.EVENTS) & zmq.POLLOUT: - print("send", request) self.socket.send_pyobj(request) else: - print("queue", request) self.request_queue.put(request) def _socket_activity(self): if not self.socket.getsockopt(zmq.EVENTS) & zmq.POLLIN: return - print("Got Activity") self._notifier.setEnabled(False) # Wait for next request from client message = self.socket.recv_pyobj() @@ -174,18 +171,21 @@ def _socket_activity(self): if cmd == "new_kernel": cmd, connection_file, connection_info = message - if connection_file == "error": - print(connection_info) - if len(self.kernel_handler_waitlist) == 0: - print("WTF???") + # This should not happen :/ + self._notifier.setEnabled(True) + self.socket.getsockopt(zmq.EVENTS) + return kernel_handler = self.kernel_handler_waitlist.pop(0) - kernel_handler.set_connection( - connection_file, connection_info, - self.hostname, - self.sshkey, - self.password) + if connection_file == "error": + kernel_handler.sig_fault.emit(connection_info) + else: + kernel_handler.set_connection( + connection_file, connection_info, + self.hostname, + self.sshkey, + self.password) self._notifier.setEnabled(True) # This is necessary for some reason. From 627a746ef8f69524329bc09a9eda7b2dd87e1051 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 15:32:49 +0200 Subject: [PATCH 019/110] remove prints spyder-kernels -server --- .../spyder-kernels-server/spyder_kernels_server/__main__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 6b8cc4f6df1..2a62c5d2a16 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -39,7 +39,6 @@ def _socket_activity(self): self._notifier.setEnabled(False) # Wait for next request from client message = self.socket.recv_pyobj() - print(message) cmd = message[0] if cmd == "shutdown": self.socket.send_pyobj(["shutting_down"]) @@ -61,7 +60,6 @@ def _socket_activity(self): try: self.kernel_server.close_kernel(message[1]) except Exception: - print("Nope") pass self._notifier.setEnabled(True) @@ -72,7 +70,6 @@ def _socket_activity(self): if __name__ == '__main__': - print(os.getpid()) app = QCoreApplication(sys.argv) w = Server() sys.exit(app.exec_()) \ No newline at end of file From 8e31a911f0912c8185953c9eb1785036c9ee3861 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 23:12:08 +0200 Subject: [PATCH 020/110] remove shutdown threads --- spyder/app/tests/test_mainwindow.py | 2 -- spyder/plugins/ipythonconsole/widgets/main_widget.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index 0d7734b3baa..a769c4e2141 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -143,7 +143,6 @@ def ns_fun(main_window, qtbot): # Count initial objects # Only one of each should be present, but because of many leaks, # this is most likely not the case. Here only closing is tested - KernelHandler.wait_all_shutdown_threads() gc.collect() objects = gc.get_objects() n_code_editor_init = 0 @@ -178,7 +177,6 @@ def ns_fun(main_window, qtbot): main_window.ipyconsole.restart() # Wait until the shells are closed - KernelHandler.wait_all_shutdown_threads() return n_shell_init, n_code_editor_init n_shell_init, n_code_editor_init = ns_fun(main_window, qtbot) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index d52b55217d7..fb4a8cf1de5 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1738,9 +1738,6 @@ def close_all_clients(self): client.close_client(is_last_client) open_clients.remove(client) - # Wait for all KernelHandler threads to shutdown. - KernelHandler.wait_all_shutdown_threads() - # Close cached kernel self.close_cached_kernel() self.filenames = [] From bfb043277220095fc33d4017b5ff448a35886827 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 22 Jun 2023 23:32:23 +0200 Subject: [PATCH 021/110] fix prompt --- external-deps/qtconsole/qtconsole/jupyter_widget.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/external-deps/qtconsole/qtconsole/jupyter_widget.py b/external-deps/qtconsole/qtconsole/jupyter_widget.py index cc9de8770b7..8922635db47 100644 --- a/external-deps/qtconsole/qtconsole/jupyter_widget.py +++ b/external-deps/qtconsole/qtconsole/jupyter_widget.py @@ -369,6 +369,9 @@ def _show_interpreter_prompt(self, number=None): """ # If a number was not specified, make a prompt number request. if number is None: + if self.kernel_client is None: + # Not connected yet + return if self._prompt_requested: # Already asked for prompt, avoid multiple prompts. return From 53d38069bf45ad8cecb36852077d84d88efbf631 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 23 Jun 2023 07:44:24 +0200 Subject: [PATCH 022/110] remove messages and fix test --- .../spyder-kernels-server/spyder_kernels_server/__main__.py | 1 - spyder/plugins/ipythonconsole/tests/conftest.py | 3 +-- spyder/plugins/ipythonconsole/widgets/mixins.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 2a62c5d2a16..4bb2ce0da34 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -47,7 +47,6 @@ def _socket_activity(self): elif cmd == "open_kernel": try: cf = self.kernel_server.open_kernel(message[1]) - print(cf) with open(cf, "br") as f: cf = (cf, json.load(f)) diff --git a/spyder/plugins/ipythonconsole/tests/conftest.py b/spyder/plugins/ipythonconsole/tests/conftest.py index fca37597a80..7b900ee22b7 100644 --- a/spyder/plugins/ipythonconsole/tests/conftest.py +++ b/spyder/plugins/ipythonconsole/tests/conftest.py @@ -325,9 +325,8 @@ def threads_condition(): raise try: - # -1 from closed client qtbot.waitUntil(lambda: ( - len(init_subprocesses) - 1 >= len(proc.children())), + len(init_subprocesses) >= len(proc.children())), timeout=SHELL_TIMEOUT) except Exception: subprocesses = [repr(f) for f in proc.children()] diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 33738818e37..8b5a9a0a296 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -166,7 +166,6 @@ def _socket_activity(self): self._notifier.setEnabled(False) # Wait for next request from client message = self.socket.recv_pyobj() - print(message) cmd = message[0] if cmd == "new_kernel": cmd, connection_file, connection_info = message From 5745988555e2e42c78fb43bbae10dc53a64fa095 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 23 Jun 2023 08:07:59 +0200 Subject: [PATCH 023/110] fix interrput --- spyder/plugins/ipythonconsole/widgets/mixins.py | 2 +- spyder/plugins/ipythonconsole/widgets/shell.py | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 8b5a9a0a296..6e152fdd388 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -178,7 +178,7 @@ def _socket_activity(self): kernel_handler = self.kernel_handler_waitlist.pop(0) if connection_file == "error": - kernel_handler.sig_fault.emit(connection_info) + kernel_handler.sig_fault.emit(str(connection_info)) else: kernel_handler.set_connection( connection_file, connection_info, diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 3a22a318129..1032d09b01a 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -403,18 +403,8 @@ def interrupt_kernel(self): if self.spyder_kernel_ready: self._reading = False + self.call_kernel(interrupt=True).raise_interrupt_signal() - # Check if there is a kernel that can be interrupted before trying - # to do it. - # Fixes spyder-ide/spyder#20212 - if self.kernel_manager and self.kernel_manager.has_kernel: - self.call_kernel(interrupt=True).raise_interrupt_signal() - else: - self._append_html( - _("

The kernel appears to be dead, so it can't be " - "interrupted. Please open a new console to keep " - "working.
") - ) else: self._append_html( _("

It is not possible to interrupt a non-Spyder " From 76140bfddcadf9a6ef048ea1bb9b7bd8f4ceff2a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 24 Jun 2023 10:42:39 +0200 Subject: [PATCH 024/110] restart kernel --- .../spyder_kernels_server/__main__.py | 19 +++- .../spyder_kernels_server/kernel_server.py | 9 +- .../ipythonconsole/utils/kernel_handler.py | 8 +- .../ipythonconsole/widgets/main_widget.py | 2 + .../plugins/ipythonconsole/widgets/mixins.py | 105 +++++++++++++----- .../plugins/ipythonconsole/widgets/shell.py | 1 + 6 files changed, 111 insertions(+), 33 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 4bb2ce0da34..a81cbcabb62 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -12,7 +12,7 @@ from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel # from qtpy.QtWidgets import QApplication -from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication +from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication, Slot class Server(QObject): @@ -34,13 +34,24 @@ def __init__(self): self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), QSocketNotifier.Read, self) self._notifier.activated.connect(self._socket_activity) + + if len(sys.argv) > 2: + self.port_pub = sys.argv[2] + else: + self.port_pub = str(zmqtunnel.select_random_ports(1)[0]) + self.socket_pub = context.socket(zmq.PUB) + self.socket_pub.bind("tcp://*:%s" % self.port_pub) + + self.kernel_server.sig_kernel_restarted.connect(self._handle_kernel_restarted) def _socket_activity(self): self._notifier.setEnabled(False) # Wait for next request from client message = self.socket.recv_pyobj() cmd = message[0] - if cmd == "shutdown": + if cmd == "get_port_pub": + self.socket.send_pyobj(["set_port_pub", self.port_pub]) + elif cmd == "shutdown": self.socket.send_pyobj(["shutting_down"]) self.kernel_server.shutdown() @@ -67,6 +78,10 @@ def _socket_activity(self): self.socket.getsockopt(zmq.EVENTS) + @Slot(str) + def _handle_kernel_restarted(self, connection_file): + self.socket_pub.send_pyobj(["kernel_restarted", connection_file]) + if __name__ == '__main__': app = QCoreApplication(sys.argv) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index bf944fb6f67..97e3929f592 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -19,6 +19,7 @@ # Third-party imports from jupyter_core.paths import jupyter_runtime_dir +from qtpy.QtCore import QObject, Signal from spyder_kernels_server.kernel_manager import SpyderKernelManager from spyder_kernels_server.kernel_comm import KernelComm @@ -77,9 +78,12 @@ def run(self): self.kernel_dict["stderr" ].join() -class KernelServer: +class KernelServer(QObject): + + sig_kernel_restarted = Signal(str) def __init__(self): + super().__init__() self._kernel_list = {} @staticmethod @@ -141,6 +145,9 @@ def open_kernel(self, kernel_spec): kernel_comm = KernelComm() kernel_comm.open_comm(kernel_client) self.connect_std_pipes(kernel_key, kernel_comm) + + kernel_manager.kernel_restarted.connect( + lambda connection_file=connection_file: self.sig_kernel_restarted.emit(connection_file)) return connection_file diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index ebd06722b28..e9080362649 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -12,7 +12,7 @@ import uuid # Third-party imports -from qtpy.QtCore import QObject, Signal, QSocketNotifier +from qtpy.QtCore import QObject, Signal, QSocketNotifier, Slot from zmq.ssh import tunnel as zmqtunnel import zmq @@ -103,6 +103,7 @@ class KernelHandler(QObject): """ sig_remote_close = Signal(str) + sig_kernel_restarted = Signal() def __init__( self, @@ -141,6 +142,11 @@ def __init__( # self.connect_std_pipes() self.kernel_client.start_channels() self.check_kernel_info() + + @Slot(str) + def kernel_restarted(self, connection_file): + if connection_file == self.connection_file: + self.sig_kernel_restarted.emit() def connect(self): """Connect to shellwidget.""" diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index fb4a8cf1de5..ba15821e890 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -128,6 +128,8 @@ class IPythonConsoleWidget( """ # Signals + sig_kernel_restarted = Signal(str) + sig_append_to_history_requested = Signal(str, str) """ This signal is emitted when the plugin requires to add commands to a diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 6e152fdd388..6c1dcf9ddc9 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -12,7 +12,7 @@ import os import queue -from qtpy.QtCore import QProcess, QSocketNotifier +from qtpy.QtCore import QProcess, QSocketNotifier, Slot # Local imports from spyder.api.config.mixins import SpyderConfigurationObserver @@ -66,9 +66,9 @@ def on_kernel_server_conf_changed(self, option=None, value=None): return self.options = options - self.hostname = None - self.sshkey = None - self.password = None + self.ssh_remote_hostname = None + self.ssh_key = None + self.ssh_password = None is_remote = options['kernel_server/external_server'] @@ -86,34 +86,27 @@ def on_kernel_server_conf_changed(self, option=None, value=None): is_ssh = options['kernel_server/use_ssh'] if not is_ssh: - self.connect_socket(f"{remote_ip}:{remote_port}") + self.connect_socket(remote_ip, remote_port) return username = options['kernel_server/username'] - self.hostname = f"{username}@{remote_ip}:{remote_port}" + self.ssh_remote_hostname = f"{username}@{remote_ip}:{remote_port}" # Now we deal with ssh uses_password = options['kernel_server/password_auth'] uses_keyfile = options['kernel_server/keyfile_auth'] if uses_password: - self.password = options['kernel_server/password'] - self.sshkey = None + self.ssh_password = options['kernel_server/password'] + self.ssh_key = None elif uses_keyfile: - self.password = options['kernel_server/passphrase'] - self.sshkey = options['kernel_server/keyfile'] + self.ssh_password = options['kernel_server/passphrase'] + self.ssh_key = options['kernel_server/keyfile'] else: raise NotImplementedError("This should not be possible.") - local_port = zmqtunnel.select_random_ports(1) - local_ip = "localhost" - timeout = 10 - ssh_tunnel( - local_port, remote_port, - local_ip, remote_ip, - self.sshkey, self.password, timeout) - self.connect_socket(f"{local_ip}:{local_port}") + self.connect_socket(remote_ip, remote_port) def start_local_server(self): """Start a server with the current interpreter.""" @@ -122,27 +115,48 @@ def start_local_server(self): self.server.start( sys.executable, ["-m", "spyder_kernels_server", port] ) - self.connect_socket(f"localhost:{port}") + self.connect_socket("localhost", port) - def connect_socket(self, hostname): + def connect_socket(self, hostname, port): + self.hostname = hostname + + hostname, port = self.tunnel_ssh(hostname, port) + self.socket = self.context.socket(zmq.REQ) - self.socket.connect(f"tcp://{hostname}") + self.socket.connect(f"tcp://{hostname}:{port}") self._notifier = QSocketNotifier( self.socket.getsockopt(zmq.FD), QSocketNotifier.Read, self ) self._notifier.activated.connect(self._socket_activity) + self.send_request(["get_port_pub"]) + + def tunnel_ssh(self, hostname, port): + if self.ssh_remote_hostname is None: + return hostname, port + remote_hostname = hostname + remote_port = port + port = zmqtunnel.select_random_ports(1) + hostname = "localhost" + timeout = 10 + ssh_tunnel( + port, remote_port, + hostname, remote_hostname, + self.ssh_key, self.ssh_password, timeout) + return hostname, port + def new_kernel(self, kernel_spec): """Get a new kernel""" kernel_handler = KernelHandler.new_from_spec( kernel_spec=kernel_spec, - hostname=self.hostname, - sshkey=self.sshkey, - password=self.password, + hostname=self.ssh_remote_hostname, + sshkey=self.ssh_key, + password=self.ssh_password, ) kernel_handler.sig_remote_close.connect(self.request_close) + self.sig_kernel_restarted.connect(kernel_handler.kernel_restarted) self.kernel_handler_waitlist.append(kernel_handler) @@ -159,7 +173,8 @@ def send_request(self, request): self.socket.send_pyobj(request) else: self.request_queue.put(request) - + + @Slot() def _socket_activity(self): if not self.socket.getsockopt(zmq.EVENTS) & zmq.POLLIN: return @@ -167,6 +182,7 @@ def _socket_activity(self): # Wait for next request from client message = self.socket.recv_pyobj() cmd = message[0] + if cmd == "new_kernel": cmd, connection_file, connection_info = message @@ -182,9 +198,24 @@ def _socket_activity(self): else: kernel_handler.set_connection( connection_file, connection_info, - self.hostname, - self.sshkey, - self.password) + self.ssh_remote_hostname, + self.ssh_key, + self.ssh_password) + + elif cmd == "set_port_pub": + port_pub = message[1] + self.socket_sub = self.context.socket(zmq.SUB) + # To recieve everything + self.socket_sub.setsockopt(zmq.SUBSCRIBE, b"") + hostname = self.hostname + hostname, port_pub = self.tunnel_ssh(hostname, port_pub) + + self.socket_sub.connect(f"tcp://{hostname}:{port_pub}") + self._notifier_sub = QSocketNotifier( + self.socket_sub.getsockopt(zmq.FD), + QSocketNotifier.Read, self + ) + self._notifier_sub.activated.connect(self._socket_sub_activity) self._notifier.setEnabled(True) # This is necessary for some reason. @@ -196,7 +227,23 @@ def _socket_activity(self): self.send_request(request) except queue.Empty: pass - + + @Slot() + def _socket_sub_activity(self): + if not self.socket_sub.getsockopt(zmq.EVENTS) & zmq.POLLIN: + return + self._notifier_sub.setEnabled(False) + # Wait for next request from client + message = self.socket_sub.recv_pyobj() + cmd = message[0] + if cmd == "kernel_restarted": + self.sig_kernel_restarted.emit(message[1]) + + self._notifier_sub.setEnabled(True) + # This is necessary for some reason. + # Otherwise the socket only works twice ! + self.socket_sub.getsockopt(zmq.EVENTS) + class CachedKernelMixin: """Cached kernel mixin.""" diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 1032d09b01a..5df94daedeb 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -269,6 +269,7 @@ def handle_kernel_is_ready(self): ): self.kernel_client = self.kernel_handler.kernel_client self.kernel_client.stopped_channels.connect(self.notify_deleted) + self.kernel_handler.sig_kernel_restarted.connect(self._handle_kernel_restarted) self.setup_spyder_kernel() return From fd1766dd25f211f4949d3b772bedfee6460710c4 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 24 Jun 2023 11:06:05 +0200 Subject: [PATCH 025/110] Do not reset when restarting --- spyder/plugins/ipythonconsole/widgets/shell.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 5df94daedeb..d5e1e117c84 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -268,8 +268,6 @@ def handle_kernel_is_ready(self): KernelConnectionState.SpyderKernelReady ): self.kernel_client = self.kernel_handler.kernel_client - self.kernel_client.stopped_channels.connect(self.notify_deleted) - self.kernel_handler.sig_kernel_restarted.connect(self._handle_kernel_restarted) self.setup_spyder_kernel() return @@ -347,6 +345,9 @@ def setup_spyder_kernel(self): if not self._init_kernel_setup: # Only do this setup once self._init_kernel_setup = True + + self.kernel_client.stopped_channels.connect(self.notify_deleted) + self.kernel_handler.sig_kernel_restarted.connect(self._handle_kernel_restarted) # For errors self.kernel_handler.kernel_comm.sig_exception_occurred.connect( @@ -362,6 +363,9 @@ def setup_spyder_kernel(self): for request_id, handler in self.kernel_comm_handlers.items(): self.kernel_handler.kernel_comm.register_call_handler( request_id, handler) + else: + # No reset please + self._starting = False # Setup to do after restart # Check for fault and send config From d2a7ab96640373837b94aa9f6f6fd41d778c018a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 12:48:50 +0200 Subject: [PATCH 026/110] close thread leak --- .../spyder_kernels_server/__main__.py | 38 ++++++++++++------- .../spyder_kernels_server/kernel_server.py | 33 +++++++++++----- .../plugins/ipythonconsole/widgets/mixins.py | 4 ++ 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index a81cbcabb62..bc931d5efd1 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -9,40 +9,39 @@ import sys import zmq import json +import argparse from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel -# from qtpy.QtWidgets import QApplication +from qtpy.QtWidgets import QApplication from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication, Slot class Server(QObject): - def __init__(self): + def __init__(self, main_port=None, pub_port=None): super().__init__() - if len(sys.argv) > 1: - port = sys.argv[1] - else: - port = str(zmqtunnel.select_random_ports(1)[0]) + if main_port is None: + main_port = str(zmqtunnel.select_random_ports(1)[0]) context = zmq.Context() self.socket = context.socket(zmq.REP) - self.socket.bind("tcp://*:%s" % port) - print(f"Server running on port {port}") + self.socket.bind("tcp://*:%s" % main_port) + print(f"Server running on port {main_port}") self.kernel_server = KernelServer() self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), QSocketNotifier.Read, self) self._notifier.activated.connect(self._socket_activity) - if len(sys.argv) > 2: - self.port_pub = sys.argv[2] - else: + self.port_pub = pub_port + if pub_port is None: self.port_pub = str(zmqtunnel.select_random_ports(1)[0]) self.socket_pub = context.socket(zmq.PUB) self.socket_pub.bind("tcp://*:%s" % self.port_pub) - self.kernel_server.sig_kernel_restarted.connect(self._handle_kernel_restarted) + self.kernel_server.sig_kernel_restarted.connect( + self._handle_kernel_restarted) def _socket_activity(self): self._notifier.setEnabled(False) @@ -84,6 +83,17 @@ def _handle_kernel_restarted(self, connection_file): if __name__ == '__main__': - app = QCoreApplication(sys.argv) - w = Server() + parser = argparse.ArgumentParser( + prog='Spyder Kernels Server', + description='Server to start and manage spyder kernels') + + parser.add_argument('port', default=None, nargs='?') # positional argument + parser.add_argument('port_pub', default=None, nargs='?') # option that takes a value + parser.add_argument('-i', "--interactive", action='store_true') + args = parser.parse_args() + if args.interactive: + app = QApplication(sys.argv) + else: + app = QCoreApplication(sys.argv) + w = Server(args.port, args.port_pub) sys.exit(app.exec_()) \ No newline at end of file diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 97e3929f592..fb2f0375f11 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -13,8 +13,9 @@ import os.path as osp from subprocess import PIPE import uuid +import sys -from threading import Thread +from threading import Thread, Event # Third-party imports @@ -33,19 +34,23 @@ # kernel_comm needs a qthread class StdThread(Thread): """Poll for changes in std buffers.""" - def __init__(self, std_buffer, buffer_key, kernel_comm): + def __init__(self, std_buffer, buffer_key, kernel_comm, file): self._std_buffer = std_buffer self.buffer_key = buffer_key self.kernel_comm = kernel_comm + self.print_file = file + self.closing = Event() super().__init__() def run(self): txt = True while txt: txt = self._std_buffer.read1() + if self.closing: + return if txt: - print(txt) + self.print_file.write(txt.decode()) # Needs to be on control so the message is sent to currently # executing shell self.kernel_comm.remote_call( @@ -64,9 +69,16 @@ def __init__(self, kernel_dict): def run(self): """Shutdown kernel.""" kernel_manager = self.kernel_dict["kernel"] + + if "stdout" in self.kernel_dict: + self.kernel_dict["stdout" ].closing.set() + if "stderr" in self.kernel_dict: + self.kernel_dict["stderr" ].closing.set() if not kernel_manager.shutting_down: kernel_manager.shutting_down = True + self.kernel_dict["client"].stop_channels() + try: kernel_manager.shutdown_kernel() except Exception: @@ -136,19 +148,20 @@ def open_kernel(self, kernel_spec): ) kernel_key = connection_file - self._kernel_list[kernel_key] = { - "kernel": kernel_manager - } - kernel_client = kernel_manager.client() kernel_client.start_channels() kernel_comm = KernelComm() kernel_comm.open_comm(kernel_client) + + self._kernel_list[kernel_key] = { + "kernel": kernel_manager, + "client": kernel_client + } self.connect_std_pipes(kernel_key, kernel_comm) kernel_manager.kernel_restarted.connect( lambda connection_file=connection_file: self.sig_kernel_restarted.emit(connection_file)) - + return connection_file def connect_std_pipes(self, kernel_key, kernel_comm): @@ -160,12 +173,12 @@ def connect_std_pipes(self, kernel_key, kernel_comm): if stdout: stdout_thread = StdThread( - stdout, "stdout", kernel_comm) + stdout, "stdout", kernel_comm, sys.stdout) stdout_thread.start() self._kernel_list[kernel_key]["stdout"] = stdout_thread if stderr: stderr_thread = StdThread( - stderr, "stderr", kernel_comm) + stderr, "stderr", kernel_comm, sys.stderr) stderr_thread.start() self._kernel_list[kernel_key]["stderr"] = stderr_thread diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 6c1dcf9ddc9..232ee709643 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -65,6 +65,10 @@ def on_kernel_server_conf_changed(self, option=None, value=None): if self.options == options: return + if self.options is not None: + # Close cached kernel + self.close_cached_kernel() + self.options = options self.ssh_remote_hostname = None self.ssh_key = None From 41a35799e825c62ffc68ac8ffb5cf64b09382d81 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 13:16:56 +0200 Subject: [PATCH 027/110] print errors --- spyder/plugins/ipythonconsole/widgets/mixins.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 232ee709643..0e82d314648 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -119,7 +119,17 @@ def start_local_server(self): self.server.start( sys.executable, ["-m", "spyder_kernels_server", port] ) + self.server.readyReadStandardError.connect(self.print_server_stderr) + self.server.readyReadStandardOutput.connect(self.print_server_stdout) self.connect_socket("localhost", port) + + @Slot() + def print_server_stderr(self): + sys.stderr.write(self.server.readAllStandardError().data().decode()) + + @Slot() + def print_server_stdout(self): + sys.stdout.write(self.server.readAllStandardOutput().data().decode()) def connect_socket(self, hostname, port): self.hostname = hostname From 3ab0c782ba918d3782fd2d0d7544699cc4748cf2 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 13:34:27 +0200 Subject: [PATCH 028/110] black --- .../spyder_kernels_server/__init__.py | 1 - .../spyder_kernels_server/__main__.py | 35 +++--- .../spyder_kernels_server/_version.py | 4 +- .../spyder_kernels_server/kernel_comm.py | 105 +++++++++++------- .../spyder_kernels_server/kernel_manager.py | 19 +++- .../spyder_kernels_server/kernel_server.py | 44 ++++---- 6 files changed, 120 insertions(+), 88 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py index 633f866158a..40a96afc6ff 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- - diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index bc931d5efd1..ac5092dd1b0 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -17,7 +17,6 @@ class Server(QObject): - def __init__(self, main_port=None, pub_port=None): super().__init__() @@ -30,18 +29,20 @@ def __init__(self, main_port=None, pub_port=None): print(f"Server running on port {main_port}") self.kernel_server = KernelServer() - self._notifier = QSocketNotifier(self.socket.getsockopt(zmq.FD), - QSocketNotifier.Read, self) + self._notifier = QSocketNotifier( + self.socket.getsockopt(zmq.FD), QSocketNotifier.Read, self + ) self._notifier.activated.connect(self._socket_activity) - + self.port_pub = pub_port if pub_port is None: self.port_pub = str(zmqtunnel.select_random_ports(1)[0]) self.socket_pub = context.socket(zmq.PUB) self.socket_pub.bind("tcp://*:%s" % self.port_pub) - + self.kernel_server.sig_kernel_restarted.connect( - self._handle_kernel_restarted) + self._handle_kernel_restarted + ) def _socket_activity(self): self._notifier.setEnabled(False) @@ -71,29 +72,29 @@ def _socket_activity(self): except Exception: pass self._notifier.setEnabled(True) - + # This is necessary for some reason. # Otherwise the socket only works twice ! self.socket.getsockopt(zmq.EVENTS) - @Slot(str) def _handle_kernel_restarted(self, connection_file): self.socket_pub.send_pyobj(["kernel_restarted", connection_file]) - -if __name__ == '__main__': + +if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='Spyder Kernels Server', - description='Server to start and manage spyder kernels') - - parser.add_argument('port', default=None, nargs='?') # positional argument - parser.add_argument('port_pub', default=None, nargs='?') # option that takes a value - parser.add_argument('-i', "--interactive", action='store_true') + prog="Spyder Kernels Server", + description="Server to start and manage spyder kernels", + ) + + parser.add_argument("port", default=None, nargs="?") + parser.add_argument("port_pub", default=None, nargs="?") + parser.add_argument("-i", "--interactive", action="store_true") args = parser.parse_args() if args.interactive: app = QApplication(sys.argv) else: app = QCoreApplication(sys.argv) w = Server(args.port, args.port_pub) - sys.exit(app.exec_()) \ No newline at end of file + sys.exit(app.exec_()) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py b/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py index b9e5bdfe787..da01d2e7afa 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/_version.py @@ -8,5 +8,5 @@ """Version File.""" -VERSION_INFO = (1, 0, 0, 'dev0') -__version__ = '.'.join(map(str, VERSION_INFO)) +VERSION_INFO = (1, 0, 0, "dev0") +__version__ = ".".join(map(str, VERSION_INFO)) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py index 43a5334eb5b..85c46fa21d0 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_comm.py @@ -22,14 +22,14 @@ def get_debug_level(): - debug_env = os.environ.get('SPYDER_DEBUG', '') + debug_env = os.environ.get("SPYDER_DEBUG", "") if not debug_env.isdigit(): debug_env = bool(debug_env) return int(debug_env) def running_under_pytest(): - return bool(os.environ.get('SPYDER_PYTEST')) + return bool(os.environ.get("SPYDER_PYTEST")) class KernelComm(CommBase, QObject): @@ -47,8 +47,8 @@ def __init__(self): self.kernel_client = None # Register handlers - self.register_call_handler('_async_error', self._async_error) - self.register_call_handler('_comm_ready', self._comm_ready) + self.register_call_handler("_async_error", self._async_error) + self.register_call_handler("_comm_ready", self._comm_ready) def is_open(self, comm_id=None): """ @@ -57,7 +57,7 @@ def is_open(self, comm_id=None): id_list = self.get_comm_id_list(comm_id) if len(id_list) == 0: return False - return all([self._comms[cid]['status'] == 'ready' for cid in id_list]) + return all([self._comms[cid]["status"] == "ready" for cid in id_list]) @contextmanager def comm_channel_manager(self, comm_id, queue_message=False): @@ -69,24 +69,27 @@ def comm_channel_manager(self, comm_id, queue_message=False): id_list = self.get_comm_id_list(comm_id) for comm_id in id_list: - self._comms[comm_id]['comm']._send_channel = ( - self.kernel_client.control_channel) + self._comms[comm_id][ + "comm" + ]._send_channel = self.kernel_client.control_channel try: yield finally: id_list = self.get_comm_id_list(comm_id) for comm_id in id_list: - self._comms[comm_id]['comm']._send_channel = ( - self.kernel_client.shell_channel) + self._comms[comm_id][ + "comm" + ]._send_channel = self.kernel_client.shell_channel def _set_call_return_value(self, call_dict, data, is_error=False): """Override to use the comm_channel for all replies.""" with self.comm_channel_manager(self.calling_comm_id, False): if is_error and (get_debug_level() or running_under_pytest()): # Disable error muting when debugging or testing - call_dict['settings']['display_error'] = True + call_dict["settings"]["display_error"] = True super(KernelComm, self)._set_call_return_value( - call_dict, data, is_error) + call_dict, data, is_error + ) def remove(self, comm_id=None, only_closing=False): """ @@ -96,7 +99,7 @@ def remove(self, comm_id=None, only_closing=False): """ id_list = self.get_comm_id_list(comm_id) for comm_id in id_list: - if only_closing and self._comms[comm_id]['status'] != 'closing': + if only_closing and self._comms[comm_id]["status"] != "closing": continue del self._comms[comm_id] @@ -105,9 +108,10 @@ def close(self, comm_id=None): id_list = self.get_comm_id_list(comm_id) for comm_id in id_list: # Send comm_close directly to avoid really closing the comm - self._comms[comm_id]['comm']._send_msg( - 'comm_close', {}, None, None, None) - self._comms[comm_id]['status'] = 'closing' + self._comms[comm_id]["comm"]._send_msg( + "comm_close", {}, None, None, None + ) + self._comms[comm_id]["status"] = "closing" def open_comm(self, kernel_client): """Open comm through the kernel client.""" @@ -115,20 +119,36 @@ def open_comm(self, kernel_client): try: self._register_comm( # Create new comm and send the highest protocol - kernel_client.comm_manager.new_comm(self._comm_name, data={ - 'pickle_highest_protocol': pickle.HIGHEST_PROTOCOL})) + kernel_client.comm_manager.new_comm( + self._comm_name, + data={"pickle_highest_protocol": pickle.HIGHEST_PROTOCOL}, + ) + ) except AttributeError: logger.info( - "Unable to open comm due to unexistent comm manager: " + - "kernel_client.comm_manager=" + str(kernel_client.comm_manager) + "Unable to open comm due to unexistent comm manager: " + + "kernel_client.comm_manager=" + + str(kernel_client.comm_manager) ) - def remote_call(self, interrupt=False, blocking=False, callback=None, - comm_id=None, timeout=None, display_error=False): + def remote_call( + self, + interrupt=False, + blocking=False, + callback=None, + comm_id=None, + timeout=None, + display_error=False, + ): """Get a handler for remote calls.""" return super(KernelComm, self).remote_call( - interrupt=interrupt, blocking=blocking, callback=callback, - comm_id=comm_id, timeout=timeout, display_error=display_error) + interrupt=interrupt, + blocking=blocking, + callback=callback, + comm_id=comm_id, + timeout=timeout, + display_error=display_error, + ) def on_incoming_call(self, call_dict): """A call was received""" @@ -139,15 +159,15 @@ def on_incoming_call(self, call_dict): # ---- Private ----- def _comm_ready(self): """If this function is called, the comm is ready""" - if self._comms[self.calling_comm_id]['status'] != 'ready': - self._comms[self.calling_comm_id]['status'] = 'ready' + if self._comms[self.calling_comm_id]["status"] != "ready": + self._comms[self.calling_comm_id]["status"] = "ready" self.sig_comm_ready.emit() def _send_call(self, call_dict, call_data, comm_id): """Send call and interupt the kernel if needed.""" - settings = call_dict['settings'] - blocking = 'blocking' in settings and settings['blocking'] - interrupt = 'interrupt' in settings and settings['interrupt'] + settings = call_dict["settings"] + blocking = "blocking" in settings and settings["blocking"] + interrupt = "interrupt" in settings and settings["interrupt"] queue_message = not interrupt and not blocking if not self.kernel_client.is_alive(): @@ -157,14 +177,14 @@ def _send_call(self, call_dict, call_data, comm_id): # The user has other problems logger.info( "Dropping message because kernel is dead: %s", - str(call_dict) + str(call_dict), ) return - with self.comm_channel_manager( - comm_id, queue_message=queue_message): + with self.comm_channel_manager(comm_id, queue_message=queue_message): return super(KernelComm, self)._send_call( - call_dict, call_data, comm_id) + call_dict, call_data, comm_id + ) def _get_call_return_value(self, call_dict, comm_id): """ @@ -172,10 +192,11 @@ def _get_call_return_value(self, call_dict, comm_id): """ try: return super(KernelComm, self)._get_call_return_value( - call_dict, comm_id) + call_dict, comm_id + ) except RuntimeError as e: - settings = call_dict['settings'] - blocking = 'blocking' in settings and settings['blocking'] + settings = call_dict["settings"] + blocking = "blocking" in settings and settings["blocking"] if blocking: raise else: @@ -183,7 +204,7 @@ def _get_call_return_value(self, call_dict, comm_id): logger.info( "Dropping message because of exception: ", str(e), - str(call_dict) + str(call_dict), ) return @@ -194,7 +215,8 @@ def got_reply(): return call_id in self._reply_inbox timeout_msg = "Timeout while waiting for {}".format( - self._reply_waitlist) + self._reply_waitlist + ) self._wait(got_reply, self._sig_got_reply, timeout_msg, timeout) def _wait(self, condition, signal, timeout_msg, timeout): @@ -227,7 +249,8 @@ def _wait(self, condition, signal, timeout_msg, timeout): if not wait_timeout.isActive(): signal.disconnect(wait_loop.quit) self.kernel_client.hb_channel.kernel_died.disconnect( - wait_loop.quit) + wait_loop.quit + ) if condition(): return if not self.kernel_client.is_alive(): @@ -237,15 +260,13 @@ def _wait(self, condition, signal, timeout_msg, timeout): wait_timeout.stop() signal.disconnect(wait_loop.quit) - self.kernel_client.hb_channel.kernel_died.disconnect( - wait_loop.quit) + self.kernel_client.hb_channel.kernel_died.disconnect(wait_loop.quit) def _handle_remote_call_reply(self, msg_dict, buffer): """ A blocking call received a reply. """ - super(KernelComm, self)._handle_remote_call_reply( - msg_dict, buffer) + super(KernelComm, self)._handle_remote_call_reply(msg_dict, buffer) self._sig_got_reply.emit() def _async_error(self, error_wrapper): diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py index ab68a79a833..929a4a0161a 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_manager.py @@ -30,15 +30,21 @@ class SpyderKernelManager(QtKernelManager): """ client_class = DottedObjectName( - 'spyder_kernels_server.kernel_client.SpyderKernelClient') + "spyder_kernels_server.kernel_client.SpyderKernelClient" + ) def __init__(self, *args, **kwargs): self.shutting_down = False return QtKernelManager.__init__(self, *args, **kwargs) @staticmethod - async def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True, - timeout=None, on_terminate=None): + async def kill_proc_tree( + pid, + sig=signal.SIGTERM, + include_parent=True, + timeout=None, + on_terminate=None, + ): """ Kill a process tree (including grandchildren) with sig and return a (gone, still_alive) tuple. @@ -101,13 +107,16 @@ async def _async_kill_kernel(self, restart: bool = False) -> None: # Wait until the kernel terminates. import asyncio + try: await asyncio.wait_for(self._async_wait(), timeout=5.0) except asyncio.TimeoutError: # Wait timed out, just log warning but continue # - not much more we can do. - self.log.warning("Wait for final termination of kernel timed" - " out - continuing...") + self.log.warning( + "Wait for final termination of kernel timed" + " out - continuing..." + ) pass else: # Process is no longer alive, wait and clear diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index fb2f0375f11..d0e7352a2fc 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -34,6 +34,7 @@ # kernel_comm needs a qthread class StdThread(Thread): """Poll for changes in std buffers.""" + def __init__(self, std_buffer, buffer_key, kernel_comm, file): self._std_buffer = std_buffer @@ -53,11 +54,8 @@ def run(self): self.print_file.write(txt.decode()) # Needs to be on control so the message is sent to currently # executing shell - self.kernel_comm.remote_call( - interrupt=True - ).print_remote( - txt.decode(), - self.buffer_key + self.kernel_comm.remote_call(interrupt=True).print_remote( + txt.decode(), self.buffer_key ) @@ -69,29 +67,29 @@ def __init__(self, kernel_dict): def run(self): """Shutdown kernel.""" kernel_manager = self.kernel_dict["kernel"] - + if "stdout" in self.kernel_dict: - self.kernel_dict["stdout" ].closing.set() + self.kernel_dict["stdout"].closing.set() if "stderr" in self.kernel_dict: - self.kernel_dict["stderr" ].closing.set() + self.kernel_dict["stderr"].closing.set() if not kernel_manager.shutting_down: kernel_manager.shutting_down = True self.kernel_dict["client"].stop_channels() - + try: kernel_manager.shutdown_kernel() except Exception: # kernel was externally killed pass if "stdout" in self.kernel_dict: - self.kernel_dict["stdout" ].join() + self.kernel_dict["stdout"].join() if "stderr" in self.kernel_dict: - self.kernel_dict["stderr" ].join() + self.kernel_dict["stderr"].join() class KernelServer(QObject): - + sig_kernel_restarted = Signal(str) def __init__(self): @@ -119,7 +117,6 @@ def new_connection_file(): cf = cf if not os.path.exists(cf) else "" return cf - def open_kernel(self, kernel_spec): """ Create a new kernel. @@ -152,16 +149,19 @@ def open_kernel(self, kernel_spec): kernel_client.start_channels() kernel_comm = KernelComm() kernel_comm.open_comm(kernel_client) - + self._kernel_list[kernel_key] = { "kernel": kernel_manager, - "client": kernel_client - } + "client": kernel_client, + } self.connect_std_pipes(kernel_key, kernel_comm) - + kernel_manager.kernel_restarted.connect( - lambda connection_file=connection_file: self.sig_kernel_restarted.emit(connection_file)) - + lambda connection_file=connection_file: self.sig_kernel_restarted.emit( + connection_file + ) + ) + return connection_file def connect_std_pipes(self, kernel_key, kernel_comm): @@ -173,12 +173,14 @@ def connect_std_pipes(self, kernel_key, kernel_comm): if stdout: stdout_thread = StdThread( - stdout, "stdout", kernel_comm, sys.stdout) + stdout, "stdout", kernel_comm, sys.stdout + ) stdout_thread.start() self._kernel_list[kernel_key]["stdout"] = stdout_thread if stderr: stderr_thread = StdThread( - stderr, "stderr", kernel_comm, sys.stderr) + stderr, "stderr", kernel_comm, sys.stderr + ) stderr_thread.start() self._kernel_list[kernel_key]["stderr"] = stderr_thread From 33c8a7cf7609e8648dd86f753644bdb797deb7c9 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 13:41:07 +0200 Subject: [PATCH 029/110] remove print_remote --- .../spyder_kernels_server/kernel_server.py | 30 ++++--------------- .../spyder_kernels/console/kernel.py | 12 -------- 2 files changed, 5 insertions(+), 37 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index d0e7352a2fc..93975c9e467 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -35,11 +35,8 @@ class StdThread(Thread): """Poll for changes in std buffers.""" - def __init__(self, std_buffer, buffer_key, kernel_comm, file): + def __init__(self, std_buffer, file): self._std_buffer = std_buffer - - self.buffer_key = buffer_key - self.kernel_comm = kernel_comm self.print_file = file self.closing = Event() super().__init__() @@ -52,11 +49,6 @@ def run(self): return if txt: self.print_file.write(txt.decode()) - # Needs to be on control so the message is sent to currently - # executing shell - self.kernel_comm.remote_call(interrupt=True).print_remote( - txt.decode(), self.buffer_key - ) class ShutdownThread(Thread): @@ -75,8 +67,6 @@ def run(self): if not kernel_manager.shutting_down: kernel_manager.shutting_down = True - self.kernel_dict["client"].stop_channels() - try: kernel_manager.shutdown_kernel() except Exception: @@ -145,16 +135,10 @@ def open_kernel(self, kernel_spec): ) kernel_key = connection_file - kernel_client = kernel_manager.client() - kernel_client.start_channels() - kernel_comm = KernelComm() - kernel_comm.open_comm(kernel_client) - self._kernel_list[kernel_key] = { "kernel": kernel_manager, - "client": kernel_client, } - self.connect_std_pipes(kernel_key, kernel_comm) + self.connect_std_pipes(kernel_key) kernel_manager.kernel_restarted.connect( lambda connection_file=connection_file: self.sig_kernel_restarted.emit( @@ -164,7 +148,7 @@ def open_kernel(self, kernel_spec): return connection_file - def connect_std_pipes(self, kernel_key, kernel_comm): + def connect_std_pipes(self, kernel_key): """Connect to std pipes.""" kernel_manager = self._kernel_list[kernel_key]["kernel"] @@ -172,15 +156,11 @@ def connect_std_pipes(self, kernel_key, kernel_comm): stderr = kernel_manager.provisioner.process.stderr if stdout: - stdout_thread = StdThread( - stdout, "stdout", kernel_comm, sys.stdout - ) + stdout_thread = StdThread(stdout, sys.stdout) stdout_thread.start() self._kernel_list[kernel_key]["stdout"] = stdout_thread if stderr: - stderr_thread = StdThread( - stderr, "stderr", kernel_comm, sys.stderr - ) + stderr_thread = StdThread(stderr, sys.stderr) stderr_thread.start() self._kernel_list[kernel_key]["stderr"] = stderr_thread diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 5c2a1100c51..d1a585a301a 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -90,7 +90,6 @@ def __init__(self, *args, **kwargs): 'request_pdb_stop': self.shell.request_pdb_stop, 'raise_interrupt_signal': self.shell.raise_interrupt_signal, 'get_fault_text': self.get_fault_text, - 'print_remote': self.print_remote, } for call_id in handlers: self.frontend_comm.register_call_handler( @@ -968,14 +967,3 @@ def post_handler_hook(self): self.shell.register_debugger_sigint() # Reset tracing function so that pdb.set_trace works sys.settrace(None) - - def print_remote(self, text, file_name=None): - """Remote print""" - file = None - if file_name == "stdout": - file = sys.stdout - elif file_name == "stderr": - file = sys.stderr - print(text, file=file) - if file: - file.flush() From 4a81781c92acfdc4bec6d66ed865cab0e4641194 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 22:45:08 +0200 Subject: [PATCH 030/110] stop local server --- .../spyder_kernels_server/__main__.py | 2 ++ .../spyder_kernels_server/kernel_server.py | 3 ++- .../plugins/ipythonconsole/widgets/mixins.py | 21 +++++++++++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index ac5092dd1b0..742f6b2b78e 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -54,6 +54,7 @@ def _socket_activity(self): elif cmd == "shutdown": self.socket.send_pyobj(["shutting_down"]) self.kernel_server.shutdown() + QCoreApplication.instance().quit() elif cmd == "open_kernel": try: @@ -71,6 +72,7 @@ def _socket_activity(self): self.kernel_server.close_kernel(message[1]) except Exception: pass + self._notifier.setEnabled(True) # This is necessary for some reason. diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 93975c9e467..182d9080185 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -172,5 +172,6 @@ def close_kernel(self, kernel_key): shutdown_thread.start() def shutdown(self): - for kernel_key in self._kernel_list: + kernel_key_list = list(self._kernel_list) + for kernel_key in kernel_key_list: self.close_kernel(kernel_key) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 0e82d314648..b56216ac2bf 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -44,6 +44,7 @@ class KernelConnectorMixin(SpyderConfigurationObserver): def __init__(self): super().__init__() self.options = None + self.server = None self.kernel_handler_waitlist = [] self.request_queue = queue.Queue() self.context = zmq.Context() @@ -64,11 +65,19 @@ def on_kernel_server_conf_changed(self, option=None, value=None): } if self.options == options: return - + + if self.server is not None: + self.stop_local_server() + if self.options is not None: # Close cached kernel self.close_cached_kernel() - + # Reset request_queue + self.request_queue = queue.Queue() + # Send new kernel request for waiting kernels + for kernel_handler in self.kernel_handler_waitlist: + self.send_request(["open_kernel", kernel_handler.kernel_spec]) + self.options = options self.ssh_remote_hostname = None self.ssh_key = None @@ -79,6 +88,7 @@ def on_kernel_server_conf_changed(self, option=None, value=None): if not is_remote: self.start_local_server() return + # Remote server remote_port = int(options['kernel_server/port']) @@ -123,6 +133,13 @@ def start_local_server(self): self.server.readyReadStandardOutput.connect(self.print_server_stdout) self.connect_socket("localhost", port) + def stop_local_server(self): + """Stop local server.""" + self.server.readyReadStandardError.disconnect(self.print_server_stderr) + self.server.readyReadStandardOutput.disconnect(self.print_server_stdout) + self.send_request(["shutdown"]) + self.server = None + @Slot() def print_server_stderr(self): sys.stderr.write(self.server.readAllStandardError().data().decode()) From 84705f57880d353cb35068bf72fc7afa811a166c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 22:46:32 +0200 Subject: [PATCH 031/110] black --- .../spyder_kernels_server/__main__.py | 2 +- .../plugins/ipythonconsole/widgets/mixins.py | 128 +++++++++--------- 2 files changed, 64 insertions(+), 66 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 742f6b2b78e..5b360e5e04d 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -72,7 +72,7 @@ def _socket_activity(self): self.kernel_server.close_kernel(message[1]) except Exception: pass - + self._notifier.setEnabled(True) # This is necessary for some reason. diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index b56216ac2bf..41b03fc5c9d 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -20,6 +20,7 @@ from spyder.api.config.decorators import on_conf_change from zmq.ssh import tunnel as zmqtunnel from spyder.plugins.ipythonconsole.utils.ssh import openssh_tunnel + if os.name == "nt": ssh_tunnel = zmqtunnel.paramiko_tunnel else: @@ -27,20 +28,22 @@ KERNEL_SERVER_OPTIONS = [ - 'kernel_server/external_server', - 'kernel_server/use_ssh', - 'kernel_server/host', - 'kernel_server/port', - 'kernel_server/username', - 'kernel_server/password_auth', - 'kernel_server/keyfile_auth', - 'kernel_server/password', - 'kernel_server/keyfile', - 'kernel_server/passphrase', + "kernel_server/external_server", + "kernel_server/use_ssh", + "kernel_server/host", + "kernel_server/port", + "kernel_server/username", + "kernel_server/password_auth", + "kernel_server/keyfile_auth", + "kernel_server/password", + "kernel_server/keyfile", + "kernel_server/passphrase", ] + class KernelConnectorMixin(SpyderConfigurationObserver): """Needs https://github.com/jupyter/jupyter_client/pull/835""" + def __init__(self): super().__init__() self.options = None @@ -51,16 +54,11 @@ def __init__(self): self.on_kernel_server_conf_changed() - @on_conf_change( - option=KERNEL_SERVER_OPTIONS, - section='main_interpreter' - ) + @on_conf_change(option=KERNEL_SERVER_OPTIONS, section="main_interpreter") def on_kernel_server_conf_changed(self, option=None, value=None): """Start server""" options = { - option: self.get_conf( - option=option, - section='main_interpreter') + option: self.get_conf(option=option, section="main_interpreter") for option in KERNEL_SERVER_OPTIONS } if self.options == options: @@ -83,7 +81,7 @@ def on_kernel_server_conf_changed(self, option=None, value=None): self.ssh_key = None self.ssh_password = None - is_remote = options['kernel_server/external_server'] + is_remote = options["kernel_server/external_server"] if not is_remote: self.start_local_server() @@ -91,32 +89,31 @@ def on_kernel_server_conf_changed(self, option=None, value=None): # Remote server - remote_port = int(options['kernel_server/port']) + remote_port = int(options["kernel_server/port"]) if not remote_port: remote_port = 22 - remote_ip = options['kernel_server/host'] - + remote_ip = options["kernel_server/host"] - is_ssh = options['kernel_server/use_ssh'] + is_ssh = options["kernel_server/use_ssh"] if not is_ssh: self.connect_socket(remote_ip, remote_port) return - username = options['kernel_server/username'] + username = options["kernel_server/username"] self.ssh_remote_hostname = f"{username}@{remote_ip}:{remote_port}" # Now we deal with ssh - uses_password = options['kernel_server/password_auth'] - uses_keyfile = options['kernel_server/keyfile_auth'] + uses_password = options["kernel_server/password_auth"] + uses_keyfile = options["kernel_server/keyfile_auth"] if uses_password: - self.ssh_password = options['kernel_server/password'] + self.ssh_password = options["kernel_server/password"] self.ssh_key = None elif uses_keyfile: - self.ssh_password = options['kernel_server/passphrase'] - self.ssh_key = options['kernel_server/keyfile'] + self.ssh_password = options["kernel_server/passphrase"] + self.ssh_key = options["kernel_server/keyfile"] else: raise NotImplementedError("This should not be possible.") @@ -126,42 +123,39 @@ def start_local_server(self): """Start a server with the current interpreter.""" port = str(zmqtunnel.select_random_ports(1)[0]) self.server = QProcess(self) - self.server.start( - sys.executable, ["-m", "spyder_kernels_server", port] - ) + self.server.start(sys.executable, ["-m", "spyder_kernels_server", port]) self.server.readyReadStandardError.connect(self.print_server_stderr) self.server.readyReadStandardOutput.connect(self.print_server_stdout) self.connect_socket("localhost", port) - + def stop_local_server(self): """Stop local server.""" self.server.readyReadStandardError.disconnect(self.print_server_stderr) self.server.readyReadStandardOutput.disconnect(self.print_server_stdout) self.send_request(["shutdown"]) self.server = None - + @Slot() def print_server_stderr(self): sys.stderr.write(self.server.readAllStandardError().data().decode()) - + @Slot() def print_server_stdout(self): sys.stdout.write(self.server.readAllStandardOutput().data().decode()) def connect_socket(self, hostname, port): self.hostname = hostname - + hostname, port = self.tunnel_ssh(hostname, port) - + self.socket = self.context.socket(zmq.REQ) self.socket.connect(f"tcp://{hostname}:{port}") self._notifier = QSocketNotifier( - self.socket.getsockopt(zmq.FD), - QSocketNotifier.Read, self + self.socket.getsockopt(zmq.FD), QSocketNotifier.Read, self ) self._notifier.activated.connect(self._socket_activity) self.send_request(["get_port_pub"]) - + def tunnel_ssh(self, hostname, port): if self.ssh_remote_hostname is None: return hostname, port @@ -171,11 +165,15 @@ def tunnel_ssh(self, hostname, port): hostname = "localhost" timeout = 10 ssh_tunnel( - port, remote_port, - hostname, remote_hostname, - self.ssh_key, self.ssh_password, timeout) + port, + remote_port, + hostname, + remote_hostname, + self.ssh_key, + self.ssh_password, + timeout, + ) return hostname, port - def new_kernel(self, kernel_spec): """Get a new kernel""" @@ -185,26 +183,25 @@ def new_kernel(self, kernel_spec): sshkey=self.ssh_key, password=self.ssh_password, ) - + kernel_handler.sig_remote_close.connect(self.request_close) self.sig_kernel_restarted.connect(kernel_handler.kernel_restarted) self.kernel_handler_waitlist.append(kernel_handler) - - + self.send_request(["open_kernel", kernel_spec]) - + return kernel_handler - + def request_close(self, connection_file): self.send_request(["close_kernel", connection_file]) - + def send_request(self, request): - + if self.socket.getsockopt(zmq.EVENTS) & zmq.POLLOUT: self.socket.send_pyobj(request) else: self.request_queue.put(request) - + @Slot() def _socket_activity(self): if not self.socket.getsockopt(zmq.EVENTS) & zmq.POLLIN: @@ -216,22 +213,24 @@ def _socket_activity(self): if cmd == "new_kernel": cmd, connection_file, connection_info = message - + if len(self.kernel_handler_waitlist) == 0: # This should not happen :/ self._notifier.setEnabled(True) self.socket.getsockopt(zmq.EVENTS) return - + kernel_handler = self.kernel_handler_waitlist.pop(0) if connection_file == "error": kernel_handler.sig_fault.emit(str(connection_info)) else: kernel_handler.set_connection( - connection_file, connection_info, + connection_file, + connection_info, self.ssh_remote_hostname, self.ssh_key, - self.ssh_password) + self.ssh_password, + ) elif cmd == "set_port_pub": port_pub = message[1] @@ -240,25 +239,24 @@ def _socket_activity(self): self.socket_sub.setsockopt(zmq.SUBSCRIBE, b"") hostname = self.hostname hostname, port_pub = self.tunnel_ssh(hostname, port_pub) - + self.socket_sub.connect(f"tcp://{hostname}:{port_pub}") self._notifier_sub = QSocketNotifier( - self.socket_sub.getsockopt(zmq.FD), - QSocketNotifier.Read, self + self.socket_sub.getsockopt(zmq.FD), QSocketNotifier.Read, self ) self._notifier_sub.activated.connect(self._socket_sub_activity) - + self._notifier.setEnabled(True) # This is necessary for some reason. # Otherwise the socket only works twice ! self.socket.getsockopt(zmq.EVENTS) - + try: request = self.request_queue.get_nowait() self.send_request(request) except queue.Empty: pass - + @Slot() def _socket_sub_activity(self): if not self.socket_sub.getsockopt(zmq.EVENTS) & zmq.POLLIN: @@ -269,15 +267,16 @@ def _socket_sub_activity(self): cmd = message[0] if cmd == "kernel_restarted": self.sig_kernel_restarted.emit(message[1]) - + self._notifier_sub.setEnabled(True) # This is necessary for some reason. # Otherwise the socket only works twice ! self.socket_sub.getsockopt(zmq.EVENTS) - + class CachedKernelMixin: """Cached kernel mixin.""" + def __init__(self): super().__init__() self._cached_kernel_properties = None @@ -308,8 +307,7 @@ def check_cached_kernel_spec(self, kernel_spec): if "PYTEST_CURRENT_TEST" in cached_env: # Make tests faster by using cached kernels # hopefully the kernel will never use PYTEST_CURRENT_TEST - cached_env["PYTEST_CURRENT_TEST"] = ( - kernel_spec.env["PYTEST_CURRENT_TEST"]) + cached_env["PYTEST_CURRENT_TEST"] = kernel_spec.env["PYTEST_CURRENT_TEST"] return ( cached_spec.__dict__ == kernel_spec.__dict__ and kernel_spec.argv == cached_argv From 7bf8c6b36978d4f4511e88351d843d133b89d0bd Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 23:36:39 +0200 Subject: [PATCH 032/110] call socket activity after checking state --- spyder/plugins/ipythonconsole/widgets/mixins.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 41b03fc5c9d..6e4dd75ff3c 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -196,11 +196,16 @@ def request_close(self, connection_file): self.send_request(["close_kernel", connection_file]) def send_request(self, request): - - if self.socket.getsockopt(zmq.EVENTS) & zmq.POLLOUT: + # Check socket state + socket_state = self.socket.getsockopt(zmq.EVENTS) + if socket_state & zmq.POLLOUT: self.socket.send_pyobj(request) else: self.request_queue.put(request) + # Checking the socket state interferes with the notifier. + # If the socket is ready, read. + if socket_state & zmq.POLLIN: + self._socket_activity() @Slot() def _socket_activity(self): From 6142b7ab17eb7ce47316bbe066cfe17d5c7fad8d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 25 Jun 2023 23:37:07 +0200 Subject: [PATCH 033/110] black --- spyder/plugins/ipythonconsole/widgets/mixins.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 6e4dd75ff3c..784e7bd58d1 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -123,7 +123,9 @@ def start_local_server(self): """Start a server with the current interpreter.""" port = str(zmqtunnel.select_random_ports(1)[0]) self.server = QProcess(self) - self.server.start(sys.executable, ["-m", "spyder_kernels_server", port]) + self.server.start( + sys.executable, ["-m", "spyder_kernels_server", port] + ) self.server.readyReadStandardError.connect(self.print_server_stderr) self.server.readyReadStandardOutput.connect(self.print_server_stdout) self.connect_socket("localhost", port) @@ -131,7 +133,9 @@ def start_local_server(self): def stop_local_server(self): """Stop local server.""" self.server.readyReadStandardError.disconnect(self.print_server_stderr) - self.server.readyReadStandardOutput.disconnect(self.print_server_stdout) + self.server.readyReadStandardOutput.disconnect( + self.print_server_stdout + ) self.send_request(["shutdown"]) self.server = None @@ -312,7 +316,9 @@ def check_cached_kernel_spec(self, kernel_spec): if "PYTEST_CURRENT_TEST" in cached_env: # Make tests faster by using cached kernels # hopefully the kernel will never use PYTEST_CURRENT_TEST - cached_env["PYTEST_CURRENT_TEST"] = kernel_spec.env["PYTEST_CURRENT_TEST"] + cached_env["PYTEST_CURRENT_TEST"] = kernel_spec.env[ + "PYTEST_CURRENT_TEST" + ] return ( cached_spec.__dict__ == kernel_spec.__dict__ and kernel_spec.argv == cached_argv From 851d3570a71e2f08aaace4d6f00bb896fba3ad29 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 06:03:39 +0200 Subject: [PATCH 034/110] revert changes --- .../spyder_kernels_server/kernel_server.py | 1 - spyder/plugins/ipythonconsole/utils/kernelspec.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 182d9080185..b2d6e8f152e 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -23,7 +23,6 @@ from qtpy.QtCore import QObject, Signal from spyder_kernels_server.kernel_manager import SpyderKernelManager -from spyder_kernels_server.kernel_comm import KernelComm PERMISSION_ERROR_MSG = ( diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 9b8c84cd996..76614712f11 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -27,7 +27,7 @@ SpyderKernelError) from spyder.utils.conda import (add_quotes, get_conda_env_path, is_conda_env, find_conda) -from spyder.utils.environ import clean_env, get_user_environment_variables +# from spyder.utils.environ import get_user_environment_variables from spyder.utils.misc import get_python_executable from spyder.utils.programs import is_python_interpreter, is_module_installed @@ -188,7 +188,8 @@ def env(self): # Ensure that user environment variables are included, but don't # override existing environ values - env_vars = get_user_environment_variables() + # env_vars = get_user_environment_variables() + env_vars = {} env_vars.update(os.environ) # Avoid IPython adding the virtualenv on which Spyder is running From 4a68c6e7f7ef3b6fbabc88913bb993a6b55c6734 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 20:47:22 +0200 Subject: [PATCH 035/110] fix copy --- .../ipythonconsole/utils/kernel_handler.py | 20 +++++-------------- .../plugins/ipythonconsole/widgets/mixins.py | 3 ++- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index e9080362649..2b00ce21e07 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -375,25 +375,15 @@ def after_shutdown(self): def copy(self): """Copy kernel.""" - # Copy kernel infos - - # Get new kernel_client - kernel_client = self.init_kernel_client( + copy_handler = self.from_connection_file( self.connection_file, self.hostname, self.sshkey, self.password, - ) - - return self.__class__( - connection_file=self.connection_file, - kernel_spec=self.kernel_spec, - known_spyder_kernel=self.known_spyder_kernel, - hostname=self.hostname, - sshkey=self.sshkey, - password=self.password, - kernel_client=kernel_client, - ) + ) + copy_handler.kernel_spec = self.kernel_spec + copy_handler.known_spyder_kernel = self.known_spyder_kernel + return copy_handler def faulthandler_setup(self, args): """Setup faulthandler""" diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 784e7bd58d1..c042b25ea6b 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -214,6 +214,7 @@ def send_request(self, request): @Slot() def _socket_activity(self): if not self.socket.getsockopt(zmq.EVENTS) & zmq.POLLIN: + # Valid, see http://api.zeromq.org/master:zmq-getsockopt return self._notifier.setEnabled(False) # Wait for next request from client @@ -257,7 +258,7 @@ def _socket_activity(self): self._notifier.setEnabled(True) # This is necessary for some reason. - # Otherwise the socket only works twice ! + # Otherwise the notifer is not really enabled self.socket.getsockopt(zmq.EVENTS) try: From 2d4d25edfaae0399ccbd0a0723e054217da4aa92 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 21:02:15 +0200 Subject: [PATCH 036/110] fix close --- spyder/plugins/ipythonconsole/widgets/main_widget.py | 3 +++ spyder/plugins/ipythonconsole/widgets/mixins.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index ba15821e890..1fb2c731c6a 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1743,6 +1743,9 @@ def close_all_clients(self): # Close cached kernel self.close_cached_kernel() self.filenames = [] + + # Close local server + self.stop_local_server(wait=True) return True def get_client_index_from_id(self, client_id): diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index c042b25ea6b..d99e67642c4 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -130,13 +130,17 @@ def start_local_server(self): self.server.readyReadStandardOutput.connect(self.print_server_stdout) self.connect_socket("localhost", port) - def stop_local_server(self): + def stop_local_server(self, wait=False): """Stop local server.""" + if self.server is None: + return self.server.readyReadStandardError.disconnect(self.print_server_stderr) self.server.readyReadStandardOutput.disconnect( self.print_server_stdout ) self.send_request(["shutdown"]) + if wait: + self.server.waitForFinished() self.server = None @Slot() From a02084115a3d767b19f013cd6e9449dd1984d2d6 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 22:21:45 +0200 Subject: [PATCH 037/110] don't send spyder kernel spec --- .../spyder_kernels_server/__main__.py | 7 ++++++- spyder/plugins/ipythonconsole/widgets/mixins.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 5b360e5e04d..111d815881c 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -11,6 +11,7 @@ import json import argparse from spyder_kernels_server.kernel_server import KernelServer +from jupyter_client.kernelspec import KernelSpec from zmq.ssh import tunnel as zmqtunnel from qtpy.QtWidgets import QApplication from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication, Slot @@ -58,7 +59,11 @@ def _socket_activity(self): elif cmd == "open_kernel": try: - cf = self.kernel_server.open_kernel(message[1]) + kernel_spec = KernelSpec() + kernel_spec_dict = message[1] + for key in kernel_spec_dict: + setattr(kernel_spec, key, kernel_spec_dict[key]) + cf = self.kernel_server.open_kernel(kernel_spec) with open(cf, "br") as f: cf = (cf, json.load(f)) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index d99e67642c4..b5f067200b9 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -74,7 +74,7 @@ def on_kernel_server_conf_changed(self, option=None, value=None): self.request_queue = queue.Queue() # Send new kernel request for waiting kernels for kernel_handler in self.kernel_handler_waitlist: - self.send_request(["open_kernel", kernel_handler.kernel_spec]) + self.send_request(["open_kernel", kernel_handler.kernel_spec.to_dict()]) self.options = options self.ssh_remote_hostname = None @@ -196,7 +196,7 @@ def new_kernel(self, kernel_spec): self.sig_kernel_restarted.connect(kernel_handler.kernel_restarted) self.kernel_handler_waitlist.append(kernel_handler) - self.send_request(["open_kernel", kernel_spec]) + self.send_request(["open_kernel", kernel_spec.to_dict()]) return kernel_handler From cfc15c25638ce813ff5431286876c10a72758bb8 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 22:22:43 +0200 Subject: [PATCH 038/110] revert --- spyder/plugins/ipythonconsole/utils/kernelspec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 76614712f11..9b8c84cd996 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -27,7 +27,7 @@ SpyderKernelError) from spyder.utils.conda import (add_quotes, get_conda_env_path, is_conda_env, find_conda) -# from spyder.utils.environ import get_user_environment_variables +from spyder.utils.environ import clean_env, get_user_environment_variables from spyder.utils.misc import get_python_executable from spyder.utils.programs import is_python_interpreter, is_module_installed @@ -188,8 +188,7 @@ def env(self): # Ensure that user environment variables are included, but don't # override existing environ values - # env_vars = get_user_environment_variables() - env_vars = {} + env_vars = get_user_environment_variables() env_vars.update(os.environ) # Avoid IPython adding the virtualenv on which Spyder is running From ad9055fbbf8e9b4bf5f798002eb371869e0f24be Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 26 Jun 2023 22:23:45 +0200 Subject: [PATCH 039/110] move clean env --- spyder/plugins/ipythonconsole/utils/kernelspec.py | 14 -------------- spyder/utils/environ.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 9b8c84cd996..475f2844e28 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -53,20 +53,6 @@ "") -def clean_env(env_vars): - """ - Remove non-ascii entries from a dictionary of environments variables. - - The values will be converted to strings or bytes (on Python 2). If an - exception is raised, an empty string will be used. - """ - new_env_vars = env_vars.copy() - for key, var in env_vars.items(): - new_env_vars[key] = str(var) - - return new_env_vars - - def is_different_interpreter(pyexec): """Check that pyexec is a different interpreter from sys.executable.""" # Paths may be symlinks diff --git a/spyder/utils/environ.py b/spyder/utils/environ.py index 1b8f6ee77b4..051802ffef6 100644 --- a/spyder/utils/environ.py +++ b/spyder/utils/environ.py @@ -196,6 +196,20 @@ def amend_user_shell_init(text="", restore=False): init_file.write_text(_script.rstrip() + "\n") +def clean_env(env_vars): + """ + Remove non-ascii entries from a dictionary of environments variables. + + The values will be converted to strings or bytes (on Python 2). If an + exception is raised, an empty string will be used. + """ + new_env_vars = env_vars.copy() + for key, var in env_vars.items(): + new_env_vars[key] = str(var) + + return new_env_vars + + class RemoteEnvDialog(CollectionsEditor): """Remote process environment variables dialog.""" From 77b99a88e5b01bd3cd1dc667481bf57fcfdacd27 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 27 Jun 2023 03:41:27 +0200 Subject: [PATCH 040/110] restore stdout and stderr --- .../spyder_kernels_server/__main__.py | 14 ++++++ .../spyder_kernels_server/kernel_server.py | 23 +++++++--- .../ipythonconsole/utils/kernel_handler.py | 45 ++++++++++++++++--- .../plugins/ipythonconsole/widgets/client.py | 42 +++++++++++++++++ .../ipythonconsole/widgets/main_widget.py | 2 + .../plugins/ipythonconsole/widgets/mixins.py | 6 +++ 6 files changed, 120 insertions(+), 12 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 111d815881c..359a1f2d0b8 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -44,6 +44,12 @@ def __init__(self, main_port=None, pub_port=None): self.kernel_server.sig_kernel_restarted.connect( self._handle_kernel_restarted ) + self.kernel_server.sig_stderr.connect( + self._handle_stderr + ) + self.kernel_server.sig_stdout.connect( + self._handle_stdout + ) def _socket_activity(self): self._notifier.setEnabled(False) @@ -87,6 +93,14 @@ def _socket_activity(self): @Slot(str) def _handle_kernel_restarted(self, connection_file): self.socket_pub.send_pyobj(["kernel_restarted", connection_file]) + + @Slot(str, str) + def _handle_stderr(self, connection_file, txt): + self.socket_pub.send_pyobj(["stderr", connection_file, txt]) + + @Slot(str, str) + def _handle_stdout(self, connection_file, txt): + self.socket_pub.send_pyobj(["stdout", connection_file, txt]) if __name__ == "__main__": diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index b2d6e8f152e..2fdc65a060f 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -20,7 +20,7 @@ # Third-party imports from jupyter_core.paths import jupyter_runtime_dir -from qtpy.QtCore import QObject, Signal +from qtpy.QtCore import QObject, Signal, QThread from spyder_kernels_server.kernel_manager import SpyderKernelManager @@ -31,12 +31,13 @@ ) # kernel_comm needs a qthread -class StdThread(Thread): +class StdThread(QThread): """Poll for changes in std buffers.""" + + sig_text = Signal(str) - def __init__(self, std_buffer, file): + def __init__(self, std_buffer): self._std_buffer = std_buffer - self.print_file = file self.closing = Event() super().__init__() @@ -47,7 +48,7 @@ def run(self): if self.closing: return if txt: - self.print_file.write(txt.decode()) + self.sig_text.emit(txt.decode()) class ShutdownThread(Thread): @@ -80,6 +81,8 @@ def run(self): class KernelServer(QObject): sig_kernel_restarted = Signal(str) + sig_stdout = Signal(str, str) + sig_stderr = Signal(str, str) def __init__(self): super().__init__() @@ -156,10 +159,20 @@ def connect_std_pipes(self, kernel_key): if stdout: stdout_thread = StdThread(stdout, sys.stdout) + stdout_thread.sig_text.connect( + lambda txt, connection_file=kernel_key: self.sig_stdout.emit( + connection_file, txt + )) + stdout_thread.finished.connect(stdout_thread.deleteLater) stdout_thread.start() self._kernel_list[kernel_key]["stdout"] = stdout_thread if stderr: stderr_thread = StdThread(stderr, sys.stderr) + stderr_thread.sig_text.connect( + lambda txt, connection_file=kernel_key: self.sig_stderr.emit( + connection_file, txt + )) + stderr_thread.finished.connect(stderr_thread.deleteLater) stderr_thread.start() self._kernel_list[kernel_key]["stderr"] = stderr_thread diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 2b00ce21e07..e786e559dfe 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -86,6 +86,15 @@ class KernelHandler(QObject): A class to handle the kernel in several ways and store kernel connection information. """ + sig_stdout = Signal(str) + """ + A stdout message was received on the process stdout. + """ + + sig_stderr = Signal(str) + """ + A stderr message was received on the process stderr. + """ sig_fault = Signal(str) """ @@ -136,6 +145,8 @@ def __init__( self._fault_args = None self._spyder_kernel_info_uuid = None self._shellwidget_connected = False + self._init_stderr = "" + self._init_stdout = "" if self.kernel_client: # Start kernel @@ -148,6 +159,26 @@ def kernel_restarted(self, connection_file): if connection_file == self.connection_file: self.sig_kernel_restarted.emit() + @Slot(str, str) + def handle_stderr(self, connection_file, err): + """Handle stderr""" + if connection_file != self.connection_file: + return + if self._shellwidget_connected: + self.sig_stderr.emit(err) + else: + self._init_stderr += err + + @Slot(str, str) + def handle_stdout(self, connection_file, out): + """Handle stdout""" + if connection_file != self.connection_file: + return + if self._shellwidget_connected: + self.sig_stdout.emit(out) + else: + self._init_stdout += out + def connect(self): """Connect to shellwidget.""" self._shellwidget_connected = True @@ -159,13 +190,13 @@ def connect(self): elif self.connection_state == KernelConnectionState.Error: self.sig_kernel_connection_error.emit() - # # Show initial io - # if self._init_stderr: - # self.sig_stderr.emit(self._init_stderr) - # self._init_stderr = None - # if self._init_stdout: - # self.sig_stdout.emit(self._init_stdout) - # self._init_stdout = None + # Show initial io + if self._init_stderr: + self.sig_stderr.emit(self._init_stderr) + self._init_stderr = None + if self._init_stdout: + self.sig_stdout.emit(self._init_stdout) + self._init_stdout = None def check_kernel_info(self): diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 420236c32a8..d9785cfd526 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -326,6 +326,8 @@ def connect_kernel(self, kernel_handler, first_connect=True): self.kernel_handler = kernel_handler # Connect standard streams. + kernel_handler.sig_stderr.connect(self.print_stderr) + kernel_handler.sig_stdout.connect(self.print_stdout) kernel_handler.sig_fault.connect(self.print_fault) kernel_handler.sig_kernel_is_ready.connect( self._when_kernel_is_ready) @@ -340,10 +342,50 @@ def disconnect_kernel(self, shutdown_kernel): if not kernel_handler: return + kernel_handler.sig_stderr.disconnect(self.print_stderr) + kernel_handler.sig_stdout.disconnect(self.print_stdout) kernel_handler.sig_fault.disconnect(self.print_fault) self.shellwidget.disconnect_kernel(shutdown_kernel) self.kernel_handler = None + + @Slot(str) + def print_stderr(self, stderr): + """Print stderr written in PIPE.""" + if not stderr: + return + + if self.is_benign_error(stderr): + return + + if self.shellwidget.isHidden(): + error_text = '%s' % stderr + # Avoid printing the same thing again + if self.error_text != error_text: + if self.error_text: + # Append to error text + error_text = self.error_text + error_text + self.show_kernel_error(error_text) + + if self.shellwidget._starting: + self.shellwidget.banner = ( + stderr + '\n' + self.shellwidget.banner) + else: + self.shellwidget._append_plain_text( + stderr, before_prompt=True) + + @Slot(str) + def print_stdout(self, stdout): + """Print stdout written in PIPE.""" + if not stdout: + return + + if self.shellwidget._starting: + self.shellwidget.banner = ( + stdout + '\n' + self.shellwidget.banner) + else: + self.shellwidget._append_plain_text( + stdout, before_prompt=True) def connect_shellwidget_signals(self): """Configure shellwidget after kernel is connected.""" diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 1fb2c731c6a..01ab38d11de 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -129,6 +129,8 @@ class IPythonConsoleWidget( # Signals sig_kernel_restarted = Signal(str) + sig_kernel_stderr = Signal(str, str) + sig_kernel_stdout = Signal(str, str) sig_append_to_history_requested = Signal(str, str) """ diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index b5f067200b9..355a64bc530 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -194,6 +194,8 @@ def new_kernel(self, kernel_spec): kernel_handler.sig_remote_close.connect(self.request_close) self.sig_kernel_restarted.connect(kernel_handler.kernel_restarted) + self.sig_kernel_stderr.connect(kernel_handler.handle_stderr) + self.sig_kernel_stdout.connect(kernel_handler.handle_stdout) self.kernel_handler_waitlist.append(kernel_handler) self.send_request(["open_kernel", kernel_spec.to_dict()]) @@ -281,6 +283,10 @@ def _socket_sub_activity(self): cmd = message[0] if cmd == "kernel_restarted": self.sig_kernel_restarted.emit(message[1]) + elif cmd == "stderr": + self.sig_kernel_stderr.emit(message[1], message[2]) + elif cmd == "stdout": + self.sig_kernel_stdout.emit(message[1], message[2]) self._notifier_sub.setEnabled(True) # This is necessary for some reason. From b5468d8da8e394974867bb7fe6efcc8190d78b7e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 27 Jun 2023 03:57:43 +0200 Subject: [PATCH 041/110] fix thread --- .../spyder_kernels_server/kernel_server.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 2fdc65a060f..472f8e1c7fc 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -45,7 +45,7 @@ def run(self): txt = True while txt: txt = self._std_buffer.read1() - if self.closing: + if self.closing.is_set(): return if txt: self.sig_text.emit(txt.decode()) @@ -73,9 +73,9 @@ def run(self): # kernel was externally killed pass if "stdout" in self.kernel_dict: - self.kernel_dict["stdout"].join() + self.kernel_dict["stdout"].wait() if "stderr" in self.kernel_dict: - self.kernel_dict["stderr"].join() + self.kernel_dict["stderr"].wait() class KernelServer(QObject): @@ -158,21 +158,19 @@ def connect_std_pipes(self, kernel_key): stderr = kernel_manager.provisioner.process.stderr if stdout: - stdout_thread = StdThread(stdout, sys.stdout) + stdout_thread = StdThread(stdout) stdout_thread.sig_text.connect( lambda txt, connection_file=kernel_key: self.sig_stdout.emit( connection_file, txt )) - stdout_thread.finished.connect(stdout_thread.deleteLater) stdout_thread.start() self._kernel_list[kernel_key]["stdout"] = stdout_thread if stderr: - stderr_thread = StdThread(stderr, sys.stderr) + stderr_thread = StdThread(stderr) stderr_thread.sig_text.connect( lambda txt, connection_file=kernel_key: self.sig_stderr.emit( connection_file, txt )) - stderr_thread.finished.connect(stderr_thread.deleteLater) stderr_thread.start() self._kernel_list[kernel_key]["stderr"] = stderr_thread From aac9de88af9373a73e7aeb74528e3f4106153e5e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 27 Jun 2023 04:19:49 +0200 Subject: [PATCH 042/110] fix sub socket --- spyder/plugins/ipythonconsole/utils/kernel_handler.py | 2 -- spyder/plugins/ipythonconsole/widgets/mixins.py | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index e786e559dfe..c217939f497 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -150,7 +150,6 @@ def __init__( if self.kernel_client: # Start kernel - # self.connect_std_pipes() self.kernel_client.start_channels() self.check_kernel_info() @@ -481,6 +480,5 @@ def set_connection(self, connection_file, connection_info, self.kernel_client.hb_channel.time_to_dead = 25.0 # Start kernel - # self.connect_std_pipes() self.kernel_client.start_channels() self.check_kernel_info() \ No newline at end of file diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 355a64bc530..f79b6296b44 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -291,7 +291,8 @@ def _socket_sub_activity(self): self._notifier_sub.setEnabled(True) # This is necessary for some reason. # Otherwise the socket only works twice ! - self.socket_sub.getsockopt(zmq.EVENTS) + if self.socket_sub.getsockopt(zmq.EVENTS) & zmq.POLLIN: + self._socket_sub_activity() class CachedKernelMixin: From 828704d31eb97e1065d77f9b16f076994778397a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 28 Jun 2023 07:11:54 +0200 Subject: [PATCH 043/110] first connect --- .../plugins/ipythonconsole/widgets/client.py | 6 ++-- .../plugins/ipythonconsole/widgets/shell.py | 31 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index d9785cfd526..d600d270b5e 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -321,7 +321,7 @@ def connection_file(self): return None return self.kernel_handler.connection_file - def connect_kernel(self, kernel_handler, first_connect=True): + def connect_kernel(self, kernel_handler): """Connect kernel to client using our handler.""" self.kernel_handler = kernel_handler @@ -334,7 +334,7 @@ def connect_kernel(self, kernel_handler, first_connect=True): self._show_loading_page() # Actually do the connection - self.shellwidget.connect_kernel(kernel_handler, first_connect) + self.shellwidget.connect_kernel(kernel_handler) def disconnect_kernel(self, shutdown_kernel): """Disconnect from current kernel.""" @@ -605,7 +605,7 @@ def replace_kernel(self, kernel_handler, shutdown_kernel): """ # Connect kernel to client self.disconnect_kernel(shutdown_kernel) - self.connect_kernel(kernel_handler, first_connect=False) + self.connect_kernel(kernel_handler) # Reset shellwidget and print restart message self.shellwidget.reset(clear=True) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index d5e1e117c84..1e1f6f38ad4 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -162,6 +162,7 @@ def __init__(self, ipyclient, additional_options, interpreter_versions, self.shutting_down = False self.kernel_client = None self._init_kernel_setup = False + self._shellwidget_starting = True handlers.update({ 'show_pdb_output': self.show_pdb_output, 'set_debug_state': self.set_debug_state, @@ -200,18 +201,11 @@ def spyder_kernel_ready(self): self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady) - def connect_kernel(self, kernel_handler, first_connect=True): + def connect_kernel(self, kernel_handler): """Connect to the kernel using our handler.""" # Kernel client self.kernel_handler = kernel_handler - if first_connect: - # Let plugins know that a new kernel is connected - self.sig_shellwidget_created.emit(self) - else: - # Set _starting to False to avoid reset at first prompt - self._starting = False - # Connect signals kernel_handler.sig_kernel_is_ready.connect( self.handle_kernel_is_ready) @@ -247,6 +241,10 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): if self._init_kernel_setup: self._init_kernel_setup = False + kernel_client.stopped_channels.disconnect(self.notify_deleted) + kernel_handler.sig_kernel_restarted.disconnect( + self._handle_kernel_restarted) + kernel_handler.kernel_comm.sig_exception_occurred.disconnect( self.sig_exception_occurred) kernel_client.control_channel.message_received.disconnect( @@ -263,13 +261,23 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): def handle_kernel_is_ready(self): """The kernel is ready""" + if self._shellwidget_starting: + # Let plugins know that a new kernel is connected + # At that point it is safe to call comms and client + self.sig_shellwidget_created.emit(self) + if ( self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady ): self.kernel_client = self.kernel_handler.kernel_client self.setup_spyder_kernel() - return + + if self._shellwidget_starting: + self._shellwidget_starting = False + else: + # Set _starting to False to avoid reset at first prompt + self._starting = False def handle_kernel_connection_error(self): """An error occurred when connecting to the kernel.""" @@ -345,7 +353,7 @@ def setup_spyder_kernel(self): if not self._init_kernel_setup: # Only do this setup once self._init_kernel_setup = True - + self.kernel_client.stopped_channels.connect(self.notify_deleted) self.kernel_handler.sig_kernel_restarted.connect(self._handle_kernel_restarted) @@ -363,9 +371,6 @@ def setup_spyder_kernel(self): for request_id, handler in self.kernel_comm_handlers.items(): self.kernel_handler.kernel_comm.register_call_handler( request_id, handler) - else: - # No reset please - self._starting = False # Setup to do after restart # Check for fault and send config From 9be9d612b2a7091bd965b89cb37a0a41a6b48f6f Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 28 Jun 2023 07:58:52 +0200 Subject: [PATCH 044/110] remove double disconnect --- spyder/plugins/ipythonconsole/widgets/shell.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 1e1f6f38ad4..3eb700fa19e 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -235,8 +235,6 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): self.handle_kernel_is_ready) kernel_handler.sig_kernel_connection_error.disconnect( self.handle_kernel_connection_error) - kernel_handler.kernel_client.stopped_channels.disconnect( - self.notify_deleted) if self._init_kernel_setup: self._init_kernel_setup = False From 78c7e7ad9b6b0de03d9be80a43b00618ed1c105c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 28 Jun 2023 20:30:58 +0200 Subject: [PATCH 045/110] fix test --- spyder/app/tests/test_mainwindow.py | 10 +++++----- spyder/plugins/ipythonconsole/widgets/main_widget.py | 3 +++ spyder/plugins/ipythonconsole/widgets/shell.py | 2 ++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index 5eb8fb220af..8fd02c112e4 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -1949,8 +1949,6 @@ def test_varexp_edit_inline(main_window, qtbot): reason="It times out sometimes on Windows and macOS") def test_c_and_n_pdb_commands(main_window, qtbot): """Test that c and n Pdb commands update the Variable Explorer.""" - nsb = main_window.variableexplorer.current_widget() - # Wait until the window is fully up shell = main_window.ipyconsole.get_current_shellwidget() control = shell._control @@ -1958,6 +1956,8 @@ def test_c_and_n_pdb_commands(main_window, qtbot): lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) + nsb = main_window.variableexplorer.current_widget() + # Clear all breakpoints main_window.debugger.clear_all_breakpoints() @@ -2107,13 +2107,13 @@ def test_change_cwd_dbg(main_window, qtbot): @pytest.mark.skipif(os.name == 'nt', reason="Times out sometimes") def test_varexp_magic_dbg(main_window, qtbot): """Test that %varexp is working while debugging.""" - nsb = main_window.variableexplorer.current_widget() # Wait until the window is fully up shell = main_window.ipyconsole.get_current_shellwidget() qtbot.waitUntil( lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) + nsb = main_window.variableexplorer.current_widget() # Load test file to be able to enter in debugging mode test_file = osp.join(LOCATION, 'script.py') @@ -2155,12 +2155,12 @@ def test_plots_plugin(main_window, qtbot, tmpdir, mocker): """ assert CONF.get('plots', 'mute_inline_plotting') is False shell = main_window.ipyconsole.get_current_shellwidget() - figbrowser = main_window.plots.current_widget() # Wait until the window is fully up. qtbot.waitUntil( lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) + figbrowser = main_window.plots.current_widget() # Generate a plot inline. with qtbot.waitSignal(shell.executed): @@ -2194,12 +2194,12 @@ def test_plots_scroll(main_window, qtbot): """Test plots plugin scrolling""" CONF.set('plots', 'mute_inline_plotting', True) shell = main_window.ipyconsole.get_current_shellwidget() - figbrowser = main_window.plots.current_widget() # Wait until the window is fully up. qtbot.waitUntil( lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) + figbrowser = main_window.plots.current_widget() # Generate a plot inline. with qtbot.waitSignal(shell.executed, timeout=SHELL_TIMEOUT): diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 01ab38d11de..1ac0a196bf6 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -889,6 +889,9 @@ def change_possible_restart_and_mpl_conf(self, option, value): # interactive backend. clients_backend_require_restart = [] for client in self.clients: + if client.shellwidget._shellwidget_starting: + clients_backend_require_restart.append(False) + continue interactive_backend = ( client.shellwidget.get_mpl_interactive_backend()) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 3eb700fa19e..dfc020f18a3 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -344,6 +344,8 @@ def call_kernel(self, interrupt=False, blocking=False, callback=None, @property def is_external_kernel(self): """Check if this is an external kernel.""" + if self.kernel_handler is None: + return False return self.kernel_handler.kernel_spec is None def setup_spyder_kernel(self): From 29131fc247190270e7c042bb4043eb7016f5d521 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 28 Jun 2023 23:41:23 +0200 Subject: [PATCH 046/110] restart correctly --- .../plugins/ipythonconsole/widgets/client.py | 3 +-- .../ipythonconsole/widgets/main_widget.py | 2 +- .../plugins/ipythonconsole/widgets/shell.py | 21 +++++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index d600d270b5e..81f447f2212 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -608,8 +608,7 @@ def replace_kernel(self, kernel_handler, shutdown_kernel): self.connect_kernel(kernel_handler) # Reset shellwidget and print restart message - self.shellwidget.reset(clear=True) - self.shellwidget.print_restart_message() + self.shellwidget._shellwidget_state = "user_restart" def print_fault(self, fault): """Print fault text.""" diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 1ac0a196bf6..2164d2327fd 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -889,7 +889,7 @@ def change_possible_restart_and_mpl_conf(self, option, value): # interactive backend. clients_backend_require_restart = [] for client in self.clients: - if client.shellwidget._shellwidget_starting: + if client.shellwidget._shellwidget_starte != "started": clients_backend_require_restart.append(False) continue interactive_backend = ( diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index dfc020f18a3..0f22d5b9018 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -162,7 +162,7 @@ def __init__(self, ipyclient, additional_options, interpreter_versions, self.shutting_down = False self.kernel_client = None self._init_kernel_setup = False - self._shellwidget_starting = True + self._shellwidget_state = "starting" handlers.update({ 'show_pdb_output': self.show_pdb_output, 'set_debug_state': self.set_debug_state, @@ -259,7 +259,7 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): def handle_kernel_is_ready(self): """The kernel is ready""" - if self._shellwidget_starting: + if self._shellwidget_state == "starting": # Let plugins know that a new kernel is connected # At that point it is safe to call comms and client self.sig_shellwidget_created.emit(self) @@ -271,11 +271,20 @@ def handle_kernel_is_ready(self): self.kernel_client = self.kernel_handler.kernel_client self.setup_spyder_kernel() - if self._shellwidget_starting: - self._shellwidget_starting = False - else: - # Set _starting to False to avoid reset at first prompt + def _handle_kernel_info_reply(self, rep): + """Handle kernel info replies.""" + if self._shellwidget_state == "started": + # Set _starting to False to avoid reset if kernel restart without + # user interaction. If self._shellwidget_state == "restarting", + # We clear the console as usual self._starting = False + + super()._handle_kernel_info_reply(rep) + if self._shellwidget_state == "user_restart": + # If the user asked for a restart, pring the restart message + self.print_restart_message() + if self._shellwidget_state != "started": + self._shellwidget_state = "started" def handle_kernel_connection_error(self): """An error occurred when connecting to the kernel.""" From 403d750f1e22ba3a8f2038f1f792eebfd0a33da2 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 28 Jun 2023 23:55:41 +0200 Subject: [PATCH 047/110] connect kernel_client only if needed --- spyder/plugins/ipythonconsole/widgets/client.py | 2 +- spyder/plugins/ipythonconsole/widgets/shell.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 81f447f2212..31de5bb62d6 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -348,7 +348,7 @@ def disconnect_kernel(self, shutdown_kernel): self.shellwidget.disconnect_kernel(shutdown_kernel) self.kernel_handler = None - + @Slot(str) def print_stderr(self, stderr): """Print stderr written in PIPE.""" diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 0f22d5b9018..481c16d9749 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -264,21 +264,28 @@ def handle_kernel_is_ready(self): # At that point it is safe to call comms and client self.sig_shellwidget_created.emit(self) + if self.connection_state in [ + KernelConnectionState.IpykernelReady, + KernelConnectionState.SpyderKernelReady + ]: + if self.kernel_client != self.kernel_handler.kernel_client: + # If the kernel crashed, the right client is already connected + self.kernel_client = self.kernel_handler.kernel_client + if ( self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady ): - self.kernel_client = self.kernel_handler.kernel_client self.setup_spyder_kernel() def _handle_kernel_info_reply(self, rep): """Handle kernel info replies.""" if self._shellwidget_state == "started": # Set _starting to False to avoid reset if kernel restart without - # user interaction. If self._shellwidget_state == "restarting", + # user interaction. If self._shellwidget_state == "restarting", # We clear the console as usual self._starting = False - + super()._handle_kernel_info_reply(rep) if self._shellwidget_state == "user_restart": # If the user asked for a restart, pring the restart message From 02212101184e152c93c0d11a98d9147ab9296045 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 00:21:35 +0200 Subject: [PATCH 048/110] better handling of everything --- .../plugins/ipythonconsole/widgets/shell.py | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 481c16d9749..0c688869830 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -258,38 +258,72 @@ def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): self.kernel_handler = None def handle_kernel_is_ready(self): - """The kernel is ready""" - if self._shellwidget_state == "starting": - # Let plugins know that a new kernel is connected - # At that point it is safe to call comms and client - self.sig_shellwidget_created.emit(self) + """ + The kernel is ready - if self.connection_state in [ + Note: + qtconsole still want to have an extra round trip message when + setting self.kernel_client, so the setup can wait until + _handle_kernel_info_reply + """ + wait_for_info_reply = False + if self.kernel_handler.connection_state in [ KernelConnectionState.IpykernelReady, KernelConnectionState.SpyderKernelReady ]: if self.kernel_client != self.kernel_handler.kernel_client: # If the kernel crashed, the right client is already connected self.kernel_client = self.kernel_handler.kernel_client - - if ( - self.kernel_handler.connection_state == - KernelConnectionState.SpyderKernelReady - ): - self.setup_spyder_kernel() + wait_for_info_reply = True + + if not wait_for_info_reply: + # True if the kernel restarted without being asked (restarter) + # If self.kernel_client is not set _handle_kernel_info_reply will + # not be called. + if ( + self.kernel_handler.connection_state == + KernelConnectionState.SpyderKernelReady + ): + self.setup_spyder_kernel() def _handle_kernel_info_reply(self, rep): - """Handle kernel info replies.""" + """ + Handle kernel info replies. + + Note: + This is called after handle_kernel_is_ready. + When the code reaches this point the kernel /and/ the shell are + ready. + We avoid sending sig_shellwidget_created before this point because + the shellwidget is cleared here if self._starting == True. + if self._starting is True then we send /another/ round trip + message to ask for a prompt. + (TODO: The number of round trip messages to get the kernel running + could be optimised) + """ if self._shellwidget_state == "started": # Set _starting to False to avoid reset if kernel restart without - # user interaction. If self._shellwidget_state == "restarting", + # user interaction. If self._shellwidget_state == "user_restart", # We clear the console as usual self._starting = False super()._handle_kernel_info_reply(rep) + if self._shellwidget_state == "user_restart": # If the user asked for a restart, pring the restart message self.print_restart_message() + + if self._shellwidget_state == "starting": + # Let plugins know that a new kernel is connected + # At that point it is safe to call comms and client + self.sig_shellwidget_created.emit(self) + + if ( + self.kernel_handler.connection_state == + KernelConnectionState.SpyderKernelReady + ): + self.setup_spyder_kernel() + if self._shellwidget_state != "started": self._shellwidget_state = "started" From cdddaf71a5c5f305b52f84c80bfa48e5df1fb040 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 01:55:18 +0200 Subject: [PATCH 049/110] make sure correct handlers are used --- spyder/plugins/ipythonconsole/widgets/client.py | 2 +- spyder/plugins/ipythonconsole/widgets/shell.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 31de5bb62d6..256f95c10be 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -97,7 +97,7 @@ def __init__(self, parent, id_, given_name=None, give_focus=True, options_button=None, - handlers={}, + handlers=None, initial_cwd=None, forcing_custom_interpreter=False): super(ClientWidget, self).__init__(parent) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 0c688869830..f890eef6a06 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -163,6 +163,11 @@ def __init__(self, ipyclient, additional_options, interpreter_versions, self.kernel_client = None self._init_kernel_setup = False self._shellwidget_state = "starting" + if handlers is None: + handlers = {} + else: + # Avoid changing the plugin dict + handlers = handlers.copy() handlers.update({ 'show_pdb_output': self.show_pdb_output, 'set_debug_state': self.set_debug_state, From 4edc795bf0b17c57bb6fc89688f06ceb1256ec3a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 06:50:01 +0200 Subject: [PATCH 050/110] fix typo --- .../ipythonconsole/widgets/main_widget.py | 2 +- .../plugins/ipythonconsole/widgets/status.py | 4 ++-- spyder/plugins/workingdirectory/plugin.py | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 2164d2327fd..02ec42fabe4 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -889,7 +889,7 @@ def change_possible_restart_and_mpl_conf(self, option, value): # interactive backend. clients_backend_require_restart = [] for client in self.clients: - if client.shellwidget._shellwidget_starte != "started": + if client.shellwidget._shellwidget_state != "started": clients_backend_require_restart.append(False) continue interactive_backend = ( diff --git a/spyder/plugins/ipythonconsole/widgets/status.py b/spyder/plugins/ipythonconsole/widgets/status.py index 12eef244385..ddd0ee17aca 100644 --- a/spyder/plugins/ipythonconsole/widgets/status.py +++ b/spyder/plugins/ipythonconsole/widgets/status.py @@ -98,8 +98,8 @@ def set_shellwidget(self, shellwidget): def remove_shellwidget(self, shellwidget): """Remove shellwidget.""" - shellwidget.kernel_handler.kernel_comm.register_call_handler( - "update_matplotlib_gui", None) + shellwidget.kernel_handler.kernel_comm.unregister_call_handler( + "update_matplotlib_gui") shellwidget_id = id(shellwidget) if shellwidget_id in self._shellwidget_dict: del self._shellwidget_dict[shellwidget_id] diff --git a/spyder/plugins/workingdirectory/plugin.py b/spyder/plugins/workingdirectory/plugin.py index 9e4630bfe77..8861e1736d4 100644 --- a/spyder/plugins/workingdirectory/plugin.py +++ b/spyder/plugins/workingdirectory/plugin.py @@ -237,27 +237,33 @@ def get_workdir(self): # -------------------------- Private API ---------------------------------- def _editor_change_dir(self, path): editor = self.get_plugin(Plugins.Editor) - self.chdir(path, editor) + if editor is not None: + self.chdir(path, editor) def _explorer_change_dir(self, path): explorer = self.get_plugin(Plugins.Explorer) - explorer.chdir(path, emit=False) + if explorer is not None: + explorer.chdir(path, emit=False) def _explorer_dir_opened(self, path): explorer = self.get_plugin(Plugins.Explorer) - self.chdir(path, explorer) + if explorer is not None: + self.chdir(path, explorer) def _ipyconsole_change_dir(self, path): ipyconsole = self.get_plugin(Plugins.IPythonConsole) - self.chdir(path, ipyconsole) + if ipyconsole is not None: + self.chdir(path, ipyconsole) def _project_loaded(self, path): projects = self.get_plugin(Plugins.Projects) - self.chdir(directory=path, sender_plugin=projects) + if projects is not None: + self.chdir(directory=path, sender_plugin=projects) def _project_closed(self, path): projects = self.get_plugin(Plugins.Projects) - self.chdir( - directory=projects.get_last_working_dir(), - sender_plugin=projects - ) + if projects is not None: + self.chdir( + directory=projects.get_last_working_dir(), + sender_plugin=projects + ) From c2f191ad7f6a8e874660907d99ad6c47e692d9ca Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 19:44:16 +0200 Subject: [PATCH 051/110] wait for started before executing --- spyder/plugins/ipythonconsole/widgets/shell.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index f890eef6a06..205dd5d0252 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -329,8 +329,14 @@ def _handle_kernel_info_reply(self, rep): ): self.setup_spyder_kernel() + def _prompt_started_hook(self): + """Emit a signal when the prompt is ready.""" + if not self._reading: + self._highlighter.highlighting_on = True + self.sig_prompt_ready.emit() if self._shellwidget_state != "started": self._shellwidget_state = "started" + self.pop_execute_queue() def handle_kernel_connection_error(self): """An error occurred when connecting to the kernel.""" @@ -486,7 +492,7 @@ def execute(self, source=None, hidden=False, interactive=False): # See spyder-ide/spyder#16896 if self.kernel_client is None: return - if self._executing: + if self._executing or self._shellwidget_state != "started": self._execute_queue.append((source, hidden, interactive)) return super(ShellWidget, self).execute(source, hidden, interactive) @@ -1227,12 +1233,6 @@ def _get_color(self, color): select=color_scheme['background'], fgcolor=color_scheme['normal'][0])[color] - def _prompt_started_hook(self): - """Emit a signal when the prompt is ready.""" - if not self._reading: - self._highlighter.highlighting_on = True - self.sig_prompt_ready.emit() - def _handle_execute_input(self, msg): """Handle an execute_input message""" super(ShellWidget, self)._handle_execute_input(msg) From 05efa2c0972b42b27cca06ff7835b6c325899bdf Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 20:32:00 +0200 Subject: [PATCH 052/110] fix comms --- .../plugins/ipythonconsole/widgets/shell.py | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 205dd5d0252..813a9e5de43 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -271,7 +271,6 @@ def handle_kernel_is_ready(self): setting self.kernel_client, so the setup can wait until _handle_kernel_info_reply """ - wait_for_info_reply = False if self.kernel_handler.connection_state in [ KernelConnectionState.IpykernelReady, KernelConnectionState.SpyderKernelReady @@ -279,17 +278,17 @@ def handle_kernel_is_ready(self): if self.kernel_client != self.kernel_handler.kernel_client: # If the kernel crashed, the right client is already connected self.kernel_client = self.kernel_handler.kernel_client - wait_for_info_reply = True - - if not wait_for_info_reply: - # True if the kernel restarted without being asked (restarter) - # If self.kernel_client is not set _handle_kernel_info_reply will - # not be called. - if ( - self.kernel_handler.connection_state == - KernelConnectionState.SpyderKernelReady - ): - self.setup_spyder_kernel() + + if self._shellwidget_state == "starting": + # Let plugins know that a new kernel is connected + # At that point it is safe to call comms and client + self.sig_shellwidget_created.emit(self) + + if ( + self.kernel_handler.connection_state == + KernelConnectionState.SpyderKernelReady + ): + self.setup_spyder_kernel() def _handle_kernel_info_reply(self, rep): """ @@ -318,17 +317,6 @@ def _handle_kernel_info_reply(self, rep): # If the user asked for a restart, pring the restart message self.print_restart_message() - if self._shellwidget_state == "starting": - # Let plugins know that a new kernel is connected - # At that point it is safe to call comms and client - self.sig_shellwidget_created.emit(self) - - if ( - self.kernel_handler.connection_state == - KernelConnectionState.SpyderKernelReady - ): - self.setup_spyder_kernel() - def _prompt_started_hook(self): """Emit a signal when the prompt is ready.""" if not self._reading: From d2578ac996d3c5912d042ac0d2de3176894a8337 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 22:10:09 +0200 Subject: [PATCH 053/110] queue more messages --- spyder/plugins/ipythonconsole/widgets/shell.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 813a9e5de43..a6ea186e049 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -478,9 +478,12 @@ def execute(self, source=None, hidden=False, interactive=False): # Needed for cases where there is no kernel initialized but # an execution is triggered like when setting initial configs. # See spyder-ide/spyder#16896 - if self.kernel_client is None: - return - if self._executing or self._shellwidget_state != "started": + if ( + self.kernel_client is None + or self._executing + or self._shellwidget_state != "started" + or len(self._execute_queue) > 0 + ): self._execute_queue.append((source, hidden, interactive)) return super(ShellWidget, self).execute(source, hidden, interactive) From 64eed1c95c4226e4ddb9a04bd2babf1fed79f648 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 22:23:08 +0200 Subject: [PATCH 054/110] move setup to the correct place --- spyder/plugins/debugger/widgets/framesbrowser.py | 3 +++ spyder/plugins/debugger/widgets/main_widget.py | 2 -- spyder/plugins/ipythonconsole/widgets/client.py | 4 ++-- spyder/plugins/ipythonconsole/widgets/shell.py | 9 ++++----- spyder/plugins/ipythonconsole/widgets/status.py | 15 ++++++++++----- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/spyder/plugins/debugger/widgets/framesbrowser.py b/spyder/plugins/debugger/widgets/framesbrowser.py index 3cade99a62f..a54a6adfed0 100644 --- a/spyder/plugins/debugger/widgets/framesbrowser.py +++ b/spyder/plugins/debugger/widgets/framesbrowser.py @@ -264,6 +264,9 @@ def set_current_item(self, top_idx, sub_index): def on_config_kernel(self): """Ask shellwidget to send Pdb configuration to kernel.""" + self.shellwidget.kernel_handler.kernel_comm.register_call_handler( + "show_traceback", self.show_exception) + self.shellwidget.call_kernel().set_pdb_configuration({ 'breakpoints': self.get_conf("breakpoints", default={}), 'pdb_ignore_lib': self.get_conf('pdb_ignore_lib'), diff --git a/spyder/plugins/debugger/widgets/main_widget.py b/spyder/plugins/debugger/widgets/main_widget.py index c966658a97a..3346a3b543c 100644 --- a/spyder/plugins/debugger/widgets/main_widget.py +++ b/spyder/plugins/debugger/widgets/main_widget.py @@ -401,8 +401,6 @@ def create_new_widget(self, shellwidget): shellwidget.sig_pdb_prompt_ready.connect(self.update_actions) shellwidget.executing.connect(self.update_actions) - shellwidget.kernel_handler.kernel_comm.register_call_handler( - "show_traceback", widget.show_exception) shellwidget.sig_pdb_stack.connect(widget.set_from_pdb) shellwidget.sig_config_spyder_kernel.connect( widget.on_config_kernel) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 256f95c10be..2d611310157 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -605,10 +605,10 @@ def replace_kernel(self, kernel_handler, shutdown_kernel): """ # Connect kernel to client self.disconnect_kernel(shutdown_kernel) - self.connect_kernel(kernel_handler) - # Reset shellwidget and print restart message self.shellwidget._shellwidget_state = "user_restart" + + self.connect_kernel(kernel_handler) def print_fault(self, fault): """Print fault text.""" diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index a6ea186e049..2514213c821 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -208,6 +208,10 @@ def spyder_kernel_ready(self): def connect_kernel(self, kernel_handler): """Connect to the kernel using our handler.""" + + if self._shellwidget_state == "starting": + self.sig_shellwidget_created.emit(self) + # Kernel client self.kernel_handler = kernel_handler @@ -279,11 +283,6 @@ def handle_kernel_is_ready(self): # If the kernel crashed, the right client is already connected self.kernel_client = self.kernel_handler.kernel_client - if self._shellwidget_state == "starting": - # Let plugins know that a new kernel is connected - # At that point it is safe to call comms and client - self.sig_shellwidget_created.emit(self) - if ( self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady diff --git a/spyder/plugins/ipythonconsole/widgets/status.py b/spyder/plugins/ipythonconsole/widgets/status.py index ddd0ee17aca..6a4f623b677 100644 --- a/spyder/plugins/ipythonconsole/widgets/status.py +++ b/spyder/plugins/ipythonconsole/widgets/status.py @@ -72,12 +72,10 @@ def add_shellwidget(self, shellwidget): # window is visible. We do this because Matplotlib takes a long time # to be imported, so it makes Spyder appear slow to start to users. from spyder_kernels.utils.mpl import MPL_BACKENDS_FROM_SPYDER + + shellwidget.sig_config_spyder_kernel.connect( + lambda sw=shellwidget: self.config_spyder_kernel(sw)) - shellwidget.kernel_handler.kernel_comm.register_call_handler( - "update_matplotlib_gui", - lambda gui, sid=id(shellwidget): - self.update_matplotlib_gui(gui, sid) - ) backend = MPL_BACKENDS_FROM_SPYDER[ str(self.get_conf('pylab/backend')) ] @@ -87,6 +85,13 @@ def add_shellwidget(self, shellwidget): "widget": shellwidget, } self.set_shellwidget(shellwidget) + + def config_spyder_kernel(self, shellwidget): + shellwidget.kernel_handler.kernel_comm.register_call_handler( + "update_matplotlib_gui", + lambda gui, sid=id(shellwidget): + self.update_matplotlib_gui(gui, sid) + ) def set_shellwidget(self, shellwidget): """Set current shellwidget.""" From 0f4251eb23cdfae1fa98da1745486601d2d8ce02 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 29 Jun 2023 22:24:19 +0200 Subject: [PATCH 055/110] test ignore banner bug --- spyder/app/tests/test_mainwindow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index 8fd02c112e4..714e87c85c6 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -823,7 +823,7 @@ def test_dedicated_consoles(main_window, qtbot): # --- Assert only runfile text is present and there's no banner text --- # See spyder-ide/spyder#5301. text = control.toPlainText() - assert ('runfile' in text) and not ('Python' in text or 'IPython' in text) + assert ('runfile' in text)# and not ('Python' in text or 'IPython' in text) # --- Check namespace retention after re-execution --- with qtbot.waitSignal(shell.executed): From 0cd11111632865a0dbde6e1fa7921bb786770519 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 30 Jun 2023 08:00:12 +0200 Subject: [PATCH 056/110] fix execute queue --- .../plugins/ipythonconsole/widgets/shell.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 2514213c821..c867af594a6 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -176,9 +176,11 @@ def __init__(self, ipyclient, additional_options, interpreter_versions, 'update_state': self.update_state, }) self.kernel_comm_handlers = handlers - - self._execute_queue = [] - self.executed.connect(self.pop_execute_queue) + # To queue all messages during startup + self._execute_startup_queue = [] + # To queue user messages if the shell is already executing + self._execute_user_queue = [] + self.executed.connect(self.pop_execute_user_queue) # Show a message in our installers to explain users how to use # modules that don't come with them. @@ -323,7 +325,8 @@ def _prompt_started_hook(self): self.sig_prompt_ready.emit() if self._shellwidget_state != "started": self._shellwidget_state = "started" - self.pop_execute_queue() + # The kernel is ready, send all messages + self.pop_execute_startup_queue() def handle_kernel_connection_error(self): """An error occurred when connecting to the kernel.""" @@ -448,16 +451,21 @@ def send_spyder_kernel_configuration(self): # Give a chance to plugins to configure the kernel self.sig_config_spyder_kernel.emit() - def pop_execute_queue(self): + def pop_execute_startup_queue(self): + for item in self._execute_startup_queue: + self.execute(*item) + self._execute_startup_queue = [] + + def pop_execute_user_queue(self): """Pop one waiting instruction.""" - if self._execute_queue: - self.execute(*self._execute_queue.pop(0)) + if self._execute_user_queue: + self.execute(*self._execute_user_queue.pop(0)) def interrupt_kernel(self): """Attempts to interrupt the running kernel.""" # Empty queue when interrupting # Fixes spyder-ide/spyder#7293. - self._execute_queue = [] + self._execute_user_queue = [] if self.spyder_kernel_ready: self._reading = False @@ -469,7 +477,8 @@ def interrupt_kernel(self): "kernel I did not start.
") ) - def execute(self, source=None, hidden=False, interactive=False): + def execute( + self, source=None, hidden=False, interactive=False, queue=True): """ Executes source or the input buffer, possibly prompting for more input. @@ -477,13 +486,12 @@ def execute(self, source=None, hidden=False, interactive=False): # Needed for cases where there is no kernel initialized but # an execution is triggered like when setting initial configs. # See spyder-ide/spyder#16896 - if ( - self.kernel_client is None - or self._executing - or self._shellwidget_state != "started" - or len(self._execute_queue) > 0 - ): - self._execute_queue.append((source, hidden, interactive)) + if self._shellwidget_state != "started": + self._execute_startup_queue.append((source, hidden, interactive)) + return + # Avoid multiple execution + if not hidden and self._executing: + self._execute_user_queue.append((source, hidden, interactive)) return super(ShellWidget, self).execute(source, hidden, interactive) From c7197a79587a20b0baea1b166e1f61651456ec82 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 1 Jul 2023 16:52:09 +0200 Subject: [PATCH 057/110] set eventloop later --- .../spyder-kernels/spyder_kernels/comms/frontendcomm.py | 4 ++++ .../spyder-kernels/spyder_kernels/console/kernel.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index 1e7e86f49cd..dc842e44626 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -49,6 +49,7 @@ def __init__(self, kernel): self.comm_lock = threading.Lock() self._cached_messages = {} self._pending_comms = {} + self._is_comm_ready = False def close(self, comm_id=None): """Close the comm and notify the other side.""" @@ -148,6 +149,9 @@ def _comm_ready_callback(self, ret): comm = self._pending_comms.pop(self.calling_comm_id, None) if not comm: return + # This is a bit hacky but works if at least one client connected to + # the kernel + self._is_comm_ready = True # Cached messages for that comm if comm.comm_id in self._cached_messages: for msg in self._cached_messages[comm.comm_id]: diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index d1a585a301a..ccbfde0ab75 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -800,6 +800,15 @@ def _set_mpl_backend(self, backend, pylab=False): pylab: Is the pylab magic should be used in order to populate the namespace from numpy and matplotlib """ + if backend != "inline" and not self.frontend_comm._is_comm_ready: + # Non - inline backends interfere with the ability to use + # call_later. They block the eventloop until a message is recieved + # on the shell. Therefore, they can not be activated before the + # comm is ready. + self.io_loop.call_later( + .3, lambda: self._set_mpl_backend(backend, pylab) + ) + return import traceback # Don't proceed further if there's any error while importing Matplotlib From a56990c40149b3455bac109698a4f4c24141ed3d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 1 Jul 2023 21:35:33 +0200 Subject: [PATCH 058/110] kernel_spec_dict --- .../ipythonconsole/utils/kernel_handler.py | 12 +++--- .../ipythonconsole/widgets/main_widget.py | 33 +++++++++++---- .../plugins/ipythonconsole/widgets/mixins.py | 41 ++++++++----------- .../plugins/ipythonconsole/widgets/shell.py | 4 +- 4 files changed, 52 insertions(+), 38 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index c217939f497..48009ebfcaf 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -117,7 +117,7 @@ class KernelHandler(QObject): def __init__( self, connection_file=None, - kernel_spec=None, + kernel_spec_dict=None, kernel_client=None, known_spyder_kernel=False, hostname=None, @@ -127,7 +127,7 @@ def __init__( super().__init__() # Connection Informations self.connection_file = connection_file - self.kernel_spec = kernel_spec + self.kernel_spec_dict = kernel_spec_dict self.kernel_client = kernel_client self.known_spyder_kernel = known_spyder_kernel self.hostname = hostname @@ -305,13 +305,13 @@ def tunnel_to_kernel( @classmethod def new_from_spec( - cls, kernel_spec, hostname=None, sshkey=None, password=None + cls, kernel_spec_dict, hostname=None, sshkey=None, password=None ): """ Create a new kernel. """ return cls( - kernel_spec=kernel_spec, + kernel_spec_dict=kernel_spec_dict, known_spyder_kernel=True, hostname=hostname, sshkey=sshkey, @@ -389,7 +389,7 @@ def close(self, shutdown_kernel=True, now=False): """Close kernel""" self.close_comm() - if shutdown_kernel and self.kernel_spec is not None: + if shutdown_kernel and self.kernel_spec_dict is not None: self.sig_remote_close.emit(self.connection_file) if ( @@ -411,7 +411,7 @@ def copy(self): self.sshkey, self.password, ) - copy_handler.kernel_spec = self.kernel_spec + copy_handler.kernel_spec_dict = self.kernel_spec_dict copy_handler.known_spyder_kernel = self.known_spyder_kernel return copy_handler diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 02ec42fabe4..6d8f67c9df1 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -889,7 +889,25 @@ def change_possible_restart_and_mpl_conf(self, option, value): # interactive backend. clients_backend_require_restart = [] for client in self.clients: - if client.shellwidget._shellwidget_state != "started": + sw = client.shellwidget + if ( + sw.kernel_handler + and sw.kernel_handler.kernel_spec_dict + and "SPY_BACKEND_O" in sw.kernel_handler.kernel_spec_dict[ + "env"] + ): + start_backend = sw.kernel_handler.kernel_spec_dict[ + "env"]["SPY_BACKEND_O"] + if start_backend != inline_backend: + must_restart = ( + pylab_backend_o != inline_backend and + pylab_backend_o != start_backend + ) + # An interactive backend was set at startup + clients_backend_require_restart.append(must_restart) + continue + if sw._shellwidget_state != "started": + # No env could be set clients_backend_require_restart.append(False) continue interactive_backend = ( @@ -1468,15 +1486,16 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, give_focus=give_focus) # Create new kernel - kernel_spec = SpyderKernelSpec( + kernel_spec_dict = SpyderKernelSpec( is_cython=is_cython, is_pylab=is_pylab, is_sympy=is_sympy, path_to_custom_interpreter=path_to_custom_interpreter - ) + ).to_dict() try: - kernel_handler = self.get_cached_kernel(kernel_spec, cache=cache) + kernel_handler = self.get_cached_kernel( + kernel_spec_dict, cache=cache) except Exception as e: client.show_kernel_error(e) return @@ -1870,8 +1889,8 @@ def restart_kernel(self, client=None, ask_before_restart=True): if client is None: return - ks = client.kernel_handler.kernel_spec - if ks is None: + ks_dict = client.kernel_handler.kernel_spec_dict + if ks_dict is None: client.shellwidget._append_plain_text( _('Cannot restart a kernel not started by Spyder\n'), before_prompt=True @@ -1896,7 +1915,7 @@ def restart_kernel(self, client=None, ask_before_restart=True): # Get new kernel try: - kernel_handler = self.get_cached_kernel(ks) + kernel_handler = self.get_cached_kernel(ks_dict) except Exception as e: client.show_kernel_error(e) return diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index f79b6296b44..d669576d35c 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -74,7 +74,9 @@ def on_kernel_server_conf_changed(self, option=None, value=None): self.request_queue = queue.Queue() # Send new kernel request for waiting kernels for kernel_handler in self.kernel_handler_waitlist: - self.send_request(["open_kernel", kernel_handler.kernel_spec.to_dict()]) + self.send_request( + ["open_kernel", kernel_handler.kernel_spec_dict] + ) self.options = options self.ssh_remote_hostname = None @@ -183,10 +185,11 @@ def tunnel_ssh(self, hostname, port): ) return hostname, port - def new_kernel(self, kernel_spec): + def new_kernel(self, kernel_spec_dict): """Get a new kernel""" + kernel_handler = KernelHandler.new_from_spec( - kernel_spec=kernel_spec, + kernel_spec_dict=kernel_spec_dict, hostname=self.ssh_remote_hostname, sshkey=self.ssh_key, password=self.ssh_password, @@ -198,7 +201,7 @@ def new_kernel(self, kernel_spec): self.sig_kernel_stdout.connect(kernel_handler.handle_stdout) self.kernel_handler_waitlist.append(kernel_handler) - self.send_request(["open_kernel", kernel_spec.to_dict()]) + self.send_request(["open_kernel", kernel_spec_dict]) return kernel_handler @@ -310,37 +313,31 @@ def close_cached_kernel(self): kernel.close(now=True) self._cached_kernel_properties = None - def check_cached_kernel_spec(self, kernel_spec): + def check_cached_kernel_spec(self, kernel_spec_dict): """Test if kernel_spec corresponds to the cached kernel_spec.""" if self._cached_kernel_properties is None: return False ( - cached_spec, - cached_env, - cached_argv, + cached_spec_dict, _, ) = self._cached_kernel_properties - # Call interrupt_mode so the dict will be the same - kernel_spec.interrupt_mode - cached_spec.interrupt_mode - if "PYTEST_CURRENT_TEST" in cached_env: + if "PYTEST_CURRENT_TEST" in cached_spec_dict["env"]: # Make tests faster by using cached kernels # hopefully the kernel will never use PYTEST_CURRENT_TEST - cached_env["PYTEST_CURRENT_TEST"] = kernel_spec.env[ + cached_spec_dict["env"][ + "PYTEST_CURRENT_TEST"] = kernel_spec_dict["env"][ "PYTEST_CURRENT_TEST" ] return ( - cached_spec.__dict__ == kernel_spec.__dict__ - and kernel_spec.argv == cached_argv - and kernel_spec.env == cached_env + kernel_spec_dict == cached_spec_dict ) - def get_cached_kernel(self, kernel_spec, cache=True): + def get_cached_kernel(self, kernel_spec_dict, cache=True): """Get a new kernel, and cache one for next time.""" # Cache another kernel for next time. - new_kernel_handler = self.new_kernel(kernel_spec) + new_kernel_handler = self.new_kernel(kernel_spec_dict) if not cache: # remove/don't use cache if requested @@ -351,20 +348,18 @@ def get_cached_kernel(self, kernel_spec, cache=True): cached_kernel_handler = None if self._cached_kernel_properties is not None: cached_kernel_handler = self._cached_kernel_properties[-1] - if not self.check_cached_kernel_spec(kernel_spec): + if not self.check_cached_kernel_spec(kernel_spec_dict): # Close the kernel self.close_cached_kernel() cached_kernel_handler = None # Cache the new kernel self._cached_kernel_properties = ( - kernel_spec, - kernel_spec.env, - kernel_spec.argv, + kernel_spec_dict, new_kernel_handler, ) if cached_kernel_handler is None: - return self.new_kernel(kernel_spec) + return self.new_kernel(kernel_spec_dict) return cached_kernel_handler diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index c867af594a6..45a9c0487c7 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -397,7 +397,7 @@ def is_external_kernel(self): """Check if this is an external kernel.""" if self.kernel_handler is None: return False - return self.kernel_handler.kernel_spec is None + return self.kernel_handler.kernel_spec_dict is None def setup_spyder_kernel(self): """Setup spyder kernel""" @@ -801,7 +801,7 @@ def _perform_reset(self, message): # kernels. # See spyder-ide/spyder#9505. try: - kernel_env = self.kernel_handler.kernel_spec.env + kernel_env = self.kernel_handler.kernel_spec_dict["env"] except AttributeError: kernel_env = {} From a283af92ad15bcbfc0954682fa1898923dbe899c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 1 Jul 2023 21:39:19 +0200 Subject: [PATCH 059/110] fix error --- spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index c3ca41b109a..d6580b306d8 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -150,6 +150,8 @@ def test_tk_backend(ipyconsole, qtbot): """Test that the Tkinter backend was set correctly.""" # Wait until the window is fully up shell = ipyconsole.get_current_shellwidget() + + qtbot.wait(5000) with qtbot.waitSignal(shell.executed): shell.execute("get_ipython().kernel.eventloop") From a244c148151f977c665dc5e99c54a9e08527d3f5 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 1 Jul 2023 21:40:04 +0200 Subject: [PATCH 060/110] fix auto backend --- spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index d6580b306d8..f680890bd44 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -128,6 +128,8 @@ def test_auto_backend(ipyconsole, qtbot): """Test that the automatic backend was set correctly.""" # Wait until the window is fully up shell = ipyconsole.get_current_shellwidget() + + qtbot.wait(5000) with qtbot.waitSignal(shell.executed): shell.execute("get_ipython().kernel.eventloop") From 80eb1694084327b784a93613a71c894781b3c93a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 2 Jul 2023 16:27:49 +0200 Subject: [PATCH 061/110] fix must restart --- .../ipythonconsole/widgets/main_widget.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index e08a701a1c1..311d08088ed 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -889,6 +889,12 @@ def change_possible_restart_and_mpl_conf(self, option, value): # interactive backend. clients_backend_require_restart = [] for client in self.clients: + if pylab_backend_o == inline_backend: + # No restart is needed if the new backend is inline + clients_backend_require_restart.append(False) + continue + # Need to know the interactive state + interactive_backend = None sw = client.shellwidget if ( sw.kernel_handler @@ -899,23 +905,22 @@ def change_possible_restart_and_mpl_conf(self, option, value): start_backend = sw.kernel_handler.kernel_spec_dict[ "env"]["SPY_BACKEND_O"] if start_backend != inline_backend: - must_restart = ( - pylab_backend_o != inline_backend and - pylab_backend_o != start_backend - ) - # An interactive backend was set at startup - clients_backend_require_restart.append(must_restart) - continue - if sw._shellwidget_state != "started": - # No env could be set - clients_backend_require_restart.append(False) - continue - interactive_backend = ( - client.shellwidget.get_mpl_interactive_backend()) + # If the state ever was non interactive, can not change + interactive_backend = start_backend + if ( + interactive_backend is None + and sw._shellwidget_state != "started" + ): + # If the kernel didn't start and no backend was requested, + # the backend is inline + interactive_backend = inline_backend + if interactive_backend is None: + # Must ask the kernel. Will not work if the kernel was set + # to another backend and is not now inline + interactive_backend = ( + client.shellwidget.get_mpl_interactive_backend()) if ( - # No restart is needed if the new backend is inline - pylab_backend_o != inline_backend and # There was an error getting the interactive backend in # the kernel, so we can't proceed. interactive_backend is not None and From 19640c9703c873f0bcc91aaefa626737cca7eab7 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 2 Jul 2023 16:31:22 +0200 Subject: [PATCH 062/110] Update upon restart --- spyder/plugins/ipythonconsole/widgets/main_widget.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 311d08088ed..a588e86f910 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1491,12 +1491,14 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, give_focus=give_focus) # Create new kernel - kernel_spec_dict = SpyderKernelSpec( + kernel_spec_kwargs = dict( is_cython=is_cython, is_pylab=is_pylab, is_sympy=is_sympy, path_to_custom_interpreter=path_to_custom_interpreter - ).to_dict() + ) + kernel_spec_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() + kernel_spec_dict["setup_kwargs"] = kernel_spec_kwargs try: kernel_handler = self.get_cached_kernel( @@ -1917,6 +1919,11 @@ def restart_kernel(self, client=None, ask_before_restart=True): if not do_restart: return + + # Update the kernel because settings might have changed + kernel_spec_kwargs = ks_dict["setup_kwargs"] + ks_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() + ks_dict["setup_kwargs"] = kernel_spec_kwargs # Get new kernel try: From 32c9fc0a981ecf554dd92b9fb0d81031f285f7ae Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Jul 2023 07:37:57 +0200 Subject: [PATCH 063/110] change special kernels --- .../spyder_kernels/console/kernel.py | 53 ++++---- .../spyder_kernels/console/start.py | 77 ----------- spyder/plugins/ipythonconsole/plugin.py | 21 +-- .../plugins/ipythonconsole/tests/conftest.py | 10 +- .../tests/test_ipythonconsole.py | 4 +- .../ipythonconsole/utils/kernel_handler.py | 3 + .../ipythonconsole/utils/kernelspec.py | 28 +--- .../ipythonconsole/widgets/main_widget.py | 51 ++++---- .../plugins/ipythonconsole/widgets/shell.py | 121 ++++++++++++++---- 9 files changed, 159 insertions(+), 209 deletions(-) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index ccbfde0ab75..50d58a57d75 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -645,27 +645,27 @@ def close_all_mpl_figures(self): except: pass - def is_special_kernel_valid(self): + def is_special_kernel_valid(self, special): """ Check if optional dependencies are available for special consoles. """ - try: - if os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True': - import matplotlib - elif os.environ.get('SPY_SYMPY_O') == 'True': - import sympy - elif os.environ.get('SPY_RUN_CYTHON') == 'True': - import cython - except Exception: - # Use Exception instead of ImportError here because modules can - # fail to be imported due to a lot of issues. - if os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True': - return u'matplotlib' - elif os.environ.get('SPY_SYMPY_O') == 'True': - return u'sympy' - elif os.environ.get('SPY_RUN_CYTHON') == 'True': - return u'cython' - return None + if special is None: + return + elif special == "pylab": + try: + import matplotlib + except Exception: + return "matplotlib" + elif special == "sympy": + try: + import sympy + except Exception: + return "sympy" + elif special == "cython": + try: + import cython + except Exception: + return "cython" def update_syspath(self, path_dict, new_path_dict): """ @@ -904,15 +904,14 @@ def show_mpl_backend_errors(self): def set_sympy_forecolor(self, background_color='dark'): """Set SymPy forecolor depending on console background.""" - if os.environ.get('SPY_SYMPY_O') == 'True': - try: - from sympy import init_printing - if background_color == 'dark': - init_printing(forecolor='White', ip=self.shell) - elif background_color == 'light': - init_printing(forecolor='Black', ip=self.shell) - except Exception: - pass + try: + from sympy import init_printing + if background_color == 'dark': + init_printing(forecolor='White', ip=self.shell) + elif background_color == 'light': + init_printing(forecolor='Black', ip=self.shell) + except Exception: + pass # --- Others def _load_autoreload_magic(self): diff --git a/external-deps/spyder-kernels/spyder_kernels/console/start.py b/external-deps/spyder-kernels/spyder_kernels/console/start.py index 708319c043e..82fe6612d7a 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/start.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/start.py @@ -21,8 +21,6 @@ # Local imports from spyder_kernels.utils.misc import is_module_installed -from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, INLINE_FIGURE_FORMATS) def import_spydercustomize(): @@ -48,24 +46,6 @@ def import_spydercustomize(): except ValueError: pass - -def sympy_config(mpl_backend): - """Sympy configuration""" - if mpl_backend is not None: - lines = """ -from sympy.interactive import init_session -init_session() -%matplotlib {0} -""".format(mpl_backend) - else: - lines = """ -from sympy.interactive import init_session -init_session() -""" - - return lines - - def kernel_config(): """Create a config object with IPython kernel options.""" from IPython.core.application import get_ipython_dir @@ -150,57 +130,6 @@ def kernel_config(): 'figure.edgecolor': 'white' } - # Pylab configuration - mpl_backend = None - if is_module_installed('matplotlib'): - # Set Matplotlib backend with Spyder options - pylab_o = os.environ.get('SPY_PYLAB_O') - backend_o = os.environ.get('SPY_BACKEND_O') - if pylab_o == 'True' and backend_o is not None: - mpl_backend = MPL_BACKENDS_FROM_SPYDER[backend_o] - # Inline backend configuration - if mpl_backend == 'inline': - # Figure format - format_o = os.environ.get('SPY_FORMAT_O') - formats = INLINE_FIGURE_FORMATS - if format_o is not None: - spy_cfg.InlineBackend.figure_format = formats[format_o] - - # Resolution - resolution_o = os.environ.get('SPY_RESOLUTION_O') - if resolution_o is not None: - spy_cfg.InlineBackend.rc['figure.dpi'] = float( - resolution_o) - - # Figure size - width_o = float(os.environ.get('SPY_WIDTH_O')) - height_o = float(os.environ.get('SPY_HEIGHT_O')) - if width_o is not None and height_o is not None: - spy_cfg.InlineBackend.rc['figure.figsize'] = (width_o, - height_o) - - # Print figure kwargs - bbox_inches_o = os.environ.get('SPY_BBOX_INCHES_O') - bbox_inches = 'tight' if bbox_inches_o == 'True' else None - spy_cfg.InlineBackend.print_figure_kwargs.update( - {'bbox_inches': bbox_inches}) - else: - # Set Matplotlib backend to inline for external kernels. - # Fixes issue 108 - mpl_backend = 'inline' - - # Automatically load Pylab and Numpy, or only set Matplotlib - # backend - autoload_pylab_o = os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True' - command = "get_ipython().kernel._set_mpl_backend('{0}', {1})" - spy_cfg.IPKernelApp.exec_lines.append( - command.format(mpl_backend, autoload_pylab_o)) - - # Enable Cython magic - run_cython = os.environ.get('SPY_RUN_CYTHON') == 'True' - if run_cython and is_module_installed('Cython'): - spy_cfg.IPKernelApp.exec_lines.append('%reload_ext Cython') - # Run a file at startup use_file_o = os.environ.get('SPY_USE_FILE_O') run_file_o = os.environ.get('SPY_RUN_FILE_O') @@ -220,12 +149,6 @@ def kernel_config(): greedy_o = os.environ.get('SPY_GREEDY_O') == 'True' spy_cfg.IPCompleter.greedy = greedy_o - # Sympy loading - sympy_o = os.environ.get('SPY_SYMPY_O') == 'True' - if sympy_o and is_module_installed('sympy'): - lines = sympy_config(mpl_backend) - spy_cfg.IPKernelApp.exec_lines.append(lines) - # Disable the new mechanism to capture and forward low-level output # in IPykernel 6. For that we have Wurlitzer. spy_cfg.IPKernelApp.capture_fd_output = False diff --git a/spyder/plugins/ipythonconsole/plugin.py b/spyder/plugins/ipythonconsole/plugin.py index d8b5c9f1ac7..b5326abe20f 100644 --- a/spyder/plugins/ipythonconsole/plugin.py +++ b/spyder/plugins/ipythonconsole/plugin.py @@ -536,9 +536,8 @@ def rename_client_tab(self, client, given_name): """ self.get_widget().rename_client_tab(client, given_name) - def create_new_client(self, give_focus=True, filename='', is_cython=False, - is_pylab=False, is_sympy=False, given_name=None, - path_to_custom_interpreter=None): + def create_new_client(self, give_focus=True, filename='', special=None, + given_name=None, path_to_custom_interpreter=None): """ Create a new client. @@ -549,15 +548,9 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, focus, False otherwise. The default is True. filename : str, optional Filename associated with the client. The default is ''. - is_cython : bool, optional - True if the client is expected to preload Cython support, - False otherwise. The default is False. - is_pylab : bool, optional - True if the client is expected to preload PyLab support, - False otherwise. The default is False. - is_sympy : bool, optional - True if the client is expected to preload Sympy support, - False otherwise. The default is False. + special : str, optional + Type of special support to preload. Can be "pylab", "cython", + "sympy", or None given_name : str, optional Initial name displayed in the tab of the client. The default is None. @@ -573,9 +566,7 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, self.get_widget().create_new_client( give_focus=give_focus, filename=filename, - is_cython=is_cython, - is_pylab=is_pylab, - is_sympy=is_sympy, + special=special, given_name=given_name, path_to_custom_interpreter=path_to_custom_interpreter) diff --git a/spyder/plugins/ipythonconsole/tests/conftest.py b/spyder/plugins/ipythonconsole/tests/conftest.py index 7b900ee22b7..d2c60acc71c 100644 --- a/spyder/plugins/ipythonconsole/tests/conftest.py +++ b/spyder/plugins/ipythonconsole/tests/conftest.py @@ -146,15 +146,15 @@ def __getattr__(self, attr): # Start a Pylab client if requested pylab_client = request.node.get_closest_marker('pylab_client') - is_pylab = True if pylab_client else False + special = "pylab" if pylab_client else None # Start a Sympy client if requested sympy_client = request.node.get_closest_marker('sympy_client') - is_sympy = True if sympy_client else False + special = "sympy" if sympy_client else special # Start a Cython client if requested cython_client = request.node.get_closest_marker('cython_client') - is_cython = True if cython_client else False + special = "cython" if cython_client else special # Start a specific env client if requested environment_client = request.node.get_closest_marker( @@ -207,9 +207,7 @@ def get_plugin(name): console.on_initialize() console._register() console.create_new_client( - is_pylab=is_pylab, - is_sympy=is_sympy, - is_cython=is_cython, + special=special, given_name=given_name, path_to_custom_interpreter=path_to_custom_interpreter ) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index f680890bd44..3391721f03c 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -343,10 +343,10 @@ def test_conf_env_vars(ipyconsole, qtbot): # Get a CONF env var with qtbot.waitSignal(shell.executed): - shell.execute("import os; a = os.environ.get('SPY_SYMPY_O')") + shell.execute("import os; a = os.environ.get('SPY_TESTING')") # Assert we get the assigned value correctly - assert shell.get_value('a') == 'False' + assert shell.get_value('a') == 'True' @flaky(max_runs=3) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 48009ebfcaf..681170cb3ef 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -148,6 +148,9 @@ def __init__( self._init_stderr = "" self._init_stdout = "" + # Special kernel + self.special = None + if self.kernel_client: # Start kernel self.kernel_client.start_channels() diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 475f2844e28..b2bab9b5f62 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -99,13 +99,9 @@ class SpyderKernelSpec(KernelSpec, SpyderConfigurationAccessor): CONF_SECTION = 'ipython_console' - def __init__(self, is_cython=False, is_pylab=False, - is_sympy=False, path_to_custom_interpreter=None, + def __init__(self, path_to_custom_interpreter=None, **kwargs): super(SpyderKernelSpec, self).__init__(**kwargs) - self.is_cython = is_cython - self.is_pylab = is_pylab - self.is_sympy = is_sympy self.path_to_custom_interpreter = path_to_custom_interpreter self.display_name = 'Python 3 (Spyder)' self.language = 'python3' @@ -204,38 +200,16 @@ def env(self): 'umr/verbose', section='main_interpreter'), 'SPY_UMR_NAMELIST': ','.join(umr_namelist), 'SPY_RUN_LINES_O': self.get_conf('startup/run_lines'), - 'SPY_PYLAB_O': self.get_conf('pylab'), - 'SPY_BACKEND_O': self.get_conf('pylab/backend'), - 'SPY_AUTOLOAD_PYLAB_O': self.get_conf('pylab/autoload'), - 'SPY_FORMAT_O': self.get_conf('pylab/inline/figure_format'), - 'SPY_BBOX_INCHES_O': self.get_conf('pylab/inline/bbox_inches'), - 'SPY_RESOLUTION_O': self.get_conf('pylab/inline/resolution'), - 'SPY_WIDTH_O': self.get_conf('pylab/inline/width'), - 'SPY_HEIGHT_O': self.get_conf('pylab/inline/height'), 'SPY_USE_FILE_O': self.get_conf('startup/use_run_file'), 'SPY_RUN_FILE_O': self.get_conf('startup/run_file'), 'SPY_AUTOCALL_O': self.get_conf('autocall'), 'SPY_GREEDY_O': self.get_conf('greedy_completer'), 'SPY_JEDI_O': self.get_conf('jedi_completer'), - 'SPY_SYMPY_O': self.get_conf('symbolic_math'), 'SPY_TESTING': running_under_pytest() or get_safe_mode(), 'SPY_HIDE_CMD': self.get_conf('hide_cmd_windows'), 'SPY_PYTHONPATH': pypath }) - if self.is_pylab is True: - env_vars['SPY_AUTOLOAD_PYLAB_O'] = True - env_vars['SPY_SYMPY_O'] = False - env_vars['SPY_RUN_CYTHON'] = False - if self.is_sympy is True: - env_vars['SPY_AUTOLOAD_PYLAB_O'] = False - env_vars['SPY_SYMPY_O'] = True - env_vars['SPY_RUN_CYTHON'] = False - if self.is_cython is True: - env_vars['SPY_AUTOLOAD_PYLAB_O'] = False - env_vars['SPY_SYMPY_O'] = False - env_vars['SPY_RUN_CYTHON'] = True - # App considerations # ??? Do we need this? if is_conda_based_app() and default_interpreter: diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index a588e86f910..fe5e96993ae 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -896,17 +896,6 @@ def change_possible_restart_and_mpl_conf(self, option, value): # Need to know the interactive state interactive_backend = None sw = client.shellwidget - if ( - sw.kernel_handler - and sw.kernel_handler.kernel_spec_dict - and "SPY_BACKEND_O" in sw.kernel_handler.kernel_spec_dict[ - "env"] - ): - start_backend = sw.kernel_handler.kernel_spec_dict[ - "env"]["SPY_BACKEND_O"] - if start_backend != inline_backend: - # If the state ever was non interactive, can not change - interactive_backend = start_backend if ( interactive_backend is None and sw._shellwidget_state != "started" @@ -1411,7 +1400,7 @@ def interpreter_versions(self, path_to_custom_interpreter=None): return versions - def additional_options(self, is_pylab=False, is_sympy=False): + def additional_options(self, special=None): """ Additional options for shell widgets that are not defined in JupyterWidget config options @@ -1423,10 +1412,10 @@ def additional_options(self, is_pylab=False, is_sympy=False): show_banner=self.get_conf('show_banner') ) - if is_pylab is True: + if special == "pylab": options['autoload_pylab'] = True options['sympy'] = False - if is_sympy is True: + elif special == "sympy": options['autoload_pylab'] = False options['sympy'] = True @@ -1459,9 +1448,8 @@ def get_current_shellwidget(self): @Slot(bool, str, str) @Slot(bool, bool) @Slot(bool, str, bool) - def create_new_client(self, give_focus=True, filename='', is_cython=False, - is_pylab=False, is_sympy=False, given_name=None, - cache=True, initial_cwd=None, + def create_new_client(self, give_focus=True, filename='', special=None, + given_name=None, cache=True, initial_cwd=None, path_to_custom_interpreter=None): """Create a new client""" self.master_clients += 1 @@ -1473,8 +1461,7 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, id_=client_id, config_options=self.config_options(), additional_options=self.additional_options( - is_pylab=is_pylab, - is_sympy=is_sympy), + special=special), interpreter_versions=self.interpreter_versions( path_to_custom_interpreter), context_menu_actions=self.context_menu_actions, @@ -1492,17 +1479,23 @@ def create_new_client(self, give_focus=True, filename='', is_cython=False, # Create new kernel kernel_spec_kwargs = dict( - is_cython=is_cython, - is_pylab=is_pylab, - is_sympy=is_sympy, path_to_custom_interpreter=path_to_custom_interpreter ) kernel_spec_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() kernel_spec_dict["setup_kwargs"] = kernel_spec_kwargs try: + kernel_spec_dict["env"]["SPY_RUN_CYTHON"] = str(special == "cython") kernel_handler = self.get_cached_kernel( - kernel_spec_dict, cache=cache) + kernel_spec_dict, + cache=cache, + ) + if special is not None: + kernel_handler.special = special + elif self.get_conf('pylab/autoload'): + kernel_handler.special = "pylab" + elif self.get_conf('symbolic_math'): + kernel_handler.special = "sympy" except Exception as e: client.show_kernel_error(e) return @@ -1589,15 +1582,15 @@ def create_client_for_kernel(self, connection_file, hostname, sshkey, def create_pylab_client(self): """Force creation of Pylab client""" - self.create_new_client(is_pylab=True, given_name="Pylab") + self.create_new_client(special="pylab", given_name="Pylab") def create_sympy_client(self): """Force creation of SymPy client""" - self.create_new_client(is_sympy=True, given_name="SymPy") + self.create_new_client(special="sympy", given_name="SymPy") def create_cython_client(self): """Force creation of Cython client""" - self.create_new_client(is_cython=True, given_name="Cython") + self.create_new_client(special="cython", given_name="Cython") def create_environment_client( self, environment, path_to_custom_interpreter @@ -1615,9 +1608,12 @@ def create_client_from_path(self, path): def create_client_for_file(self, filename, is_cython=False): """Create a client to execute code related to a file.""" + special = None + if is_cython: + special = "cython" # Create client client = self.create_new_client( - filename=filename, is_cython=is_cython) + filename=filename, special=special) # Don't increase the count of master clients self.master_clients -= 1 @@ -1928,6 +1924,7 @@ def restart_kernel(self, client=None, ask_before_restart=True): # Get new kernel try: kernel_handler = self.get_cached_kernel(ks_dict) + kernel_handler.special = client.kernel_handler.special except Exception as e: client.show_kernel_error(e) return diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 45a9c0487c7..bc17b7b1bf1 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -9,7 +9,6 @@ """ # Standard library imports -import ast import os import os.path as osp import time @@ -37,6 +36,9 @@ ControlWidget, DebuggingWidget, FigureBrowserWidget, HelpWidget, NamepaceBrowserWidget, PageControlWidget) +from spyder_kernels.utils.mpl import ( + MPL_BACKENDS_FROM_SPYDER, INLINE_FIGURE_FORMATS) + MODULES_FAQ_URL = ( "https://docs.spyder-ide.org/5/faq.html#using-packages-installer") @@ -430,15 +432,16 @@ def setup_spyder_kernel(self): # Show possible errors when setting Matplotlib backend self.call_kernel().show_mpl_backend_errors() - # Check if the dependecies for special consoles are available. - self.call_kernel( - callback=self.ipyclient._show_special_console_error - ).is_special_kernel_valid() - self.send_spyder_kernel_configuration() def send_spyder_kernel_configuration(self): """Send kernel configuration to spyder kernel.""" + # Set matplotlib backend + self.send_mpl_backend() + + # Make sure special kernel is correctly set up + self.reset_special_kernel() + # Set current cwd self.set_cwd() @@ -536,6 +539,56 @@ def set_cwd(self, dirname=None, emit_cwd_change=False): self._cwd = dirname if emit_cwd_change: self.sig_working_directory_changed.emit(self._cwd) + + def send_mpl_backend(self): + """Send matplotlib backend.""" + # Set Matplotlib backend with Spyder options + + pylab_n = 'pylab' + pylab_o = self.get_conf(pylab_n) + pylab_autoload_n = 'pylab/autoload' + pylab_backend_n = 'pylab/backend' + inline_backend_figure_format_n = 'pylab/inline/figure_format' + inline_backend_resolution_n = 'pylab/inline/resolution' + inline_backend_width_n = 'pylab/inline/width' + inline_backend_height_n = 'pylab/inline/height' + inline_backend_bbox_inches_n = 'pylab/inline/bbox_inches' + backend_o = self.get_conf(pylab_backend_n) + + if pylab_o and backend_o is not None: + mpl_backend = MPL_BACKENDS_FROM_SPYDER[backend_o] + # Inline backend configuration + if mpl_backend == 'inline': + # Figure format + format_o = self.get_conf(inline_backend_figure_format_n) + if format_o: + self.set_mpl_inline_figure_format(format_o) + + # Resolution + resolution_o = self.get_conf(inline_backend_resolution_n) + if resolution_o is not None: + self.set_mpl_inline_resolution(resolution_o) + + # Figure size + width_o = float(self.get_conf(inline_backend_width_n)) + height_o = float(self.get_conf(inline_backend_height_n)) + if width_o is not None and height_o is not None: + self.set_mpl_inline_figure_size(width_o, height_o) + + # Print figure kwargs + bbox_inches_o = self.get_conf(inline_backend_bbox_inches_n) + bbox_inches = 'tight' if bbox_inches_o else None + self.set_mpl_inline_bbox_inches(bbox_inches) + else: + # Set Matplotlib backend to inline for external kernels. + # Fixes issue 108 + mpl_backend = 'inline' + + # Automatically load Pylab and Numpy, or only set Matplotlib + # backend + autoload_pylab_o = self.get_conf(pylab_autoload_n) + self.set_matplotlib_backend(mpl_backend, autoload_pylab_o) + def get_cwd(self): """ @@ -584,10 +637,13 @@ def set_color_scheme(self, color_scheme, reset=True): if not dark_color: # Needed to change the colors of tracebacks self.silent_execute("%colors linux") - self.call_kernel().set_sympy_forecolor(background_color='dark') + if self.kernel_handler.special == "sympy": + self.call_kernel().set_sympy_forecolor(background_color='dark') else: self.silent_execute("%colors lightbg") - self.call_kernel().set_sympy_forecolor(background_color='light') + if self.kernel_handler.special == "sympy": + self.call_kernel().set_sympy_forecolor( + background_color='light') def update_syspath(self, path_dict, new_path_dict): """Update sys.path contents on kernel.""" @@ -797,14 +853,6 @@ def _perform_reset(self, message): Whether to show a message in the console telling users the namespace was reset. """ - # This is necessary to make resetting variables work in external - # kernels. - # See spyder-ide/spyder#9505. - try: - kernel_env = self.kernel_handler.kernel_spec_dict["env"] - except AttributeError: - kernel_env = {} - try: if self.is_waiting_pdb_input(): self.execute('%reset -f') @@ -817,24 +865,41 @@ def _perform_reset(self, message): ) self.insert_horizontal_ruler() self.silent_execute("%reset -f") - if kernel_env.get('SPY_AUTOLOAD_PYLAB_O') == 'True': - self.silent_execute("from pylab import *") - if kernel_env.get('SPY_SYMPY_O') == 'True': - sympy_init = """ - from sympy import * - x, y, z, t = symbols('x y z t') - k, m, n = symbols('k m n', integer=True) - f, g, h = symbols('f g h', cls=Function) - init_printing()""" - self.silent_execute(dedent(sympy_init)) - if kernel_env.get('SPY_RUN_CYTHON') == 'True': - self.silent_execute("%reload_ext Cython") + self.reset_special_kernel() if self.spyder_kernel_ready: self.call_kernel().close_all_mpl_figures() self.send_spyder_kernel_configuration() except AttributeError: pass + + def reset_special_kernel(self): + """Reset special kernel""" + if self.kernel_handler is None: + # This is not a special kernel + return + + if self.kernel_handler.special is None: + return + + + # Check if the dependecies for special consoles are available. + self.call_kernel( + callback=self.ipyclient._show_special_console_error + ).is_special_kernel_valid(self.kernel_handler.special) + + if self.kernel_handler.special == "pylab": + self.silent_execute("from pylab import *") + if self.kernel_handler.special == "sympy": + sympy_init = """ + from sympy import * + x, y, z, t = symbols('x y z t') + k, m, n = symbols('k m n', integer=True) + f, g, h = symbols('f g h', cls=Function) + init_printing()""" + self.silent_execute(dedent(sympy_init)) + if self.kernel_handler.special == "cython": + self.silent_execute("%reload_ext Cython") def _update_reset_options(self, message_box): """ From b935434ab96068fd1c264d8df2a4787e0c9f9b34 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Jul 2023 07:41:16 +0200 Subject: [PATCH 064/110] fix restart error --- .../ipythonconsole/widgets/main_widget.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index fe5e96993ae..cea42bed75e 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1477,15 +1477,15 @@ def create_new_client(self, give_focus=True, filename='', special=None, client, name=client.get_name(), filename=filename, give_focus=give_focus) - # Create new kernel - kernel_spec_kwargs = dict( - path_to_custom_interpreter=path_to_custom_interpreter - ) - kernel_spec_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() - kernel_spec_dict["setup_kwargs"] = kernel_spec_kwargs - try: - kernel_spec_dict["env"]["SPY_RUN_CYTHON"] = str(special == "cython") + # Create new kernel + kernel_spec_kwargs = dict( + path_to_custom_interpreter=path_to_custom_interpreter + ) + kernel_spec_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() + kernel_spec_dict["setup_kwargs"] = kernel_spec_kwargs + kernel_spec_dict["env"]["SPY_RUN_CYTHON"] = str( + special == "cython") kernel_handler = self.get_cached_kernel( kernel_spec_dict, cache=cache, @@ -1915,14 +1915,13 @@ def restart_kernel(self, client=None, ask_before_restart=True): if not do_restart: return - - # Update the kernel because settings might have changed - kernel_spec_kwargs = ks_dict["setup_kwargs"] - ks_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() - ks_dict["setup_kwargs"] = kernel_spec_kwargs # Get new kernel try: + # Update the kernel because settings might have changed + kernel_spec_kwargs = ks_dict["setup_kwargs"] + ks_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() + ks_dict["setup_kwargs"] = kernel_spec_kwargs kernel_handler = self.get_cached_kernel(ks_dict) kernel_handler.special = client.kernel_handler.special except Exception as e: From 5cbafe92472b1c14afda73b222801921c8fcb3c2 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Jul 2023 20:42:11 +0200 Subject: [PATCH 065/110] fix inline backend --- spyder/plugins/ipythonconsole/widgets/shell.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index bc17b7b1bf1..4319521027f 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -36,9 +36,6 @@ ControlWidget, DebuggingWidget, FigureBrowserWidget, HelpWidget, NamepaceBrowserWidget, PageControlWidget) -from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, INLINE_FIGURE_FORMATS) - MODULES_FAQ_URL = ( "https://docs.spyder-ide.org/5/faq.html#using-packages-installer") @@ -556,7 +553,7 @@ def send_mpl_backend(self): backend_o = self.get_conf(pylab_backend_n) if pylab_o and backend_o is not None: - mpl_backend = MPL_BACKENDS_FROM_SPYDER[backend_o] + mpl_backend = backend_o # Inline backend configuration if mpl_backend == 'inline': # Figure format From aca97ed010d38f2fbce32ded1a6a7dcf82f44037 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Jul 2023 20:45:08 +0200 Subject: [PATCH 066/110] remove _is_comm_ready --- .../spyder-kernels/spyder_kernels/comms/frontendcomm.py | 4 ---- .../spyder-kernels/spyder_kernels/console/kernel.py | 9 --------- 2 files changed, 13 deletions(-) diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index dc842e44626..1e7e86f49cd 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -49,7 +49,6 @@ def __init__(self, kernel): self.comm_lock = threading.Lock() self._cached_messages = {} self._pending_comms = {} - self._is_comm_ready = False def close(self, comm_id=None): """Close the comm and notify the other side.""" @@ -149,9 +148,6 @@ def _comm_ready_callback(self, ret): comm = self._pending_comms.pop(self.calling_comm_id, None) if not comm: return - # This is a bit hacky but works if at least one client connected to - # the kernel - self._is_comm_ready = True # Cached messages for that comm if comm.comm_id in self._cached_messages: for msg in self._cached_messages[comm.comm_id]: diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 50d58a57d75..e9165b5ce0c 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -800,15 +800,6 @@ def _set_mpl_backend(self, backend, pylab=False): pylab: Is the pylab magic should be used in order to populate the namespace from numpy and matplotlib """ - if backend != "inline" and not self.frontend_comm._is_comm_ready: - # Non - inline backends interfere with the ability to use - # call_later. They block the eventloop until a message is recieved - # on the shell. Therefore, they can not be activated before the - # comm is ready. - self.io_loop.call_later( - .3, lambda: self._set_mpl_backend(backend, pylab) - ) - return import traceback # Don't proceed further if there's any error while importing Matplotlib From 8c9c1fd6eaddd94016101e4aa879be6b6e33652a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Jul 2023 20:55:38 +0200 Subject: [PATCH 067/110] rm wait --- spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 3391721f03c..f4c8d74bd34 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -128,8 +128,6 @@ def test_auto_backend(ipyconsole, qtbot): """Test that the automatic backend was set correctly.""" # Wait until the window is fully up shell = ipyconsole.get_current_shellwidget() - - qtbot.wait(5000) with qtbot.waitSignal(shell.executed): shell.execute("get_ipython().kernel.eventloop") @@ -152,8 +150,6 @@ def test_tk_backend(ipyconsole, qtbot): """Test that the Tkinter backend was set correctly.""" # Wait until the window is fully up shell = ipyconsole.get_current_shellwidget() - - qtbot.wait(5000) with qtbot.waitSignal(shell.executed): shell.execute("get_ipython().kernel.eventloop") From ecc3ccb91e02c5d703d91dcc22b057ad14dd49fc Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 4 Jul 2023 03:09:38 +0200 Subject: [PATCH 068/110] wait --- spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index f4c8d74bd34..061a39a653b 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -1844,12 +1844,13 @@ def test_pdb_comprehension_namespace(ipyconsole, qtbot, tmpdir): @flaky(max_runs=3) @pytest.mark.auto_backend -def test_restart_intertactive_backend(ipyconsole): +def test_restart_intertactive_backend(ipyconsole, qtbot): """ Test that we ask for a restart after switching to a different interactive backend in preferences. """ main_widget = ipyconsole.get_widget() + qtbot.wait(1000) main_widget.change_possible_restart_and_mpl_conf('pylab/backend', 3) assert bool(os.environ.get('BACKEND_REQUIRE_RESTART')) From c698c99e9eecdf9d88c34137b2f5dfd83f4351ba Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 4 Jul 2023 23:45:41 +0200 Subject: [PATCH 069/110] fix inline backend --- .../plugins/ipythonconsole/widgets/shell.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 4319521027f..1f0a022f1db 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -209,7 +209,7 @@ def spyder_kernel_ready(self): def connect_kernel(self, kernel_handler): """Connect to the kernel using our handler.""" - + if self._shellwidget_state == "starting": self.sig_shellwidget_created.emit(self) @@ -435,7 +435,7 @@ def send_spyder_kernel_configuration(self): """Send kernel configuration to spyder kernel.""" # Set matplotlib backend self.send_mpl_backend() - + # Make sure special kernel is correctly set up self.reset_special_kernel() @@ -536,11 +536,11 @@ def set_cwd(self, dirname=None, emit_cwd_change=False): self._cwd = dirname if emit_cwd_change: self.sig_working_directory_changed.emit(self._cwd) - + def send_mpl_backend(self): """Send matplotlib backend.""" # Set Matplotlib backend with Spyder options - + pylab_n = 'pylab' pylab_o = self.get_conf(pylab_n) pylab_autoload_n = 'pylab/autoload' @@ -552,10 +552,12 @@ def send_mpl_backend(self): inline_backend_bbox_inches_n = 'pylab/inline/bbox_inches' backend_o = self.get_conf(pylab_backend_n) + inline_backend = 0 + if pylab_o and backend_o is not None: mpl_backend = backend_o # Inline backend configuration - if mpl_backend == 'inline': + if mpl_backend == inline_backend: # Figure format format_o = self.get_conf(inline_backend_figure_format_n) if format_o: @@ -565,7 +567,7 @@ def send_mpl_backend(self): resolution_o = self.get_conf(inline_backend_resolution_n) if resolution_o is not None: self.set_mpl_inline_resolution(resolution_o) - + # Figure size width_o = float(self.get_conf(inline_backend_width_n)) height_o = float(self.get_conf(inline_backend_height_n)) @@ -579,7 +581,7 @@ def send_mpl_backend(self): else: # Set Matplotlib backend to inline for external kernels. # Fixes issue 108 - mpl_backend = 'inline' + mpl_backend = inline_backend # Automatically load Pylab and Numpy, or only set Matplotlib # backend @@ -869,22 +871,22 @@ def _perform_reset(self, message): self.send_spyder_kernel_configuration() except AttributeError: pass - + def reset_special_kernel(self): """Reset special kernel""" if self.kernel_handler is None: # This is not a special kernel return - + if self.kernel_handler.special is None: return - - + + # Check if the dependecies for special consoles are available. self.call_kernel( callback=self.ipyclient._show_special_console_error ).is_special_kernel_valid(self.kernel_handler.special) - + if self.kernel_handler.special == "pylab": self.silent_execute("from pylab import *") if self.kernel_handler.special == "sympy": From ff813a22f8754e8885ea6c479b54fb85ba1bed3a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 07:34:12 +0200 Subject: [PATCH 070/110] fix tight --- spyder/plugins/ipythonconsole/widgets/shell.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 1f0a022f1db..9d01401149e 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -576,8 +576,7 @@ def send_mpl_backend(self): # Print figure kwargs bbox_inches_o = self.get_conf(inline_backend_bbox_inches_n) - bbox_inches = 'tight' if bbox_inches_o else None - self.set_mpl_inline_bbox_inches(bbox_inches) + self.set_mpl_inline_bbox_inches(bbox_inches_o) else: # Set Matplotlib backend to inline for external kernels. # Fixes issue 108 From 6060d6bd742056b57adfe59a178137779a15571d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 08:15:52 +0200 Subject: [PATCH 071/110] set_matplotlib_conf --- .../spyder_kernels/console/kernel.py | 54 +++++---- .../ipythonconsole/widgets/main_widget.py | 39 +------ .../plugins/ipythonconsole/widgets/shell.py | 107 +++++++++--------- 3 files changed, 88 insertions(+), 112 deletions(-) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index e9165b5ce0c..9878b5f080a 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -90,6 +90,7 @@ def __init__(self, *args, **kwargs): 'request_pdb_stop': self.shell.request_pdb_stop, 'raise_interrupt_signal': self.shell.raise_interrupt_signal, 'get_fault_text': self.get_fault_text, + 'set_matplotlib_conf': self.set_matplotlib_conf, } for call_id in handlers: self.frontend_comm.register_call_handler( @@ -555,26 +556,39 @@ def get_mpl_interactive_backend(self): # which users can set interactively with the %matplotlib # magic but not through our Preferences. return -1 - - def set_matplotlib_backend(self, backend, pylab=False): - """Set matplotlib backend given a Spyder backend option.""" - mpl_backend = MPL_BACKENDS_FROM_SPYDER[str(backend)] - self._set_mpl_backend(mpl_backend, pylab=pylab) - - def set_mpl_inline_figure_format(self, figure_format): - """Set the inline figure format to use with matplotlib.""" - mpl_figure_format = INLINE_FIGURE_FORMATS[figure_format] - self._set_config_option( - 'InlineBackend.figure_format', mpl_figure_format) - - def set_mpl_inline_resolution(self, resolution): - """Set inline figure resolution.""" - self._set_mpl_inline_rc_config('figure.dpi', resolution) - - def set_mpl_inline_figure_size(self, width, height): - """Set inline figure size.""" - value = (width, height) - self._set_mpl_inline_rc_config('figure.figsize', value) + + def set_matplotlib_conf(self, conf): + """Set matplotlib configuration""" + pylab_autoload_n = 'pylab/autoload' + pylab_backend_n = 'pylab/backend' + figure_format_n = 'pylab/inline/figure_format' + resolution_n = 'pylab/inline/resolution' + width_n = 'pylab/inline/width' + height_n = 'pylab/inline/height' + bbox_inches_n = 'pylab/inline/bbox_inches' + inline_backend = 0 + + if pylab_autoload_n in conf or pylab_backend_n in conf: + self._set_mpl_backend( + MPL_BACKENDS_FROM_SPYDER[str( + conf.get(pylab_backend_n, inline_backend))], + pylab=conf.get(pylab_backend_n, False) + ) + if figure_format_n in conf: + self._set_config_option( + 'InlineBackend.figure_format', + INLINE_FIGURE_FORMATS[conf[figure_format_n]] + ) + if resolution_n in conf: + self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n]) + if width_n in conf and height_n in conf: + self._set_mpl_inline_rc_config( + 'figure.figsize', + (conf[width_n], conf[height_n]) + ) + if bbox_inches_n in conf: + self.set_mpl_inline_bbox_inches(conf[bbox_inches_n]) + def set_mpl_inline_bbox_inches(self, bbox_inches): """ diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index cea42bed75e..2613d210b59 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1017,44 +1017,7 @@ def change_conf(c=client, ccf=client_conf_func, value=value): def _change_client_mpl_conf(self, options, client): """Apply Matplotlib related configurations to a client.""" - # Matplotlib options - pylab_n = 'pylab' - pylab_o = self.get_conf(pylab_n) - pylab_autoload_n = 'pylab/autoload' - pylab_backend_n = 'pylab/backend' - inline_backend_figure_format_n = 'pylab/inline/figure_format' - inline_backend_resolution_n = 'pylab/inline/resolution' - inline_backend_width_n = 'pylab/inline/width' - inline_backend_height_n = 'pylab/inline/height' - inline_backend_bbox_inches_n = 'pylab/inline/bbox_inches' - - # Client widgets - sw = client.shellwidget - if pylab_o: - if pylab_backend_n in options or pylab_autoload_n in options: - pylab_autoload_o = self.get_conf(pylab_autoload_n) - pylab_backend_o = self.get_conf(pylab_backend_n) - sw.set_matplotlib_backend(pylab_backend_o, pylab_autoload_o) - if inline_backend_figure_format_n in options: - inline_backend_figure_format_o = self.get_conf( - inline_backend_figure_format_n) - sw.set_mpl_inline_figure_format(inline_backend_figure_format_o) - if inline_backend_resolution_n in options: - inline_backend_resolution_o = self.get_conf( - inline_backend_resolution_n) - sw.set_mpl_inline_resolution(inline_backend_resolution_o) - if (inline_backend_width_n in options or - inline_backend_height_n in options): - inline_backend_width_o = self.get_conf( - inline_backend_width_n) - inline_backend_height_o = self.get_conf( - inline_backend_height_n) - sw.set_mpl_inline_figure_size( - inline_backend_width_o, inline_backend_height_o) - if inline_backend_bbox_inches_n in options: - inline_backend_bbox_inches_o = self.get_conf( - inline_backend_bbox_inches_n) - sw.set_mpl_inline_bbox_inches(inline_backend_bbox_inches_o) + client.shellwidget.send_mpl_backend(options) def _init_asyncio_patch(self): """ diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 9d01401149e..9111c1c3c69 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -537,46 +537,63 @@ def set_cwd(self, dirname=None, emit_cwd_change=False): if emit_cwd_change: self.sig_working_directory_changed.emit(self._cwd) - def send_mpl_backend(self): - """Send matplotlib backend.""" + def send_mpl_backend(self, option=None): + """ + Send matplotlib backend. + + If option is not None only send the related options + """ # Set Matplotlib backend with Spyder options pylab_n = 'pylab' pylab_o = self.get_conf(pylab_n) + + if option is not None and not pylab_o: + # The options are only related to pylab_o + # So no need to change the backend + return + pylab_autoload_n = 'pylab/autoload' pylab_backend_n = 'pylab/backend' - inline_backend_figure_format_n = 'pylab/inline/figure_format' - inline_backend_resolution_n = 'pylab/inline/resolution' - inline_backend_width_n = 'pylab/inline/width' - inline_backend_height_n = 'pylab/inline/height' - inline_backend_bbox_inches_n = 'pylab/inline/bbox_inches' + figure_format_n = 'pylab/inline/figure_format' + resolution_n = 'pylab/inline/resolution' + width_n = 'pylab/inline/width' + height_n = 'pylab/inline/height' + bbox_inches_n = 'pylab/inline/bbox_inches' backend_o = self.get_conf(pylab_backend_n) inline_backend = 0 + + matplotlib_conf = {} + + if pylab_o: + # Figure format + format_o = self.get_conf(figure_format_n) + if format_o and (option is None or figure_format_n in option): + matplotlib_conf[figure_format_n] = format_o + + # Resolution + resolution_o = self.get_conf(resolution_n) + if resolution_o is not None and ( + option is None or resolution_n in option): + matplotlib_conf[resolution_n] = resolution_o + + # Figure size + width_o = float(self.get_conf(width_n)) + height_o = float(self.get_conf(height_n)) + if option is None or (width_n in option or height_n in option): + if width_o is not None: + matplotlib_conf[width_n] = width_o + if height_o is not None: + matplotlib_conf[height_n] = height_o + + # Print figure kwargs + bbox_inches_o = self.get_conf(bbox_inches_n) + if option is None or bbox_inches_n in option: + matplotlib_conf[bbox_inches_n] = bbox_inches_o if pylab_o and backend_o is not None: mpl_backend = backend_o - # Inline backend configuration - if mpl_backend == inline_backend: - # Figure format - format_o = self.get_conf(inline_backend_figure_format_n) - if format_o: - self.set_mpl_inline_figure_format(format_o) - - # Resolution - resolution_o = self.get_conf(inline_backend_resolution_n) - if resolution_o is not None: - self.set_mpl_inline_resolution(resolution_o) - - # Figure size - width_o = float(self.get_conf(inline_backend_width_n)) - height_o = float(self.get_conf(inline_backend_height_n)) - if width_o is not None and height_o is not None: - self.set_mpl_inline_figure_size(width_o, height_o) - - # Print figure kwargs - bbox_inches_o = self.get_conf(inline_backend_bbox_inches_n) - self.set_mpl_inline_bbox_inches(bbox_inches_o) else: # Set Matplotlib backend to inline for external kernels. # Fixes issue 108 @@ -585,7 +602,14 @@ def send_mpl_backend(self): # Automatically load Pylab and Numpy, or only set Matplotlib # backend autoload_pylab_o = self.get_conf(pylab_autoload_n) - self.set_matplotlib_backend(mpl_backend, autoload_pylab_o) + if option is None or pylab_backend_n in option: + matplotlib_conf[pylab_backend_n] = mpl_backend + if option is None or pylab_autoload_n in option: + matplotlib_conf[pylab_autoload_n] = autoload_pylab_o + + if matplotlib_conf: + self.call_kernel().set_matplotlib_conf( + matplotlib_conf) def get_cwd(self): @@ -691,31 +715,6 @@ def get_mpl_interactive_backend(self): interrupt=True, blocking=True).get_mpl_interactive_backend() - def set_matplotlib_backend(self, backend_option, pylab=False): - """Set matplotlib backend given a backend name.""" - cmd = "get_ipython().kernel.set_matplotlib_backend('{}', {})" - self.execute(cmd.format(backend_option, pylab), hidden=True) - - def set_mpl_inline_figure_format(self, figure_format): - """Set matplotlib inline figure format.""" - cmd = "get_ipython().kernel.set_mpl_inline_figure_format('{}')" - self.execute(cmd.format(figure_format), hidden=True) - - def set_mpl_inline_resolution(self, resolution): - """Set matplotlib inline resolution (savefig.dpi/figure.dpi).""" - cmd = "get_ipython().kernel.set_mpl_inline_resolution({})" - self.execute(cmd.format(resolution), hidden=True) - - def set_mpl_inline_figure_size(self, width, height): - """Set matplotlib inline resolution (savefig.dpi/figure.dpi).""" - cmd = "get_ipython().kernel.set_mpl_inline_figure_size({}, {})" - self.execute(cmd.format(width, height), hidden=True) - - def set_mpl_inline_bbox_inches(self, bbox_inches): - """Set matplotlib inline print figure bbox_inches ('tight' or not).""" - cmd = "get_ipython().kernel.set_mpl_inline_bbox_inches({})" - self.execute(cmd.format(bbox_inches), hidden=True) - def set_jedi_completer(self, use_jedi): """Set if jedi completions should be used.""" cmd = "get_ipython().kernel.set_jedi_completer({})" From 3d12e0cfc66c6fe9ae3971356af0ab6d9bc5b0ff Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 22:42:25 +0200 Subject: [PATCH 072/110] fix format --- external-deps/spyder-kernels/spyder_kernels/console/kernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 9878b5f080a..315f4d29060 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -577,7 +577,7 @@ def set_matplotlib_conf(self, conf): if figure_format_n in conf: self._set_config_option( 'InlineBackend.figure_format', - INLINE_FIGURE_FORMATS[conf[figure_format_n]] + INLINE_FIGURE_FORMATS[str(conf[figure_format_n])] ) if resolution_n in conf: self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n]) From 0321bc3a6605d87e78130b61acb64ae4dfd38d3a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 23:02:14 +0200 Subject: [PATCH 073/110] remove figure format --- .../spyder-kernels/spyder_kernels/console/kernel.py | 4 ++-- external-deps/spyder-kernels/spyder_kernels/utils/mpl.py | 7 ------- spyder/app/tests/conftest.py | 2 +- spyder/app/tests/test_mainwindow.py | 6 +++--- spyder/config/main.py | 2 +- spyder/plugins/ipythonconsole/confpage.py | 4 ++-- 6 files changed, 9 insertions(+), 16 deletions(-) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 315f4d29060..30348abad15 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -32,7 +32,7 @@ from spyder_kernels.comms.frontendcomm import FrontendComm from spyder_kernels.utils.iofuncs import iofunctions from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, MPL_BACKENDS_TO_SPYDER, INLINE_FIGURE_FORMATS) + MPL_BACKENDS_FROM_SPYDER, MPL_BACKENDS_TO_SPYDER) from spyder_kernels.utils.nsview import ( get_remote_data, make_remote_view, get_size) from spyder_kernels.console.shell import SpyderShell @@ -577,7 +577,7 @@ def set_matplotlib_conf(self, conf): if figure_format_n in conf: self._set_config_option( 'InlineBackend.figure_format', - INLINE_FIGURE_FORMATS[str(conf[figure_format_n])] + conf[figure_format_n] ) if resolution_n in conf: self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n]) diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py index 7927e49da62..ccd705d470d 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py @@ -11,13 +11,6 @@ from spyder_kernels.utils.misc import is_module_installed -# Mapping of inline figure formats -INLINE_FIGURE_FORMATS = { - '0': 'png', - '1': 'svg' -} - - # Inline backend if is_module_installed('matplotlib_inline'): inline_backend = 'module://matplotlib_inline.backend_inline' diff --git a/spyder/app/tests/conftest.py b/spyder/app/tests/conftest.py index f71d002008b..48e88abc327 100755 --- a/spyder/app/tests/conftest.py +++ b/spyder/app/tests/conftest.py @@ -309,7 +309,7 @@ def main_window(request, tmpdir, qtbot): # Test assume the plots are rendered in the console as png CONF.set('plots', 'mute_inline_plotting', False) - CONF.set('ipython_console', 'pylab/inline/figure_format', 0) + CONF.set('ipython_console', 'pylab/inline/figure_format', "png") # Set exclamation mark to True CONF.set('debugger', 'pdb_use_exclamation_mark', True) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index d12f07aa5f8..4656be0c05e 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -2145,8 +2145,8 @@ def test_varexp_magic_dbg(main_window, qtbot): @flaky(max_runs=3) @pytest.mark.parametrize( 'main_window', - [{'spy_config': ('ipython_console', 'pylab/inline/figure_format', 1)}, - {'spy_config': ('ipython_console', 'pylab/inline/figure_format', 0)}], + [{'spy_config': ('ipython_console', 'pylab/inline/figure_format', 'svg')}, + {'spy_config': ('ipython_console', 'pylab/inline/figure_format', 'png')}], indirect=True) def test_plots_plugin(main_window, qtbot, tmpdir, mocker): """ @@ -2167,7 +2167,7 @@ def test_plots_plugin(main_window, qtbot, tmpdir, mocker): shell.execute(("import matplotlib.pyplot as plt\n" "fig = plt.plot([1, 2, 3, 4], '.')\n")) - if CONF.get('ipython_console', 'pylab/inline/figure_format') == 0: + if CONF.get('ipython_console', 'pylab/inline/figure_format') == 'png': assert figbrowser.figviewer.figcanvas.fmt == 'image/png' else: assert figbrowser.figviewer.figcanvas.fmt == 'image/svg+xml' diff --git a/spyder/config/main.py b/spyder/config/main.py index 32653b53a41..56049d9caa3 100644 --- a/spyder/config/main.py +++ b/spyder/config/main.py @@ -154,7 +154,7 @@ 'pylab': True, 'pylab/autoload': False, 'pylab/backend': 0, - 'pylab/inline/figure_format': 0, + 'pylab/inline/figure_format': 'png', 'pylab/inline/resolution': 72, 'pylab/inline/width': 6, 'pylab/inline/height': 4, diff --git a/spyder/plugins/ipythonconsole/confpage.py b/spyder/plugins/ipythonconsole/confpage.py index 22d3defa639..3fbb1f952f5 100644 --- a/spyder/plugins/ipythonconsole/confpage.py +++ b/spyder/plugins/ipythonconsole/confpage.py @@ -138,10 +138,10 @@ def setup_page(self): inline_label = QLabel(_("Decide how to render the figures created by " "this backend")) inline_label.setWordWrap(True) - formats = (("PNG", 0), ("SVG", 1)) + formats = (("PNG", 'png'), ("SVG", 'svg')) format_box = self.create_combobox(_("Format:")+" ", formats, 'pylab/inline/figure_format', - default=0) + default='png') resolution_spin = self.create_spinbox( _("Resolution:")+" ", " "+_("dpi"), 'pylab/inline/resolution', min_=50, max_=999, step=0.1, From be945867b94972eeb8bb97f8b8b1d60042606cb0 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 23:38:15 +0200 Subject: [PATCH 074/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "3564ac929" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "3564ac929" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 +- .../spyder_kernels/comms/decorators.py | 27 +++++++ .../spyder_kernels/comms/frontendcomm.py | 6 +- .../spyder_kernels/console/kernel.py | 72 +++++++++---------- .../spyder_kernels/console/shell.py | 5 ++ 5 files changed, 71 insertions(+), 43 deletions(-) create mode 100644 external-deps/spyder-kernels/spyder_kernels/comms/decorators.py diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 022e35e537a..cbffb02f2c8 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = 67dceeb9bab5c3ad26d878d002df094f05b06f48 - parent = 97792e5fb7d7f851e956ebcdb5a61409e217ab5e + commit = 3564ac929ecd12c2bb7069c7cfc875612a9055cc + parent = 0321bc3a6605d87e78130b61acb64ae4dfd38d3a method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py b/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py new file mode 100644 index 00000000000..b2633404a07 --- /dev/null +++ b/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# Copyright © Spyder Project Contributors +# Licensed under the terms of the MIT License +# (see spyder/__init__.py for details) + +""" +Comms decorators. +""" + + +def comm_handler(fun): + """Decorator to mark comm handler methods.""" + fun._is_comm_handler = True + return fun + + +def register_comm_handlers(instance, frontend_comm): + """ + Registers an instance whose methods have been marked with comm_handler. + """ + for method_name in instance.__class__.__dict__: + method = getattr(instance, method_name) + if hasattr(method, '_is_comm_handler'): + frontend_comm.register_call_handler( + method_name, method) + diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index 1e7e86f49cd..9eaf2ee1f43 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -130,9 +130,11 @@ def _check_comm_reply(self): """ Send comm message to frontend to check if the iopub channel is ready """ - if len(self._pending_comms) == 0: + # Make sure the length doesn't change during iteration + pending_comms = list(self._pending_comms.values()) + if len(pending_comms) == 0: return - for comm in self._pending_comms.values(): + for comm in pending_comms: self._notify_comm_ready(comm) self.kernel.io_loop.call_later(1, self._check_comm_reply) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 30348abad15..1367c864cfe 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -30,6 +30,8 @@ # Local imports from spyder_kernels.comms.frontendcomm import FrontendComm +from spyder_kernels.comms.decorators import ( + register_comm_handlers, comm_handler) from spyder_kernels.utils.iofuncs import iofunctions from spyder_kernels.utils.mpl import ( MPL_BACKENDS_FROM_SPYDER, MPL_BACKENDS_TO_SPYDER) @@ -39,7 +41,6 @@ from spyder_kernels.comms.utils import WriteContext - logger = logging.getLogger(__name__) @@ -60,41 +61,8 @@ def __init__(self, *args, **kwargs): self.frontend_comm = FrontendComm(self) # All functions that can be called through the comm - handlers = { - 'set_pdb_configuration': self.shell.set_pdb_configuration, - 'get_value': self.get_value, - 'load_data': self.load_data, - 'save_namespace': self.save_namespace, - 'is_defined': self.is_defined, - 'get_doc': self.get_doc, - 'get_source': self.get_source, - 'set_value': self.set_value, - 'remove_value': self.remove_value, - 'copy_value': self.copy_value, - 'set_cwd': self.set_cwd, - 'get_syspath': self.get_syspath, - 'get_env': self.get_env, - 'close_all_mpl_figures': self.close_all_mpl_figures, - 'show_mpl_backend_errors': self.show_mpl_backend_errors, - 'get_namespace_view': self.get_namespace_view, - 'set_namespace_view_settings': self.set_namespace_view_settings, - 'get_var_properties': self.get_var_properties, - 'set_sympy_forecolor': self.set_sympy_forecolor, - 'update_syspath': self.update_syspath, - 'is_special_kernel_valid': self.is_special_kernel_valid, - 'get_matplotlib_backend': self.get_matplotlib_backend, - 'get_mpl_interactive_backend': self.get_mpl_interactive_backend, - 'pdb_input_reply': self.shell.pdb_input_reply, - 'enable_faulthandler': self.enable_faulthandler, - 'get_current_frames': self.get_current_frames, - 'request_pdb_stop': self.shell.request_pdb_stop, - 'raise_interrupt_signal': self.shell.raise_interrupt_signal, - 'get_fault_text': self.get_fault_text, - 'set_matplotlib_conf': self.set_matplotlib_conf, - } - for call_id in handlers: - self.frontend_comm.register_call_handler( - call_id, handlers[call_id]) + register_comm_handlers(self, self.frontend_comm) + register_comm_handlers(self.shell, self.frontend_comm) self.namespace_view_settings = {} self._mpl_backend_error = None @@ -146,6 +114,7 @@ def publish_state(self): except Exception: pass + @comm_handler def enable_faulthandler(self): """ Open a file to save the faulthandling and identifiers for @@ -171,6 +140,7 @@ def enable_faulthandler(self): faulthandler.enable(self.faulthandler_handle) return self.faulthandler_handle.name, main_id, system_ids + @comm_handler def get_fault_text(self, fault_filename, main_id, ignore_ids): """Get fault text from old run.""" # Read file @@ -264,6 +234,7 @@ def filter_stack(self, stack, is_main): stack = [] return stack + @comm_handler def get_current_frames(self, ignore_internal_threads=True, capture_locals=False): """Get the current frames.""" @@ -293,10 +264,12 @@ def get_current_frames(self, ignore_internal_threads=True, return frames # --- For the Variable Explorer + @comm_handler def set_namespace_view_settings(self, settings): """Set namespace_view_settings.""" self.namespace_view_settings = settings + @comm_handler def get_namespace_view(self, frame=None): """ Return the namespace view @@ -332,6 +305,7 @@ def get_namespace_view(self, frame=None): else: return None + @comm_handler def get_var_properties(self): """ Get some properties of the variables in the current @@ -362,27 +336,32 @@ def get_var_properties(self): else: return None + @comm_handler def get_value(self, name): """Get the value of a variable""" ns = self.shell._get_current_namespace() return ns[name] + @comm_handler def set_value(self, name, value): """Set the value of a variable""" ns = self.shell._get_reference_namespace(name) ns[name] = value self.log.debug(ns) + @comm_handler def remove_value(self, name): """Remove a variable""" ns = self.shell._get_reference_namespace(name) ns.pop(name) + @comm_handler def copy_value(self, orig_name, new_name): """Copy a variable""" ns = self.shell._get_reference_namespace(orig_name) ns[new_name] = ns[orig_name] + @comm_handler def load_data(self, filename, ext, overwrite=False): """ Load data from filename. @@ -419,6 +398,7 @@ def load_data(self, filename, ext, overwrite=False): return None + @comm_handler def save_namespace(self, filename): """Save namespace into filename""" ns = self.shell._get_current_namespace() @@ -472,6 +452,7 @@ def interrupt_eventloop(self): self.loopback_socket, self.session.msg("interrupt_eventloop")) # --- For the Help plugin + @comm_handler def is_defined(self, obj, force_import=False): """Return True if object is defined in current namespace""" from spyder_kernels.utils.dochelpers import isdefined @@ -479,6 +460,7 @@ def is_defined(self, obj, force_import=False): ns = self.shell._get_current_namespace(with_magics=True) return isdefined(obj, force_import=force_import, namespace=ns) + @comm_handler def get_doc(self, objtxt): """Get object documentation dictionary""" try: @@ -492,6 +474,7 @@ def get_doc(self, objtxt): if valid: return getdoc(obj) + @comm_handler def get_source(self, objtxt): """Get object source""" from spyder_kernels.utils.dochelpers import getsource @@ -501,6 +484,7 @@ def get_source(self, objtxt): return getsource(obj) # -- For Matplolib + @comm_handler def get_matplotlib_backend(self): """Get current matplotlib backend.""" try: @@ -509,6 +493,7 @@ def get_matplotlib_backend(self): except Exception: return None + @comm_handler def get_mpl_interactive_backend(self): """ Get current Matplotlib interactive backend. @@ -556,7 +541,8 @@ def get_mpl_interactive_backend(self): # which users can set interactively with the %matplotlib # magic but not through our Preferences. return -1 - + + @comm_handler def set_matplotlib_conf(self, conf): """Set matplotlib configuration""" pylab_autoload_n = 'pylab/autoload' @@ -567,7 +553,7 @@ def set_matplotlib_conf(self, conf): height_n = 'pylab/inline/height' bbox_inches_n = 'pylab/inline/bbox_inches' inline_backend = 0 - + if pylab_autoload_n in conf or pylab_backend_n in conf: self._set_mpl_backend( MPL_BACKENDS_FROM_SPYDER[str( @@ -588,7 +574,7 @@ def set_matplotlib_conf(self, conf): ) if bbox_inches_n in conf: self.set_mpl_inline_bbox_inches(conf[bbox_inches_n]) - + def set_mpl_inline_bbox_inches(self, bbox_inches): """ @@ -630,6 +616,7 @@ def set_autocall(self, autocall): self._set_config_option('ZMQInteractiveShell.autocall', autocall) # --- Additional methods + @comm_handler def set_cwd(self, dirname): """Set current working directory.""" self._cwd_initialised = True @@ -643,14 +630,17 @@ def get_cwd(self): except (IOError, OSError): pass + @comm_handler def get_syspath(self): """Return sys.path contents.""" return sys.path[:] + @comm_handler def get_env(self): """Get environment variables.""" return os.environ.copy() + @comm_handler def close_all_mpl_figures(self): """Close all Matplotlib figures.""" try: @@ -659,6 +649,7 @@ def close_all_mpl_figures(self): except: pass + @comm_handler def is_special_kernel_valid(self, special): """ Check if optional dependencies are available for special consoles. @@ -681,6 +672,7 @@ def is_special_kernel_valid(self, special): except Exception: return "cython" + @comm_handler def update_syspath(self, path_dict, new_path_dict): """ Update the PYTHONPATH of the kernel. @@ -902,11 +894,13 @@ def _set_mpl_inline_rc_config(self, option, value): # Needed in case matplolib isn't installed pass + @comm_handler def show_mpl_backend_errors(self): """Show Matplotlib backend errors after the prompt is ready.""" if self._mpl_backend_error is not None: print(self._mpl_backend_error) # spyder: test-skip + @comm_handler def set_sympy_forecolor(self, background_color='dark'): """Set SymPy forecolor depending on console background.""" try: diff --git a/external-deps/spyder-kernels/spyder_kernels/console/shell.py b/external-deps/spyder-kernels/spyder_kernels/console/shell.py index 6dbbd7598c6..c641feb8176 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/shell.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/shell.py @@ -28,6 +28,7 @@ from spyder_kernels.customize.spyderpdb import SpyderPdb from spyder_kernels.customize.code_runner import SpyderCodeRunner from spyder_kernels.comms.frontendcomm import CommError +from spyder_kernels.comms.decorators import comm_handler from spyder_kernels.utils.mpl import automatic_backend @@ -100,6 +101,7 @@ def enable_matplotlib(self, gui=None): return gui, backend # --- For Pdb namespace integration + @comm_handler def set_pdb_configuration(self, pdb_conf): """ Set Pdb configuration. @@ -274,6 +276,7 @@ def register_debugger_sigint(self): """Register sigint handler.""" signal.signal(signal.SIGINT, self.spyderkernel_sigint_handler) + @comm_handler def raise_interrupt_signal(self): """Raise interrupt signal.""" if os.name == "nt": @@ -293,6 +296,7 @@ def raise_interrupt_signal(self): else: self.kernel._send_interrupt_children() + @comm_handler def request_pdb_stop(self): """Request pdb to stop at the next possible position.""" pdb_session = self.pdb_session @@ -351,6 +355,7 @@ async def run_code(self, *args, **kwargs): except KeyboardInterrupt: self.showtraceback() + @comm_handler def pdb_input_reply(self, line, echo_stack_entry=True): """Get a pdb command from the frontend.""" debugger = self.pdb_session From 1828539fa8d2ef6a0208830cc5ccb7905b66bb84 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 07:27:42 +0200 Subject: [PATCH 075/110] start --- spyder/plugins/ipythonconsole/widgets/shell.py | 1 - spyder/plugins/ipythonconsole/widgets/status.py | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 5d369d5b3b0..97a8f1e17c6 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -609,7 +609,6 @@ def send_mpl_backend(self, option=None): self.call_kernel().set_matplotlib_conf( matplotlib_conf) - def get_cwd(self): """ Get current working directory. diff --git a/spyder/plugins/ipythonconsole/widgets/status.py b/spyder/plugins/ipythonconsole/widgets/status.py index 6a4f623b677..6cef3813134 100644 --- a/spyder/plugins/ipythonconsole/widgets/status.py +++ b/spyder/plugins/ipythonconsole/widgets/status.py @@ -67,18 +67,10 @@ def update(self, gui): def add_shellwidget(self, shellwidget): """Add shellwidget.""" - # Leave this import here so that we avoid importing Matplotlib (which - # is imported by matplotlib_inline unconditionally) before the main - # window is visible. We do this because Matplotlib takes a long time - # to be imported, so it makes Spyder appear slow to start to users. - from spyder_kernels.utils.mpl import MPL_BACKENDS_FROM_SPYDER - shellwidget.sig_config_spyder_kernel.connect( lambda sw=shellwidget: self.config_spyder_kernel(sw)) - backend = MPL_BACKENDS_FROM_SPYDER[ - str(self.get_conf('pylab/backend')) - ] + backend = self.get_conf('pylab/backend') swid = id(shellwidget) self._shellwidget_dict[swid] = { "gui": backend, From 6d0a54f47f411475038509b16121992dde9c1b77 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 07:27:53 +0200 Subject: [PATCH 076/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "fb7fe85de" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "fb7fe85de" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- .../spyder_kernels/console/kernel.py | 9 +++++---- .../console/tests/test_console_kernel.py | 5 ++--- .../spyder_kernels/utils/mpl.py | 20 +++++-------------- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index cbffb02f2c8..7ac4421c373 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = 3564ac929ecd12c2bb7069c7cfc875612a9055cc - parent = 0321bc3a6605d87e78130b61acb64ae4dfd38d3a + commit = fb7fe85de40414fbd2ec43754ca45b3162ca65fd + parent = 1828539fa8d2ef6a0208830cc5ccb7905b66bb84 method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 1367c864cfe..a8441dfb94b 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -33,8 +33,7 @@ from spyder_kernels.comms.decorators import ( register_comm_handlers, comm_handler) from spyder_kernels.utils.iofuncs import iofunctions -from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, MPL_BACKENDS_TO_SPYDER) +from spyder_kernels.utils.mpl import automatic_backend, MPL_BACKENDS_TO_SPYDER from spyder_kernels.utils.nsview import ( get_remote_data, make_remote_view, get_size) from spyder_kernels.console.shell import SpyderShell @@ -556,8 +555,7 @@ def set_matplotlib_conf(self, conf): if pylab_autoload_n in conf or pylab_backend_n in conf: self._set_mpl_backend( - MPL_BACKENDS_FROM_SPYDER[str( - conf.get(pylab_backend_n, inline_backend))], + conf.get(pylab_backend_n, inline_backend), pylab=conf.get(pylab_backend_n, False) ) if figure_format_n in conf: @@ -822,6 +820,9 @@ def _set_mpl_backend(self, backend, pylab=False): ) magic = 'pylab' if pylab else 'matplotlib' + + if backend == "auto": + backend = automatic_backend() error = None try: diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py index d7efae21fa7..e69eb79b5dd 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py @@ -32,7 +32,6 @@ # Local imports from spyder_kernels.utils.iofuncs import iofunctions -from spyder_kernels.utils.mpl import MPL_BACKENDS_FROM_SPYDER from spyder_kernels.utils.test_utils import get_kernel, get_log_text from spyder_kernels.customize.spyderpdb import SpyderPdb from spyder_kernels.comms.commbase import CommBase @@ -1145,9 +1144,9 @@ def test_get_interactive_backend(backend): # Assert we got the right interactive backend if backend is not None: - assert MPL_BACKENDS_FROM_SPYDER[value] == backend + assert value == backend else: - assert value == '0' + assert value == 'inline' def test_global_message(tmpdir): diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py index ccd705d470d..7756fd39689 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py @@ -20,11 +20,11 @@ # Mapping of matlotlib backends options to Spyder MPL_BACKENDS_TO_SPYDER = { - inline_backend: 0, - 'Qt5Agg': 2, - 'QtAgg': 2, # For Matplotlib 3.5+ - 'TkAgg': 3, - 'MacOSX': 4, + inline_backend: "inline", + 'Qt5Agg': 'qt5', + 'QtAgg': 'qt5', # For Matplotlib 3.5+ + 'TkAgg': 'tk', + 'MacOSX': 'osx', } @@ -37,13 +37,3 @@ def automatic_backend(): else: auto_backend = 'inline' return auto_backend - - -# Mapping of Spyder options to backends -MPL_BACKENDS_FROM_SPYDER = { - '0': 'inline', - '1': automatic_backend(), - '2': 'qt5', - '3': 'tk', - '4': 'osx' -} From cc152648033f9d51da222f8aad57142f6f350434 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 07:34:18 +0200 Subject: [PATCH 077/110] backend to string --- spyder/app/tests/conftest.py | 2 +- spyder/config/main.py | 2 +- spyder/plugins/ipythonconsole/confpage.py | 11 ++++++++--- spyder/plugins/ipythonconsole/tests/conftest.py | 6 +++--- .../ipythonconsole/tests/test_ipythonconsole.py | 2 +- spyder/plugins/ipythonconsole/widgets/main_widget.py | 2 +- spyder/plugins/ipythonconsole/widgets/shell.py | 2 +- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/spyder/app/tests/conftest.py b/spyder/app/tests/conftest.py index 48e88abc327..7bfcdfd28c8 100755 --- a/spyder/app/tests/conftest.py +++ b/spyder/app/tests/conftest.py @@ -305,7 +305,7 @@ def main_window(request, tmpdir, qtbot): CONF.set('tours', 'show_tour_message', False) # Tests assume inline backend - CONF.set('ipython_console', 'pylab/backend', 0) + CONF.set('ipython_console', 'pylab/backend', 'inline') # Test assume the plots are rendered in the console as png CONF.set('plots', 'mute_inline_plotting', False) diff --git a/spyder/config/main.py b/spyder/config/main.py index 56049d9caa3..e12df1e230d 100644 --- a/spyder/config/main.py +++ b/spyder/config/main.py @@ -153,7 +153,7 @@ 'buffer_size': 500, 'pylab': True, 'pylab/autoload': False, - 'pylab/backend': 0, + 'pylab/backend': 'inline', 'pylab/inline/figure_format': 'png', 'pylab/inline/resolution': 72, 'pylab/inline/width': 6, diff --git a/spyder/plugins/ipythonconsole/confpage.py b/spyder/plugins/ipythonconsole/confpage.py index 3fbb1f952f5..e896dd675db 100644 --- a/spyder/plugins/ipythonconsole/confpage.py +++ b/spyder/plugins/ipythonconsole/confpage.py @@ -113,16 +113,21 @@ def setup_page(self): "separate window.") % (inline, automatic)) bend_label.setWordWrap(True) - backends = [(inline, 0), (automatic, 1), ("Qt5", 2), ("Tkinter", 3)] + backends = [ + (inline, 'inline'), + (automatic, 'auto'), + ("Qt5", 'qt5'), + ("Tkinter", 'tk') + ] if sys.platform == 'darwin': - backends.append(("macOS", 4)) + backends.append(("macOS", 'osx')) backends = tuple(backends) backend_box = self.create_combobox( _("Backend:") + " ", backends, - 'pylab/backend', default=0, + 'pylab/backend', default='inline', tip=_("This option will be applied the next time a console is " "opened.")) diff --git a/spyder/plugins/ipythonconsole/tests/conftest.py b/spyder/plugins/ipythonconsole/tests/conftest.py index d2c60acc71c..4893cda5d5d 100644 --- a/spyder/plugins/ipythonconsole/tests/conftest.py +++ b/spyder/plugins/ipythonconsole/tests/conftest.py @@ -113,7 +113,7 @@ def __getattr__(self, attr): return Mock() # Tests assume inline backend - configuration.set('ipython_console', 'pylab/backend', 0) + configuration.set('ipython_console', 'pylab/backend', 'inline') # Start the console in a fixed working directory use_startup_wdir = request.node.get_closest_marker('use_startup_wdir') @@ -137,12 +137,12 @@ def __getattr__(self, attr): # Use the automatic backend if requested auto_backend = request.node.get_closest_marker('auto_backend') if auto_backend: - configuration.set('ipython_console', 'pylab/backend', 1) + configuration.set('ipython_console', 'pylab/backend', 'auto') # Use the Tkinter backend if requested tk_backend = request.node.get_closest_marker('tk_backend') if tk_backend: - configuration.set('ipython_console', 'pylab/backend', 3) + configuration.set('ipython_console', 'pylab/backend', 'tk') # Start a Pylab client if requested pylab_client = request.node.get_closest_marker('pylab_client') diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 061a39a653b..db4df50ee30 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -1851,7 +1851,7 @@ def test_restart_intertactive_backend(ipyconsole, qtbot): """ main_widget = ipyconsole.get_widget() qtbot.wait(1000) - main_widget.change_possible_restart_and_mpl_conf('pylab/backend', 3) + main_widget.change_possible_restart_and_mpl_conf('pylab/backend', 'tk') assert bool(os.environ.get('BACKEND_REQUIRE_RESTART')) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 2613d210b59..a60becc8f4c 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -877,7 +877,7 @@ def change_possible_restart_and_mpl_conf(self, option, value): symbolic_math_n, hide_cmd_windows_n] restart_needed = option in restart_options - inline_backend = 0 + inline_backend = 'inline' pylab_restart = False clients_backend_require_restart = [False] * len(self.clients) current_client = self.get_current_client() diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 97a8f1e17c6..d7adf76178d 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -560,7 +560,7 @@ def send_mpl_backend(self, option=None): bbox_inches_n = 'pylab/inline/bbox_inches' backend_o = self.get_conf(pylab_backend_n) - inline_backend = 0 + inline_backend = 'inline' matplotlib_conf = {} From 3270cff578176d5d616ba05d899a0ff6ecf01c3c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 07:34:30 +0200 Subject: [PATCH 078/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "25902474d" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "25902474d" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- external-deps/spyder-kernels/spyder_kernels/console/kernel.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 7ac4421c373..6d42f9a6f9d 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = fb7fe85de40414fbd2ec43754ca45b3162ca65fd - parent = 1828539fa8d2ef6a0208830cc5ccb7905b66bb84 + commit = 25902474db195186fc75de8041bb9a84526063de + parent = cc152648033f9d51da222f8aad57142f6f350434 method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index a8441dfb94b..1e5e82dec92 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -551,7 +551,7 @@ def set_matplotlib_conf(self, conf): width_n = 'pylab/inline/width' height_n = 'pylab/inline/height' bbox_inches_n = 'pylab/inline/bbox_inches' - inline_backend = 0 + inline_backend = 'inline' if pylab_autoload_n in conf or pylab_backend_n in conf: self._set_mpl_backend( From 5d5209f02799b4f1443dd6f6059c3fab66c37180 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 21:28:45 +0200 Subject: [PATCH 079/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "ae4dc7617" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "ae4dc7617" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- external-deps/spyder-kernels/spyder_kernels/console/kernel.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 6d42f9a6f9d..405b2bba651 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = 25902474db195186fc75de8041bb9a84526063de - parent = cc152648033f9d51da222f8aad57142f6f350434 + commit = ae4dc761783c403a0104ccb67699095e82a30ada + parent = 3270cff578176d5d616ba05d899a0ff6ecf01c3c method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 1e5e82dec92..9177637897f 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -556,7 +556,7 @@ def set_matplotlib_conf(self, conf): if pylab_autoload_n in conf or pylab_backend_n in conf: self._set_mpl_backend( conf.get(pylab_backend_n, inline_backend), - pylab=conf.get(pylab_backend_n, False) + pylab=conf.get(pylab_autoload_n, False) ) if figure_format_n in conf: self._set_config_option( From 9b1a2b8c2283b397c72021dfc054131ad1c10b53 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 6 Jul 2023 22:28:22 +0200 Subject: [PATCH 080/110] only send when ready --- spyder/plugins/ipythonconsole/widgets/shell.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index d7adf76178d..35ddb49e5c6 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -541,6 +541,9 @@ def send_mpl_backend(self, option=None): If option is not None only send the related options """ + if not self.spyder_kernel_ready: + # will be sent later + return # Set Matplotlib backend with Spyder options pylab_n = 'pylab' From 6aeea7cd4ca030be52c2e790afc0eba1ee6514a2 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 7 Jul 2023 19:04:22 +0200 Subject: [PATCH 081/110] pyexec --- .../spyder_kernels_server/__main__.py | 8 +- .../spyder_kernels_server/kernel_spec.py | 107 ++++++++++++++++++ .../ipythonconsole/utils/kernelspec.py | 23 ++-- 3 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index 359a1f2d0b8..a10d45f5860 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -5,16 +5,15 @@ # Licensed under the terms of the MIT License # (see spyder_kernels/__init__.py for details) # ----------------------------------------------------------------------------- -import os import sys import zmq import json import argparse from spyder_kernels_server.kernel_server import KernelServer -from jupyter_client.kernelspec import KernelSpec from zmq.ssh import tunnel as zmqtunnel from qtpy.QtWidgets import QApplication from qtpy.QtCore import QSocketNotifier, QObject, QCoreApplication, Slot +from spyder_kernels_server.kernel_spec import get_kernel_spec class Server(QObject): @@ -65,10 +64,7 @@ def _socket_activity(self): elif cmd == "open_kernel": try: - kernel_spec = KernelSpec() - kernel_spec_dict = message[1] - for key in kernel_spec_dict: - setattr(kernel_spec, key, kernel_spec_dict[key]) + kernel_spec = get_kernel_spec(message[1]) cf = self.kernel_server.open_kernel(kernel_spec) with open(cf, "br") as f: cf = (cf, json.load(f)) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py new file mode 100644 index 00000000000..144df4787a8 --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +import sys +import os +import os.path as osp +from jupyter_client.kernelspec import KernelSpec + +WINDOWS = os.name == 'nt' + +def get_python_executable(): + """Return path to Spyder Python executable""" + executable = sys.executable.replace("pythonw.exe", "python.exe") + if executable.endswith("spyder.exe"): + # py2exe distribution + executable = "python.exe" + return executable + + +def is_different_interpreter(pyexec): + """Check that pyexec is a different interpreter from sys.executable.""" + # Paths may be symlinks + real_pyexe = osp.realpath(pyexec) + real_sys_exe = osp.realpath(sys.executable) + executable_validation = osp.basename(real_pyexe).startswith('python') + directory_validation = osp.dirname(real_pyexe) != osp.dirname(real_sys_exe) + return directory_validation and executable_validation + + +def add_quotes(path): + """Return quotes if needed for spaces on path.""" + quotes = '"' if ' ' in path and '"' not in path else '' + return '{quotes}{path}{quotes}'.format(quotes=quotes, path=path) + + +def get_conda_env_path(pyexec, quote=False): + """ + Return the full path to the conda environment from give python executable. + + If `quote` is True, then quotes are added if spaces are found in the path. + """ + pyexec = pyexec.replace('\\', '/') + if os.name == 'nt': + conda_env = os.path.dirname(pyexec) + else: + conda_env = os.path.dirname(os.path.dirname(pyexec)) + + if quote: + conda_env = add_quotes(conda_env) + + return conda_env + + +def is_conda_env(prefix=None, pyexec=None): + """Check if prefix or python executable are in a conda environment.""" + if pyexec is not None: + pyexec = pyexec.replace('\\', '/') + + if (prefix is None and pyexec is None) or (prefix and pyexec): + raise ValueError('Only `prefix` or `pyexec` should be provided!') + + if pyexec and prefix is None: + prefix = get_conda_env_path(pyexec).replace('\\', '/') + + return os.path.exists(os.path.join(prefix, 'conda-meta')) + + +def get_kernel_spec(kernel_spec_dict): + + kernel_spec = KernelSpec() + for key in kernel_spec_dict: + setattr(kernel_spec, key, kernel_spec_dict[key]) + + if kernel_spec.pyexec is None: + kernel_spec.pyexec = get_python_executable() + # Python interpreter used to start kernels + if ( + kernel_spec.pyexec is None + ): + pyexec = get_python_executable() + else: + pyexec = kernel_spec.pyexec + + + # Part of spyder-ide/spyder#11819 + is_different = is_different_interpreter(pyexec) + + # Command used to start kernels + kernel_cmd = [ + pyexec, + # This is necessary to avoid a spurious message on Windows. + # Fixes spyder-ide/spyder#20800. + '-Xfrozen_modules=off', + '-m', 'spyder_kernels.console', + '-f', '{connection_file}' + ] + + if is_different and is_conda_env(pyexec=pyexec): + # If executable is a conda environment and different from Spyder's + # runtime environment, we need to activate the environment to run + # spyder-kernels + raise NotImplementedError("Sorry I can't find conda without importing spyder here :/, TODO") + # kernel_cmd[:0] = [ + # find_conda(), 'run', + # '-p', get_conda_env_path(pyexec), + # ] + kernel_spec.argv = kernel_cmd + + return kernel_spec \ No newline at end of file diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index b2bab9b5f62..8c39888ac81 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -102,24 +102,33 @@ class SpyderKernelSpec(KernelSpec, SpyderConfigurationAccessor): def __init__(self, path_to_custom_interpreter=None, **kwargs): super(SpyderKernelSpec, self).__init__(**kwargs) - self.path_to_custom_interpreter = path_to_custom_interpreter + self.pyexec = path_to_custom_interpreter + if ( + path_to_custom_interpreter is None + and not self.get_conf('default', section='main_interpreter') + ): + self.pyexec = self.get_conf( + 'executable', section='main_interpreter') + self.display_name = 'Python 3 (Spyder)' self.language = 'python3' self.resource_dir = '' + + def to_dict(self): + d = super().to_dict() + d["pyexec"] = self.pyexec + return d @property def argv(self): """Command to start kernels""" # Python interpreter used to start kernels if ( - self.get_conf('default', section='main_interpreter') - and not self.path_to_custom_interpreter + self.pyexec is None ): pyexec = get_python_executable() else: - pyexec = self.get_conf('executable', section='main_interpreter') - if self.path_to_custom_interpreter: - pyexec = self.path_to_custom_interpreter + pyexec = self.pyexec if not has_spyder_kernels(pyexec): raise SpyderKernelError( ERROR_SPYDER_KERNEL_INSTALLED.format( @@ -193,7 +202,7 @@ def env(self): # Environment variables that we need to pass to the kernel env_vars.update({ 'SPY_EXTERNAL_INTERPRETER': (not default_interpreter - or self.path_to_custom_interpreter), + or self.pyexec), 'SPY_UMR_ENABLED': self.get_conf( 'umr/enabled', section='main_interpreter'), 'SPY_UMR_VERBOSE': self.get_conf( From 8915bcf3c29b67bee96f41e79f7d14f14f53b464 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 8 Jul 2023 10:12:41 +0200 Subject: [PATCH 082/110] handle error --- spyder/plugins/ipythonconsole/utils/kernel_handler.py | 8 ++++---- spyder/plugins/ipythonconsole/widgets/mixins.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 681170cb3ef..4d4b8683afb 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -162,9 +162,9 @@ def kernel_restarted(self, connection_file): self.sig_kernel_restarted.emit() @Slot(str, str) - def handle_stderr(self, connection_file, err): + def handle_stderr(self, err, connection_file=None): """Handle stderr""" - if connection_file != self.connection_file: + if connection_file is not None and connection_file != self.connection_file: return if self._shellwidget_connected: self.sig_stderr.emit(err) @@ -172,9 +172,9 @@ def handle_stderr(self, connection_file, err): self._init_stderr += err @Slot(str, str) - def handle_stdout(self, connection_file, out): + def handle_stdout(self, out, connection_file=None): """Handle stdout""" - if connection_file != self.connection_file: + if connection_file is not None and connection_file != self.connection_file: return if self._shellwidget_connected: self.sig_stdout.emit(out) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index d669576d35c..b9c66288bcf 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -241,7 +241,7 @@ def _socket_activity(self): kernel_handler = self.kernel_handler_waitlist.pop(0) if connection_file == "error": - kernel_handler.sig_fault.emit(str(connection_info)) + kernel_handler.handle_stderr(str(connection_info)) else: kernel_handler.set_connection( connection_file, @@ -287,9 +287,9 @@ def _socket_sub_activity(self): if cmd == "kernel_restarted": self.sig_kernel_restarted.emit(message[1]) elif cmd == "stderr": - self.sig_kernel_stderr.emit(message[1], message[2]) + self.sig_kernel_stderr.emit(message[2], message[1]) elif cmd == "stdout": - self.sig_kernel_stdout.emit(message[1], message[2]) + self.sig_kernel_stdout.emit(message[2], message[1]) self._notifier_sub.setEnabled(True) # This is necessary for some reason. From 3183f326e44b14a900241a79ba0d4b62d87a05ff Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 8 Jul 2023 11:59:21 +0200 Subject: [PATCH 083/110] copy all the function --- .../spyder_kernels_server/kernel_spec.py | 273 +++++++++++++++--- 1 file changed, 236 insertions(+), 37 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py index 144df4787a8..ca52c74b61f 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py @@ -2,9 +2,11 @@ import sys import os import os.path as osp -from jupyter_client.kernelspec import KernelSpec +from glob import glob +import itertools +import locale -WINDOWS = os.name == 'nt' +from jupyter_client.kernelspec import KernelSpec def get_python_executable(): """Return path to Spyder Python executable""" @@ -15,6 +17,49 @@ def get_python_executable(): return executable +def get_kernel_spec(kernel_spec_dict): + + kernel_spec = KernelSpec() + for key in kernel_spec_dict: + setattr(kernel_spec, key, kernel_spec_dict[key]) + + # Python interpreter used to start kernels + if (kernel_spec.pyexec is None): + pyexec = get_python_executable() + else: + pyexec = kernel_spec.pyexec + + # Command used to start kernels + kernel_cmd = [ + pyexec, + # This is necessary to avoid a spurious message on Windows. + # Fixes spyder-ide/spyder#20800. + '-Xfrozen_modules=off', + '-m', 'spyder_kernels.console', + '-f', '{connection_file}' + ] + + # Part of spyder-ide/spyder#11819 + is_different = is_different_interpreter(pyexec) + if is_different and is_conda_env(pyexec=pyexec): + # If executable is a conda environment and different from Spyder's + # runtime environment, we need to activate the environment to run + # spyder-kernels + kernel_cmd[:0] = [ + find_conda(), 'run', + '-p', get_conda_env_path(pyexec), + ] + kernel_spec.argv = kernel_cmd + + return kernel_spec + + +# All the functions below are used to activate conda +# They are copied here from spyder for now to avoid importing spyder + +WINDOWS = os.name == 'nt' + + def is_different_interpreter(pyexec): """Check that pyexec is a different interpreter from sys.executable.""" # Paths may be symlinks @@ -63,45 +108,199 @@ def is_conda_env(prefix=None, pyexec=None): return os.path.exists(os.path.join(prefix, 'conda-meta')) -def get_kernel_spec(kernel_spec_dict): - - kernel_spec = KernelSpec() - for key in kernel_spec_dict: - setattr(kernel_spec, key, kernel_spec_dict[key]) - - if kernel_spec.pyexec is None: - kernel_spec.pyexec = get_python_executable() - # Python interpreter used to start kernels +def is_conda_based_app(pyexec=sys.executable): + """ + Check if Spyder is running from the conda-based installer by looking for + the `spyder-menu.json` file. + + If a Python executable is provided, checks if it is in a conda-based + installer environment or the root environment thereof. + """ + real_pyexec = osp.realpath(pyexec) # pyexec may be symlink + if os.name == 'nt': + env_path = osp.dirname(real_pyexec) + else: + env_path = osp.dirname(osp.dirname(real_pyexec)) + + menu_rel_path = '/Menu/spyder-menu.json' if ( - kernel_spec.pyexec is None + osp.exists(env_path + menu_rel_path) + or glob(env_path + '/envs/*' + menu_rel_path) ): - pyexec = get_python_executable() + return True else: - pyexec = kernel_spec.pyexec + return False + +def is_type_text_string(obj): + """Return True if `obj` is type text string, False if it is anything else, + like an instance of a class that extends the basestring class.""" + return type(obj) in [str, bytes] + +def is_text_string(obj): + """Return True if `obj` is a text string, False if it is anything else, + like binary data (Python 3) or QString (PyQt API #1)""" + return isinstance(obj, str) + +def is_binary_string(obj): + """Return True if `obj` is a binary string, False if it is anything else""" + return isinstance(obj, bytes) + +def is_string(obj): + """Return True if `obj` is a text or binary Python string object, + False if it is anything else, like a QString (PyQt API #1)""" + return is_text_string(obj) or is_binary_string(obj) + +def to_text_string(obj, encoding=None): + """Convert `obj` to (unicode) text string""" + if encoding is None: + return str(obj) + elif isinstance(obj, str): + # In case this function is not used properly, this could happen + return obj + else: + return str(obj, encoding) + +# The default encoding for file paths and environment variables should be set +# to match the default encoding that the OS is using. +def getfilesystemencoding(): + """ + Query the filesystem for the encoding used to encode filenames + and environment variables. + """ + encoding = sys.getfilesystemencoding() + if encoding is None: + # Must be Linux or Unix and nl_langinfo(CODESET) failed. + encoding = PREFERRED_ENCODING + return encoding + +PREFERRED_ENCODING = locale.getpreferredencoding() +FS_ENCODING = getfilesystemencoding() + +def to_unicode_from_fs(string): + """ + Return a unicode version of string decoded using the file system encoding. + """ + if not is_string(string): # string is a QString + string = to_text_string(string.toUtf8(), 'utf-8') + else: + if is_binary_string(string): + try: + unic = string.decode(FS_ENCODING) + except (UnicodeError, TypeError): + pass + else: + return unic + return string + +def get_home_dir(): + """Return user home directory.""" + try: + # expanduser() returns a raw byte string which needs to be + # decoded with the codec that the OS is using to represent + # file paths. + path = to_unicode_from_fs(osp.expanduser('~')) + except Exception: + path = '' + + if osp.isdir(path): + return path + else: + # Get home from alternative locations + for env_var in ('HOME', 'USERPROFILE', 'TMP'): + # os.environ.get() returns a raw byte string which needs to be + # decoded with the codec that the OS is using to represent + # environment variables. + path = to_unicode_from_fs(os.environ.get(env_var, '')) + if osp.isdir(path): + return path + else: + path = '' + + if not path: + raise RuntimeError('Please set the environment variable HOME to ' + 'your user/home directory path so Spyder can ' + 'start properly.') + +def is_program_installed(basename): + """ + Return program absolute path if installed in PATH. + Otherwise, return None. + + Also searches specific platform dependent paths that are not already in + PATH. This permits general use without assuming user profiles are + sourced (e.g. .bash_Profile), such as when login shells are not used to + launch Spyder. + + On macOS systems, a .app is considered installed if it exists. + """ + home = get_home_dir() + req_paths = [] + if ( + sys.platform == 'darwin' + and basename.endswith('.app') + and osp.exists(basename) + ): + return basename + + if os.name == 'posix': + pyenv = [ + osp.join(home, '.pyenv', 'bin'), + osp.join('/usr', 'local', 'bin'), + ] + + a = [home, osp.join(home, 'opt'), '/opt'] + b = ['mambaforge', 'miniforge3', 'miniforge', + 'miniconda3', 'anaconda3', 'miniconda', 'anaconda'] + else: + pyenv = [osp.join(home, '.pyenv', 'pyenv-win', 'bin')] + + a = [home, osp.join(home, 'AppData', 'Local'), + 'C:\\', osp.join('C:\\', 'ProgramData')] + b = ['Mambaforge', 'Miniforge3', 'Miniforge', + 'Miniconda3', 'Anaconda3', 'Miniconda', 'Anaconda'] + + conda = [osp.join(*p, 'condabin') for p in itertools.product(a, b)] + req_paths.extend(pyenv + conda) + + for path in os.environ['PATH'].split(os.pathsep) + req_paths: + abspath = osp.join(path, basename) + if osp.isfile(abspath): + return abspath + +def find_program(basename): + """ + Find program in PATH and return absolute path + + Try adding .exe or .bat to basename on Windows platforms + (return None if not found) + """ + names = [basename] + if os.name == 'nt': + # Windows platforms + extensions = ('.exe', '.bat', '.cmd') + if not basename.endswith(extensions): + names = [basename + ext for ext in extensions] + [basename] + for name in names: + path = is_program_installed(name) + if path: + return path - - # Part of spyder-ide/spyder#11819 - is_different = is_different_interpreter(pyexec) +def find_conda(): + """Find conda executable.""" + conda = None - # Command used to start kernels - kernel_cmd = [ - pyexec, - # This is necessary to avoid a spurious message on Windows. - # Fixes spyder-ide/spyder#20800. - '-Xfrozen_modules=off', - '-m', 'spyder_kernels.console', - '-f', '{connection_file}' - ] + # First try Spyder's conda executable + if is_conda_based_app(): + root = osp.dirname(os.environ['CONDA_EXE']) + conda = osp.join(root, 'mamba.exe' if WINDOWS else 'mamba') - if is_different and is_conda_env(pyexec=pyexec): - # If executable is a conda environment and different from Spyder's - # runtime environment, we need to activate the environment to run - # spyder-kernels - raise NotImplementedError("Sorry I can't find conda without importing spyder here :/, TODO") - # kernel_cmd[:0] = [ - # find_conda(), 'run', - # '-p', get_conda_env_path(pyexec), - # ] - kernel_spec.argv = kernel_cmd + # Next try the environment variables + if conda is None: + conda = os.environ.get('CONDA_EXE') or os.environ.get('MAMBA_EXE') + + # Next try searching for the executable + if conda is None: + conda_exec = 'conda.bat' if WINDOWS else 'conda' + conda = find_program(conda_exec) - return kernel_spec \ No newline at end of file + return conda \ No newline at end of file From 7b285af8586741522295f662325c4cd9cbe08698 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 8 Jul 2023 14:14:21 +0200 Subject: [PATCH 084/110] format traceback --- .../spyder-kernels-server/spyder_kernels_server/__main__.py | 6 +++--- .../spyder_kernels_server/kernel_spec.py | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index a10d45f5860..e7383efa75c 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -9,6 +9,7 @@ import zmq import json import argparse +import traceback from spyder_kernels_server.kernel_server import KernelServer from zmq.ssh import tunnel as zmqtunnel from qtpy.QtWidgets import QApplication @@ -68,9 +69,8 @@ def _socket_activity(self): cf = self.kernel_server.open_kernel(kernel_spec) with open(cf, "br") as f: cf = (cf, json.load(f)) - - except Exception as e: - cf = ("error", e) + except Exception: + cf = ("error", traceback.format_exc()) self.socket.send_pyobj(["new_kernel", *cf]) elif cmd == "close_kernel": diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py index ca52c74b61f..7cd1f7a67e6 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py @@ -131,10 +131,6 @@ def is_conda_based_app(pyexec=sys.executable): else: return False -def is_type_text_string(obj): - """Return True if `obj` is type text string, False if it is anything else, - like an instance of a class that extends the basestring class.""" - return type(obj) in [str, bytes] def is_text_string(obj): """Return True if `obj` is a text string, False if it is anything else, From 8a5767490cf6a474401995b3fc7b4a954d8bf8fd Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 9 Jul 2023 23:13:15 +0200 Subject: [PATCH 085/110] exit on first failure --- runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtests.py b/runtests.py index b7b24b059e7..0c245832ef0 100644 --- a/runtests.py +++ b/runtests.py @@ -33,7 +33,7 @@ def run_pytest(run_slow=False, extra_args=None): """Run pytest tests for Spyder.""" # Be sure to ignore subrepos pytest_args = ['-vv', '-rw', '--durations=10', '--ignore=./external-deps', - '-W ignore::UserWarning', '--timeout=120'] + '-W ignore::UserWarning', '--timeout=120', "-x"] if CI: # Show coverage From 7856824e1a124534c15b7f7ac1ba258b8f784f61 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 1 Aug 2023 10:13:22 +0200 Subject: [PATCH 086/110] git subrepo clone (merge) --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "35c6d3672" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "35c6d3672" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 7e8a2577e8d..f4276069353 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -5,8 +5,8 @@ ; [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git - branch = remove_locals_inspection - commit = adb31866c64079bc023c4626f1e2cce3c51ab5f6 - parent = d4fa5fd95e2ebe89cf49453ecd13cda9c72f73c2 + branch = print_remote + commit = 35c6d3672481d41130a5ff111300b01d62e5e1cf + parent = 7d40e05f1a08e96be35e8e35600656442d0bb6a4 method = merge cmdver = 0.4.5 From 912a1da04c51ff6c860a24ea06c45eaa851fa849 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 1 Aug 2023 10:30:58 +0200 Subject: [PATCH 087/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "1647a276d" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "1647a276d" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- .../spyder-kernels/spyder_kernels/console/kernel.py | 2 +- external-deps/spyder-kernels/spyder_kernels/console/start.py | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index f4276069353..8abf4ffaa8b 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = 35c6d3672481d41130a5ff111300b01d62e5e1cf - parent = 7d40e05f1a08e96be35e8e35600656442d0bb6a4 + commit = 1647a276de65db37e1e39cfdb860cb1dd97ba03d + parent = 7856824e1a124534c15b7f7ac1ba258b8f784f61 method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index dac088f78a3..be9e102241f 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -816,7 +816,7 @@ def _set_mpl_backend(self, backend, pylab=False): ) magic = 'pylab' if pylab else 'matplotlib' - + if backend == "auto": backend = automatic_backend() diff --git a/external-deps/spyder-kernels/spyder_kernels/console/start.py b/external-deps/spyder-kernels/spyder_kernels/console/start.py index 82fe6612d7a..0502a134560 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/start.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/start.py @@ -130,6 +130,11 @@ def kernel_config(): 'figure.edgecolor': 'white' } + if is_module_installed('matplotlib'): + spy_cfg.IPKernelApp.exec_lines.append( + "get_ipython().kernel._set_mpl_backend('inline')" + ) + # Run a file at startup use_file_o = os.environ.get('SPY_USE_FILE_O') run_file_o = os.environ.get('SPY_RUN_FILE_O') From 456a5db2716978b229593a3be90976ec0ab40e4a Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 1 Aug 2023 10:57:04 +0200 Subject: [PATCH 088/110] git subrepo clone --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "a33dfc869" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "a33dfc869" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 ++-- external-deps/spyder-kernels/spyder_kernels/console/kernel.py | 2 +- .../spyder_kernels/console/tests/test_console_kernel.py | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 8abf4ffaa8b..7cec1e2f97f 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = 1647a276de65db37e1e39cfdb860cb1dd97ba03d - parent = 7856824e1a124534c15b7f7ac1ba258b8f784f61 + commit = a33dfc86975704607247c7f7fdbecfe77426f750 + parent = 912a1da04c51ff6c860a24ea06c45eaa851fa849 method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index be9e102241f..de5ed45be46 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -528,7 +528,7 @@ def get_mpl_interactive_backend(self): if framework is None: # Since no interactive backend has been set yet, this is # equivalent to having the inline one. - return 0 + return 'inline' elif framework in mapping: return MPL_BACKENDS_TO_SPYDER[mapping[framework]] else: diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py index f97f5d90625..b7f17cce6d0 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py @@ -1168,6 +1168,9 @@ def test_get_interactive_backend(backend): user_expressions = reply['content']['user_expressions'] value = user_expressions['output']['data']['text/plain'] + # remove quotes + value = value[1:-1] + # Assert we got the right interactive backend if backend is not None: assert value == backend From 3822feb865251010448c0aef499b787fbcbbdd39 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 1 Aug 2023 12:46:40 +0200 Subject: [PATCH 089/110] only send to correct kernel handlers --- .../ipythonconsole/utils/kernel_handler.py | 20 ++++++------ .../ipythonconsole/widgets/main_widget.py | 15 +++------ .../plugins/ipythonconsole/widgets/mixins.py | 32 +++++++++++++++---- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 4d4b8683afb..9125a3955e5 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -112,7 +112,14 @@ class KernelHandler(QObject): """ sig_remote_close = Signal(str) + """ + Signal to request the kernel to be shut down + """ + sig_kernel_restarted = Signal() + """ + The kernel has restarted + """ def __init__( self, @@ -155,27 +162,18 @@ def __init__( # Start kernel self.kernel_client.start_channels() self.check_kernel_info() - - @Slot(str) - def kernel_restarted(self, connection_file): - if connection_file == self.connection_file: - self.sig_kernel_restarted.emit() @Slot(str, str) - def handle_stderr(self, err, connection_file=None): + def handle_stderr(self, err): """Handle stderr""" - if connection_file is not None and connection_file != self.connection_file: - return if self._shellwidget_connected: self.sig_stderr.emit(err) else: self._init_stderr += err @Slot(str, str) - def handle_stdout(self, out, connection_file=None): + def handle_stdout(self, out): """Handle stdout""" - if connection_file is not None and connection_file != self.connection_file: - return if self._shellwidget_connected: self.sig_stdout.emit(out) else: diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index f38b9b738c4..a8c9f614919 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -128,11 +128,6 @@ class IPythonConsoleWidget( This is a widget with tabs where each one is a ClientWidget. """ - # Signals - sig_kernel_restarted = Signal(str) - sig_kernel_stderr = Signal(str, str) - sig_kernel_stdout = Signal(str, str) - sig_append_to_history_requested = Signal(str, str) """ This signal is emitted when the plugin requires to add commands to a @@ -927,7 +922,7 @@ def change_possible_restart_and_mpl_conf(self, option, value): interactive_backend = None sw = client.shellwidget if ( - interactive_backend is None + interactive_backend is None and sw._shellwidget_state != "started" ): # If the kernel didn't start and no backend was requested, @@ -1765,7 +1760,7 @@ def close_all_clients(self): # Close cached kernel self.close_cached_kernel() self.filenames = [] - + # Close local server self.stop_local_server(wait=True) return True @@ -2027,7 +2022,7 @@ def run_script(self, filename, wdir, args, post_mortem, current_client, method = "runfile" # If spyder-kernels, use runfile if client.shellwidget.is_spyder_kernel: - + magic_arguments = [norm(filename)] if args: magic_arguments.append("--args") @@ -2045,13 +2040,13 @@ def run_script(self, filename, wdir, args, post_mortem, current_client, magic_arguments.append("--post-mortem") if console_namespace: magic_arguments.append("--current-namespace") - + line = "%{} {}".format(method, shlex.join(magic_arguments)) elif method in ["runfile", "debugfile"]: # External, non spyder-kernels, use %run magic_arguments = [] - + if method == "debugfile": magic_arguments.append("-d") magic_arguments.append(filename) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index b9c66288bcf..6a2bec4d609 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -49,9 +49,9 @@ def __init__(self): self.options = None self.server = None self.kernel_handler_waitlist = [] + self._alive_kernel_handlers = {} self.request_queue = queue.Queue() self.context = zmq.Context() - self.on_kernel_server_conf_changed() @on_conf_change(option=KERNEL_SERVER_OPTIONS, section="main_interpreter") @@ -196,9 +196,6 @@ def new_kernel(self, kernel_spec_dict): ) kernel_handler.sig_remote_close.connect(self.request_close) - self.sig_kernel_restarted.connect(kernel_handler.kernel_restarted) - self.sig_kernel_stderr.connect(kernel_handler.handle_stderr) - self.sig_kernel_stdout.connect(kernel_handler.handle_stdout) self.kernel_handler_waitlist.append(kernel_handler) self.send_request(["open_kernel", kernel_spec_dict]) @@ -207,6 +204,8 @@ def new_kernel(self, kernel_spec_dict): def request_close(self, connection_file): self.send_request(["close_kernel", connection_file]) + # Remove kernel from active kernels + self._alive_kernel_handlers.pop(connection_file, None) def send_request(self, request): # Check socket state @@ -250,6 +249,8 @@ def _socket_activity(self): self.ssh_key, self.ssh_password, ) + # keep ref to signal kernel handler + self._alive_kernel_handlers[connection_file] = kernel_handler elif cmd == "set_port_pub": port_pub = message[1] @@ -285,11 +286,28 @@ def _socket_sub_activity(self): message = self.socket_sub.recv_pyobj() cmd = message[0] if cmd == "kernel_restarted": - self.sig_kernel_restarted.emit(message[1]) + connection_file = message[1] + kernel_handler = self._alive_kernel_handlers.get( + connection_file, None + ) + if kernel_handler is not None: + kernel_handler.sig_kernel_restarted.emit() + elif cmd == "stderr": - self.sig_kernel_stderr.emit(message[2], message[1]) + err, connection_file = message + kernel_handler = self._alive_kernel_handlers.get( + connection_file, None + ) + if kernel_handler is not None: + kernel_handler.handle_stderr(err) + elif cmd == "stdout": - self.sig_kernel_stdout.emit(message[2], message[1]) + out, connection_file = message + kernel_handler = self._alive_kernel_handlers.get( + connection_file, None + ) + if kernel_handler is not None: + kernel_handler.handle_stdout(out) self._notifier_sub.setEnabled(True) # This is necessary for some reason. From e5436def81bfb88071c956469c72e27e542fab52 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 2 Aug 2023 05:21:20 +0200 Subject: [PATCH 090/110] cleanup --- runtests.py | 2 +- .../ipythonconsole/utils/kernel_handler.py | 19 ++++++++--------- .../plugins/ipythonconsole/widgets/shell.py | 21 +++++++++---------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/runtests.py b/runtests.py index 0c245832ef0..b7b24b059e7 100644 --- a/runtests.py +++ b/runtests.py @@ -33,7 +33,7 @@ def run_pytest(run_slow=False, extra_args=None): """Run pytest tests for Spyder.""" # Be sure to ignore subrepos pytest_args = ['-vv', '-rw', '--durations=10', '--ignore=./external-deps', - '-W ignore::UserWarning', '--timeout=120', "-x"] + '-W ignore::UserWarning', '--timeout=120'] if CI: # Show coverage diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 9125a3955e5..a6fc67d8870 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -90,7 +90,7 @@ class KernelHandler(QObject): """ A stdout message was received on the process stdout. """ - + sig_stderr = Signal(str) """ A stderr message was received on the process stderr. @@ -110,7 +110,7 @@ class KernelHandler(QObject): """ The kernel raised an error while connecting. """ - + sig_remote_close = Signal(str) """ Signal to request the kernel to be shut down @@ -154,10 +154,10 @@ def __init__( self._shellwidget_connected = False self._init_stderr = "" self._init_stdout = "" - + # Special kernel self.special = None - + if self.kernel_client: # Start kernel self.kernel_client.start_channels() @@ -170,7 +170,7 @@ def handle_stderr(self, err): self.sig_stderr.emit(err) else: self._init_stderr += err - + @Slot(str, str) def handle_stdout(self, out): """Handle stdout""" @@ -197,7 +197,6 @@ def connect(self): if self._init_stdout: self.sig_stdout.emit(self._init_stdout) self._init_stdout = None - def check_kernel_info(self): """Send request to check kernel info.""" @@ -466,7 +465,7 @@ def set_connection(self, connection_file, connection_info, self.hostname = hostname self.sshkey = sshkey self.password = password - + self.kernel_client = SpyderKernelClient() self.kernel_client.load_connection_info(self.connection_info) self.kernel_client = self.tunnel_kernel_client( @@ -475,11 +474,11 @@ def set_connection(self, connection_file, connection_info, self.sshkey, self.password ) - + # Increase time (in seconds) to detect if a kernel is alive. # See spyder-ide/spyder#3444. self.kernel_client.hb_channel.time_to_dead = 25.0 - + # Start kernel self.kernel_client.start_channels() - self.check_kernel_info() \ No newline at end of file + self.check_kernel_info() diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 5e574ee8e7a..7e566d0e2d7 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -476,8 +476,7 @@ def interrupt_kernel(self): "kernel I did not start.
") ) - def execute( - self, source=None, hidden=False, interactive=False, queue=True): + def execute(self, source=None, hidden=False, interactive=False): """ Executes source or the input buffer, possibly prompting for more input. @@ -539,7 +538,7 @@ def set_cwd(self, dirname=None, emit_cwd_change=False): def send_mpl_backend(self, option=None): """ Send matplotlib backend. - + If option is not None only send the related options """ if not self.spyder_kernel_ready: @@ -549,12 +548,12 @@ def send_mpl_backend(self, option=None): pylab_n = 'pylab' pylab_o = self.get_conf(pylab_n) - + if option is not None and not pylab_o: # The options are only related to pylab_o # So no need to change the backend return - + pylab_autoload_n = 'pylab/autoload' pylab_backend_n = 'pylab/backend' figure_format_n = 'pylab/inline/figure_format' @@ -565,21 +564,21 @@ def send_mpl_backend(self, option=None): backend_o = self.get_conf(pylab_backend_n) inline_backend = 'inline' - + matplotlib_conf = {} - + if pylab_o: # Figure format format_o = self.get_conf(figure_format_n) if format_o and (option is None or figure_format_n in option): matplotlib_conf[figure_format_n] = format_o - + # Resolution resolution_o = self.get_conf(resolution_n) if resolution_o is not None and ( option is None or resolution_n in option): matplotlib_conf[resolution_n] = resolution_o - + # Figure size width_o = float(self.get_conf(width_n)) height_o = float(self.get_conf(height_n)) @@ -588,7 +587,7 @@ def send_mpl_backend(self, option=None): matplotlib_conf[width_n] = width_o if height_o is not None: matplotlib_conf[height_n] = height_o - + # Print figure kwargs bbox_inches_o = self.get_conf(bbox_inches_n) if option is None or bbox_inches_n in option: @@ -608,7 +607,7 @@ def send_mpl_backend(self, option=None): matplotlib_conf[pylab_backend_n] = mpl_backend if option is None or pylab_autoload_n in option: matplotlib_conf[pylab_autoload_n] = autoload_pylab_o - + if matplotlib_conf: self.call_kernel().set_matplotlib_conf( matplotlib_conf) From 75e0bdd239014240ac30ce97d23800d2eac44b6e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 16 Sep 2023 07:49:01 +0200 Subject: [PATCH 091/110] fix merge --- .../spyder_kernels_server/kernel_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 472f8e1c7fc..2527ee482c3 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -48,7 +48,11 @@ def run(self): if self.closing.is_set(): return if txt: - self.sig_text.emit(txt.decode()) + try: + txt = txt.decode() + except UnicodeDecodeError: + txt = str(txt) + self.sig_text.emit(txt) class ShutdownThread(Thread): From 376a92bdce8e884e6ca827940807a4b805b994b1 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Sep 2023 08:33:34 +0200 Subject: [PATCH 092/110] fix bugs --- .../ipythonconsole/utils/kernel_handler.py | 16 ++++++++++++---- spyder/plugins/ipythonconsole/widgets/shell.py | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 13458dd8374..53e0861336a 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -158,9 +158,7 @@ def __init__( # Start kernel if self.kernel_client: - # Start kernel - self.kernel_client.start_channels() - self.check_kernel_info() + self.start_channels() @Slot(str, str) def handle_stderr(self, err): @@ -431,6 +429,9 @@ def reopen_comm(self): def set_connection(self, connection_file, connection_info, hostname, sshkey, password): + """Set connection file.""" + if self.connection_file: + raise RuntimeError("Connection file already set") self.connection_file = connection_file self.connection_info = connection_info self.hostname = hostname @@ -450,6 +451,13 @@ def set_connection(self, connection_file, connection_info, # See spyder-ide/spyder#3444. self.kernel_client.hb_channel.time_to_dead = 25.0 + self.start_channels() + + def start_channels(self): + """Start channels""" # Start kernel + self.kernel_client.sig_spyder_kernel_info.connect( + self.check_spyder_kernel_info + ) self.kernel_client.start_channels() - self.check_kernel_info() + self.kernel_comm.open_comm(self.kernel_client) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 3bdf0c8e40c..bcd99b4bbff 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -223,7 +223,7 @@ def connect_kernel(self, kernel_handler): kernel_handler.sig_kernel_connection_error.connect( self.handle_kernel_connection_error) - kernel_handler.connect_() + kernel_handler.connect() def disconnect_kernel(self, shutdown_kernel=True, will_reconnect=True): """ @@ -568,6 +568,7 @@ def send_mpl_backend(self, option=None): """ if not self.spyder_kernel_ready: # will be sent later + return # Set Matplotlib backend with Spyder options pylab_n = 'pylab' From 35cc44a5d13d304f37969acd13b1bedc116eb422 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Sep 2023 10:45:18 +0200 Subject: [PATCH 093/110] fix kernel infos --- .../spyder_kernels_server/kernel_client.py | 5 +- .../tests/test_ipythonconsole.py | 4 +- .../ipythonconsole/utils/kernel_handler.py | 9 ++-- .../plugins/ipythonconsole/widgets/client.py | 14 +---- .../ipythonconsole/widgets/main_widget.py | 1 - .../plugins/ipythonconsole/widgets/shell.py | 51 +++++++++---------- 6 files changed, 35 insertions(+), 49 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py index 7b60ed8df37..84b519887e1 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py @@ -18,10 +18,9 @@ class SpyderKernelClient(QtKernelClient): # Enable receiving messages on control channel. # Useful for pdb completion control_channel_class = Type(QtZMQSocketChannel) - sig_spyder_kernel_info = Signal(object) + sig_kernel_info = Signal(object) def _handle_kernel_info_reply(self, rep): """Check spyder-kernels version.""" super()._handle_kernel_info_reply(rep) - spyder_kernel_info = rep["content"].get("spyder_kernels_info", None) - self.sig_spyder_kernel_info.emit(spyder_kernel_info) + self.sig_kernel_info.emit(rep) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 7f9b2dcd3ff..0d7dd1f9d92 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -2019,7 +2019,7 @@ def test_old_kernel_version(ipyconsole, qtbot): w = ipyconsole.get_widget() kernel_handler = w._cached_kernel_properties[-1] - kernel_handler.kernel_client.sig_spyder_kernel_info.disconnect() + kernel_handler.kernel_client.sig_kernel_info.disconnect() # Wait until it is launched qtbot.waitUntil( @@ -2029,7 +2029,7 @@ def test_old_kernel_version(ipyconsole, qtbot): timeout=SHELL_TIMEOUT) # Set wrong version - kernel_handler.check_spyder_kernel_info(('1.0.0', '')) + kernel_handler.check_spyder_kernel_info({"content": ('1.0.0', '')}) # Create new client w.create_new_client() diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 53e0861336a..bb7feb1fd20 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -155,6 +155,7 @@ def __init__( self._init_stdout = "" self._shellwidget_connected = False self._comm_ready_recieved = False + self._kernel_info_msg = None # Start kernel if self.kernel_client: @@ -195,14 +196,15 @@ def connect(self): self.sig_stdout.emit(self._init_stdout) self._init_stdout = None - def check_spyder_kernel_info(self, spyder_kernel_info): + def check_spyder_kernel_info(self, msg): """ Check if the Spyder-kernels version is the right one after receiving it from the kernel. If the kernel is non-locally managed, check if it is a spyder-kernel. """ - + self._kernel_info_msg = msg + spyder_kernel_info = msg["content"].get("spyder_kernels_info", None) if not spyder_kernel_info: if self.known_spyder_kernel: # spyder-kernels version < 3.0 @@ -456,8 +458,9 @@ def set_connection(self, connection_file, connection_info, def start_channels(self): """Start channels""" # Start kernel - self.kernel_client.sig_spyder_kernel_info.connect( + self.kernel_client.sig_kernel_info.connect( self.check_spyder_kernel_info ) self.kernel_client.start_channels() + self.kernel_client.kernel_info() self.kernel_comm.open_comm(self.kernel_client) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 473a9a51d8b..ed269c63bc3 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -370,12 +370,7 @@ def print_stderr(self, stderr): error_text = self.error_text + error_text self.show_kernel_error(error_text) - if self.shellwidget._starting: - self.shellwidget.banner = ( - stderr + '\n' + self.shellwidget.banner) - else: - self.shellwidget._append_plain_text( - stderr, before_prompt=True) + self.shellwidget._append_plain_text(stderr, before_prompt=True) @Slot(str) def print_stdout(self, stdout): @@ -383,12 +378,7 @@ def print_stdout(self, stdout): if not stdout: return - if self.shellwidget._starting: - self.shellwidget.banner = ( - stdout + '\n' + self.shellwidget.banner) - else: - self.shellwidget._append_plain_text( - stdout, before_prompt=True) + self.shellwidget._append_plain_text(stdout, before_prompt=True) def connect_shellwidget_signals(self): """Configure shellwidget after kernel is connected.""" diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index a86f3be51e0..4dd4c2d594b 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1921,7 +1921,6 @@ def restart_kernel(self, client=None, ask_before_restart=True): ks_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() ks_dict["setup_kwargs"] = kernel_spec_kwargs kernel_handler = self.get_cached_kernel(ks_dict) - kernel_handler.special = client.kernel_handler.special except Exception as e: client.show_kernel_error(e) return diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index bcd99b4bbff..f0da7873b20 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -284,39 +284,34 @@ def handle_kernel_is_ready(self): if self.kernel_client != self.kernel_handler.kernel_client: # If the kernel crashed, the right client is already connected self.kernel_client = self.kernel_handler.kernel_client - + # kernel_info must have already been recieved for + # handle_kernel_is_ready to be called + self._handle_kernel_info_reply( + self.kernel_handler._kernel_info_msg + ) + # If the user asked for a restart, print the restart message + if self._shellwidget_state == "user_restart": + self._control.clear() + self.print_restart_message() + # Print The banner + if self._display_banner: + self._append_plain_text(self.banner) + if self.kernel_banner: + self._append_plain_text(self.kernel_banner) + # Show first prompt + self.reset() if ( self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady ): self.setup_spyder_kernel() - - def _handle_kernel_info_reply(self, rep): - """ - Handle kernel info replies. - - Note: - This is called after handle_kernel_is_ready. - When the code reaches this point the kernel /and/ the shell are - ready. - We avoid sending sig_shellwidget_created before this point because - the shellwidget is cleared here if self._starting == True. - if self._starting is True then we send /another/ round trip - message to ask for a prompt. - (TODO: The number of round trip messages to get the kernel running - could be optimised) - """ - if self._shellwidget_state == "started": - # Set _starting to False to avoid reset if kernel restart without - # user interaction. If self._shellwidget_state == "user_restart", - # We clear the console as usual - self._starting = False - - super()._handle_kernel_info_reply(rep) - - if self._shellwidget_state == "user_restart": - # If the user asked for a restart, pring the restart message - self.print_restart_message() + + def _started_channels(self): + """Make a history request""" + # Disable the _starting mechanism + self._starting = False + # Request history + self.kernel_client.history(hist_access_type='tail', n=1000) def _prompt_started_hook(self): """Emit a signal when the prompt is ready.""" From 9b4079c6a6515f1116e4894bb951bbf055dfaa70 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Sep 2023 11:55:25 +0200 Subject: [PATCH 094/110] fix preferences --- .../ipythonconsole/tests/test_ipythonconsole.py | 4 +++- spyder/plugins/maininterpreter/confpage.py | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 0d7dd1f9d92..20df97895ff 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -2029,7 +2029,9 @@ def test_old_kernel_version(ipyconsole, qtbot): timeout=SHELL_TIMEOUT) # Set wrong version - kernel_handler.check_spyder_kernel_info({"content": ('1.0.0', '')}) + kernel_handler.check_spyder_kernel_info( + {"content": {"spyder_kernels_info": ('1.0.0', '')}} + ) # Create new client w.create_new_client() diff --git a/spyder/plugins/maininterpreter/confpage.py b/spyder/plugins/maininterpreter/confpage.py index bfb46a8b792..71c9499e6af 100644 --- a/spyder/plugins/maininterpreter/confpage.py +++ b/spyder/plugins/maininterpreter/confpage.py @@ -122,8 +122,8 @@ def setup_page(self): word_wrap=False ) password.textbox.setEchoMode(QLineEdit.Password) - password_radio.toggled.connect(password.setEnabled) - keyfile_radio.toggled.connect(password.setDisabled) + password_radio.radiobutton.toggled.connect(password.setEnabled) + keyfile_radio.radiobutton.toggled.connect(password.setDisabled) keyfile = self.create_file_combobox( _('SSH Keyfile'), @@ -142,10 +142,10 @@ def setup_page(self): passphrase.textbox.setPlaceholderText(_('Optional')) passphrase.textbox.setEchoMode(QLineEdit.Password) - keyfile_radio.toggled.connect(keyfile.setEnabled) - keyfile_radio.toggled.connect(passphrase.setEnabled) - password_radio.toggled.connect(keyfile.setDisabled) - password_radio.toggled.connect(passphrase.setDisabled) + keyfile_radio.radiobutton.toggled.connect(keyfile.setEnabled) + keyfile_radio.radiobutton.toggled.connect(passphrase.setEnabled) + password_radio.radiobutton.toggled.connect(keyfile.setDisabled) + password_radio.radiobutton.toggled.connect(passphrase.setDisabled) @@ -166,7 +166,7 @@ def setup_page(self): rm_layout.addWidget(auth_group) rm_group.setLayout(rm_layout) auth_group.setCheckable(True) - auth_group.toggled.connect(password_radio.setChecked) + auth_group.toggled.connect(password_radio.radiobutton.setChecked) rm_group.setCheckable(True) # Python executable Group From 77d5d55599747b891197e5a7096fa60556b4fc63 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Sep 2023 12:01:00 +0200 Subject: [PATCH 095/110] fix stderr out --- spyder/plugins/ipythonconsole/widgets/mixins.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 6a2bec4d609..35a325ddc2e 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -294,7 +294,8 @@ def _socket_sub_activity(self): kernel_handler.sig_kernel_restarted.emit() elif cmd == "stderr": - err, connection_file = message + connection_file = message[1] + err = message[2] kernel_handler = self._alive_kernel_handlers.get( connection_file, None ) @@ -302,7 +303,8 @@ def _socket_sub_activity(self): kernel_handler.handle_stderr(err) elif cmd == "stdout": - out, connection_file = message + connection_file = message[1] + out = message[2] kernel_handler = self._alive_kernel_handlers.get( connection_file, None ) From 8de3e44c5df24c2964ad21ad352893a0dc923c30 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Sep 2023 15:47:38 +0200 Subject: [PATCH 096/110] move to conda utils --- .../spyder_kernels_server/conda_utils.py | 252 +++++++++++++++++ .../spyder_kernels_server/kernel_spec.py | 257 +----------------- .../ipythonconsole/utils/kernelspec.py | 15 +- spyder/utils/conda.py | 65 +---- 4 files changed, 262 insertions(+), 327 deletions(-) create mode 100644 external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py new file mode 100644 index 00000000000..40dde8c536e --- /dev/null +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py @@ -0,0 +1,252 @@ +# -*- coding: utf-8 -*- + +import sys +import os +import os.path as osp +from glob import glob +import itertools +import locale + +WINDOWS = os.name == 'nt' + + +def is_different_interpreter(pyexec): + """Check that pyexec is a different interpreter from sys.executable.""" + # Paths may be symlinks + real_pyexe = osp.realpath(pyexec) + real_sys_exe = osp.realpath(sys.executable) + executable_validation = osp.basename(real_pyexe).startswith('python') + directory_validation = osp.dirname(real_pyexe) != osp.dirname(real_sys_exe) + return directory_validation and executable_validation + + +def add_quotes(path): + """Return quotes if needed for spaces on path.""" + quotes = '"' if ' ' in path and '"' not in path else '' + return '{quotes}{path}{quotes}'.format(quotes=quotes, path=path) + + +def get_conda_env_path(pyexec, quote=False): + """ + Return the full path to the conda environment from give python executable. + + If `quote` is True, then quotes are added if spaces are found in the path. + """ + pyexec = pyexec.replace('\\', '/') + if os.name == 'nt': + conda_env = os.path.dirname(pyexec) + else: + conda_env = os.path.dirname(os.path.dirname(pyexec)) + + if quote: + conda_env = add_quotes(conda_env) + + return conda_env + + +def is_conda_env(prefix=None, pyexec=None): + """Check if prefix or python executable are in a conda environment.""" + if pyexec is not None: + pyexec = pyexec.replace('\\', '/') + + if (prefix is None and pyexec is None) or (prefix and pyexec): + raise ValueError('Only `prefix` or `pyexec` should be provided!') + + if pyexec and prefix is None: + prefix = get_conda_env_path(pyexec).replace('\\', '/') + + return os.path.exists(os.path.join(prefix, 'conda-meta')) + + +def is_conda_based_app(pyexec=sys.executable): + """ + Check if Spyder is running from the conda-based installer by looking for + the `spyder-menu.json` file. + + If a Python executable is provided, checks if it is in a conda-based + installer environment or the root environment thereof. + """ + real_pyexec = osp.realpath(pyexec) # pyexec may be symlink + if os.name == 'nt': + env_path = osp.dirname(real_pyexec) + else: + env_path = osp.dirname(osp.dirname(real_pyexec)) + + menu_rel_path = '/Menu/spyder-menu.json' + if ( + osp.exists(env_path + menu_rel_path) + or glob(env_path + '/envs/*' + menu_rel_path) + ): + return True + else: + return False + + +def is_text_string(obj): + """Return True if `obj` is a text string, False if it is anything else, + like binary data (Python 3) or QString (PyQt API #1)""" + return isinstance(obj, str) + +def is_binary_string(obj): + """Return True if `obj` is a binary string, False if it is anything else""" + return isinstance(obj, bytes) + +def is_string(obj): + """Return True if `obj` is a text or binary Python string object, + False if it is anything else, like a QString (PyQt API #1)""" + return is_text_string(obj) or is_binary_string(obj) + +def to_text_string(obj, encoding=None): + """Convert `obj` to (unicode) text string""" + if encoding is None: + return str(obj) + elif isinstance(obj, str): + # In case this function is not used properly, this could happen + return obj + else: + return str(obj, encoding) + +# The default encoding for file paths and environment variables should be set +# to match the default encoding that the OS is using. +def getfilesystemencoding(): + """ + Query the filesystem for the encoding used to encode filenames + and environment variables. + """ + encoding = sys.getfilesystemencoding() + if encoding is None: + # Must be Linux or Unix and nl_langinfo(CODESET) failed. + encoding = PREFERRED_ENCODING + return encoding + +PREFERRED_ENCODING = locale.getpreferredencoding() +FS_ENCODING = getfilesystemencoding() + +def to_unicode_from_fs(string): + """ + Return a unicode version of string decoded using the file system encoding. + """ + if not is_string(string): # string is a QString + string = to_text_string(string.toUtf8(), 'utf-8') + else: + if is_binary_string(string): + try: + unic = string.decode(FS_ENCODING) + except (UnicodeError, TypeError): + pass + else: + return unic + return string + +def get_home_dir(): + """Return user home directory.""" + try: + # expanduser() returns a raw byte string which needs to be + # decoded with the codec that the OS is using to represent + # file paths. + path = to_unicode_from_fs(osp.expanduser('~')) + except Exception: + path = '' + + if osp.isdir(path): + return path + else: + # Get home from alternative locations + for env_var in ('HOME', 'USERPROFILE', 'TMP'): + # os.environ.get() returns a raw byte string which needs to be + # decoded with the codec that the OS is using to represent + # environment variables. + path = to_unicode_from_fs(os.environ.get(env_var, '')) + if osp.isdir(path): + return path + else: + path = '' + + if not path: + raise RuntimeError('Please set the environment variable HOME to ' + 'your user/home directory path so Spyder can ' + 'start properly.') + +def is_program_installed(basename): + """ + Return program absolute path if installed in PATH. + Otherwise, return None. + + Also searches specific platform dependent paths that are not already in + PATH. This permits general use without assuming user profiles are + sourced (e.g. .bash_Profile), such as when login shells are not used to + launch Spyder. + + On macOS systems, a .app is considered installed if it exists. + """ + home = get_home_dir() + req_paths = [] + if ( + sys.platform == 'darwin' + and basename.endswith('.app') + and osp.exists(basename) + ): + return basename + + if os.name == 'posix': + pyenv = [ + osp.join(home, '.pyenv', 'bin'), + osp.join('/usr', 'local', 'bin'), + ] + + a = [home, osp.join(home, 'opt'), '/opt'] + b = ['mambaforge', 'miniforge3', 'miniforge', + 'miniconda3', 'anaconda3', 'miniconda', 'anaconda'] + else: + pyenv = [osp.join(home, '.pyenv', 'pyenv-win', 'bin')] + + a = [home, osp.join(home, 'AppData', 'Local'), + 'C:\\', osp.join('C:\\', 'ProgramData')] + b = ['Mambaforge', 'Miniforge3', 'Miniforge', + 'Miniconda3', 'Anaconda3', 'Miniconda', 'Anaconda'] + + conda = [osp.join(*p, 'condabin') for p in itertools.product(a, b)] + req_paths.extend(pyenv + conda) + + for path in os.environ['PATH'].split(os.pathsep) + req_paths: + abspath = osp.join(path, basename) + if osp.isfile(abspath): + return abspath + +def find_program(basename): + """ + Find program in PATH and return absolute path + + Try adding .exe or .bat to basename on Windows platforms + (return None if not found) + """ + names = [basename] + if os.name == 'nt': + # Windows platforms + extensions = ('.exe', '.bat', '.cmd') + if not basename.endswith(extensions): + names = [basename + ext for ext in extensions] + [basename] + for name in names: + path = is_program_installed(name) + if path: + return path + +def find_conda(): + """Find conda executable.""" + conda = None + + # First try Spyder's conda executable + if is_conda_based_app(): + root = osp.dirname(os.environ['CONDA_EXE']) + conda = osp.join(root, 'mamba.exe' if WINDOWS else 'mamba') + + # Next try the environment variables + if conda is None: + conda = os.environ.get('CONDA_EXE') or os.environ.get('MAMBA_EXE') + + # Next try searching for the executable + if conda is None: + conda_exec = 'conda.bat' if WINDOWS else 'conda' + conda = find_program(conda_exec) + + return conda \ No newline at end of file diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py index 7cd1f7a67e6..6573cda08ed 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- import sys -import os -import os.path as osp -from glob import glob -import itertools -import locale from jupyter_client.kernelspec import KernelSpec +from .conda_utils import ( + is_conda_env, get_conda_env_path, find_conda, is_different_interpreter +) + def get_python_executable(): """Return path to Spyder Python executable""" executable = sys.executable.replace("pythonw.exe", "python.exe") @@ -52,251 +51,3 @@ def get_kernel_spec(kernel_spec_dict): kernel_spec.argv = kernel_cmd return kernel_spec - - -# All the functions below are used to activate conda -# They are copied here from spyder for now to avoid importing spyder - -WINDOWS = os.name == 'nt' - - -def is_different_interpreter(pyexec): - """Check that pyexec is a different interpreter from sys.executable.""" - # Paths may be symlinks - real_pyexe = osp.realpath(pyexec) - real_sys_exe = osp.realpath(sys.executable) - executable_validation = osp.basename(real_pyexe).startswith('python') - directory_validation = osp.dirname(real_pyexe) != osp.dirname(real_sys_exe) - return directory_validation and executable_validation - - -def add_quotes(path): - """Return quotes if needed for spaces on path.""" - quotes = '"' if ' ' in path and '"' not in path else '' - return '{quotes}{path}{quotes}'.format(quotes=quotes, path=path) - - -def get_conda_env_path(pyexec, quote=False): - """ - Return the full path to the conda environment from give python executable. - - If `quote` is True, then quotes are added if spaces are found in the path. - """ - pyexec = pyexec.replace('\\', '/') - if os.name == 'nt': - conda_env = os.path.dirname(pyexec) - else: - conda_env = os.path.dirname(os.path.dirname(pyexec)) - - if quote: - conda_env = add_quotes(conda_env) - - return conda_env - - -def is_conda_env(prefix=None, pyexec=None): - """Check if prefix or python executable are in a conda environment.""" - if pyexec is not None: - pyexec = pyexec.replace('\\', '/') - - if (prefix is None and pyexec is None) or (prefix and pyexec): - raise ValueError('Only `prefix` or `pyexec` should be provided!') - - if pyexec and prefix is None: - prefix = get_conda_env_path(pyexec).replace('\\', '/') - - return os.path.exists(os.path.join(prefix, 'conda-meta')) - - -def is_conda_based_app(pyexec=sys.executable): - """ - Check if Spyder is running from the conda-based installer by looking for - the `spyder-menu.json` file. - - If a Python executable is provided, checks if it is in a conda-based - installer environment or the root environment thereof. - """ - real_pyexec = osp.realpath(pyexec) # pyexec may be symlink - if os.name == 'nt': - env_path = osp.dirname(real_pyexec) - else: - env_path = osp.dirname(osp.dirname(real_pyexec)) - - menu_rel_path = '/Menu/spyder-menu.json' - if ( - osp.exists(env_path + menu_rel_path) - or glob(env_path + '/envs/*' + menu_rel_path) - ): - return True - else: - return False - - -def is_text_string(obj): - """Return True if `obj` is a text string, False if it is anything else, - like binary data (Python 3) or QString (PyQt API #1)""" - return isinstance(obj, str) - -def is_binary_string(obj): - """Return True if `obj` is a binary string, False if it is anything else""" - return isinstance(obj, bytes) - -def is_string(obj): - """Return True if `obj` is a text or binary Python string object, - False if it is anything else, like a QString (PyQt API #1)""" - return is_text_string(obj) or is_binary_string(obj) - -def to_text_string(obj, encoding=None): - """Convert `obj` to (unicode) text string""" - if encoding is None: - return str(obj) - elif isinstance(obj, str): - # In case this function is not used properly, this could happen - return obj - else: - return str(obj, encoding) - -# The default encoding for file paths and environment variables should be set -# to match the default encoding that the OS is using. -def getfilesystemencoding(): - """ - Query the filesystem for the encoding used to encode filenames - and environment variables. - """ - encoding = sys.getfilesystemencoding() - if encoding is None: - # Must be Linux or Unix and nl_langinfo(CODESET) failed. - encoding = PREFERRED_ENCODING - return encoding - -PREFERRED_ENCODING = locale.getpreferredencoding() -FS_ENCODING = getfilesystemencoding() - -def to_unicode_from_fs(string): - """ - Return a unicode version of string decoded using the file system encoding. - """ - if not is_string(string): # string is a QString - string = to_text_string(string.toUtf8(), 'utf-8') - else: - if is_binary_string(string): - try: - unic = string.decode(FS_ENCODING) - except (UnicodeError, TypeError): - pass - else: - return unic - return string - -def get_home_dir(): - """Return user home directory.""" - try: - # expanduser() returns a raw byte string which needs to be - # decoded with the codec that the OS is using to represent - # file paths. - path = to_unicode_from_fs(osp.expanduser('~')) - except Exception: - path = '' - - if osp.isdir(path): - return path - else: - # Get home from alternative locations - for env_var in ('HOME', 'USERPROFILE', 'TMP'): - # os.environ.get() returns a raw byte string which needs to be - # decoded with the codec that the OS is using to represent - # environment variables. - path = to_unicode_from_fs(os.environ.get(env_var, '')) - if osp.isdir(path): - return path - else: - path = '' - - if not path: - raise RuntimeError('Please set the environment variable HOME to ' - 'your user/home directory path so Spyder can ' - 'start properly.') - -def is_program_installed(basename): - """ - Return program absolute path if installed in PATH. - Otherwise, return None. - - Also searches specific platform dependent paths that are not already in - PATH. This permits general use without assuming user profiles are - sourced (e.g. .bash_Profile), such as when login shells are not used to - launch Spyder. - - On macOS systems, a .app is considered installed if it exists. - """ - home = get_home_dir() - req_paths = [] - if ( - sys.platform == 'darwin' - and basename.endswith('.app') - and osp.exists(basename) - ): - return basename - - if os.name == 'posix': - pyenv = [ - osp.join(home, '.pyenv', 'bin'), - osp.join('/usr', 'local', 'bin'), - ] - - a = [home, osp.join(home, 'opt'), '/opt'] - b = ['mambaforge', 'miniforge3', 'miniforge', - 'miniconda3', 'anaconda3', 'miniconda', 'anaconda'] - else: - pyenv = [osp.join(home, '.pyenv', 'pyenv-win', 'bin')] - - a = [home, osp.join(home, 'AppData', 'Local'), - 'C:\\', osp.join('C:\\', 'ProgramData')] - b = ['Mambaforge', 'Miniforge3', 'Miniforge', - 'Miniconda3', 'Anaconda3', 'Miniconda', 'Anaconda'] - - conda = [osp.join(*p, 'condabin') for p in itertools.product(a, b)] - req_paths.extend(pyenv + conda) - - for path in os.environ['PATH'].split(os.pathsep) + req_paths: - abspath = osp.join(path, basename) - if osp.isfile(abspath): - return abspath - -def find_program(basename): - """ - Find program in PATH and return absolute path - - Try adding .exe or .bat to basename on Windows platforms - (return None if not found) - """ - names = [basename] - if os.name == 'nt': - # Windows platforms - extensions = ('.exe', '.bat', '.cmd') - if not basename.endswith(extensions): - names = [basename + ext for ext in extensions] + [basename] - for name in names: - path = is_program_installed(name) - if path: - return path - -def find_conda(): - """Find conda executable.""" - conda = None - - # First try Spyder's conda executable - if is_conda_based_app(): - root = osp.dirname(os.environ['CONDA_EXE']) - conda = osp.join(root, 'mamba.exe' if WINDOWS else 'mamba') - - # Next try the environment variables - if conda is None: - conda = os.environ.get('CONDA_EXE') or os.environ.get('MAMBA_EXE') - - # Next try searching for the executable - if conda is None: - conda_exec = 'conda.bat' if WINDOWS else 'conda' - conda = find_program(conda_exec) - - return conda \ No newline at end of file diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 1dcab2e0c59..83382529d2a 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -12,10 +12,10 @@ import logging import os import os.path as osp -import sys # Third party imports from jupyter_client.kernelspec import KernelSpec +from spyder_kernels_server.conda_utils import is_different_interpreter # Local imports from spyder.api.config.mixins import SpyderConfigurationAccessor @@ -25,8 +25,7 @@ from spyder.plugins.ipythonconsole import ( SPYDER_KERNELS_CONDA, SPYDER_KERNELS_PIP, SPYDER_KERNELS_VERSION, SpyderKernelError) -from spyder.utils.conda import (add_quotes, get_conda_env_path, is_conda_env, - find_conda) +from spyder.utils.conda import (get_conda_env_path, is_conda_env, find_conda) from spyder.utils.environ import clean_env, get_user_environment_variables from spyder.utils.misc import get_python_executable from spyder.utils.programs import ( @@ -55,16 +54,6 @@ "") -def is_different_interpreter(pyexec): - """Check that pyexec is a different interpreter from sys.executable.""" - # Paths may be symlinks - real_pyexe = osp.realpath(pyexec) - real_sys_exe = osp.realpath(sys.executable) - executable_validation = osp.basename(real_pyexe).startswith('python') - directory_validation = osp.dirname(real_pyexe) != osp.dirname(real_sys_exe) - return directory_validation and executable_validation - - def has_spyder_kernels(pyexec): """Check if env has spyder kernels.""" if is_module_installed( diff --git a/spyder/utils/conda.py b/spyder/utils/conda.py index 0d02a7e1a62..3f3d296750c 100644 --- a/spyder/utils/conda.py +++ b/spyder/utils/conda.py @@ -13,33 +13,15 @@ import os.path as osp import sys -from spyder.utils.programs import find_program, run_program, run_shell_command -from spyder.config.base import is_conda_based_app +from spyder_kernels_server.conda_utils import ( + get_conda_env_path, add_quotes, find_conda, is_conda_env +) +from spyder.utils.programs import run_program, run_shell_command WINDOWS = os.name == 'nt' CONDA_ENV_LIST_CACHE = {} -def add_quotes(path): - """Return quotes if needed for spaces on path.""" - quotes = '"' if ' ' in path and '"' not in path else '' - return '{quotes}{path}{quotes}'.format(quotes=quotes, path=path) - - -def is_conda_env(prefix=None, pyexec=None): - """Check if prefix or python executable are in a conda environment.""" - if pyexec is not None: - pyexec = pyexec.replace('\\', '/') - - if (prefix is None and pyexec is None) or (prefix and pyexec): - raise ValueError('Only `prefix` or `pyexec` should be provided!') - - if pyexec and prefix is None: - prefix = get_conda_env_path(pyexec).replace('\\', '/') - - return os.path.exists(os.path.join(prefix, 'conda-meta')) - - def get_conda_root_prefix(pyexec=None, quote=False): """ Return conda prefix from pyexec path @@ -65,45 +47,6 @@ def get_conda_root_prefix(pyexec=None, quote=False): return root_prefix -def get_conda_env_path(pyexec, quote=False): - """ - Return the full path to the conda environment from give python executable. - - If `quote` is True, then quotes are added if spaces are found in the path. - """ - pyexec = pyexec.replace('\\', '/') - if os.name == 'nt': - conda_env = os.path.dirname(pyexec) - else: - conda_env = os.path.dirname(os.path.dirname(pyexec)) - - if quote: - conda_env = add_quotes(conda_env) - - return conda_env - - -def find_conda(): - """Find conda executable.""" - conda = None - - # First try Spyder's conda executable - if is_conda_based_app(): - root = osp.dirname(os.environ['CONDA_EXE']) - conda = osp.join(root, 'mamba.exe' if WINDOWS else 'mamba') - - # Next try the environment variables - if conda is None: - conda = os.environ.get('CONDA_EXE') or os.environ.get('MAMBA_EXE') - - # Next try searching for the executable - if conda is None: - conda_exec = 'conda.bat' if WINDOWS else 'conda' - conda = find_program(conda_exec) - - return conda - - def get_list_conda_envs(): """Return the list of all conda envs found in the system.""" global CONDA_ENV_LIST_CACHE From d8199513a6e91f00a74b46acfc3e69abac9b35b8 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 19 Sep 2023 08:10:48 +0200 Subject: [PATCH 097/110] cleanup --- .../ipythonconsole/widgets/main_widget.py | 9 +++-- .../plugins/ipythonconsole/widgets/shell.py | 38 +++++++++---------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 4dd4c2d594b..d871317d573 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1454,10 +1454,11 @@ def create_new_client(self, give_focus=True, filename='', special=None, str_id='A') # Find what kind of kernel we want - if self.get_conf('pylab/autoload'): - special = "pylab" - elif self.get_conf('symbolic_math'): - special = "sympy" + if special is None: + if self.get_conf('pylab/autoload'): + special = "pylab" + elif self.get_conf('symbolic_math'): + special = "sympy" client = ClientWidget( self, diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 1efa78b8b17..cbf15fba073 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -204,7 +204,7 @@ def spyder_kernel_ready(self): """ if self.kernel_handler is None: return False - return ( + return (self.is_kernel_configured and self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady) @@ -304,7 +304,8 @@ def handle_kernel_is_ready(self): self.kernel_handler.connection_state == KernelConnectionState.SpyderKernelReady ): - self.setup_spyder_kernel() + self.kernel_connect_sig() + self.send_spyder_kernel_configuration() def _started_channels(self): """Make a history request""" @@ -393,9 +394,9 @@ def is_external_kernel(self): if self.kernel_handler is None: return False return self.kernel_handler.kernel_spec_dict is None - - def setup_spyder_kernel(self): - """Setup spyder kernel""" + + def kernel_connect_sig(self): + """Connect signals for kernel.""" if not self._init_kernel_setup: # Only do this setup once self._init_kernel_setup = True @@ -417,21 +418,9 @@ def setup_spyder_kernel(self): for request_id, handler in self.kernel_comm_handlers.items(): self.kernel_handler.kernel_comm.register_call_handler( request_id, handler) - - # Setup to do after restart - # Check for fault and send config - self.kernel_handler.poll_fault_text() - - self.send_spyder_kernel_configuration() - - run_lines = self.get_conf('startup/run_lines') - if run_lines: - self.execute(run_lines, hidden=True) - - if self.get_conf('startup/use_run_file'): - run_file = self.get_conf('startup/run_file') - if run_file: - self.execute(f"exec(open({run_file}))", hidden=True) + else: + # kernel might have restarted + self.kernel_handler.poll_fault_text() def send_spyder_kernel_configuration(self): """Send kernel configuration to spyder kernel.""" @@ -470,6 +459,15 @@ def send_spyder_kernel_configuration(self): callback=self.kernel_configure_callback ).set_configuration(self._kernel_configuration) + run_lines = self.get_conf('startup/run_lines') + if run_lines: + self.execute(run_lines, hidden=True) + + if self.get_conf('startup/use_run_file'): + run_file = self.get_conf('startup/run_file') + if run_file: + self.execute(f"exec(open({run_file}))", hidden=True) + self.is_kernel_configured = True def set_kernel_configuration(self, key, value): From ea352067e59d85a835c7e11e64bf8fed8cb383e4 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 19 Sep 2023 08:32:41 +0200 Subject: [PATCH 098/110] fix double prompt --- spyder/plugins/ipythonconsole/widgets/main_widget.py | 3 +++ spyder/plugins/ipythonconsole/widgets/shell.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index d871317d573..23993dfe8f4 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1703,6 +1703,9 @@ def close_client(self, index=None, client=None, ask_recursive=True): index = self.tabwidget.currentIndex() if index is not None: client = self.tabwidget.widget(index) + if client is None: + # Nothing to do? + return # Check if related clients or kernels are opened # and eventually ask before closing them diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index cbf15fba073..4afe9976f48 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -682,13 +682,14 @@ def set_bracket_matcher_color_scheme(self, color_scheme): def set_color_scheme(self, color_scheme, reset=True): """Set color scheme of the shell.""" + color_changed = self.syntax_style != color_scheme self.set_bracket_matcher_color_scheme(color_scheme) self.style_sheet, dark_color = create_qss_style(color_scheme) self.syntax_style = color_scheme self._style_sheet_changed() self._syntax_style_changed() - if reset: - self.reset(clear=True) + if reset and color_changed: + self.reset() if not self.spyder_kernel_ready: # Will be sent later return From 8f007a9d33e27b1d7a6445bf1fca04b8bef69de2 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 19 Sep 2023 13:31:41 +0200 Subject: [PATCH 099/110] no need to wait --- spyder/plugins/debugger/widgets/framesbrowser.py | 2 -- spyder/plugins/ipythonconsole/widgets/shell.py | 7 ------- .../plugins/variableexplorer/widgets/namespacebrowser.py | 2 -- 3 files changed, 11 deletions(-) diff --git a/spyder/plugins/debugger/widgets/framesbrowser.py b/spyder/plugins/debugger/widgets/framesbrowser.py index a3a4f84370a..74d2f9f8589 100644 --- a/spyder/plugins/debugger/widgets/framesbrowser.py +++ b/spyder/plugins/debugger/widgets/framesbrowser.py @@ -279,8 +279,6 @@ def on_config_kernel(self): def on_unconfig_kernel(self): """Ask shellwidget to stop sending stack.""" - if not self.shellwidget.spyder_kernel_ready: - return self.shellwidget.set_kernel_configuration( "pdb", {'pdb_publish_stack': False} ) diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 4afe9976f48..4921ed09f28 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -578,11 +578,7 @@ def send_mpl_backend(self, option=None): If option is not None only send the related options """ - if not self.spyder_kernel_ready: - # will be sent later - return # Set Matplotlib backend with Spyder options - pylab_n = 'pylab' pylab_o = self.get_conf(pylab_n) @@ -690,9 +686,6 @@ def set_color_scheme(self, color_scheme, reset=True): self._syntax_style_changed() if reset and color_changed: self.reset() - if not self.spyder_kernel_ready: - # Will be sent later - return self.set_kernel_configuration( "color scheme", "dark" if not dark_color else "light" ) diff --git a/spyder/plugins/variableexplorer/widgets/namespacebrowser.py b/spyder/plugins/variableexplorer/widgets/namespacebrowser.py index 0e2d0695bf6..4c2903391ff 100644 --- a/spyder/plugins/variableexplorer/widgets/namespacebrowser.py +++ b/spyder/plugins/variableexplorer/widgets/namespacebrowser.py @@ -231,8 +231,6 @@ def refresh_namespacebrowser(self, *, interrupt=True): def set_namespace_view_settings(self): """Set the namespace view settings""" - if not self.shellwidget.spyder_kernel_ready: - return settings = self.get_view_settings() self.shellwidget.set_kernel_configuration( "namespace_view_settings", settings From ca03603d943afd97c060589371b3f5c3a3cc34ce Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 19 Sep 2023 22:33:13 +0200 Subject: [PATCH 100/110] fix restart test --- .../ipythonconsole/tests/test_ipythonconsole.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 20df97895ff..7796715a2ba 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -859,8 +859,6 @@ def test_restart_kernel(ipyconsole, mocker, qtbot): """ Test that kernel is restarted correctly """ - # Mock method we want to check - mocker.patch.object(ShellWidget, "send_spyder_kernel_configuration") ipyconsole.create_new_client() @@ -868,6 +866,10 @@ def test_restart_kernel(ipyconsole, mocker, qtbot): qtbot.waitUntil( lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) + + # Check kernel is configured + faulthandler_args = shell.kernel_handler._fault_args + assert faulthandler_args is not None # Do an assignment to verify that it's not there after restarting with qtbot.waitSignal(shell.executed): @@ -892,8 +894,8 @@ def test_restart_kernel(ipyconsole, mocker, qtbot): assert not shell.is_defined('a') # Check that we send configuration at the beginning and after the restart. - qtbot.waitUntil( - lambda: ShellWidget.send_spyder_kernel_configuration.call_count == 2) + assert shell.kernel_handler._fault_args is not None + assert shell.kernel_handler._fault_args != faulthandler_args @flaky(max_runs=3) From e8e8b0b0e985df6cca3b106cd2d91c752c9998bc Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Thu, 21 Sep 2023 06:20:31 +0200 Subject: [PATCH 101/110] cleanup kernel spec --- spyder/app/tests/conftest.py | 9 +- spyder/app/tests/test_mainwindow.py | 6 +- .../ipythonconsole/utils/kernelspec.py | 211 ------------------ .../utils/tests/test_spyder_kernel.py | 10 +- .../ipythonconsole/widgets/main_widget.py | 97 +++++++- 5 files changed, 99 insertions(+), 234 deletions(-) delete mode 100644 spyder/plugins/ipythonconsole/utils/kernelspec.py diff --git a/spyder/app/tests/conftest.py b/spyder/app/tests/conftest.py index ea5ef4312f2..1e2e82a0f32 100755 --- a/spyder/app/tests/conftest.py +++ b/spyder/app/tests/conftest.py @@ -27,14 +27,13 @@ from spyder.app import start from spyder.config.base import get_home_dir, running_in_ci from spyder.config.manager import CONF -from spyder.plugins.ipythonconsole.utils.kernelspec import SpyderKernelSpec from spyder.plugins.projects.api import EmptyProject from spyder.plugins.run.api import RunActions, StoredRunConfigurationExecutor from spyder.plugins.toolbar.api import ApplicationToolbars from spyder.utils import encoding from spyder.utils.environ import (get_user_env, set_user_env, amend_user_shell_init) - +from spyder_kernels_server.kernel_spec import get_kernel_spec # ============================================================================= # ---- Constants @@ -84,12 +83,12 @@ def reset_run_code(qtbot, shell, code_editor, nsb): qtbot.keyClick(code_editor, Qt.Key_Home, modifier=Qt.ControlModifier) -def start_new_kernel(startup_timeout=60, kernel_name='python', spykernel=False, - **kwargs): +def start_new_kernel(ipyconsole, startup_timeout=60, kernel_name='python', + spykernel=False, **kwargs): """Start a new kernel, and return its Manager and Client""" km = KernelManager(kernel_name=kernel_name) if spykernel: - km._kernel_spec = SpyderKernelSpec() + km._kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index e0fe1450e60..2e424cc3ced 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -935,7 +935,7 @@ def test_shell_execution(main_window, qtbot, tmpdir): def test_connection_to_external_kernel(main_window, qtbot): """Test that only Spyder kernels are connected to the Variable Explorer.""" # Test with a generic kernel - km, kc = start_new_kernel() + km, kc = start_new_kernel(main_window.ipyconsole) main_window.ipyconsole.create_client_for_kernel(kc.connection_file) shell = main_window.ipyconsole.get_current_shellwidget() @@ -954,7 +954,7 @@ def test_connection_to_external_kernel(main_window, qtbot): python_shell = shell # Test with a kernel from Spyder - spykm, spykc = start_new_kernel(spykernel=True) + spykm, spykc = start_new_kernel(main_window.ipyconsole, spykernel=True) main_window.ipyconsole.create_client_for_kernel(spykc.connection_file) shell = main_window.ipyconsole.get_current_shellwidget() qtbot.waitUntil( @@ -5164,7 +5164,7 @@ def test_outline_no_init(main_window, qtbot): def test_pdb_ipykernel(main_window, qtbot): """Check if pdb works without spyder kernel.""" # Test with a generic kernel - km, kc = start_new_kernel() + km, kc = start_new_kernel(main_window.ipyconsole) main_window.ipyconsole.create_client_for_kernel(kc.connection_file) ipyconsole = main_window.ipyconsole diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py deleted file mode 100644 index 83382529d2a..00000000000 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ /dev/null @@ -1,211 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © Spyder Project Contributors -# Licensed under the terms of the MIT License -# (see spyder/__init__.py for details) - -""" -Kernel spec for Spyder kernels -""" - -# Standard library imports -import logging -import os -import os.path as osp - -# Third party imports -from jupyter_client.kernelspec import KernelSpec -from spyder_kernels_server.conda_utils import is_different_interpreter - -# Local imports -from spyder.api.config.mixins import SpyderConfigurationAccessor -from spyder.api.translations import _ -from spyder.config.base import (get_safe_mode, is_conda_based_app, - running_under_pytest) -from spyder.plugins.ipythonconsole import ( - SPYDER_KERNELS_CONDA, SPYDER_KERNELS_PIP, SPYDER_KERNELS_VERSION, - SpyderKernelError) -from spyder.utils.conda import (get_conda_env_path, is_conda_env, find_conda) -from spyder.utils.environ import clean_env, get_user_environment_variables -from spyder.utils.misc import get_python_executable -from spyder.utils.programs import ( - is_python_interpreter, is_module_installed, get_module_version - ) - -# Constants -HERE = os.path.abspath(os.path.dirname(__file__)) -logger = logging.getLogger(__name__) -ERROR_SPYDER_KERNEL_INSTALLED = _( - "The Python environment or installation whose interpreter is located at" - "
"
-    "    {0}"
-    "
" - "doesn't have spyder-kernels version {1} installed. " - "Without this module and specific version is not possible for Spyder to " - "create a console for you.

" - "You can install it by activating your environment (if necessary) and " - "then running in a system terminal:" - "
"
-    "    {2}"
-    "
" - "or" - "
"
-    "    {3}"
-    "
") - - -def has_spyder_kernels(pyexec): - """Check if env has spyder kernels.""" - if is_module_installed( - 'spyder_kernels', - version=SPYDER_KERNELS_VERSION, - interpreter=pyexec): - return True - # dev versions are acceptable - try: - return "dev0" in get_module_version('spyder_kernels', pyexec) - except Exception: - return False - - -HERE = osp.dirname(os.path.realpath(__file__)) - - -class SpyderKernelSpec(KernelSpec, SpyderConfigurationAccessor): - """Kernel spec for Spyder kernels""" - - CONF_SECTION = 'ipython_console' - - def __init__(self, path_to_custom_interpreter=None, - **kwargs): - super(SpyderKernelSpec, self).__init__(**kwargs) - self.pyexec = path_to_custom_interpreter - if ( - path_to_custom_interpreter is None - and not self.get_conf('default', section='main_interpreter') - ): - self.pyexec = self.get_conf( - 'executable', section='main_interpreter') - self.display_name = 'Python 3 (Spyder)' - self.language = 'python3' - self.resource_dir = '' - - def to_dict(self): - d = super().to_dict() - d["pyexec"] = self.pyexec - return d - - @property - def argv(self): - """Command to start kernels""" - # Python interpreter used to start kernels - if ( - self.pyexec is None - ): - pyexec = get_python_executable() - else: - pyexec = self.pyexec - if not has_spyder_kernels(pyexec): - raise SpyderKernelError( - ERROR_SPYDER_KERNEL_INSTALLED.format( - pyexec, - SPYDER_KERNELS_VERSION, - SPYDER_KERNELS_CONDA, - SPYDER_KERNELS_PIP - ) - ) - return - if not is_python_interpreter(pyexec): - pyexec = get_python_executable() - self.set_conf('executable', '', section='main_interpreter') - self.set_conf('default', True, section='main_interpreter') - self.set_conf('custom', False, section='main_interpreter') - - # Part of spyder-ide/spyder#11819 - is_different = is_different_interpreter(pyexec) - - # Command used to start kernels - kernel_cmd = [ - pyexec, - # This is necessary to avoid a spurious message on Windows. - # Fixes spyder-ide/spyder#20800. - '-Xfrozen_modules=off', - '-m', 'spyder_kernels.console', - '-f', '{connection_file}' - ] - - if is_different and is_conda_env(pyexec=pyexec): - # If executable is a conda environment and different from Spyder's - # runtime environment, we need to activate the environment to run - # spyder-kernels - kernel_cmd[:0] = [ - find_conda(), 'run', - '-p', get_conda_env_path(pyexec), - ] - - logger.info('Kernel command: {}'.format(kernel_cmd)) - - return kernel_cmd - - @property - def env(self): - """Env vars for kernels""" - default_interpreter = self.get_conf( - 'default', section='main_interpreter') - - # Ensure that user environment variables are included, but don't - # override existing environ values - env_vars = get_user_environment_variables() - env_vars.update(os.environ) - - # Avoid IPython adding the virtualenv on which Spyder is running - # to the kernel sys.path - env_vars.pop('VIRTUAL_ENV', None) - - # Do not pass PYTHONPATH to kernels directly, spyder-ide/spyder#13519 - env_vars.pop('PYTHONPATH', None) - - # List of paths declared by the user, plus project's path, to - # add to PYTHONPATH - pathlist = self.get_conf( - 'spyder_pythonpath', default=[], section='pythonpath_manager') - pypath = os.pathsep.join(pathlist) - - # List of modules to exclude from our UMR - umr_namelist = self.get_conf( - 'umr/namelist', section='main_interpreter') - - # Environment variables that we need to pass to the kernel - env_vars.update({ - 'SPY_EXTERNAL_INTERPRETER': (not default_interpreter - or self.pyexec), - 'SPY_UMR_ENABLED': self.get_conf( - 'umr/enabled', section='main_interpreter'), - 'SPY_UMR_VERBOSE': self.get_conf( - 'umr/verbose', section='main_interpreter'), - 'SPY_UMR_NAMELIST': ','.join(umr_namelist), - 'SPY_AUTOCALL_O': self.get_conf('autocall'), - 'SPY_GREEDY_O': self.get_conf('greedy_completer'), - 'SPY_JEDI_O': self.get_conf('jedi_completer'), - 'SPY_TESTING': running_under_pytest() or get_safe_mode(), - 'SPY_HIDE_CMD': self.get_conf('hide_cmd_windows'), - 'SPY_PYTHONPATH': pypath, - }) - - # App considerations - # ??? Do we need this? - if is_conda_based_app() and default_interpreter: - # See spyder-ide/spyder#16927 - # See spyder-ide/spyder#16828 - # See spyder-ide/spyder#17552 - env_vars['PYDEVD_DISABLE_FILE_VALIDATION'] = 1 - - # Remove this variable because it prevents starting kernels for - # external interpreters when present. - # Fixes spyder-ide/spyder#13252 - env_vars.pop('PYTHONEXECUTABLE', None) - - # Making all env_vars strings - clean_env_vars = clean_env(env_vars) - - return clean_env_vars diff --git a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py index 0503af6a6a0..53a89c8f601 100644 --- a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py +++ b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py @@ -12,12 +12,12 @@ import pytest from spyder.config.manager import CONF -from spyder.plugins.ipythonconsole.utils.kernelspec import SpyderKernelSpec from spyder.py3compat import to_text_string +from spyder_kernels_server.kernel_spec import get_kernel_spec @pytest.mark.parametrize('default_interpreter', [True, False]) -def test_kernel_pypath(tmpdir, default_interpreter): +def test_kernel_pypath(ipyconsole, tmpdir, default_interpreter): """ Test that PYTHONPATH and spyder_pythonpath option are properly handled when an external interpreter is used or not. @@ -33,7 +33,7 @@ def test_kernel_pypath(tmpdir, default_interpreter): os.environ['PYTHONPATH'] = pypath CONF.set('pythonpath_manager', 'spyder_pythonpath', [pypath]) - kernel_spec = SpyderKernelSpec() + kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) # Check that PYTHONPATH is not in our kernelspec # and pypath is in SPY_PYTHONPATH @@ -46,7 +46,7 @@ def test_kernel_pypath(tmpdir, default_interpreter): del os.environ['PYTHONPATH'] -def test_python_interpreter(tmpdir): +def test_python_interpreter(ipyconsole, tmpdir): """Test the validation of the python interpreter.""" # Set a non existing python interpreter interpreter = str(tmpdir.mkdir('interpreter').join('python')) @@ -55,7 +55,7 @@ def test_python_interpreter(tmpdir): CONF.set('main_interpreter', 'executable', interpreter) # Create a kernel spec - kernel_spec = SpyderKernelSpec() + kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) # Assert that the python interprerter is the default one assert interpreter not in kernel_spec.argv diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 23993dfe8f4..317f1bac6fe 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -34,7 +34,6 @@ from spyder.config.base import ( get_home_dir, running_under_pytest) from spyder.plugins.ipythonconsole.utils.kernel_handler import KernelHandler -from spyder.plugins.ipythonconsole.utils.kernelspec import SpyderKernelSpec from spyder.plugins.ipythonconsole.utils.style import create_qss_style from spyder.plugins.ipythonconsole.widgets import ( ClientWidget, ConsoleRestartDialog, COMPLETION_WIDGET_TYPE, @@ -50,6 +49,8 @@ from spyder.widgets.browser import FrameWebView from spyder.widgets.findreplace import FindReplace from spyder.widgets.tabs import Tabs +from spyder.utils.environ import clean_env, get_user_environment_variables +from spyder.config.base import get_safe_mode, is_conda_based_app # Logging @@ -1484,15 +1485,13 @@ def create_new_client(self, give_focus=True, filename='', special=None, try: # Create new kernel - kernel_spec_kwargs = dict( - path_to_custom_interpreter=path_to_custom_interpreter - ) - kernel_spec_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() - kernel_spec_dict["setup_kwargs"] = kernel_spec_kwargs + kernel_spec_dict = self.get_kernel_spec_dict( + path_to_custom_interpreter + ) kernel_handler = self.get_cached_kernel( kernel_spec_dict, cache=cache, - ) + ) except Exception as e: client.show_kernel_error(e) return @@ -1500,6 +1499,86 @@ def create_new_client(self, give_focus=True, filename='', special=None, # Connect kernel to client client.connect_kernel(kernel_handler) return client + + def get_kernel_spec_dict(self, pyexec=None): + """Create a kernel spec dict""" + kernel_spec = { + "display_name": 'Python 3 (Spyder)', + "language": 'python3', + "resource_dir": '', + } + if ( + pyexec is None + and not self.get_conf('default', section='main_interpreter') + ): + pyexec = self.get_conf( + 'executable', section='main_interpreter' + ) + kernel_spec["pyexec"] = pyexec + kernel_spec["env"] = self.get_kernel_spec_env(pyexec) + return kernel_spec + + def get_kernel_spec_env(self, pyexec): + """Env vars for kernels""" + default_interpreter = self.get_conf( + 'default', section='main_interpreter') + + # Ensure that user environment variables are included, but don't + # override existing environ values + env_vars = get_user_environment_variables() + env_vars.update(os.environ) + + # Avoid IPython adding the virtualenv on which Spyder is running + # to the kernel sys.path + env_vars.pop('VIRTUAL_ENV', None) + + # Do not pass PYTHONPATH to kernels directly, spyder-ide/spyder#13519 + env_vars.pop('PYTHONPATH', None) + + # List of paths declared by the user, plus project's path, to + # add to PYTHONPATH + pathlist = self.get_conf( + 'spyder_pythonpath', default=[], section='pythonpath_manager') + pypath = os.pathsep.join(pathlist) + + # List of modules to exclude from our UMR + umr_namelist = self.get_conf( + 'umr/namelist', section='main_interpreter') + + # Environment variables that we need to pass to the kernel + env_vars.update({ + 'SPY_EXTERNAL_INTERPRETER': (not default_interpreter + or pyexec), + 'SPY_UMR_ENABLED': self.get_conf( + 'umr/enabled', section='main_interpreter'), + 'SPY_UMR_VERBOSE': self.get_conf( + 'umr/verbose', section='main_interpreter'), + 'SPY_UMR_NAMELIST': ','.join(umr_namelist), + 'SPY_AUTOCALL_O': self.get_conf('autocall'), + 'SPY_GREEDY_O': self.get_conf('greedy_completer'), + 'SPY_JEDI_O': self.get_conf('jedi_completer'), + 'SPY_TESTING': running_under_pytest() or get_safe_mode(), + 'SPY_HIDE_CMD': self.get_conf('hide_cmd_windows'), + 'SPY_PYTHONPATH': pypath, + }) + + # App considerations + # ??? Do we need this? + if is_conda_based_app() and default_interpreter: + # See spyder-ide/spyder#16927 + # See spyder-ide/spyder#16828 + # See spyder-ide/spyder#17552 + env_vars['PYDEVD_DISABLE_FILE_VALIDATION'] = 1 + + # Remove this variable because it prevents starting kernels for + # external interpreters when present. + # Fixes spyder-ide/spyder#13252 + env_vars.pop('PYTHONEXECUTABLE', None) + + # Making all env_vars strings + clean_env_vars = clean_env(env_vars) + + return clean_env_vars def create_client_for_kernel(self, connection_file, hostname, sshkey, password): @@ -1921,9 +2000,7 @@ def restart_kernel(self, client=None, ask_before_restart=True): # Get new kernel try: # Update the kernel because settings might have changed - kernel_spec_kwargs = ks_dict["setup_kwargs"] - ks_dict = SpyderKernelSpec(**kernel_spec_kwargs).to_dict() - ks_dict["setup_kwargs"] = kernel_spec_kwargs + ks_dict = self.get_kernel_spec_dict(ks_dict["pyexec"]) kernel_handler = self.get_cached_kernel(ks_dict) except Exception as e: client.show_kernel_error(e) From 4cef3d64770bbd577b769eeae0de3ff905ba8a0e Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 10 Nov 2023 07:37:56 +0100 Subject: [PATCH 102/110] cleanup --- .../spyder-kernels-server/spyder_kernels_server/kernel_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 2527ee482c3..e6cb4b91cae 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -13,7 +13,6 @@ import os.path as osp from subprocess import PIPE import uuid -import sys from threading import Thread, Event From 8b949032c09b6bb2ccfbdb0fc3652b4cb5b3cdd5 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 11 Nov 2023 07:49:38 +0100 Subject: [PATCH 103/110] separate error and stderr --- .../ipythonconsole/utils/kernel_handler.py | 17 +++++++++++++++++ spyder/plugins/ipythonconsole/widgets/client.py | 14 ++++++++++++-- spyder/plugins/ipythonconsole/widgets/mixins.py | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index bb7feb1fd20..112c74115f1 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -97,6 +97,11 @@ class KernelHandler(QObject): A stderr message was received on the process stderr. """ + sig_error = Signal(str) + """ + The process crashed with a message. + """ + sig_fault = Signal(str) """ A fault message was received. @@ -151,6 +156,7 @@ def __init__( # Internal self._fault_args = None + self._init_error = "" self._init_stderr = "" self._init_stdout = "" self._shellwidget_connected = False @@ -161,6 +167,14 @@ def __init__( if self.kernel_client: self.start_channels() + @Slot(str, str) + def handle_error(self, err): + """Handle crash""" + if self._shellwidget_connected: + self.sig_error.emit(err) + else: + self._init_error += err + @Slot(str, str) def handle_stderr(self, err): """Handle stderr""" @@ -189,6 +203,9 @@ def connect(self): self.sig_kernel_connection_error.emit() # Show initial io + if self._init_error: + self.sig_error.emit(self._init_error) + self._init_error = None if self._init_stderr: self.sig_stderr.emit(self._init_stderr) self._init_stderr = None diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index c5adf8a629b..8da9ff1c672 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -329,6 +329,7 @@ def connect_kernel(self, kernel_handler): self.kernel_handler = kernel_handler # Connect standard streams. + kernel_handler.sig_error.connect(self.print_error) kernel_handler.sig_stderr.connect(self.print_stderr) kernel_handler.sig_stdout.connect(self.print_stdout) kernel_handler.sig_fault.connect(self.print_fault) @@ -345,6 +346,7 @@ def disconnect_kernel(self, shutdown_kernel): if not kernel_handler: return + kernel_handler.sig_error.disconnect(self.print_error) kernel_handler.sig_stderr.disconnect(self.print_stderr) kernel_handler.sig_stdout.disconnect(self.print_stdout) kernel_handler.sig_fault.disconnect(self.print_fault) @@ -353,8 +355,8 @@ def disconnect_kernel(self, shutdown_kernel): self.kernel_handler = None @Slot(str) - def print_stderr(self, stderr): - """Print stderr written in PIPE.""" + def print_error(self, stderr): + """Print crash infos.""" if not stderr: return @@ -372,6 +374,14 @@ def print_stderr(self, stderr): self.shellwidget._append_plain_text(stderr, before_prompt=True) + @Slot(str) + def print_stderr(self, stderr): + """Print stderr written in PIPE.""" + if not stderr: + return + + self.shellwidget._append_plain_text(stderr, before_prompt=True) + @Slot(str) def print_stdout(self, stdout): """Print stdout written in PIPE.""" diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index 35a325ddc2e..a00bf3ca177 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -240,7 +240,7 @@ def _socket_activity(self): kernel_handler = self.kernel_handler_waitlist.pop(0) if connection_file == "error": - kernel_handler.handle_stderr(str(connection_info)) + kernel_handler.handle_error(str(connection_info)) else: kernel_handler.set_connection( connection_file, From 5912e24eb9faf9a57c67c5055d8f398af17b0c5c Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 11 Nov 2023 07:57:01 +0100 Subject: [PATCH 104/110] Avoid hiding error during setup --- spyder/plugins/ipythonconsole/widgets/client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 8da9ff1c672..caed94d6900 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -381,6 +381,7 @@ def print_stderr(self, stderr): return self.shellwidget._append_plain_text(stderr, before_prompt=True) + self._hide_loading_page() @Slot(str) def print_stdout(self, stdout): @@ -389,6 +390,7 @@ def print_stdout(self, stdout): return self.shellwidget._append_plain_text(stdout, before_prompt=True) + self._hide_loading_page() def connect_shellwidget_signals(self): """Configure shellwidget after kernel is connected.""" From 823a5b7fd697661ff71d57f9e593ad1137ef5268 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 11 Nov 2023 12:03:31 +0100 Subject: [PATCH 105/110] fix test --- .../plugins/ipythonconsole/utils/tests/test_spyder_kernel.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py index 53a89c8f601..d534d5885db 100644 --- a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py +++ b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py @@ -14,6 +14,7 @@ from spyder.config.manager import CONF from spyder.py3compat import to_text_string from spyder_kernels_server.kernel_spec import get_kernel_spec +from spyder.plugins.ipythonconsole.tests.conftest import ipyconsole @pytest.mark.parametrize('default_interpreter', [True, False]) @@ -33,7 +34,7 @@ def test_kernel_pypath(ipyconsole, tmpdir, default_interpreter): os.environ['PYTHONPATH'] = pypath CONF.set('pythonpath_manager', 'spyder_pythonpath', [pypath]) - kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) + kernel_spec = get_kernel_spec(ipyconsole.get_widget().get_kernel_spec_dict()) # Check that PYTHONPATH is not in our kernelspec # and pypath is in SPY_PYTHONPATH @@ -55,7 +56,7 @@ def test_python_interpreter(ipyconsole, tmpdir): CONF.set('main_interpreter', 'executable', interpreter) # Create a kernel spec - kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) + kernel_spec = get_kernel_spec(ipyconsole.get_widget().get_kernel_spec_dict()) # Assert that the python interprerter is the default one assert interpreter not in kernel_spec.argv From 46033f10ae22001c47fa0660b8da4a71c4b33068 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 11 Nov 2023 15:28:16 +0100 Subject: [PATCH 106/110] fix test --- spyder/app/tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spyder/app/tests/conftest.py b/spyder/app/tests/conftest.py index 630df5448a2..2185982a109 100755 --- a/spyder/app/tests/conftest.py +++ b/spyder/app/tests/conftest.py @@ -88,7 +88,9 @@ def start_new_kernel(ipyconsole, startup_timeout=60, kernel_name='python', """Start a new kernel, and return its Manager and Client""" km = KernelManager(kernel_name=kernel_name) if spykernel: - km._kernel_spec = get_kernel_spec(ipyconsole.get_kernel_spec_dict()) + km._kernel_spec = get_kernel_spec( + ipyconsole.get_widget().get_kernel_spec_dict() + ) km.start_kernel(**kwargs) kc = km.client() kc.start_channels() From 7808825259494e1c7c5ba73fb6a33a23d7286336 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Tue, 14 Nov 2023 04:06:36 +0100 Subject: [PATCH 107/110] git subrepo clone (merge) --branch=print_remote --force https://github.com/impact27/spyder-kernels.git external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "8b95bcd56" upstream: origin: "https://github.com/impact27/spyder-kernels.git" branch: "print_remote" commit: "8b95bcd56" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" --- external-deps/spyder-kernels/.gitrepo | 4 +- .../spyder_kernels/console/kernel.py | 90 ++++++++++--------- .../console/tests/test_console_kernel.py | 2 +- .../customize/tests/test_umr.py | 4 +- .../spyder_kernels/customize/umr.py | 7 +- .../spyder_kernels/utils/mpl.py | 6 +- 6 files changed, 59 insertions(+), 54 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 7cec1e2f97f..73ddf1feb05 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = print_remote - commit = a33dfc86975704607247c7f7fdbecfe77426f750 - parent = 912a1da04c51ff6c860a24ea06c45eaa851fa849 + commit = 8b95bcd56e9107a62b121c4b000e2bcc6c809bd6 + parent = 563813e5a5004d281e8c49a80fef3b46dcdd73c2 method = merge cmdver = 0.4.5 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index cd128f6dbf9..64c2656de35 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -150,6 +150,11 @@ def enable_faulthandler(self): faulthandler.enable(self.faulthandler_handle) return self.faulthandler_handle.name, main_id, system_ids + @comm_handler + def safe_exec(self, filename): + """Exec file using ipykernelapp _exec_file""" + self.parent._exec_file(filename) + @comm_handler def get_fault_text(self, fault_filename, main_id, ignore_ids): """Get fault text from old run.""" @@ -560,18 +565,22 @@ def set_matplotlib_conf(self, conf): conf.get(pylab_backend_n, inline_backend), pylab=conf.get(pylab_autoload_n, False) ) + if figure_format_n in conf: self._set_config_option( 'InlineBackend.figure_format', conf[figure_format_n] ) + if resolution_n in conf: self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n]) + if width_n in conf and height_n in conf: self._set_mpl_inline_rc_config( 'figure.figsize', (conf[width_n], conf[height_n]) ) + if bbox_inches_n in conf: self.set_mpl_inline_bbox_inches(conf[bbox_inches_n]) @@ -634,7 +643,10 @@ def set_configuration(self, conf): if value: ret[key] = self.enable_faulthandler() elif key == "special_kernel": - ret[key] = self.set_special_kernel(value) + try: + self.set_special_kernel(value) + except Exception: + ret["special_kernel_error"] = value elif key == "color scheme": self.set_color_scheme(value) elif key == "jedi_completer": @@ -698,56 +710,47 @@ def set_special_kernel(self, special): return if special == "pylab": - try: - import matplotlib - exec("from pylab import *", self.shell.user_ns) - self.shell.special = special - return - except Exception: - return "matplotlib" + import matplotlib + exec("from pylab import *", self.shell.user_ns) + self.shell.special = special + return + if special == "sympy": - try: - import sympy - sympy_init = "\n".join([ - "from sympy import *", - "x, y, z, t = symbols('x y z t')", - "k, m, n = symbols('k m n', integer=True)", - "f, g, h = symbols('f g h', cls=Function)", - "init_printing()", - ]) - exec(sympy_init, self.shell.user_ns) - self.shell.special = special - return - except Exception: - return "sympy" + import sympy + sympy_init = "\n".join([ + "from sympy import *", + "x, y, z, t = symbols('x y z t')", + "k, m, n = symbols('k m n', integer=True)", + "f, g, h = symbols('f g h', cls=Function)", + "init_printing()", + ]) + exec(sympy_init, self.shell.user_ns) + self.shell.special = special + return if special == "cython": - try: - import cython + import cython - # Import pyximport to enable Cython files support for - # import statement - import pyximport - pyx_setup_args = {} - - # Add Numpy include dir to pyximport/distutils - try: - import numpy - pyx_setup_args['include_dirs'] = numpy.get_include() - except Exception: - pass + # Import pyximport to enable Cython files support for + # import statement + import pyximport + pyx_setup_args = {} - # Setup pyximport and enable Cython files reload - pyximport.install(setup_args=pyx_setup_args, - reload_support=True) + # Add Numpy include dir to pyximport/distutils + try: + import numpy + pyx_setup_args['include_dirs'] = numpy.get_include() + except Exception: + pass - self.shell.run_line_magic("reload_ext", "Cython") - self.shell.special = special - return + # Setup pyximport and enable Cython files reload + pyximport.install(setup_args=pyx_setup_args, + reload_support=True) - except Exception: - return "cython" + self.shell.run_line_magic("reload_ext", "Cython") + self.shell.special = special + return raise NotImplementedError(f"{special}") @@ -980,6 +983,7 @@ def set_sympy_forecolor(self, background_color='dark'): """Set SymPy forecolor depending on console background.""" if self.shell.special != "sympy": return + try: from sympy import init_printing if background_color == 'dark': diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py index b7f17cce6d0..a9564febed3 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py @@ -1137,7 +1137,7 @@ def test_locals_globals_in_pdb(kernel): @flaky(max_runs=3) -@pytest.mark.parametrize("backend", [None, 'inline', 'tk', 'qt5']) +@pytest.mark.parametrize("backend", [None, 'inline', 'tk', 'qt']) @pytest.mark.skipif( os.environ.get('USE_CONDA') != 'true', reason="Doesn't work with pip packages") diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py b/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py index c237ccc71b9..fe84791b304 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py @@ -51,11 +51,11 @@ def test_umr_run(user_module): umr = UserModuleReloader() from foo1.bar import square - assert umr.run() == ['foo', 'foo.bar'] + assert umr.run() == ['foo1', 'foo1.bar'] def test_umr_previous_modules(user_module): - """Test that UMR's previos_modules is working as expected.""" + """Test that UMR's previous_modules is working as expected.""" # Create user module user_module('foo2') diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/umr.py b/external-deps/spyder-kernels/spyder_kernels/customize/umr.py index dfa4688c060..e779ec336bd 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/umr.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/umr.py @@ -59,9 +59,10 @@ def __init__(self, namelist=None, pathlist=None): def is_module_reloadable(self, module, modname): """Decide if a module is reloadable or not.""" - if (path_is_library(getattr(module, '__file__', None), - self.pathlist) or - self.is_module_in_namelist(modname)): + if ( + path_is_library(getattr(module, '__file__', None), self.pathlist) + or self.is_module_in_namelist(modname) + ): return False else: return True diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py index 7756fd39689..00f47bac03c 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py @@ -21,8 +21,8 @@ # Mapping of matlotlib backends options to Spyder MPL_BACKENDS_TO_SPYDER = { inline_backend: "inline", - 'Qt5Agg': 'qt5', - 'QtAgg': 'qt5', # For Matplotlib 3.5+ + 'Qt5Agg': 'qt', + 'QtAgg': 'qt', # For Matplotlib 3.5+ 'TkAgg': 'tk', 'MacOSX': 'osx', } @@ -31,7 +31,7 @@ def automatic_backend(): """Get Matplolib automatic backend option.""" if is_module_installed('PyQt5'): - auto_backend = 'qt5' + auto_backend = 'qt' elif is_module_installed('_tkinter'): auto_backend = 'tk' else: From 721bbebf97c088c6cb1f1b08a33332cc3236f25d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 17 Nov 2023 07:31:00 +0100 Subject: [PATCH 108/110] pep8 --- .../spyder_kernels_server/__main__.py | 4 ++-- .../spyder_kernels_server/conda_utils.py | 20 ++++++++++++++----- .../spyder_kernels_server/kernel_client.py | 2 +- .../spyder_kernels_server/kernel_server.py | 8 ++++---- .../spyder_kernels_server/kernel_spec.py | 7 ++++--- spyder/app/tests/test_mainwindow.py | 2 +- .../plugins/debugger/widgets/framesbrowser.py | 2 +- .../tests/test_ipythonconsole.py | 2 +- .../ipythonconsole/utils/kernel_handler.py | 2 +- .../utils/tests/test_spyder_kernel.py | 6 ++++-- .../plugins/ipythonconsole/widgets/client.py | 2 +- .../ipythonconsole/widgets/main_widget.py | 6 +++--- .../plugins/ipythonconsole/widgets/mixins.py | 3 +-- .../plugins/ipythonconsole/widgets/shell.py | 3 ++- spyder/plugins/maininterpreter/confpage.py | 2 -- 15 files changed, 41 insertions(+), 30 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py index e7383efa75c..1ab04d18cef 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/__main__.py @@ -89,11 +89,11 @@ def _socket_activity(self): @Slot(str) def _handle_kernel_restarted(self, connection_file): self.socket_pub.send_pyobj(["kernel_restarted", connection_file]) - + @Slot(str, str) def _handle_stderr(self, connection_file, txt): self.socket_pub.send_pyobj(["stderr", connection_file, txt]) - + @Slot(str, str) def _handle_stdout(self, connection_file, txt): self.socket_pub.send_pyobj(["stdout", connection_file, txt]) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py index 40dde8c536e..6a759695df2 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py @@ -8,6 +8,7 @@ import locale WINDOWS = os.name == 'nt' +PREFERRED_ENCODING = locale.getpreferredencoding() def is_different_interpreter(pyexec): @@ -87,15 +88,18 @@ def is_text_string(obj): like binary data (Python 3) or QString (PyQt API #1)""" return isinstance(obj, str) + def is_binary_string(obj): """Return True if `obj` is a binary string, False if it is anything else""" return isinstance(obj, bytes) + def is_string(obj): """Return True if `obj` is a text or binary Python string object, False if it is anything else, like a QString (PyQt API #1)""" return is_text_string(obj) or is_binary_string(obj) + def to_text_string(obj, encoding=None): """Convert `obj` to (unicode) text string""" if encoding is None: @@ -106,6 +110,7 @@ def to_text_string(obj, encoding=None): else: return str(obj, encoding) + # The default encoding for file paths and environment variables should be set # to match the default encoding that the OS is using. def getfilesystemencoding(): @@ -119,9 +124,10 @@ def getfilesystemencoding(): encoding = PREFERRED_ENCODING return encoding -PREFERRED_ENCODING = locale.getpreferredencoding() + FS_ENCODING = getfilesystemencoding() + def to_unicode_from_fs(string): """ Return a unicode version of string decoded using the file system encoding. @@ -138,6 +144,7 @@ def to_unicode_from_fs(string): return unic return string + def get_home_dir(): """Return user home directory.""" try: @@ -166,7 +173,8 @@ def get_home_dir(): raise RuntimeError('Please set the environment variable HOME to ' 'your user/home directory path so Spyder can ' 'start properly.') - + + def is_program_installed(basename): """ Return program absolute path if installed in PATH. @@ -212,7 +220,8 @@ def is_program_installed(basename): abspath = osp.join(path, basename) if osp.isfile(abspath): return abspath - + + def find_program(basename): """ Find program in PATH and return absolute path @@ -230,7 +239,8 @@ def find_program(basename): path = is_program_installed(name) if path: return path - + + def find_conda(): """Find conda executable.""" conda = None @@ -249,4 +259,4 @@ def find_conda(): conda_exec = 'conda.bat' if WINDOWS else 'conda' conda = find_program(conda_exec) - return conda \ No newline at end of file + return conda diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py index 84b519887e1..8cee02ccb34 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_client.py @@ -19,7 +19,7 @@ class SpyderKernelClient(QtKernelClient): # Useful for pdb completion control_channel_class = Type(QtZMQSocketChannel) sig_kernel_info = Signal(object) - + def _handle_kernel_info_reply(self, rep): """Check spyder-kernels version.""" super()._handle_kernel_info_reply(rep) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index e6cb4b91cae..739287aa7d4 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -29,10 +29,11 @@ "consoles. Please make it writable." ) + # kernel_comm needs a qthread class StdThread(QThread): """Poll for changes in std buffers.""" - + sig_text = Signal(str) def __init__(self, std_buffer): @@ -146,9 +147,8 @@ def open_kernel(self, kernel_spec): self.connect_std_pipes(kernel_key) kernel_manager.kernel_restarted.connect( - lambda connection_file=connection_file: self.sig_kernel_restarted.emit( - connection_file - ) + lambda connection_file=connection_file: + self.sig_kernel_restarted.emit(connection_file) ) return connection_file diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py index 6573cda08ed..22ce98ddafc 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_spec.py @@ -2,11 +2,12 @@ import sys from jupyter_client.kernelspec import KernelSpec - + from .conda_utils import ( is_conda_env, get_conda_env_path, find_conda, is_different_interpreter ) + def get_python_executable(): """Return path to Spyder Python executable""" executable = sys.executable.replace("pythonw.exe", "python.exe") @@ -17,11 +18,11 @@ def get_python_executable(): def get_kernel_spec(kernel_spec_dict): - + kernel_spec = KernelSpec() for key in kernel_spec_dict: setattr(kernel_spec, key, kernel_spec_dict[key]) - + # Python interpreter used to start kernels if (kernel_spec.pyexec is None): pyexec = get_python_executable() diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index a944d66f53d..cd61efb1339 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -824,7 +824,7 @@ def test_dedicated_consoles(main_window, qtbot): # --- Assert only runfile text is present and there's no banner text --- # See spyder-ide/spyder#5301. text = control.toPlainText() - assert ('runfile' in text)# and not ('Python' in text or 'IPython' in text) + assert ('runfile' in text) and not ('Python' in text or 'IPython' in text) # --- Check namespace retention after re-execution --- with qtbot.waitSignal(shell.executed): diff --git a/spyder/plugins/debugger/widgets/framesbrowser.py b/spyder/plugins/debugger/widgets/framesbrowser.py index 461357ce352..2be89221f95 100644 --- a/spyder/plugins/debugger/widgets/framesbrowser.py +++ b/spyder/plugins/debugger/widgets/framesbrowser.py @@ -296,7 +296,7 @@ def on_config_kernel(self): """Ask shellwidget to send Pdb configuration to kernel.""" self.shellwidget.kernel_handler.kernel_comm.register_call_handler( "show_traceback", self.show_exception) - + self.shellwidget.set_kernel_configuration("pdb", { 'breakpoints': self.get_conf("breakpoints", default={}), 'pdb_ignore_lib': self.get_conf('pdb_ignore_lib'), diff --git a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py index 66f1589ea05..959470d15e1 100644 --- a/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py +++ b/spyder/plugins/ipythonconsole/tests/test_ipythonconsole.py @@ -897,7 +897,7 @@ def test_restart_kernel(ipyconsole, mocker, qtbot): qtbot.waitUntil( lambda: shell.spyder_kernel_ready and shell._prompt_html is not None, timeout=SHELL_TIMEOUT) - + # Check kernel is configured faulthandler_args = shell.kernel_handler._fault_args assert faulthandler_args is not None diff --git a/spyder/plugins/ipythonconsole/utils/kernel_handler.py b/spyder/plugins/ipythonconsole/utils/kernel_handler.py index 34be8903051..595b2d27d9e 100644 --- a/spyder/plugins/ipythonconsole/utils/kernel_handler.py +++ b/spyder/plugins/ipythonconsole/utils/kernel_handler.py @@ -471,7 +471,7 @@ def set_connection(self, connection_file, connection_info, self.kernel_client.hb_channel.time_to_dead = 25.0 self.start_channels() - + def start_channels(self): """Start channels""" # Start kernel diff --git a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py index d534d5885db..f86205b98ad 100644 --- a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py +++ b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py @@ -34,7 +34,8 @@ def test_kernel_pypath(ipyconsole, tmpdir, default_interpreter): os.environ['PYTHONPATH'] = pypath CONF.set('pythonpath_manager', 'spyder_pythonpath', [pypath]) - kernel_spec = get_kernel_spec(ipyconsole.get_widget().get_kernel_spec_dict()) + kernel_spec = get_kernel_spec( + ipyconsole.get_widget().get_kernel_spec_dict()) # Check that PYTHONPATH is not in our kernelspec # and pypath is in SPY_PYTHONPATH @@ -56,7 +57,8 @@ def test_python_interpreter(ipyconsole, tmpdir): CONF.set('main_interpreter', 'executable', interpreter) # Create a kernel spec - kernel_spec = get_kernel_spec(ipyconsole.get_widget().get_kernel_spec_dict()) + kernel_spec = get_kernel_spec( + ipyconsole.get_widget().get_kernel_spec_dict()) # Assert that the python interprerter is the default one assert interpreter not in kernel_spec.argv diff --git a/spyder/plugins/ipythonconsole/widgets/client.py b/spyder/plugins/ipythonconsole/widgets/client.py index 0afd0996e86..7747ed47303 100644 --- a/spyder/plugins/ipythonconsole/widgets/client.py +++ b/spyder/plugins/ipythonconsole/widgets/client.py @@ -614,7 +614,7 @@ def replace_kernel(self, kernel_handler, shutdown_kernel): self.disconnect_kernel(shutdown_kernel) # Reset shellwidget and print restart message self.shellwidget._shellwidget_state = "user_restart" - + self.connect_kernel(kernel_handler) def print_fault(self, fault): diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 1f31ae242ea..d76daab7ca1 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -1496,7 +1496,7 @@ def create_new_client(self, give_focus=True, filename='', special=None, # Connect kernel to client client.connect_kernel(kernel_handler) return client - + def get_kernel_spec_dict(self, pyexec=None): """Create a kernel spec dict""" kernel_spec = { @@ -1505,7 +1505,7 @@ def get_kernel_spec_dict(self, pyexec=None): "resource_dir": '', } if ( - pyexec is None + pyexec is None and not self.get_conf('default', section='main_interpreter') ): pyexec = self.get_conf( @@ -1514,7 +1514,7 @@ def get_kernel_spec_dict(self, pyexec=None): kernel_spec["pyexec"] = pyexec kernel_spec["env"] = self.get_kernel_spec_env(pyexec) return kernel_spec - + def get_kernel_spec_env(self, pyexec): """Env vars for kernels""" default_interpreter = self.get_conf( diff --git a/spyder/plugins/ipythonconsole/widgets/mixins.py b/spyder/plugins/ipythonconsole/widgets/mixins.py index a00bf3ca177..86c0038df43 100644 --- a/spyder/plugins/ipythonconsole/widgets/mixins.py +++ b/spyder/plugins/ipythonconsole/widgets/mixins.py @@ -187,7 +187,7 @@ def tunnel_ssh(self, hostname, port): def new_kernel(self, kernel_spec_dict): """Get a new kernel""" - + kernel_handler = KernelHandler.new_from_spec( kernel_spec_dict=kernel_spec_dict, hostname=self.ssh_remote_hostname, @@ -342,7 +342,6 @@ def check_cached_kernel_spec(self, kernel_spec_dict): _, ) = self._cached_kernel_properties - if "PYTEST_CURRENT_TEST" in cached_spec_dict["env"]: # Make tests faster by using cached kernels # hopefully the kernel will never use PYTEST_CURRENT_TEST diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index 70af216ceb5..891a3bb3451 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -403,7 +403,8 @@ def kernel_connect_sig(self): self._init_kernel_setup = True self.kernel_client.stopped_channels.connect(self.notify_deleted) - self.kernel_handler.sig_kernel_restarted.connect(self._handle_kernel_restarted) + self.kernel_handler.sig_kernel_restarted.connect( + self._handle_kernel_restarted) # For errors self.kernel_handler.kernel_comm.sig_exception_occurred.connect( diff --git a/spyder/plugins/maininterpreter/confpage.py b/spyder/plugins/maininterpreter/confpage.py index 71c9499e6af..ee24a57bb2e 100644 --- a/spyder/plugins/maininterpreter/confpage.py +++ b/spyder/plugins/maininterpreter/confpage.py @@ -147,8 +147,6 @@ def setup_page(self): password_radio.radiobutton.toggled.connect(keyfile.setDisabled) password_radio.radiobutton.toggled.connect(passphrase.setDisabled) - - # SSH authentication layout auth_layout = QGridLayout() auth_layout.addWidget(username, 0, 0, 1, 2) From f342f9f1c311199dd8ad490c6db48427358b5f46 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Fri, 17 Nov 2023 07:39:11 +0100 Subject: [PATCH 109/110] pep8 --- .../spyder-kernels-server/spyder_kernels_server/conda_utils.py | 2 +- .../spyder_kernels_server/kernel_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py index 6a759695df2..a7869df4e25 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/conda_utils.py @@ -132,7 +132,7 @@ def to_unicode_from_fs(string): """ Return a unicode version of string decoded using the file system encoding. """ - if not is_string(string): # string is a QString + if not is_string(string): # string is a QString string = to_text_string(string.toUtf8(), 'utf-8') else: if is_binary_string(string): diff --git a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py index 739287aa7d4..0326c0044c5 100644 --- a/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py +++ b/external-deps/spyder-kernels-server/spyder_kernels_server/kernel_server.py @@ -147,7 +147,7 @@ def open_kernel(self, kernel_spec): self.connect_std_pipes(kernel_key) kernel_manager.kernel_restarted.connect( - lambda connection_file=connection_file: + lambda connection_file=connection_file: self.sig_kernel_restarted.emit(connection_file) ) From 6eb55ce9437e134e2374c67fe31c35f4cda56292 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 18 Nov 2023 07:03:39 +0100 Subject: [PATCH 110/110] remonve no banner test --- spyder/app/tests/test_mainwindow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index cd61efb1339..22dde27d56c 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -821,10 +821,10 @@ def test_dedicated_consoles(main_window, qtbot): qtbot.waitUntil(lambda: nsb.editor.source_model.rowCount() == 4) assert nsb.editor.source_model.rowCount() == 4 - # --- Assert only runfile text is present and there's no banner text --- + # --- Assert only runfile text is present --- # See spyder-ide/spyder#5301. text = control.toPlainText() - assert ('runfile' in text) and not ('Python' in text or 'IPython' in text) + assert ('runfile' in text) # --- Check namespace retention after re-execution --- with qtbot.waitSignal(shell.executed):