From 2388ce2f1688a576f0028466bbdc834b6e027b57 Mon Sep 17 00:00:00 2001 From: Jonathan McPherson Date: Wed, 18 Dec 2024 16:38:12 -0800 Subject: [PATCH 1/4] attempt to reconnect socket if session didn't exit --- .../src/KallichoreAdapterApi.ts | 38 ++++++++++++++----- .../src/KallichoreSession.ts | 9 +++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts index 7efbe2602dd..0414296997e 100644 --- a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts +++ b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts @@ -13,7 +13,7 @@ import { findAvailablePort } from './PortFinder'; import { KallichoreAdapterApi } from './positron-supervisor'; import { JupyterKernelExtra, JupyterKernelSpec, JupyterLanguageRuntimeSession } from './jupyter-adapter'; import { KallichoreSession } from './KallichoreSession'; -import { Barrier, PromiseHandles } from './async'; +import { Barrier, PromiseHandles, withTimeout } from './async'; import { LogStreamer } from './LogStreamer'; import { createUniqueId, summarizeHttpError } from './util'; @@ -479,15 +479,31 @@ export class KCApi implements KallichoreAdapterApi { * @param session The session to add the disconnect handler to */ private addDisconnectHandler(session: KallichoreSession) { - session.disconnected.event(async (state: positron.RuntimeState) => { + this._disposables.push(session.disconnected.event(async (state: positron.RuntimeState) => { if (state !== positron.RuntimeState.Exited) { // The websocket disconnected while the session was still // running. This could signal a problem with the supervisor; we // should see if it's still running. this._log.appendLine(`Session '${session.metadata.sessionName}' disconnected while in state '${state}'. This is unexpected; checking server status.`); - await this.testServerExited(); + + // If the server did not exit, and the session also appears to + // still be running, try to reconnect the websocket. It's + // possible the connection just got dropped or interrupted. + const exited = await this.testServerExited(); + if (!exited) { + this._log.appendLine(`The server is still running; attempting to reconnect to session ${session.metadata.sessionId}`); + try { + await withTimeout(session.connect(), 2000, `Timed out reconnecting to session ${session.metadata.sessionId}`); + this._log.appendLine(`Successfully restored connection to ${session.metadata.sessionId}`); + } catch (err) { + // The session could not be reconnected; mark it as + // offline and explain to the user what happened. + session.markOffline('Lost connection to the session WebSocket event stream'); + vscode.window.showErrorMessage(vscode.l10n.t('Unable to re-establish connection to {0}: {1}', session.metadata.sessionName, err)); + } + } } - }); + })); } /** @@ -501,13 +517,14 @@ export class KCApi implements KallichoreAdapterApi { * unresponsive. * * @returns A promise that resolves when the server has been confirmed to be - * running or has been restarted. + * running or has been restarted. Resolves with `true` if the server did in fact exit, `false` otherwise. */ - private async testServerExited() { + private async testServerExited(): Promise { // If we're currently starting, it doesn't make sense to test the // server status since we're already in the process of starting it. if (this._starting) { - return this._starting.promise; + await this._starting.promise; + return false; } // Load the server state so we can check the process ID @@ -517,7 +534,7 @@ export class KCApi implements KallichoreAdapterApi { // If there's no server state, return as we can't check its status if (!serverState) { this._log.appendLine(`No Kallichore server state found; cannot test server process`); - return; + return false; } // Test the process ID to see if the server is still running. @@ -536,7 +553,7 @@ export class KCApi implements KallichoreAdapterApi { // The server is still running; nothing to do if (serverRunning) { - return; + return false; } // Clean up the state so we don't try to reconnect to a server that @@ -568,6 +585,9 @@ export class KCApi implements KallichoreAdapterApi { vscode.window.showInformationMessage( vscode.l10n.t('The process supervising the interpreters has exited unexpectedly and could not automatically restarted: ' + err)); } + + // The server did exit. + return true; } /** diff --git a/extensions/positron-supervisor/src/KallichoreSession.ts b/extensions/positron-supervisor/src/KallichoreSession.ts index 3b1fd2053bf..703ee0c0364 100644 --- a/extensions/positron-supervisor/src/KallichoreSession.ts +++ b/extensions/positron-supervisor/src/KallichoreSession.ts @@ -1165,6 +1165,15 @@ export class KallichoreSession implements JupyterLanguageRuntimeSession { this.onExited(exitCode); } + /** + * Marks the kernel as offline. + * + * @param reason The reason for the kernel going offline + */ + markOffline(reason: string) { + this.onStateChange(positron.RuntimeState.Offline, reason); + } + private onExited(exitCode: number) { if (this._restarting) { // If we're restarting, wait for the kernel to start up again From 245155223714c6602532f3164a1f04161515c445 Mon Sep 17 00:00:00 2001 From: Jonathan McPherson Date: Wed, 18 Dec 2024 16:48:40 -0800 Subject: [PATCH 2/4] clean up some eslint --- extensions/positron-supervisor/src/KallichoreAdapterApi.ts | 4 ++-- extensions/positron-supervisor/src/KallichoreSession.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts index 0414296997e..f6eece178fd 100644 --- a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts +++ b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts @@ -216,7 +216,7 @@ export class KCApi implements KallichoreAdapterApi { // Start the server in a new terminal this._log.appendLine(`Starting Kallichore server ${shellPath} on port ${port}`); - const terminal = vscode.window.createTerminal({ + const terminal = vscode.window.createTerminal({ name: 'Kallichore', shellPath: wrapperPath, shellArgs: [ @@ -231,7 +231,7 @@ export class KCApi implements KallichoreAdapterApi { message: `*** Kallichore Server (${shellPath}) ***`, hideFromUser: !showTerminal, isTransient: false - }); + } satisfies vscode.TerminalOptions); // Flag to track if the terminal exited before the start barrier opened let exited = false; diff --git a/extensions/positron-supervisor/src/KallichoreSession.ts b/extensions/positron-supervisor/src/KallichoreSession.ts index 703ee0c0364..8ff1c37fef8 100644 --- a/extensions/positron-supervisor/src/KallichoreSession.ts +++ b/extensions/positron-supervisor/src/KallichoreSession.ts @@ -571,8 +571,8 @@ export class KallichoreSession implements JupyterLanguageRuntimeSession { type === positron.RuntimeClientType.IPyWidgetControl) { const msg: JupyterCommOpen = { - target_name: type, // eslint-disable-line - comm_id: id, // eslint-disable-line + target_name: type, + comm_id: id, data: params }; const commOpen = new CommOpenCommand(msg, metadata); From e29a4291de80e8798a0b01f1d36efcd9160149f4 Mon Sep 17 00:00:00 2001 From: Jonathan McPherson Date: Thu, 19 Dec 2024 08:19:33 -0800 Subject: [PATCH 3/4] add a debug command to force disconnect --- extensions/positron-supervisor/package.json | 7 ++++ .../positron-supervisor/package.nls.json | 3 +- .../src/KallichoreAdapterApi.ts | 35 +++++++++++++++++++ .../src/KallichoreSession.ts | 7 ++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/extensions/positron-supervisor/package.json b/extensions/positron-supervisor/package.json index 1e55ea4d228..41e3b39fdb8 100644 --- a/extensions/positron-supervisor/package.json +++ b/extensions/positron-supervisor/package.json @@ -83,6 +83,13 @@ "category": "%command.positron.supervisor.category%", "title": "%command.showKernelSupervisorLog.title%", "shortTitle": "%command.showKernelSupervisorLog.title%" + }, + { + "command": "positron.supervisor.reconnectSession", + "category": "%command.positron.supervisor.category%", + "title": "%command.reconnectSession.title%", + "shortTitle": "%command.reconnectSession.title%", + "enablement": "isDevelopment" } ] }, diff --git a/extensions/positron-supervisor/package.nls.json b/extensions/positron-supervisor/package.nls.json index 8f4b97ae176..b37e03e37a8 100644 --- a/extensions/positron-supervisor/package.nls.json +++ b/extensions/positron-supervisor/package.nls.json @@ -13,5 +13,6 @@ "configuration.attachOnStartup.description": "Run before starting up Jupyter kernel (when supported)", "configuration.sleepOnStartup.description": "Sleep for n seconds before starting up Jupyter kernel (when supported)", "command.positron.supervisor.category": "Kernel Supervisor", - "command.showKernelSupervisorLog.title": "Show the Kernel Supervisor Log" + "command.showKernelSupervisorLog.title": "Show the Kernel Supervisor Log", + "command.reconnectSession.title": "Reconnect the current session" } diff --git a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts index f6eece178fd..b62efee595c 100644 --- a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts +++ b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts @@ -93,6 +93,10 @@ export class KCApi implements KallichoreAdapterApi { this._log.appendLine(`Failed to start Kallichore server: ${err}`); }); } + + _context.subscriptions.push(vscode.commands.registerCommand('positron.supervisor.reconnectSession', () => { + this.reconnectActiveSession(); + })); } /** @@ -725,4 +729,35 @@ export class KCApi implements KallichoreAdapterApi { throw new Error(`Kallichore server not found (expected at ${embeddedBinary})`); } + + /** + * Reconnects to the active session, if one exists. Primarily useful as a + * troubleshooting tool. + */ + async reconnectActiveSession() { + // Get the foreground session from the Positron API + const session = await positron.runtime.getForegroundSession(); + if (!session) { + vscode.window.showInformationMessage(vscode.l10n.t('No active session to reconnect to')); + return; + } + + // Find the session in our list + const kallichoreSession = this._sessions.find(s => s.metadata.sessionId === session.metadata.sessionId); + if (!kallichoreSession) { + vscode.window.showInformationMessage(vscode.l10n.t('Active session {0} not managed by the kernel supervisor', session.metadata.sessionName)); + return; + } + + // Ensure the session is still active + if (kallichoreSession.runtimeState === positron.RuntimeState.Exited) { + vscode.window.showInformationMessage(vscode.l10n.t('Session {0} is not running', session.metadata.sessionName)); + return; + } + + // Disconnect the session; since the session is active, this should + // trigger a reconnect. + kallichoreSession.log('Disconnecting by user request', vscode.LogLevel.Info); + kallichoreSession.disconnect(); + } } diff --git a/extensions/positron-supervisor/src/KallichoreSession.ts b/extensions/positron-supervisor/src/KallichoreSession.ts index 8ff1c37fef8..f925bfe1f8a 100644 --- a/extensions/positron-supervisor/src/KallichoreSession.ts +++ b/extensions/positron-supervisor/src/KallichoreSession.ts @@ -1044,6 +1044,13 @@ export class KallichoreSession implements JupyterLanguageRuntimeSession { this._disposables = []; } + /** + * Disconnect the session + */ + public disconnect() { + this._socket?.ws.close(); + } + /** * Main entry point for handling messages delivered over the websocket from * the Kallichore server. From 950cdbae2edb967cca488b3bd7de7811caaf6130 Mon Sep 17 00:00:00 2001 From: Jonathan McPherson Date: Thu, 19 Dec 2024 08:25:40 -0800 Subject: [PATCH 4/4] small tweaks --- extensions/positron-supervisor/src/KallichoreAdapterApi.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts index b62efee595c..58482250fe1 100644 --- a/extensions/positron-supervisor/src/KallichoreAdapterApi.ts +++ b/extensions/positron-supervisor/src/KallichoreAdapterApi.ts @@ -502,7 +502,7 @@ export class KCApi implements KallichoreAdapterApi { } catch (err) { // The session could not be reconnected; mark it as // offline and explain to the user what happened. - session.markOffline('Lost connection to the session WebSocket event stream'); + session.markOffline('Lost connection to the session WebSocket event stream and could not restore it: ' + err); vscode.window.showErrorMessage(vscode.l10n.t('Unable to re-establish connection to {0}: {1}', session.metadata.sessionName, err)); } } @@ -520,8 +520,9 @@ export class KCApi implements KallichoreAdapterApi { * handle the case where the server process is running but it's become * unresponsive. * - * @returns A promise that resolves when the server has been confirmed to be - * running or has been restarted. Resolves with `true` if the server did in fact exit, `false` otherwise. + * @returns A promise that resolves when the server has been confirmed to + * be running or has been restarted. Resolves with `true` if the server did + * in fact exit, `false` otherwise. */ private async testServerExited(): Promise { // If we're currently starting, it doesn't make sense to test the