Skip to content

Commit

Permalink
debug
Browse files Browse the repository at this point in the history
  • Loading branch information
@DerThorsten
DerThorsten committed Dec 11, 2023
1 parent d552808 commit 0758e4a
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 48 deletions.
88 changes: 57 additions & 31 deletions jupyterlite_xeus/add_on.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
"""a JupyterLite addon for creating the env for xeus-python"""
"""a JupyterLite addon for creating the env for xeus kernels"""
import json
import os
from pathlib import Path
from tempfile import TemporaryDirectory
import warnings

from jupyterlite_core.addons.federated_extensions import FederatedExtensionAddon
from jupyterlite_core.constants import (
Expand All @@ -17,13 +18,12 @@
from .prefix_bundler import get_prefix_bundler
from .create_conda_env import create_conda_env_from_yaml

EXTENSION_NAME = "xeus-python-kernel"
EXTENSION_NAME = "xeus"
STATIC_DIR = Path("@jupyterlite") / EXTENSION_NAME / "static"


def get_kernel_binaries(path):
""" return path to the kernel binary (js and wasm) if they exist, else None"""
print("considering", path)
json_file = path / "kernel.json"
if json_file.exists():

Expand All @@ -33,27 +33,18 @@ def get_kernel_binaries(path):

kernel_binary_js = Path(kernel_binary+".js")
kernel_binary_wasm = Path(kernel_binary+".wasm")



if kernel_binary_js.exists() and kernel_binary_wasm.exists():
return kernel_binary_js, kernel_binary_wasm
else:
print("kernel binary files not found")
print("kernel_binary_js.exists()", kernel_binary_js.exists())
print("kernel_binary_wasm.exists()", kernel_binary_wasm.exists())
warnings.warn(f"kernel binaries not found for {path.name}")

else:
print("no kernel.json found")
warnings.warn(f"kernel.json not found for {path.name}")

return None


class PackagesList(List):
def from_string(self, s):
return s.split(",")




class XeusAddon(FederatedExtensionAddon):
__all__ = ["post_build"]
Expand All @@ -78,30 +69,61 @@ def __init__(self, *args, **kwargs):

def post_build(self, manager):

# from prefix has higher priority than from environment file
if self.prefix:
# from existing prefix
yield from self.copy_kernels_from_prefix()
elif self.environment_file:
# from environment file
yield from self.create_and_copy_from_env()
else:
# check that either prefix or environment_file is set
if not self.prefix and not self.environment_file:
raise ValueError("Either prefix or environment_file must be set")

# create the prefix if it does not exist
if not self.prefix:
yield from self.create_prefix()

# copy the kernels from the prefix
yield from self.copy_kernels_from_prefix()

def create_and_copy_from_env(self):
# copy the jupyterlab extensions
#yield from self.copy_jupyterlab_extensions_from_prefix()

def create_prefix(self):
print("environment_file", self.environment_file)
# read the environment file
root_prefix = Path(self.cwd.name) / "env"
env_name = "xeus-python"
env_name = "xeus-env"
env_prefix = root_prefix / "envs" / env_name
self.prefix = str(env_prefix)
create_conda_env_from_yaml(
env_name=env_name,
root_prefix=root_prefix,
env_file=self.environment_file,
)
yield from self.copy_kernels_from_prefix()





def copy_jupyterlab_extensions_from_prefix(self):
# Find the federated extensions in the emscripten-env and install them
prefix = Path(self.prefix)
for pkg_json in self.env_extensions(prefix / SHARE_LABEXTENSIONS):
print("pkg_json", pkg_json)
yield from self.safe_copy_extension(pkg_json)


def safe_copy_extension(self, pkg_json):
"""Copy a labextension, and overwrite it
if it's already in the output
"""
pkg_path = pkg_json.parent
stem = json.loads(pkg_json.read_text(**UTF8))["name"]
dest = self.output_extensions / stem
file_dep = [
p for p in pkg_path.rglob("*") if not (p.is_dir() or self.is_ignored_sourcemap(p.name))
]

yield dict(
name=f"xeus:copy:ext:{stem}",
file_dep=file_dep,
actions=[(self.copy_one, [pkg_path, dest])],
)



def copy_kernels_from_prefix(self):
Expand Down Expand Up @@ -174,7 +196,7 @@ def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js):

# this part is a bit more complicated:
# Some kernels expect certain files to be at a certain places on the hard drive.
# Ie python (even pure python without additional packages) expects to find certail *.py
# Ie python (even pure python without additional packages) expects to find certain *.py
# files in a dir like $PREFIX/lib/python3.11/... .
# Since the kernels run in the browser we need a way to take the needed files from the
# $PREFIX of the emscripten-32 wasm env, bundle them into smth like tar.gz file(s) and
Expand All @@ -191,10 +213,14 @@ def copy_kernel(self, kernel_dir, kernel_wasm, kernel_js):
prefix_bundler_kwargs = kernel_spec["metadata"].get("prefix_bundler_kwargs", dict())


# THIS WILL BE REMOVED ONCE THE NEXT VERSION OF XPYTHON IS RELEASED
# (and the kernel.json file contains the prefix_bundler info)

if language == "python":
prefix_bundler_name = "empack"
# we can also drop the "if" above and just always use empack.
# but this will make the build a bit slower.
# Besides that, there should not be any harm in using empack for all kernels.
# If a kernel does not support empack, it will still just work and will
# **not ** do any extra work at runtime / kernel startup time.
prefix_bundler_name = "empack"



Expand Down
17 changes: 10 additions & 7 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ import { IKernel, IKernelSpecs } from '@jupyterlite/kernel';

import { WebWorkerKernel } from './web_worker_kernel';

const rel_path = '../extensions/@jupyterlite/xeus-python-kernel/static/';
const EXTENSION_NAME = 'xeus';
const EXTENSION_STATIC_DIR = `../extensions/@jupyterlite/${EXTENSION_NAME}/static/`;


// helper function to fetch json
function getPkgJson(url: string) {
const json_url = rel_path + url;
const json_url = EXTENSION_STATIC_DIR + url;
const xhr = new XMLHttpRequest();
xhr.open('GET', json_url, false);
xhr.send(null);
Expand All @@ -43,9 +45,9 @@ const kernel_specs = kernel_dir.map(kernel_dir => {
spec.dir = kernel_dir;
spec.resources = {
'logo-32x32':
rel_path + 'share/jupyter/kernels/' + kernel_dir + '/logo-32x32.png',
EXTENSION_STATIC_DIR + 'share/jupyter/kernels/' + kernel_dir + '/logo-32x32.png',
'logo-64x64':
rel_path + 'share/jupyter/kernels/' + kernel_dir + '/logo-64x64.png'
EXTENSION_STATIC_DIR + 'share/jupyter/kernels/' + kernel_dir + '/logo-64x64.png'
};
return spec;
});
Expand All @@ -68,9 +70,10 @@ const server_kernels = kernel_specs.map(spec => {
kernelspecs.register({
spec: spec,
create: async (options: IKernel.IOptions): Promise<IKernel> => {
const mountDrive = !!(
serviceWorker?.enabled && broadcastChannel?.enabled
);
// const mountDrive = !!(
// serviceWorker?.enabled && broadcastChannel?.enabled
// );
const mountDrive = false;

if (mountDrive) {
console.info(
Expand Down
61 changes: 51 additions & 10 deletions src/web_worker_kernel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,50 +23,89 @@ interface IXeusKernel {
processMessage(msg: any): Promise<void>;
}


export class WebWorkerKernel implements IKernel {
/**
* Instantiate a new WebWorkerKernel
*
* @param options The instantiation options for a new WebWorkerKernel
*/
constructor(options: WebWorkerKernel.IOptions, spec: any) {
console.log('constructing WebWorkerKernel kernel');
const { id, name, sendMessage, location } = options;
this._id = id;
this._name = name;
this._location = location;
this._spec = spec;
this._sendMessage = sendMessage;
console.log('constructing WebWorkerKernel worker');
this._worker = new Worker(new URL('./worker.js', import.meta.url), {
type: 'module'
});
console.log('constructing WebWorkerKernel done');

this._worker.onmessage = e => {
this._processWorkerMessage(e.data);
};

console.log("wrap");
this._remote = wrap(this._worker);
this._remote.processMessage({
msg: {
header: {
msg_type: 'initialize'
}
},
spec: this._spec
});
this.initFileSystem(options);
console.log("wrap done");

// this._remote.processMessage({
// msg: {
// header: {
// msg_type: 'initialize'
// }
// },
// spec: this._spec
// });

if(false){
console.log('init filesystem');
this.initFileSystem(options);

}
console.log('constructing WebWorkerKernel done2');
}

async handleMessage(msg: KernelMessage.IMessage): Promise<void> {
console.log('handleMessage', msg);
this._parent = msg;
this._parentHeader = msg.header;
console.log("send message to worker");
await this._sendMessageToWorker(msg);
console.log("send message to worker awaiting done");
}

private async _sendMessageToWorker(msg: any): Promise<void> {

if(this._first_message){
this._first_message = false;
console.log('first message');
await this._remote.ready();
console.log("waited for ready");

await this._remote.processMessage({
msg: {
header: {
msg_type: 'initialize'
}
},
spec: this._spec
});
console.log('first message done');
}


// TODO Remove this??
if (msg.header.msg_type !== 'input_reply') {
this._executeDelegate = new PromiseDelegate<void>();
}

console.log(' this._remote.processMessage({ msg, parent: this.parent });');
await this._remote.processMessage({ msg, parent: this.parent });
console.log(' this._remote.processMessage({ msg, parent: this.parent }); done');
if (msg.header.msg_type !== 'input_reply') {
return await this._executeDelegate.promise;
}
Expand Down Expand Up @@ -101,6 +140,7 @@ export class WebWorkerKernel implements IKernel {
* @param msg The worker message to process.
*/
private _processWorkerMessage(msg: any): void {
console.log('processWorkerMessage', msg);
if (!msg.header) {
return;
}
Expand Down Expand Up @@ -182,12 +222,13 @@ export class WebWorkerKernel implements IKernel {

await this._remote.ready();

if (options.mountDrive) {
if (false || options.mountDrive) {
await this._remote.mount(driveName, '/drive', PageConfig.getBaseUrl());
await this._remote.cd(localPath);
}
}

private _first_message: boolean = true;
private _spec: any;
private _id: string;
private _name: string;
Expand Down
9 changes: 9 additions & 0 deletions src/worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@
// Copyright (c) JupyterLite Contributors
// Distributed under the terms of the Modified BSD License.

console.log('worker loaded');

import { expose } from 'comlink';



import {
DriveFS,
DriveFSEmscriptenNodeOps,
Expand All @@ -15,6 +19,8 @@ declare function createXeusModule(options: any): any;

globalThis.Module = {};

console.log('worker here');

// const WASM_KERNEL_FILE = 'kernels/xlite/xlite.js';
// const WASM_FILE = 'kernels/xlite/xlite.wasm';
// TODO Remove this. This is to ensure we always perform node ops on Nodes and
Expand Down Expand Up @@ -99,6 +105,7 @@ async function get_stdin() {

class XeusKernel {
constructor(resolve: any) {
console.log('constructing kernel');
this._resolve = resolve;
}

Expand All @@ -107,6 +114,7 @@ class XeusKernel {
}

mount(driveName: string, mountpoint: string, baseUrl: string): void {
console.log('mounting drive');
const { FS, PATH, ERRNO_CODES } = globalThis.Module;

if (!FS) {
Expand Down Expand Up @@ -241,5 +249,6 @@ class XeusKernel {
}

globalThis.ready = new Promise(resolve => {
console.log('expose(new XeusKernel(resolve));');
expose(new XeusKernel(resolve));
});

0 comments on commit 0758e4a

Please sign in to comment.