Skip to content

Commit

Permalink
add startup script for local jupyter connecting through ocean-spark-c…
Browse files Browse the repository at this point in the history
…onnect (#16)
  • Loading branch information
sigmarkarl authored Feb 8, 2024
1 parent 8d2da7b commit bc9817d
Showing 1 changed file with 10 additions and 4 deletions.
14 changes: 10 additions & 4 deletions jupyter_server/gateway/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,17 @@ async def start_kernel(self, *, kernel_id=None, path=None, **kwargs):
kwargs["kernel_name"] = "python3"
kwargs["local"] = True
env = kwargs["env"]
app = kernel_name[3:]
startup_file = f"/tmp/{app}.py"
if kernel_name.startswith('sc-'):
startup_content = f"""from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('{kernel_name}').remote('sc://{app}.spark-apps.svc.cluster.local').getOrCreate()
app = kernel_name
startup_file = f"/tmp/{app}.py"
if os.getenv("CHOWN_HOME", "no") == "yes":
startup_content = f"""from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('{kernel_name}').remote('sc://{app}-driver-svc.spark-apps.svc.cluster.local').getOrCreate()
"""
else:
app_short = app[0:-17]
startup_content = f"""from ocean_spark_connect.ocean_spark_session import OceanSparkSession
spark = OceanSparkSession.Builder().cluster_id("osc-739db584").appid("{app_short}").profile("default").getOrCreate()
"""
env["PYTHONSTARTUP"] = startup_file
with open(startup_file, "w") as f:
Expand Down

0 comments on commit bc9817d

Please sign in to comment.