Don't embed Python packages in workerd. (#1580)

* Don't embed Python packages in workerd. * Fixes --------- Co-authored-by: Hood Chatham <[email protected]>
cloudflare · Jan 29, 2024 · fa6bac1 · fa6bac1
1 parent 5d27f8f
commit fa6bac1
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 9 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -114,9 +114,9 @@ http_archive(
 http_archive(
     name = "pyodide_packages",
     build_file = "//:build/BUILD.pyodide_packages",
-    sha256 = "048d42372928aaef4dc565f507ec39fe9a4954e62bdb9163aaee1e106cffb6d8",
+    sha256 = "f9aa1e044567f1f3e36c3516d066481093dbc116032c45294eee400628d8b4a2",
     type = "zip",
-    urls = ["https://github.com/dom96/pyodide_packages/releases/download/v0.11/pyodide_packages.tar.zip"],
+    urls = ["https://github.com/dom96/pyodide_packages/releases/download/just-micropip/pyodide_packages.tar.zip"],
 )
 
 # ========================================================================================

diff --git a/samples/pyodide-fastapi/config.capnp b/samples/pyodide-fastapi/config.capnp
@@ -23,7 +23,8 @@ const config :Workerd.Config = (
 const mainWorker :Workerd.Worker = (
   modules = [
     (name = "worker", pythonModule = embed "./worker.py"),
-    (name = "fastapi", pythonRequirement = "fastapi"),
+    (name = "fastapi==0.103.2", pythonRequirement = ""),
+    (name = "ssl", pythonRequirement = ""),
   ],
   compatibilityDate = "2023-12-18",
   compatibilityFlags = ["experimental"],

diff --git a/samples/pyodide-fastapi/worker.py b/samples/pyodide-fastapi/worker.py
@@ -6,8 +6,6 @@ async def fetch(request):
 
 def test():
     import fastapi
-    import langchain
-    import asgi
 
 
 # Set up fastapi app

diff --git a/samples/pyodide-langchain/config.capnp b/samples/pyodide-langchain/config.capnp
@@ -0,0 +1,35 @@
+using Workerd = import "/workerd/workerd.capnp";
+
+const config :Workerd.Config = (
+  services = [
+    (name = "main", worker = .mainWorker),
+  ],
+
+  sockets = [
+    # Serve HTTP on port 8080.
+    ( name = "http",
+      address = "*:8080",
+      http = (),
+      service = "main"
+    ),
+  ],
+  autogates = [
+    # Pyodide is included as a builtin wasm module so it requires the
+    # corresponding autogate flag.
+    "workerd-autogate-builtin-wasm-modules",
+  ]
+);
+
+const mainWorker :Workerd.Worker = (
+  modules = [
+    (name = "worker", pythonModule = embed "./worker.py"),
+    (name = "aiohttp", pythonRequirement = "aiohttp"),
+    (name = "ssl", pythonRequirement = "ssl"),
+    (name = "langchain==0.0.339", pythonRequirement = ""),
+    (name = "openai==0.28.1", pythonRequirement = ""),
+  ],
+  compatibilityDate = "2023-12-18",
+  compatibilityFlags = ["experimental"],
+  # Learn more about compatibility dates at:
+  # https://developers.cloudflare.com/workers/platform/compatibility-dates/
+);
diff --git a/samples/pyodide-langchain/worker.py b/samples/pyodide-langchain/worker.py
@@ -0,0 +1,16 @@
+from js import Response
+
+
+def fetch(request):
+    return Response.new("hello world")
+
+
+from langchain.chat_models import ChatOpenAI
+import openai
+
+API_KEY = "sk-abcdefgh"
+
+
+def test():
+    ChatOpenAI(openai_api_key=API_KEY)
+    print("OK?")
diff --git a/samples/pyodide/worker.py b/samples/pyodide/worker.py
@@ -2,4 +2,8 @@
 
 
 def fetch(request):
-  return Response.new("hello world")
+    return Response.new("hello world")
+
+
+def test():
+    print("Hi there, this is a test")
diff --git a/src/pyodide/python-entrypoint-helper.js b/src/pyodide/python-entrypoint-helper.js
@@ -12,8 +12,8 @@ function initializePackageIndex(pyodide) {
     );
   }
   const API = pyodide._api;
-  API.config.indexURL = "https://cdn.jsdelivr.net/pyodide/v0.25.0a2/full/";
-  globalThis.location = "https://cdn.jsdelivr.net/pyodide/v0.25.0a2/full/";
+  API.config.indexURL = "https://cdn.jsdelivr.net/pyodide/v0.25.0/full/";
+  globalThis.location = "https://cdn.jsdelivr.net/pyodide/v0.25.0/full/";
   API.lockfile_info = lockfile.info;
   API.lockfile_packages = lockfile.packages;
   API.repodata_packages = lockfile.packages;
@@ -49,7 +49,7 @@ function initializePackageIndex(pyodide) {
   };
 }
 
-// These packages are currently embedded inside workerd and so don't need to
+// These packages are currently embedded inside EW and so don't need to
 // be separately installed.
 const EMBEDDED_PYTHON_PACKAGES = [
   "aiohttp",
@@ -133,6 +133,7 @@ function transformMetadata(metadata) {
 async function setupPackages(pyodide) {
   // The metadata is a JSON-serialised WorkerBundle (defined in pipeline.capnp).
   const metadata = transformMetadata(origMetadata);
+  const isWorkerd = metadata.modules !== undefined;
 
   initializePackageIndex(pyodide);
 
@@ -150,6 +151,13 @@ async function setupPackages(pyodide) {
 
     if (value.pythonRequirement !== undefined) {
       requirements.push(name);
+      // Packages are not embedded in workerd.
+      // TODO: Improve package loading in workerd.
+      if (isWorkerd) {
+        micropipRequirements.push(name);
+        continue;
+      }
+
       if (!EMBEDDED_PYTHON_PACKAGES.includes(name)) {
         pythonRequirements.push(name);
       }
@@ -163,6 +171,7 @@ async function setupPackages(pyodide) {
   if (micropipRequirements.length > 0) {
     // Micropip and ssl packages are pre-loaded via the packages tarball. This means
     // we should be able to load micropip directly now.
+
     const micropip = pyodide.pyimport("micropip");
     await micropip.install(micropipRequirements);
   }