Added automatic requirements installation from docstring

main.py

@@ -26,6 +26,9 @@
 import json
 import uuid
 import sys
+import ast
+import re
+import subprocess
 
 
 from config import API_KEY, PIPELINES_DIR
@@ -106,7 +109,31 @@ def get_all_pipelines():
     return pipelines
 
 
+def get_docstring(module_path):
+    with open(module_path, "r") as file:
+        tree = ast.parse(file.read())
+        return ast.get_docstring(tree)
+
+
+def find_requirements(docstring):
+    result = re.search(r"requirements:(.*)", docstring)
+    if result is None:
+        return ""
+    return result.group(1)
+
+
+def pip_install_requirements(requirements):
+    requirements = requirements.split()
+    if len(requirements) <= 0:
+        return
+    pip = subprocess.Popen(["pip", "install"] + requirements, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, _ = pip.communicate()
+
+
 async def load_module_from_path(module_name, module_path):
+    docstring = get_docstring(module_path)
+    requirements = find_requirements(docstring)
+    pip_install_requirements(requirements)
     spec = importlib.util.spec_from_file_location(module_name, module_path)
     module = importlib.util.module_from_spec(spec)
 

start.sh

@@ -81,30 +81,6 @@ download_pipelines() {
   fi
 }
 
-# Function to parse and install requirements from frontmatter
-install_frontmatter_requirements() {
-  local file=$1
-  local file_content=$(cat "$1")
-  # Extract the first triple-quoted block
-  local first_block=$(echo "$file_content" | awk '/"""/{flag=!flag; if(flag) count++; if(count == 2) {exit}} flag' )
-
-  # Check if the block contains requirements
-  local requirements=$(echo "$first_block" | grep -i 'requirements:')
-
-  if [ -n "$requirements" ]; then
-    # Extract the requirements list
-    requirements=$(echo "$requirements" | awk -F': ' '{print $2}' | tr ',' ' ' | tr -d '\r')
-
-    # Construct and echo the pip install command
-    local pip_command="pip install $requirements"
-    echo "$pip_command"
-    pip install $requirements
-  else
-    echo "No requirements found in frontmatter of $file."
-  fi
-}
-
-
 
 # Check if PIPELINES_URLS environment variable is set and non-empty
 if [[ -n "$PIPELINES_URLS" ]]; then
@@ -116,12 +92,6 @@ if [[ -n "$PIPELINES_URLS" ]]; then
   for path in "${ADDR[@]}"; do
     download_pipelines "$path" "$pipelines_dir"
   done
-
-  for file in "$pipelines_dir"/*; do
-    if [[ -f "$file" ]]; then
-      install_frontmatter_requirements "$file"
-    fi
-  done
 else
   echo "PIPELINES_URLS not specified. Skipping pipelines download and installation."
 fi