diff --git a/main.py b/main.py index 7c6d7e99..b70fb12c 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,9 @@ import json import uuid import sys +import ast +import re +import subprocess from config import API_KEY, PIPELINES_DIR @@ -106,7 +109,31 @@ def get_all_pipelines(): return pipelines +def get_docstring(module_path): + with open(module_path, "r") as file: + tree = ast.parse(file.read()) + return ast.get_docstring(tree) + + +def find_requirements(docstring): + result = re.search(r"requirements:(.*)", docstring) + if result is None: + return "" + return result.group(1) + + +def pip_install_requirements(requirements): + requirements = requirements.split() + if len(requirements) <= 0: + return + pip = subprocess.Popen(["pip", "install"] + requirements, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, _ = pip.communicate() + + async def load_module_from_path(module_name, module_path): + docstring = get_docstring(module_path) + requirements = find_requirements(docstring) + pip_install_requirements(requirements) spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) diff --git a/start.sh b/start.sh index a6fd1f63..cfeafb51 100755 --- a/start.sh +++ b/start.sh @@ -81,30 +81,6 @@ download_pipelines() { fi } -# Function to parse and install requirements from frontmatter -install_frontmatter_requirements() { - local file=$1 - local file_content=$(cat "$1") - # Extract the first triple-quoted block - local first_block=$(echo "$file_content" | awk '/"""/{flag=!flag; if(flag) count++; if(count == 2) {exit}} flag' ) - - # Check if the block contains requirements - local requirements=$(echo "$first_block" | grep -i 'requirements:') - - if [ -n "$requirements" ]; then - # Extract the requirements list - requirements=$(echo "$requirements" | awk -F': ' '{print $2}' | tr ',' ' ' | tr -d '\r') - - # Construct and echo the pip install command - local pip_command="pip install $requirements" - echo "$pip_command" - pip install $requirements - else - echo "No requirements found in frontmatter of $file." - fi -} - - # Check if PIPELINES_URLS environment variable is set and non-empty if [[ -n "$PIPELINES_URLS" ]]; then @@ -116,12 +92,6 @@ if [[ -n "$PIPELINES_URLS" ]]; then for path in "${ADDR[@]}"; do download_pipelines "$path" "$pipelines_dir" done - - for file in "$pipelines_dir"/*; do - if [[ -f "$file" ]]; then - install_frontmatter_requirements "$file" - fi - done else echo "PIPELINES_URLS not specified. Skipping pipelines download and installation." fi