kubernetes

.github/workflows/deploy-to-kubernetes.yml

@@ -0,0 +1,29 @@
+name: Deploy to Kubernetes
+
+on:
+  push:
+    branches:
+      - master
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Kubectl
+      uses: azure/setup-kubectl@v1
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Update Kubernetes Deployment
+      run: |
+        kubectl apply -f path/to/cogvlm-deployment.yml
+        kubectl rollout status deployment/cogvlm-deployment

.gitignore

@@ -19,6 +19,7 @@ eggs/
 .eggs/
 lib/
 lib64/
+lambda/node_modules/
 errors.txt
 # Local .terraform directories
 **/.terraform/*

Dockerfiles/Dockerfilecogvlm

@@ -0,0 +1,40 @@
+FROM nvidia/cuda:12.1.1-devel-ubuntu22.04 as builder
+
+# Set environment variables
+ENV BASE_IMG=nvidia/cuda:12.1.1-devel-ubuntu22.04
+ENV DOCKER_BUILDKIT=1
+ENV WORLD_SIZE=4
+ENV ARTIFACTS_PATH=/app/artifacts
+ENV STORAGE_PATH=/app/storage
+ENV HF_HUB_ENABLE_HF_TRANSFER=True
+
+# Set the working directory to the root
+WORKDIR /
+
+# Install Python 3.10 and other necessary system packages
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    software-properties-common \
+    && add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    python3.10 python3.10-dev python3.10-distutils python3-pip python3.10-venv openmpi-bin libopenmpi-dev \
+    && python3.10 -m pip install --no-cache-dir --upgrade pip setuptools wheel
+
+# Copy the requirements.txt file into the container
+COPY requirements.txt .
+
+# Install Python dependencies from requirements.txt
+RUN python3.10 -m pip install -r requirements.txt
+
+# Adjust the working directory to where your application's code will reside
+WORKDIR /swarms-cloud/servers
+
+# Assuming your application's entire directory structure needs to be copied,
+# Adjust the COPY command to ensure the entire application is available in the container
+COPY . /swarms-cloud
+
+# Expose the port the app runs on
+EXPOSE 8000
+
+# Command to run the application
+CMD ["python3.10", "-m", "uvicorn", "cogvlm:app", "--host", "0.0.0.0", "--port", "8000"]

lambda/index.js

@@ -0,0 +1,34 @@
+const express = require('express');
+const axios = require('axios');
+const awsServerlessExpress = require('aws-serverless-express');
+const app = express();
+app.use(express.json());
+
+const modelEndpoints = {
+  'cogvlm-chat-17b': process.env.COGVLM_ENDPOINT,
+  'qwenvl-chat': process.env.QWENVL_ENDPOINT,
+  // Add more models and their endpoints as necessary
+};
+
+app.post('/v1/chat/completions', async (req, res) => {
+  const modelName = req.body.model;
+  const endpoint = modelEndpoints[modelName];
+
+  if (!endpoint) {
+    return res.status(404).send({ error: 'Model not found' });
+  }
+
+  try {
+    const modelResponse = await axios.post(endpoint, req.body);
+    res.json(modelResponse.data);
+  } catch (error) {
+    res.status(500).send({ error: 'Error forwarding request to model' });
+  }
+});
+
+const server = awsServerlessExpress.createServer(app);
+
+exports.handler = (event, context) => {
+  context.callbackWaitsForEmptyEventLoop = false; // Set to false to speed up successive invocations
+  awsServerlessExpress.proxy(server, event, context);
+};