From 7b3fd75b0e6b07ff58557798b2b33c40c8eb6133 Mon Sep 17 00:00:00 2001 From: Kye Date: Wed, 3 Apr 2024 08:12:48 -0400 Subject: [PATCH] [CLEANUP] --- delete_all_services.sh | 177 +++++++++++++++++++++++++++++++++++++++ servers/cogvlm/cogvlm.py | 68 +++++++-------- 2 files changed, 209 insertions(+), 36 deletions(-) create mode 100755 delete_all_services.sh diff --git a/delete_all_services.sh b/delete_all_services.sh new file mode 100755 index 0000000..2ead776 --- /dev/null +++ b/delete_all_services.sh @@ -0,0 +1,177 @@ +#!/bin/bash + +# This script is intended to delete resources for EC2, ECS, and EKS across all regions. Use with extreme caution. + +# Function to delete EC2 instances in all regions +delete_ec2_instances() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 instances in region $region..." + instances=$(aws ec2 describe-instances --region $region --query "Reservations[*].Instances[*].InstanceId" --output text --filters "Name=instance-state-name,Values=running") + if [ -n "$instances" ]; then + aws ec2 terminate-instances --instance-ids $instances --region $region + echo "EC2 instances terminated in region $region" + else + echo "No running EC2 instances to delete in region $region" + fi + done +} + +# Function to delete ECS clusters in all regions +delete_ecs_clusters() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting ECS clusters in region $region..." + clusters=$(aws ecs list-clusters --region $region --query "clusterArns[]" --output text) + for cluster in $clusters; do + aws ecs update-cluster-settings --cluster $cluster --region $region --settings name=containerInsights,value=disabled + services=$(aws ecs list-services --cluster $cluster --region $region --query "serviceArns[]" --output text) + if [ -n "$services" ]; then + aws ecs update-service --cluster $cluster --service $services --desired-count 0 --region $region + aws ecs delete-service --cluster $cluster --service $services --force --region $region + fi + aws ecs delete-cluster --cluster $cluster --region $region + echo "ECS cluster deleted in region $region" + done + done +} + +# Function to delete EKS clusters in all regions +delete_eks_clusters() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EKS clusters in region $region..." + clusters=$(aws eks list-clusters --region $region --query "clusters[]" --output text) + for cluster in $clusters; do + aws eks delete-cluster --name $cluster --region $region + echo "EKS cluster $cluster deleted in region $region" + done + done +} + +# Function to delete ECR repositories in all regions +delete_ecr_repositories() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting ECR repositories in region $region..." + repositories=$(aws ecr describe-repositories --region $region --query "repositories[].repositoryName" --output text) + for repo in $repositories; do + images=$(aws ecr list-images --repository-name $repo --region $region --query 'imageIds[*]' --output text) + if [ -n "$images" ]; then + aws ecr batch-delete-image --repository-name $repo --image-ids imageTag=$images --region $region + echo "Images deleted from $repo in region $region" + fi + aws ecr delete-repository --repository-name $repo --region $region --force + echo "ECR repository $repo deleted in region $region" + done + done +} + + +# Function to delete Auto Scaling groups in all regions +delete_auto_scaling_groups() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting Auto Scaling groups in region $region..." + asgs=$(aws autoscaling describe-auto-scaling-groups --region $region --query "AutoScalingGroups[].AutoScalingGroupName" --output text) + for asg in $asgs; do + aws autoscaling update-auto-scaling-group --auto-scaling-group-name $asg --min-size 0 --max-size 0 --desired-capacity 0 --region $region + aws autoscaling delete-auto-scaling-group --auto-scaling-group-name $asg --force-delete --region $region + echo "Auto Scaling group $asg deleted in region $region" + done + done +} + +# Function to delete Load Balancers (ELB and ALB) in all regions +delete_load_balancers() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting Load Balancers in region $region..." + # Classic Load Balancers + clbs=$(aws elb describe-load-balancers --region $region --query "LoadBalancerDescriptions[].LoadBalancerName" --output text) + for clb in $clbs; do + aws elb delete-load-balancer --load-balancer-name $clb --region $region + echo "Classic Load Balancer $clb deleted in region $region" + done + # Application and Network Load Balancers + anlbs=$(aws elbv2 describe-load-balancers --region $region --query "LoadBalancers[].LoadBalancerArn" --output text) + for anlb in $anlbs; do + aws elbv2 delete-load-balancer --load-balancer-arn $anlb --region $region + echo "Application/Network Load Balancer $anlb deleted in region $region" + done + done +} + + +######## ------ Delete all services +delete_ec2_volumes() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 volumes in region $region..." + volumes=$(aws ec2 describe-volumes --region $region --query "Volumes[].VolumeId" --output text) + for volume in $volumes; do + aws ec2 delete-volume --volume-id $volume --region $region + echo "Volume $volume deleted in region $region" + done + done +} + +delete_ec2_snapshots() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 snapshots in region $region..." + snapshots=$(aws ec2 describe-snapshots --owner-ids $(aws sts get-caller-identity --query "Account" --output text) --region $region --query "Snapshots[].SnapshotId" --output text) + for snapshot in $snapshots; do + aws ec2 delete-snapshot --snapshot-id $snapshot --region $region + echo "Snapshot $snapshot deleted in region $region" + done + done +} + +delete_ec2_key_pairs() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 key pairs in region $region..." + key_pairs=$(aws ec2 describe-key-pairs --region $region --query "KeyPairs[].KeyName" --output text) + for key_pair in $key_pairs; do + aws ec2 delete-key-pair --key-name $key_pair --region $region + echo "Key pair $key_pair deleted in region $region" + done + done +} + +delete_ec2_security_groups() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 security groups in region $region..." + # Exclude default security group + security_groups=$(aws ec2 describe-security-groups --region $region --query "SecurityGroups[?GroupName != 'default'].GroupId" --output text) + for sg in $security_groups; do + aws ec2 delete-security-group --group-id $sg --region $region + echo "Security group $sg deleted in region $region" + done + done +} + +delete_ec2_amis() { + for region in $(aws ec2 describe-regions --query "Regions[].RegionName" --output text); do + echo "Deleting EC2 AMIs in region $region..." + amis=$(aws ec2 describe-images --owners self --region $region --query "Images[].ImageId" --output text) + for ami in $amis; do + aws ec2 deregister-image --image-id $ami --region $region + echo "AMI $ami deregistered in region $region" + done + done +} + +# Make sure to call these functions in your script + + + + + +# Call functions +delete_ec2_instances +delete_ecs_clusters +delete_eks_clusters +delete_ecr_repositories +delete_auto_scaling_groups +delete_load_balancers +delete_ec2_volumes +delete_ec2_snapshots +delete_ec2_key_pairs +delete_ec2_security_groups +delete_ec2_amis + + +echo "Deletion script completed." diff --git a/servers/cogvlm/cogvlm.py b/servers/cogvlm/cogvlm.py index 0a020dc..aac3e7f 100644 --- a/servers/cogvlm/cogvlm.py +++ b/servers/cogvlm/cogvlm.py @@ -6,7 +6,7 @@ import torch import uvicorn -from fastapi import Depends, FastAPI, HTTPException +from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from loguru import logger from PIL import Image @@ -20,7 +20,6 @@ TextIteratorStreamer, ) -from swarms_cloud.auth_with_swarms_cloud import authenticate_user from swarms_cloud.schema.cog_vlm_schemas import ( ChatCompletionRequest, ChatCompletionResponse, @@ -35,10 +34,7 @@ TextContent, UsageInfo, ) -from swarms_cloud.calculate_pricing import calculate_pricing, count_tokens -from swarms_cloud.auth_with_swarms_cloud import fetch_api_key_info -from swarms_cloud.log_api_request_to_supabase import log_to_supabase, ModelAPILogEntry -from exa.structs.parallelize_models_gpus import prepare_model_for_ddp_inference +# from exa.structs.parallelize_models_gpus import prepare_model_for_ddp_inference # Load environment variables from .env file load_dotenv() @@ -89,9 +85,9 @@ torch_dtype=torch_type, low_cpu_mem_usage=True, quantization_config=bnb_config, -) # .eval() +).eval() -model = prepare_model_for_ddp_inference(model) +# model = prepare_model_for_ddp_inference(model) # Torch type if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8: @@ -114,13 +110,13 @@ async def list_models(): @app.post("/v1/chat/completions", response_model=ChatCompletionResponse) async def create_chat_completion( - request: ChatCompletionRequest, token: str = Depends(authenticate_user) + request: ChatCompletionRequest, # token: str = Depends(authenticate_user) ): try: if len(request.messages) < 1 or request.messages[-1].role == "assistant": raise HTTPException(status_code=400, detail="Invalid request") - print(f"Request: {request}") + # print(f"Request: {request}") gen_params = dict( messages=request.messages, @@ -146,33 +142,33 @@ async def create_chat_completion( content=response["text"], ) - # # Log the entry to supabase - entry = ModelAPILogEntry( - user_id=fetch_api_key_info(token), - model_id="41a2869c-5f8d-403f-83bb-1f06c56bad47", - input_tokens=count_tokens(request.messages, tokenizer, request.model), - output_tokens=count_tokens(response["text"], tokenizer, request.model), - all_cost=calculate_pricing( - texts=[message.content], tokenizer=tokenizer, rate_per_million=15.0 - ), - input_cost=calculate_pricing( - texts=[message.content], tokenizer=tokenizer, rate_per_million=15.0 - ), - output_cost=calculate_pricing( - texts=response["text"], tokenizer=tokenizer, rate_per_million=15.0 - ) - * 5, - messages=request.messages, - # temperature=request.temperature, - top_p=request.top_p, - # echo=request.echo, - stream=request.stream, - repetition_penalty=request.repetition_penalty, - max_tokens=request.max_tokens, - ) + # # # Log the entry to supabase + # entry = ModelAPILogEntry( + # user_id=fetch_api_key_info(token), + # model_id="41a2869c-5f8d-403f-83bb-1f06c56bad47", + # input_tokens=count_tokens(request.messages, tokenizer, request.model), + # output_tokens=count_tokens(response["text"], tokenizer, request.model), + # all_cost=calculate_pricing( + # texts=[message.content], tokenizer=tokenizer, rate_per_million=15.0 + # ), + # input_cost=calculate_pricing( + # texts=[message.content], tokenizer=tokenizer, rate_per_million=15.0 + # ), + # output_cost=calculate_pricing( + # texts=response["text"], tokenizer=tokenizer, rate_per_million=15.0 + # ) + # * 5, + # messages=request.messages, + # # temperature=request.temperature, + # top_p=request.top_p, + # # echo=request.echo, + # stream=request.stream, + # repetition_penalty=request.repetition_penalty, + # max_tokens=request.max_tokens, + # ) - # Log the entry to supabase - log_to_supabase(entry=entry) + # # Log the entry to supabase + # log_to_supabase(entry=entry) # ChatCompletionResponseChoice logger.debug(f"==== message ====\n{message}")