From adcbe3b54faa59a339c244014773dade16d6063a Mon Sep 17 00:00:00 2001 From: Eylon Goor Date: Thu, 4 Jan 2024 18:54:15 +0200 Subject: [PATCH] push deployment methods Change-Id: I3881642bc43a4eb8544cb1fb7c7a851a940547c6 --- README.md | 18 ++ app.json | 4 +- cloud_function/{cf_agent.py => main.py} | 0 setup/deploy.sh | 65 +++++++ setup/main.tf | 222 ++++++++++++++++++++++++ setup/prebuild.sh | 33 +++- 6 files changed, 337 insertions(+), 5 deletions(-) rename cloud_function/{cf_agent.py => main.py} (100%) create mode 100644 setup/deploy.sh create mode 100644 setup/main.tf diff --git a/README.md b/README.md index 2f985cd..4fafa30 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Shrinkify uses the VertexAI PaLM API to shorten long headlines to a certain numb ## Installation +### Using Cloud Run Button 1. Click the big blue button to deploy: @@ -21,6 +22,23 @@ Shrinkify uses the VertexAI PaLM API to shorten long headlines to a certain numb 1. Once installation is finished you will recieve your tool's URL. Save it. +### Using Terraform +1. In your cloud project, open the cloud shell + +1. Clone this repository + `git clone <>` + +1. cd into the cloned directory + `cd shrinkify` + +1. Grant permissions to deployment script + `chmod 777 setup/deploy.sh` + +1. Run deployment script + `./setup/deploy.sh deploy_all` + +1. Once deployment is completed go "Cloud Run" and find the service called "shrinkify_service" + ## Usage diff --git a/app.json b/app.json index 760ae93..4c92cb5 100644 --- a/app.json +++ b/app.json @@ -1,7 +1,7 @@ { - "name": "Shrinkify", + "name": "shrinkify_service", "options": { - "allow-unauthenticated": false, + "allow-unauthenticated": true, "memory": "512Mi", "cpu": "1" }, diff --git a/cloud_function/cf_agent.py b/cloud_function/main.py similarity index 100% rename from cloud_function/cf_agent.py rename to cloud_function/main.py diff --git a/setup/deploy.sh b/setup/deploy.sh new file mode 100644 index 0000000..c6b3847 --- /dev/null +++ b/setup/deploy.sh @@ -0,0 +1,65 @@ +#!/bin/bash +COLOR='\033[0;36m' # Cyan +NC='\033[0m' # No Color +# Variables +PROJECT_ID=$(gcloud config get-value project 2> /dev/null) +PROJECT_NAME=$(gcloud projects describe $PROJECT_ID --format="value(name)") +PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID | grep projectNumber | sed "s/.* '//;s/'//g") +SERVICE_ACCOUNT=$PROJECT_NUMBER-compute@developer.gserviceaccount.com +IMAGE_NAME="shrinkify" +TAG="latest" # e.g., "latest" + + +enable_apis() { + echo -e "${COLOR}Enabling APIs...${NC}" + gcloud services enable storage-component.googleapis.com + gcloud services enable artifactregistry.googleapis.com \ + run.googleapis.com \ + iamcredentials.googleapis.com \ + cloudbuild.googleapis.com \ + cloudfunctions.googleapis.com \ + aiplatform.googleapis.com \ + pubsub.googleapis.com \ + eventarc.googleapis.com \ + bigquery.googleapis.com \ + compute.googleapis.com +} + +zip_cf_source() { + echo -e "${COLOR}Zipping cloud function source...${NC}" + zip -j setup/shrinkify_cf.zip cloud_function/main.py cloud_function/requirements.txt +} + +create_image() { + echo "Enabling container deployment..." + gcloud auth configure-docker + echo -e "${COLOR}Creating Image...${NC}" + docker build -t ${IMAGE_NAME} ../ + docker tag ${IMAGE_NAME} gcr.io/${PROJECT_ID}/${IMAGE_NAME}:${TAG} + docker push gcr.io/${PROJECT_ID}/${IMAGE_NAME}:${TAG} + echo "Image pushed to GCP Container Registry successfully." +} + +run_tf() { + cd setup + terraform init + echo -e "${COLOR}Creating Infra...${NC}" + terraform apply -var "project_id=$PROJECT_ID" -var "project_number=$PROJECT_NUMBER" -auto-approve + echo -e "${COLOR}Infra Created!${NC}" +} + +deploy_all() { + enable_apis + zip_cf_source + create_image + run_tf +} + +for i in "$@"; do + "$i" + exitcode=$? + if [ $exitcode -ne 0 ]; then + echo "Breaking script as command '$i' failed" + exit $exitcode + fi +done \ No newline at end of file diff --git a/setup/main.tf b/setup/main.tf new file mode 100644 index 0000000..faf30b4 --- /dev/null +++ b/setup/main.tf @@ -0,0 +1,222 @@ +############################################# +# Vars +############################################# + +variable "project_id" { + type = string +} + +variable "project_number" { + type = string +} + +variable "region" { + type = string + default = "us-central1" +} + +variable "location" { + type = string + default = "US" +} + +variable "service_name" { + type = string + default = "shrinkify" +} + +variable "dataset_id" { + type = string + default = "shrinkify_output" +} + +variable "table_id" { + type = string + default = "results_*" +} + +locals { + service_account = "${var.project_number}-compute@developer.gserviceaccount.com" +} + +provider "google" { + project = var.project_id + region = var.region +} + +############################################# +# IAM +############################################# + +resource "google_project_iam_member" "storage_admin" { + project = var.project_id + role = "roles/storage.admin" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "bigquery_admin" { + project = var.project_id + role = "roles/bigquery.admin" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "run_invoker" { + project = var.project_id + role = "roles/run.invoker" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "function_viewer" { + project = var.project_id + role = "roles/cloudfunctions.viewer" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "service_account_user" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "eventarc_event_receiver" { + project = var.project_id + role = "roles/eventarc.eventReceiver" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "service_account_token_creator" { + project = var.project_id + role = "roles/iam.serviceAccountTokenCreator" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "eventarc_admin" { + project = var.project_id + role = "roles/eventarc.admin" + member = "serviceAccount:${local.service_account}" +} + +resource "google_project_iam_member" "pubsub_service_account_token_creator" { + project = var.project_id + role = "roles/iam.serviceAccountTokenCreator" + member = "serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com" +} + +############################################# +# Cloud Run +############################################# + +data "google_container_registry_image" "shrinkify_image" { + name = "gcr.io/${var.project_id}/${var.service_name}:latest" +} + +resource "google_cloud_run_service" "shrinkify_service" { + name = "${var.service_name}-service" + location = var.region + template { + spec { + containers { + image = data.google_container_registry_image.shrinkify_image.name + env { + name = "PROJECT_NAME" + value = var.project_id + } + env { + name = "PROJECT_NUMBER" + value = var.project_number + } + } + } + } + traffic { + percent = 100 + latest_revision = true + } +} +data "google_iam_policy" "public_access" { + binding { + role = "roles/run.invoker" + members = [ + "allUsers", + ] + } +} +resource "google_cloud_run_service_iam_policy" "public_access" { + location = google_cloud_run_service.shrinkify_service.location + project = google_cloud_run_service.shrinkify_service.project + service = google_cloud_run_service.shrinkify_service.name + policy_data = data.google_iam_policy.public_access.policy_data +} + +############################################# +# Cloud Function +############################################# + +# Create GCS Bucket and upload zipped CF code + +resource "google_storage_bucket" "cf_zip_bucket" { + name = "${var.project_id}-shrinkify" + location = "US" + project = var.project_id + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_object" "source" { + name = "shrinkify_cf.zip" + source = "shrinkify_cf.zip" + bucket = google_storage_bucket.cf_zip_bucket.name +} + +# Deploy Cloud Function + +resource "google_cloudfunctions2_function" "function" { + depends_on = [ + google_project_iam_member.bigquery_admin, + google_project_iam_member.eventarc_admin, + google_project_iam_member.eventarc_event_receiver, + google_project_iam_member.function_viewer, + google_project_iam_member.pubsub_service_account_token_creator, + google_project_iam_member.run_invoker, + google_project_iam_member.service_account_token_creator, + google_project_iam_member.service_account_user, + google_project_iam_member.storage_admin, + ] + name = "shrinkify_cf" + location = var.region + project = var.project_id + build_config { + runtime = "python310" + entry_point = "cloud_agent" # Set the entry point in the code + source { + storage_source { + bucket = google_storage_bucket.cf_zip_bucket.name + object = google_storage_bucket_object.source.name + } + } + } + service_config { + max_instance_count = 1 + available_memory = "512M" + timeout_seconds = 540 + service_account_email = local.service_account + } + event_trigger { + trigger_region = var.region + event_type = "google.cloud.audit.log.v1.written" + retry_policy = "RETRY_POLICY_DO_NOT_RETRY" + service_account_email = local.service_account + event_filters { + attribute = "serviceName" + value = "bigquery.googleapis.com" + } + event_filters { + attribute = "methodName" + value = "google.cloud.bigquery.v2.JobService.InsertJob" + } + event_filters { + attribute = "resourceName" + value = "/projects/${var.project_id}/datasets/${var.dataset_id}/tables/${var.table_id}" + operator = "match-path-pattern" # This allows path patterns to be used in the value field + } + } +} \ No newline at end of file diff --git a/setup/prebuild.sh b/setup/prebuild.sh index 9ed3466..05bb0c7 100644 --- a/setup/prebuild.sh +++ b/setup/prebuild.sh @@ -12,14 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +project_number=$(gcloud projects describe ${GOOGLE_CLOUD_PROJECT} --format="value(projectNumber)") +service_account="serviceAccount:${project_number}-compute@developer.gserviceaccount.com" +cf_name="shrinkify-cf" + echo "Setting Project ID: ${GOOGLE_CLOUD_PROJECT}" gcloud config set project ${GOOGLE_CLOUD_PROJECT} +echo "Enabling container deployment..." +gcloud auth configure-docker + echo "Enabling APIs..." -gcloud services enable artifactregistry.googleapis.com cloudbuild.googleapis.com cloudfunctions.googleapis.com aiplatform.googleapis.com pubsub.googleapis.com eventarc.googleapis.com bigquery.googleapis.com compute.googleapis.com +gcloud services enable artifactregistry.googleapis.com \ + run.googleapis.com \ + iamcredentials.googleapis.com \ + cloudbuild.googleapis.com \ + cloudfunctions.googleapis.com \ + aiplatform.googleapis.com \ + pubsub.googleapis.com \ + eventarc.googleapis.com \ + bigquery.googleapis.com \ + compute.googleapis.com + +echo "Granting service account eventarc permissions..." +gcloud projects add-iam-policy-binding ${GOOGLE_CLOUD_PROJECT} \ + --member=$service_account \ + --role=roles/eventarc.eventReceiver echo "Creating cloud function..." -gcloud functions deploy shrinkify-cloud-agent \ +gcloud functions deploy $cf_name \ --gen2 \ --region=us-central1 \ --runtime=python39 \ @@ -30,4 +51,10 @@ gcloud functions deploy shrinkify-cloud-agent \ --trigger-event-filters="serviceName=bigquery.googleapis.com" \ --trigger-event-filters="methodName=google.cloud.bigquery.v2.JobService.InsertJob" \ --trigger-event-filters-path-pattern="resourceName=/projects/${GOOGLE_CLOUD_PROJECT}/datasets/shrinkify_output/tables/results_*" \ ---timeout=3600s +--timeout=520s + +echo "Setting service account permissions..." +gcloud run services add-iam-policy-binding $cf_name \ + --member=$service_account \ + --role='roles/run.invoker' \ + --region=${GOOGLE_CLOUD_REGION} \ No newline at end of file