push deployment methods

Change-Id: I3881642bc43a4eb8544cb1fb7c7a851a940547c6

README.md

@@ -12,6 +12,7 @@ Shrinkify uses the VertexAI PaLM API to shorten long headlines to a certain numb
 
 
 ## Installation
+### Using Cloud Run Button
 
 1. Click the big blue button to deploy:
 
@@ -21,6 +22,23 @@ Shrinkify uses the VertexAI PaLM API to shorten long headlines to a certain numb
 
 1. Once installation is finished you will recieve your tool's URL. Save it.
 
+### Using Terraform
+1. In your cloud project, open the cloud shell
+
+1. Clone this repository 
+   `git clone <<ADD>>`
+
+1. cd into the cloned directory
+   `cd shrinkify`
+
+1. Grant permissions to deployment script
+   `chmod 777 setup/deploy.sh`
+
+1. Run deployment script
+   `./setup/deploy.sh deploy_all`
+
+1. Once deployment is completed go "Cloud Run" and find the service called "shrinkify_service"
+
 
 ## Usage
 

app.json

@@ -1,7 +1,7 @@
 {
-    "name": "Shrinkify",
+    "name": "shrinkify_service",
     "options": {
-      "allow-unauthenticated": false,
+      "allow-unauthenticated": true,
       "memory": "512Mi",
       "cpu": "1"
     },

setup/deploy.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+COLOR='\033[0;36m' # Cyan
+NC='\033[0m' # No Color
+# Variables
+PROJECT_ID=$(gcloud config get-value project 2> /dev/null)
+PROJECT_NAME=$(gcloud projects describe $PROJECT_ID --format="value(name)")
+PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID | grep projectNumber | sed "s/.* '//;s/'//g")
+SERVICE_ACCOUNT=$PROJECT_NUMBER[email protected]
+IMAGE_NAME="shrinkify"
+TAG="latest" # e.g., "latest"
+
+
+enable_apis() {
+  echo -e "${COLOR}Enabling APIs...${NC}"
+  gcloud services enable storage-component.googleapis.com
+  gcloud services enable artifactregistry.googleapis.com \
+    run.googleapis.com \
+    iamcredentials.googleapis.com \
+    cloudbuild.googleapis.com \
+    cloudfunctions.googleapis.com \
+    aiplatform.googleapis.com \
+    pubsub.googleapis.com \
+    eventarc.googleapis.com \
+    bigquery.googleapis.com \
+    compute.googleapis.com
+}
+
+zip_cf_source() {
+    echo -e "${COLOR}Zipping cloud function source...${NC}"
+    zip -j setup/shrinkify_cf.zip cloud_function/main.py cloud_function/requirements.txt
+}
+
+create_image() {
+    echo "Enabling container deployment..."
+    gcloud auth configure-docker
+    echo -e "${COLOR}Creating Image...${NC}"
+    docker build -t ${IMAGE_NAME} ../
+    docker tag ${IMAGE_NAME} gcr.io/${PROJECT_ID}/${IMAGE_NAME}:${TAG}
+    docker push gcr.io/${PROJECT_ID}/${IMAGE_NAME}:${TAG}
+    echo "Image pushed to GCP Container Registry successfully."
+}
+
+run_tf() {
+    cd setup
+    terraform init
+    echo -e "${COLOR}Creating Infra...${NC}"
+    terraform apply -var "project_id=$PROJECT_ID" -var "project_number=$PROJECT_NUMBER" -auto-approve
+    echo -e "${COLOR}Infra Created!${NC}"
+}
+
+deploy_all() {
+    enable_apis
+    zip_cf_source
+    create_image
+    run_tf
+}
+
+for i in "$@"; do
+    "$i"
+    exitcode=$?
+    if [ $exitcode -ne 0 ]; then
+        echo "Breaking script as command '$i' failed"
+        exit $exitcode
+    fi
+done

setup/main.tf

@@ -0,0 +1,222 @@
+#############################################
+# Vars
+#############################################
+
+variable "project_id" {
+  type = string
+}
+
+variable "project_number" {
+  type = string
+}
+
+variable "region" {
+  type    = string
+  default = "us-central1"
+}
+
+variable "location" {
+  type    = string
+  default = "US"
+}
+
+variable "service_name" {
+  type    = string
+  default = "shrinkify"
+}
+
+variable "dataset_id" {
+  type = string
+  default = "shrinkify_output"
+}
+
+variable "table_id" {
+  type = string
+  default = "results_*"
+}
+
+locals {
+  service_account = "${var.project_number}[email protected]"
+}
+
+provider "google" {
+  project = var.project_id
+  region  = var.region
+}
+
+#############################################
+# IAM
+#############################################
+
+resource "google_project_iam_member" "storage_admin" {
+  project = var.project_id
+  role    = "roles/storage.admin"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "bigquery_admin" {
+  project = var.project_id
+  role    = "roles/bigquery.admin"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "run_invoker" {
+  project = var.project_id
+  role    = "roles/run.invoker"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "function_viewer" {
+  project = var.project_id
+  role    = "roles/cloudfunctions.viewer"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "service_account_user" {
+  project = var.project_id
+  role    = "roles/iam.serviceAccountUser"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "eventarc_event_receiver" {
+  project = var.project_id
+  role    = "roles/eventarc.eventReceiver"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "service_account_token_creator" {
+  project = var.project_id
+  role    = "roles/iam.serviceAccountTokenCreator"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "eventarc_admin" {
+  project = var.project_id
+  role    = "roles/eventarc.admin"
+  member  = "serviceAccount:${local.service_account}"
+}
+
+resource "google_project_iam_member" "pubsub_service_account_token_creator" {
+  project = var.project_id
+  role    = "roles/iam.serviceAccountTokenCreator"
+  member  = "serviceAccount:service-${var.project_number}@gcp-sa-pubsub.iam.gserviceaccount.com"
+}
+
+#############################################
+# Cloud Run
+#############################################
+
+data "google_container_registry_image" "shrinkify_image" {
+  name = "gcr.io/${var.project_id}/${var.service_name}:latest"
+}
+
+resource "google_cloud_run_service" "shrinkify_service" {
+  name     = "${var.service_name}-service"
+  location = var.region
+  template {
+    spec {
+      containers {
+        image = data.google_container_registry_image.shrinkify_image.name
+        env {
+          name = "PROJECT_NAME"
+          value = var.project_id
+        }
+        env {
+          name = "PROJECT_NUMBER"
+          value = var.project_number
+        }
+      }
+    }
+  }
+  traffic {
+    percent         = 100
+    latest_revision = true
+  }
+}
+data "google_iam_policy" "public_access" {
+  binding {
+    role = "roles/run.invoker"
+    members = [
+      "allUsers",
+    ]
+  }
+}
+resource "google_cloud_run_service_iam_policy" "public_access" {
+  location    = google_cloud_run_service.shrinkify_service.location
+  project     = google_cloud_run_service.shrinkify_service.project
+  service     = google_cloud_run_service.shrinkify_service.name
+  policy_data = data.google_iam_policy.public_access.policy_data
+}
+
+#############################################
+# Cloud Function 
+#############################################
+
+# Create GCS Bucket and upload zipped CF code
+
+resource "google_storage_bucket" "cf_zip_bucket" {
+  name          = "${var.project_id}-shrinkify"
+  location      = "US"
+  project       = var.project_id
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_object" "source" {
+  name   = "shrinkify_cf.zip"
+  source = "shrinkify_cf.zip"
+  bucket = google_storage_bucket.cf_zip_bucket.name
+}
+
+# Deploy Cloud Function 
+
+resource "google_cloudfunctions2_function" "function" {
+  depends_on = [
+    google_project_iam_member.bigquery_admin,
+    google_project_iam_member.eventarc_admin,
+    google_project_iam_member.eventarc_event_receiver,
+    google_project_iam_member.function_viewer,
+    google_project_iam_member.pubsub_service_account_token_creator,
+    google_project_iam_member.run_invoker,
+    google_project_iam_member.service_account_token_creator,
+    google_project_iam_member.service_account_user,
+    google_project_iam_member.storage_admin,
+  ]
+  name = "shrinkify_cf"
+  location = var.region
+  project = var.project_id
+  build_config {
+    runtime     = "python310"
+    entry_point = "cloud_agent" # Set the entry point in the code
+    source {
+      storage_source {
+        bucket = google_storage_bucket.cf_zip_bucket.name
+        object = google_storage_bucket_object.source.name
+      }
+    }
+  }
+  service_config {
+    max_instance_count  = 1
+    available_memory    = "512M"
+    timeout_seconds     = 540
+    service_account_email = local.service_account
+  }
+  event_trigger {
+    trigger_region = var.region
+    event_type = "google.cloud.audit.log.v1.written"
+    retry_policy = "RETRY_POLICY_DO_NOT_RETRY"
+    service_account_email = local.service_account
+    event_filters {
+      attribute = "serviceName"
+      value = "bigquery.googleapis.com"
+    }
+    event_filters {
+      attribute = "methodName"
+      value = "google.cloud.bigquery.v2.JobService.InsertJob"
+    }
+    event_filters {
+      attribute = "resourceName"
+      value = "/projects/${var.project_id}/datasets/${var.dataset_id}/tables/${var.table_id}"
+      operator = "match-path-pattern" # This allows path patterns to be used in the value field
+    }
+  }
+}

setup/prebuild.sh

@@ -12,14 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+project_number=$(gcloud projects describe ${GOOGLE_CLOUD_PROJECT} --format="value(projectNumber)")
+service_account="serviceAccount:${project_number}[email protected]"
+cf_name="shrinkify-cf"
+
 echo "Setting Project ID: ${GOOGLE_CLOUD_PROJECT}"
 gcloud config set project ${GOOGLE_CLOUD_PROJECT}
 
+echo "Enabling container deployment..."
+gcloud auth configure-docker
+
 echo "Enabling APIs..."
-gcloud services enable artifactregistry.googleapis.com cloudbuild.googleapis.com cloudfunctions.googleapis.com aiplatform.googleapis.com pubsub.googleapis.com eventarc.googleapis.com bigquery.googleapis.com compute.googleapis.com
+gcloud services enable artifactregistry.googleapis.com \
+ run.googleapis.com \
+ iamcredentials.googleapis.com \
+ cloudbuild.googleapis.com \
+ cloudfunctions.googleapis.com \
+ aiplatform.googleapis.com \
+ pubsub.googleapis.com \
+ eventarc.googleapis.com \
+ bigquery.googleapis.com \
+ compute.googleapis.com
+
+echo "Granting service account eventarc permissions..."
+gcloud projects add-iam-policy-binding ${GOOGLE_CLOUD_PROJECT} \
+    --member=$service_account \
+    --role=roles/eventarc.eventReceiver
 
 echo "Creating cloud function..."
-gcloud functions deploy shrinkify-cloud-agent \
+gcloud functions deploy $cf_name \
 --gen2 \
 --region=us-central1 \
 --runtime=python39 \
@@ -30,4 +51,10 @@ gcloud functions deploy shrinkify-cloud-agent \
 --trigger-event-filters="serviceName=bigquery.googleapis.com" \
 --trigger-event-filters="methodName=google.cloud.bigquery.v2.JobService.InsertJob" \
 --trigger-event-filters-path-pattern="resourceName=/projects/${GOOGLE_CLOUD_PROJECT}/datasets/shrinkify_output/tables/results_*" \
---timeout=3600s 
+--timeout=520s 
+
+echo "Setting service account permissions..."
+gcloud run services add-iam-policy-binding $cf_name \
+  --member=$service_account \
+  --role='roles/run.invoker' \
+  --region=${GOOGLE_CLOUD_REGION}