Support for GCP cloud storage

serverpod · Feb 28, 2023 · b2a7313 · b2a7313
1 parent 47a8ce7
commit b2a7313
Show file tree

Hide file tree

Showing 35 changed files with 1,959 additions and 37 deletions.
diff --git a/gcp_client/lib/src/protocol/client.dart b/gcp_client/lib/src/protocol/client.dart
@@ -22,6 +22,12 @@ class _EndpointExample extends _i1.EndpointRef {
         'hello',
         {'name': name},
       );
+
+  _i2.Future<bool> verifyUpload() => caller.callServerEndpoint<bool>(
+        'example',
+        'verifyUpload',
+        {},
+      );
 }
 
 class Client extends _i1.ServerpodClient {

diff --git a/gcp_flutter/lib/main.dart b/gcp_flutter/lib/main.dart
@@ -1,3 +1,5 @@
+import 'dart:typed_data';
+
 import 'package:gcp_client/gcp_client.dart';
 import 'package:flutter/material.dart';
 import 'package:serverpod_flutter/serverpod_flutter.dart';
@@ -51,7 +53,17 @@ class MyHomePageState extends State<MyHomePage> {
   // is successful.
   void _callHello() async {
     try {
-      final result = await client.example.hello(_textEditingController.text);
+      final uploadDescription =
+          await client.example.hello(_textEditingController.text);
+
+      var buffer = Int8List(64);
+      var byteData = ByteData.view(buffer.buffer);
+
+      var uploader = FileUploader(uploadDescription);
+      var fileUploaded = await uploader.uploadByteData(byteData);
+      print('File uploaded: $fileUploaded');
+      var result = '${await client.example.verifyUpload()}';
+
       setState(() {
         _resultMessage = result;
       });

diff --git a/gcp_server/gcp/terraform/modules/serverpod/dns.tf b/gcp_server/gcp/terraform/modules/serverpod/dns.tf
@@ -1,6 +1,6 @@
 resource "google_dns_managed_zone" "private-dns" {
-  name        = "serverpod-${var.runmode}-private"
-  dns_name    = "private-${var.runmode}.${var.top_domain}."
+  name     = "serverpod-${var.runmode}-private"
+  dns_name = "private-${var.runmode}.${var.top_domain}."
 
   visibility = "private"
 
@@ -11,6 +11,24 @@ resource "google_dns_managed_zone" "private-dns" {
   }
 }
 
+resource "google_dns_record_set" "database-private" {
+  name         = "database.private-${var.runmode}.${var.top_domain}."
+  managed_zone = "serverpod-${var.runmode}-private"
+  type         = "A"
+  ttl          = 60
+  rrdatas      = [google_sql_database_instance.serverpod.private_ip_address]
+}
+
+resource "google_dns_record_set" "redis-private" {
+  name         = "redis.private-${var.runmode}.${var.top_domain}."
+  managed_zone = "serverpod-${var.runmode}-private"
+  type         = "A"
+  ttl          = 60
+  rrdatas      = [google_redis_instance.serverpod.host]
+}
+
+
+
 resource "google_dns_record_set" "api" {
   name         = "api.${var.top_domain}."
   managed_zone = "examplepod"
@@ -28,33 +46,26 @@ resource "google_dns_record_set" "insights" {
 }
 
 resource "google_dns_record_set" "web" {
-  name         = "${var.top_domain}."
+  name         = "app.${var.top_domain}."
   managed_zone = "examplepod"
   type         = "A"
   ttl          = 60
   rrdatas      = [google_compute_global_forwarding_rule.web.ip_address]
 }
 
-resource "google_dns_record_set" "database" {
-  name         = "database.${var.top_domain}."
+resource "google_dns_record_set" "storage" {
+  name         = "storage.${var.top_domain}."
   managed_zone = "examplepod"
   type         = "A"
   ttl          = 60
-  rrdatas      = [google_sql_database_instance.serverpod.public_ip_address]
+  rrdatas      = [google_compute_global_forwarding_rule.storage.ip_address]
 }
 
-resource "google_dns_record_set" "database-private" {
-  name         = "database.private-${var.runmode}.${var.top_domain}."
-  managed_zone = "serverpod-${var.runmode}-private"
+resource "google_dns_record_set" "database" {
+  name         = "database.${var.top_domain}."
+  managed_zone = "examplepod"
   type         = "A"
   ttl          = 60
-  rrdatas      = [google_sql_database_instance.serverpod.private_ip_address]
+  rrdatas      = [google_sql_database_instance.serverpod.public_ip_address]
 }
 
-resource "google_dns_record_set" "redis-private" {
-  name         = "redis.private-${var.runmode}.${var.top_domain}."
-  managed_zone = "serverpod-${var.runmode}-private"
-  type         = "A"
-  ttl          = 60
-  rrdatas      = [google_redis_instance.serverpod.host]
-}
diff --git a/gcp_server/gcp/terraform/modules/serverpod/instance_group.tf b/gcp_server/gcp/terraform/modules/serverpod/instance_group.tf
@@ -80,9 +80,9 @@ resource "google_compute_autoscaler" "serverpod" {
 }
 
 resource "google_compute_health_check" "serverpod-instance-group" {
-  name               = "serverpod-${var.runmode}-group-health-check"
-  timeout_sec        = 5
-  check_interval_sec = 30
+  name                = "serverpod-${var.runmode}-group-health-check"
+  timeout_sec         = 5
+  check_interval_sec  = 30
   healthy_threshold   = 2
   unhealthy_threshold = 3
 

diff --git a/gcp_server/gcp/terraform/modules/serverpod/load_balancer.tf b/gcp_server/gcp/terraform/modules/serverpod/load_balancer.tf
@@ -54,6 +54,20 @@ resource "google_compute_target_http_proxy" "web" {
   # ssl_certificates = [google_compute_managed_ssl_certificate.api.id]
 }
 
+resource "google_compute_global_forwarding_rule" "storage" {
+  name                  = "serverpod-${var.runmode}-storage"
+  ip_protocol           = "TCP"
+  load_balancing_scheme = "EXTERNAL"
+  port_range            = "80"
+  target                = google_compute_target_http_proxy.storage.self_link
+}
+
+resource "google_compute_target_http_proxy" "storage" {
+  name    = "serverpod-${var.runmode}-proxy-storage"
+  url_map = google_compute_url_map.serverpod.id
+  # ssl_certificates = [google_compute_managed_ssl_certificate.api.id]
+}
+
 resource "google_compute_url_map" "serverpod" {
   name            = "serverpod-${var.runmode}-balancer"
   default_service = google_compute_backend_service.web.id
@@ -66,11 +80,6 @@ resource "google_compute_url_map" "serverpod" {
   path_matcher {
     name            = "api"
     default_service = google_compute_backend_service.api.id
-
-    # path_rule {
-    #   paths   = ["/*"]
-    #   service = google_compute_region_backend_service.default.id
-    # }
   }
 
   host_rule {
@@ -81,11 +90,16 @@ resource "google_compute_url_map" "serverpod" {
   path_matcher {
     name            = "insights"
     default_service = google_compute_backend_service.insights.id
+  }
+
+  host_rule {
+    hosts        = ["storage.${var.top_domain}"]
+    path_matcher = "storage"
+  }
 
-    # path_rule {
-    #   paths   = ["/*"]
-    #   service = google_compute_region_backend_service.default.id
-    # }
+  path_matcher {
+    name            = "storage"
+    default_service = google_compute_backend_bucket.storage.id
   }
 }
 
@@ -137,6 +151,12 @@ resource "google_compute_backend_service" "web" {
   port_name = "web"
 }
 
+resource "google_compute_backend_bucket" "storage" {
+  name        = "serverpod-${var.runmode}-backend-storage"
+  bucket_name = google_storage_bucket.public.name
+  enable_cdn  = false
+}
+
 resource "google_compute_health_check" "serverpod-balancer" {
   name               = "serverpod-${var.runmode}-health-check"
   timeout_sec        = 5

diff --git a/gcp_server/gcp/terraform/modules/serverpod/redis.tf b/gcp_server/gcp/terraform/modules/serverpod/redis.tf
@@ -1,7 +1,7 @@
 resource "google_redis_instance" "serverpod" {
-    name = "serverpod-${var.runmode}-cache"
-    tier = "BASIC"
-    memory_size_gb = 1
+  name           = "serverpod-${var.runmode}-cache"
+  tier           = "BASIC"
+  memory_size_gb = 1
 
-    authorized_network = google_compute_network.serverpod.id
+  authorized_network = google_compute_network.serverpod.id
 }
diff --git a/gcp_server/gcp/terraform/modules/serverpod/storage.tf b/gcp_server/gcp/terraform/modules/serverpod/storage.tf
@@ -0,0 +1,35 @@
+resource "google_storage_bucket" "public" {
+  name                        = "storage.${var.top_domain}"
+  location                    = "US"
+  uniform_bucket_level_access = false
+
+  force_destroy = true
+
+  cors {
+    origin          = ["*"]
+    method          = ["*"]
+    response_header = ["*"]
+    max_age_seconds = 3600
+  }
+}
+
+resource "google_storage_bucket_access_control" "public" {
+  bucket = google_storage_bucket.public.id
+  role   = "READER"
+  entity = "allUsers"
+}
+
+resource "google_storage_bucket" "private" {
+  name                        = "private-storage.${var.top_domain}"
+  location                    = "US"
+  uniform_bucket_level_access = false
+
+  force_destroy = true
+
+  cors {
+    origin          = ["*"]
+    method          = ["*"]
+    response_header = ["*"]
+    max_age_seconds = 3600
+  }
+}
diff --git a/gcp_server/generated/protocol.yaml b/gcp_server/generated/protocol.yaml
@@ -1,2 +1,3 @@
 example:
   - hello:
+  - verifyUpload:
diff --git a/gcp_server/lib/server.dart b/gcp_server/lib/server.dart
@@ -1,6 +1,8 @@
 import 'package:serverpod/serverpod.dart';
 
 import 'package:gcp_server/src/web/routes/root.dart';
+import 'package:serverpod_cloud_storage_gcp/serverpod_cloud_storage_gcp.dart'
+    as gcp;
 
 import 'src/generated/protocol.dart';
 import 'src/generated/endpoints.dart';
@@ -29,6 +31,25 @@ void run(List<String> args) async {
     '/*',
   );
 
+  pod.addCloudStorage(gcp.GoogleCloudStorage(
+    serverpod: pod,
+    storageId: 'public',
+    public: true,
+    region: 'us',
+    bucket: 'storage.examplepod.com',
+    publicHost: 'storage.examplepod.com',
+  ));
+
+  pod.addCloudStorage(gcp.GoogleCloudStorage(
+    serverpod: pod,
+    storageId: 'private',
+    public: false,
+    region: 'us',
+    bucket: 'private-storage.examplepod.com',
+  ));
+
+  // Configure storage
+
   // Start the server.
   await pod.start();
 }
diff --git a/gcp_server/lib/src/endpoints/example_endpoint.dart b/gcp_server/lib/src/endpoints/example_endpoint.dart
@@ -16,6 +16,43 @@ class ExampleEndpoint extends Endpoint {
   // supported. The `session` object provides access to the database, logging,
   // passwords, and information about the request being made to the server.
   Future<String> hello(Session session, String name) async {
-    return 'Hello $name';
+    var uploadDescription =
+        await session.storage.createDirectFileUploadDescription(
+      storageId: 'public',
+      path: 'testfile.bin',
+    );
+
+    print('uploadDescription: $uploadDescription');
+
+    var exists = await session.storage
+        .fileExists(storageId: 'public', path: 'testfile.bin');
+    print('File already exists: $exists');
+
+    var file = await session.storage
+        .retrieveFile(storageId: 'public', path: 'testfile.bin');
+    print('retrieved: ${file?.lengthInBytes}');
+
+    if (file != null) {
+      await session.storage.storeFile(
+        storageId: 'public',
+        path: 'testfile2.bin',
+        byteData: file,
+      );
+
+      await session.storage.storeFile(
+        storageId: 'private',
+        path: 'testfile.bin',
+        byteData: file,
+      );
+    }
+
+    return uploadDescription!;
+  }
+
+  Future<bool> verifyUpload(Session session) async {
+    return await session.storage.verifyDirectFileUpload(
+      storageId: 'public',
+      path: 'testfile.bin',
+    );
   }
 }
diff --git a/gcp_server/lib/src/generated/endpoints.dart b/gcp_server/lib/src/generated/endpoints.dart
@@ -41,7 +41,17 @@ class Endpoints extends _i1.EndpointDispatch {
             session,
             params['name'],
           ),
-        )
+        ),
+        'verifyUpload': _i1.MethodConnector(
+          name: 'verifyUpload',
+          params: {},
+          call: (
+            _i1.Session session,
+            Map<String, dynamic> params,
+          ) async =>
+              (endpoints['example'] as _i2.ExampleEndpoint)
+                  .verifyUpload(session),
+        ),
       },
     );
   }