GCS connector (#336)

* GCS connector

* fix token + do not use external files for GCS

* clean up + increment version

apps/desktop/src/app/sync/sources/gcs.ts

@@ -0,0 +1,100 @@
+import {
+  ConnectorSettings,
+  ISourceConnector,
+  SourceConnectorDefinition,
+  SyncItem,
+  SearchResults,
+  ConnectorParameters,
+  Field,
+  FileStatus,
+} from '../models';
+import { Observable, of, from, switchMap, map } from 'rxjs';
+import { GoogleBaseImpl } from './google.base';
+
+const BUCKET_KEY = 'GCS_BUCKET';
+
+const MAX_PAGE_SIZE = 1000;
+
+export const GCSConnector: SourceConnectorDefinition = {
+  id: 'gcs',
+  title: 'Google Cloud',
+  logo: 'assets/logos/gcs.svg',
+  description: 'File storage service developed by Google',
+  factory: (data?: ConnectorSettings) => of(new GCSImpl(data)),
+};
+
+class GCSImpl extends GoogleBaseImpl implements ISourceConnector {
+  override DISCOVERY_DOCS = ['https://www.googleapis.com/discovery/v1/apis/storage/v1/rest'];
+  isExternal = false;
+  resumable = false;
+
+  constructor(data?: ConnectorSettings) {
+    super(data);
+  }
+
+  getParameters(): Observable<Field[]> {
+    return of([
+      {
+        id: 'bucket',
+        label: 'Bucket',
+        type: 'text',
+        required: true,
+      },
+    ]);
+  }
+
+  handleParameters(params: ConnectorParameters) {
+    localStorage.setItem(BUCKET_KEY, params.bucket);
+  }
+
+  getFiles(query?: string, pageSize?: number): Observable<SearchResults> {
+    if (query) {
+      // GCS API doesn't have any command to filter by keywords.
+      // As a workaround we retrieve all the objects and do the filtering ourselves.
+      pageSize = MAX_PAGE_SIZE;
+    }
+    return this._getFiles(query, pageSize);
+  }
+
+  private _getFiles(query?: string, pageSize: number = 50, nextPage?: string | number): Observable<SearchResults> {
+    const regexp = query ? new RegExp(`(${query})`, 'i') : null;
+    const bucket = localStorage.getItem(BUCKET_KEY);
+    if (!bucket) {
+      return of({ items: [] as SyncItem[] });
+    } else {
+      return from(
+        fetch(
+          `https://storage.googleapis.com/storage/v1/b/${bucket}/o?maxResults=${pageSize}&pageToken=${nextPage || ''}`,
+          {
+            headers: { Authorization: 'Bearer ' + localStorage.getItem(this.TOKEN) },
+          },
+        ),
+      ).pipe(
+        switchMap((res) => res.json()),
+        map((res) => ({
+          items: (res.items as any[]).map(this.mapResult).filter((item) => !regexp || regexp.test(item.title)),
+          nextPage: res.nextPageToken ? this._getFiles(query, pageSize, res.nextPageToken) : undefined,
+        })),
+      );
+    }
+  }
+
+  /* eslint-disable  @typescript-eslint/no-explicit-any */
+  private mapResult(result: any): SyncItem {
+    return {
+      originalId: result.id,
+      title: result.name,
+      uuid: '',
+      metadata: {
+        mediaLink: result.mediaLink,
+      },
+      status: FileStatus.PENDING,
+    };
+  }
+
+  download(resource: SyncItem): Observable<Blob> {
+    return from(
+      fetch(resource.metadata.mediaLink, { headers: { Authorization: 'Bearer ' + localStorage.getItem(this.TOKEN) } }),
+    ).pipe(switchMap((res) => res.blob()));
+  }
+}

apps/desktop/src/app/sync/sources/gdrive.ts

@@ -10,16 +10,12 @@ import {
   SyncItem,
   SearchResults,
 } from '../models';
-import { BehaviorSubject, filter, from, map, Observable, of, concatMap, take } from 'rxjs';
+import { filter, from, map, Observable, of, concatMap, take } from 'rxjs';
 import { injectScript } from '@flaps/core';
-import { environment } from '../../../environments/environment';
+import { GoogleBaseImpl } from './google.base';
 
 declare var gapi: any;
 
-const DISCOVERY_DOCS = ['https://www.googleapis.com/discovery/v1/apis/drive/v3/rest'];
-
-const TOKEN = 'GDRIVE_TOKEN';
-
 export const GDrive: SourceConnectorDefinition = {
   id: 'gdrive',
   title: 'Google Drive',
@@ -28,76 +24,19 @@ export const GDrive: SourceConnectorDefinition = {
   factory: (data?: ConnectorSettings) => of(new GDriveImpl(data)),
 };
 
-class GDriveImpl implements ISourceConnector {
-  hasServerSideAuth = true;
+class GDriveImpl extends GoogleBaseImpl implements ISourceConnector {
+  override DISCOVERY_DOCS = ['https://www.googleapis.com/discovery/v1/apis/drive/v3/rest'];
   isExternal = false;
   resumable = true;
-  private isAuthenticated: BehaviorSubject<boolean> = new BehaviorSubject<boolean>(false);
-  API_KEY: string;
 
   constructor(data?: ConnectorSettings) {
-    this.API_KEY = data?.API_KEY || '';
+    super(data);
   }
 
   getParameters() {
     return of([]);
   }
 
-  goToOAuth(reset?: boolean) {
-    if (reset) {
-      localStorage.removeItem(TOKEN);
-    }
-    const token = localStorage.getItem(TOKEN);
-    if (token) {
-      injectScript('https://apis.google.com/js/api.js').subscribe(() =>
-        gapi.load('client', () => {
-          gapi.client.init({
-            apiKey: this.API_KEY,
-            discoveryDocs: DISCOVERY_DOCS,
-          });
-          gapi.client.setToken({ access_token: token });
-          this.isAuthenticated.next(true);
-        }),
-      );
-    } else {
-      if ((window as any)['electron']) {
-        (window as any)['electron'].openExternal(
-          `${environment.connectors.gdrive.endpoint}?redirect=nuclia-desktop://`,
-        );
-      } else {
-        location.href = `${environment.connectors.gdrive.endpoint}?redirect=http://localhost:4200`;
-      }
-    }
-  }
-
-  authenticate(): Observable<boolean> {
-    if (!this.isAuthenticated.getValue()) {
-      injectScript('https://apis.google.com/js/api.js').subscribe(() => {
-        gapi.load('client', () => {
-          gapi.client.init({
-            apiKey: this.API_KEY,
-            discoveryDocs: DISCOVERY_DOCS,
-          });
-          const interval = setInterval(() => {
-            const deeplink = (window as any)['deeplink'] || location.search;
-            if (deeplink && deeplink.includes('?')) {
-              const params = new URLSearchParams(deeplink.split('?')[1]);
-              const isGoogle = params.get('google');
-              if (isGoogle) {
-                const token = params.get('token') || '';
-                localStorage.setItem(TOKEN, token);
-                gapi.client.setToken({ access_token: token });
-                clearInterval(interval);
-                this.isAuthenticated.next(true);
-              }
-            }
-          }, 500);
-        });
-      });
-    }
-    return this.isAuthenticated.asObservable();
-  }
-
   getFiles(query?: string, pageSize?: number) {
     return this._getFiles(query, pageSize);
   }
@@ -119,7 +58,7 @@ class GDriveImpl implements ISourceConnector {
             .then(
               (res: any) => res,
               () => {
-                localStorage.removeItem(TOKEN);
+                localStorage.removeItem(this.TOKEN);
                 throw new Error('Unauthorized');
               },
             ),
@@ -155,7 +94,7 @@ class GDriveImpl implements ISourceConnector {
           gapi.client
             .init({
               apiKey: this.API_KEY,
-              discoveryDocs: DISCOVERY_DOCS,
+              discoveryDocs: this.DISCOVERY_DOCS,
             })
             .then(() =>
               fetch(request, {

apps/desktop/src/app/sync/sources/google.base.ts

@@ -0,0 +1,77 @@
+/// <reference path="../../../../../../node_modules/@types/gapi/index.d.ts" />
+/// <reference path="../../../../../../node_modules/@types/gapi.auth2/index.d.ts" />
+/// <reference path="../../../../../../node_modules/@types/gapi.client.drive/index.d.ts" />
+
+import { ConnectorSettings } from '../models';
+import { BehaviorSubject, Observable } from 'rxjs';
+import { injectScript } from '@flaps/core';
+import { environment } from '../../../environments/environment';
+
+declare var gapi: any;
+
+export class GoogleBaseImpl {
+  TOKEN = 'GOOGLE_TOKEN';
+  DISCOVERY_DOCS: string[] = [];
+  hasServerSideAuth = true;
+  private isAuthenticated: BehaviorSubject<boolean> = new BehaviorSubject<boolean>(false);
+  API_KEY: string;
+
+  constructor(data?: ConnectorSettings) {
+    this.API_KEY = data?.API_KEY || '';
+  }
+
+  goToOAuth(reset?: boolean) {
+    if (reset) {
+      localStorage.removeItem(this.TOKEN);
+    }
+    const token = localStorage.getItem(this.TOKEN);
+    if (token) {
+      injectScript('https://apis.google.com/js/api.js').subscribe(() =>
+        gapi.load('client', () => {
+          gapi.client.init({
+            apiKey: this.API_KEY,
+            discoveryDocs: this.DISCOVERY_DOCS,
+          });
+          gapi.client.setToken({ access_token: token });
+          this.isAuthenticated.next(true);
+        }),
+      );
+    } else {
+      if ((window as any)['electron']) {
+        (window as any)['electron'].openExternal(
+          `${environment.connectors.google.endpoint}?redirect=nuclia-desktop://`,
+        );
+      } else {
+        location.href = `${environment.connectors.google.endpoint}?redirect=http://localhost:4200`;
+      }
+    }
+  }
+
+  authenticate(): Observable<boolean> {
+    if (!this.isAuthenticated.getValue()) {
+      injectScript('https://apis.google.com/js/api.js').subscribe(() => {
+        gapi.load('client', () => {
+          gapi.client.init({
+            apiKey: this.API_KEY,
+            discoveryDocs: this.DISCOVERY_DOCS,
+          });
+          const interval = setInterval(() => {
+            const deeplink = (window as any)['deeplink'] || location.search;
+            if (deeplink && deeplink.includes('?')) {
+              const params = new URLSearchParams(deeplink.split('?')[1]);
+              const isGoogle = params.get('google');
+              if (isGoogle) {
+                const token = params.get('token') || '';
+                localStorage.setItem(this.TOKEN, token);
+                gapi.client.setToken({ access_token: token });
+                clearInterval(interval);
+                this.isAuthenticated.next(true);
+              }
+            }
+          }, 500);
+        });
+      });
+    }
+    return this.isAuthenticated.asObservable();
+  }
+}

apps/desktop/src/app/sync/sync.service.ts

@@ -37,6 +37,7 @@ import { FolderConnector } from './sources/folder';
 import { S3Connector } from './sources/s3';
 import { ProcessingPullResponse } from '@nuclia/core';
 import { convertDataURIToBinary, NucliaProtobufConverter } from './protobuf';
+import { GCSConnector } from './sources/gcs';
 
 const ACCOUNT_KEY = 'NUCLIA_ACCOUNT';
 const QUEUE_KEY = 'NUCLIA_QUEUE';
@@ -64,10 +65,11 @@ export class SyncService {
       instance?: ReplaySubject<ISourceConnector>;
     };
   } = {
-    gdrive: { definition: GDrive, settings: environment.connectors.gdrive },
+    gdrive: { definition: GDrive, settings: environment.connectors.google },
     dropbox: { definition: DropboxConnector, settings: environment.connectors.dropbox },
     folder: { definition: FolderConnector, settings: {} },
     s3: { definition: S3Connector, settings: {} },
+    gcs: { definition: GCSConnector, settings: environment.connectors.google },
   };
   destinations: { [id: string]: { definition: DestinationConnectorDefinition; settings: ConnectorSettings } } = {
     nucliacloud: {

apps/desktop/src/environments/environment.prod.ts

@@ -11,7 +11,7 @@ export const environment = {
     dropbox: {
       CLIENT_ID: '__DROPBOX_KEY__',
     },
-    gdrive: {
+    google: {
       endpoint: 'https://nuclia.cloud/api/external_auth/gdrive/authorize',
       API_KEY: '__GOOGLE_API_KEY__',
     },

apps/desktop/src/environments/environment.ts

@@ -11,7 +11,7 @@ export const environment = {
     dropbox: {
       CLIENT_ID: 'FAKE',
     },
-    gdrive: {
+    google: {
       endpoint: 'https://stashify.cloud/api/external_auth/gdrive/authorize',
       API_KEY: 'FAKE',
     },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nuclia",
-  "version": "1.0.17",
+  "version": "1.0.18",
   "license": "MIT",
   "author": "Nuclia.cloud",
   "description": "Nuclia frontend apps and libs",