Google sheets to v3

.eslintrc.json

@@ -61,7 +61,8 @@
             "@typescript-eslint/no-unused-vars": [ "error", { "ignoreRestSiblings": true, "argsIgnorePattern": "^_" } ],
             "@typescript-eslint/no-explicit-any": "off",
             "@typescript-eslint/ban-ts-comment": "warn",
-            "eol-last": [ "error", "always" ]
+            "eol-last": [ "error", "always" ], 
+            "@typescript-eslint/no-empty-interface": "off"
           }
         }
       ]

.github/workflows/main.yml

@@ -39,12 +39,12 @@ jobs:
       matrix:
         database: [
                    "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
-                   "spanner",
                    "mysql", "mysql5",
                    "mssql", "mssql17",
+                   spanner,
                    "mongo", "mongo4",
-                   "firestore",
                    "dynamodb",
+                   "firestore",
                    "google-sheets"
                    ]
 

apps/velo-external-db/src/app.ts

@@ -3,12 +3,10 @@ import { create, readCommonConfig } from '@wix-velo/external-db-config'
 import { ExternalDbRouter, Hooks } from '@wix-velo/velo-external-db-core'
 import { engineConnectorFor } from './storage/factory'
 
-
-const initConnector = async(hooks?: Hooks) => {
-    const { vendor, type: adapterType, hideAppInfo } = readCommonConfig()
-
+const initConnector = async(wixDataBaseUrl?: string, hooks?: Hooks) => {
+    const { vendor, type: adapterType, externalDatabaseId, allowedMetasites, hideAppInfo } = readCommonConfig()
     const configReader = create()
-    const { authorization, secretKey, ...dbConfig } = await configReader.readConfig()
+    const { authorization, ...dbConfig } = await configReader.readConfig()
 
     const { connector: engineConnector, providers, cleanup } = await engineConnectorFor(adapterType, dbConfig)
 
@@ -18,23 +16,25 @@ const initConnector = async(hooks?: Hooks) => {
             authorization: {
                 roleConfig: authorization
             },
-            secretKey,
+            externalDatabaseId,
+            allowedMetasites,
             vendor,
             adapterType,
             commonExtended: true,
-            hideAppInfo
+            hideAppInfo,
+            wixDataBaseUrl: wixDataBaseUrl || 'https://www.wixapis.com/wix-data'
         },
         hooks,
     })
 
     return { externalDbRouter, cleanup: async() => await cleanup(), schemaProvider: providers.schemaProvider }
 }
 
-export const createApp = async() => {
+export const createApp = async(wixDataBaseUrl?: string) => {
     const app = express()
-    const initConnectorResponse = await initConnector()
+    const initConnectorResponse = await initConnector(wixDataBaseUrl)
     app.use(initConnectorResponse.externalDbRouter.router)
     const server = app.listen(8080, () => console.log('Connector listening on port 8080'))
 
-    return { server, ...initConnectorResponse, reload: () => initConnector() }
+    return { server, ...initConnectorResponse, reload: () => initConnector(wixDataBaseUrl) }
 }

apps/velo-external-db/src/storage/factory.ts

@@ -31,18 +31,18 @@ export const engineConnectorFor = async(_type: string, config: any): Promise<Dat
             const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
             return await googleSheetFactory(config)
         }
-        case 'airtable': {
-            const { airtableFactory } = require('@wix-velo/external-db-airtable')
-            return await airtableFactory(config)
-        }
+        // case 'airtable': {
+        //     const { airtableFactory } = require('@wix-velo/external-db-airtable')
+        //     return await airtableFactory(config)
+        // }
         case 'dynamodb': {
             const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
             return await dynamoDbFactory(config)
         }
-        case 'bigquery': {
-            const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
-            return await bigqueryFactory(config)
-        }
+        // case 'bigquery': {
+        //     const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
+        //     return await bigqueryFactory(config)
+        // }
         default: {
             const { stubFactory } = require('./stub-db/stub-connector')
             return await stubFactory(type, config)

apps/velo-external-db/test/drivers/data_api_rest_test_support.ts

@@ -1,9 +1,70 @@
+import axios from 'axios'
 import { Item } from '@wix-velo/velo-external-db-types'
+import { dataSpi } from '@wix-velo/velo-external-db-core'
+import { streamToArray } from '@wix-velo/test-commons'
 
-const axios = require('axios').create({
+const axiosInstance = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
-export const givenItems = async(items: Item[], collectionName: string, auth: any) => await axios.post('/data/insert/bulk', { collectionName: collectionName, items: items }, auth)
+export const insertRequest = (collectionName: string, items: Item[], overwriteExisting: boolean): dataSpi.InsertRequest => ({
+    collectionId: collectionName,
+    items: items,
+    overwriteExisting,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const updateRequest = (collectionName: string, items: Item[]): dataSpi.UpdateRequest => ({
+    collectionId: collectionName,
+    items: items,
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    }
+})
+
+export const countRequest = (collectionName: string, filter?: dataSpi.Filter): dataSpi.CountRequest => ({
+    collectionId: collectionName,
+    filter: filter ?? '',
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+})
+
+export const queryRequest = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], filter?: dataSpi.Filter): dataSpi.QueryRequest => ({
+    collectionId: collectionName,
+    query: {
+        filter: filter ?? '',
+        sort: sort,
+        fields: fields,
+        fieldsets: undefined,
+        paging: {
+            limit: 25,
+            offset: 0,
+        },
+        cursorPaging: null
+    },
+    includeReferencedItems: [],
+    options: {
+        consistentRead: false,
+        appOptions: {},
+    },
+    omitTotalCount: false
+})
+
+
+export const queryCollectionAsArray = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], auth: any, filter?: dataSpi.Filter) =>
+    axiosInstance.post('/data/query',
+        queryRequest(collectionName, sort, fields, filter), { responseType: 'stream', transformRequest: auth.transformRequest })
+        .then(response => streamToArray(response.data))
+
+
+export const pagingMetadata = (count: number, total?: number): dataSpi.QueryResponsePart => ({ pagingMetadata: { count: count, offset: 0, total: total, tooManyToCount: false } })
+
 
-export const expectAllDataIn = async(collectionName: string, auth: any) => (await axios.post('/data/find', { collectionName: collectionName, filter: '', sort: '', skip: 0, limit: 25 }, auth)).data
+export const givenItems = async(items: Item[], collectionName: string, auth: any) =>
+    await axiosInstance.post('/data/insert', insertRequest(collectionName, items, false), { responseType: 'stream', transformRequest: auth.transformRequest })

apps/velo-external-db/test/drivers/hooks_test_support_v3.ts

@@ -0,0 +1,34 @@
+import { ExternalDbRouter } from '@wix-velo/velo-external-db-core'
+import { Item } from '@wix-velo/velo-external-db-types'
+
+
+export const requestBodyWith = (collectionId: string, items: Item[]) => ({
+    ...writeRequestBodyWith(collectionId, items), ...readRequestBodyWith(collectionId)
+})
+
+export const writeRequestBodyWith = (collectionId: string, items: Item[]) => ({
+    collectionId, items, item: items[0], itemId: items[0]?._id, itemIds: items.map((item: { _id: any }) => item._id), overWriteExisting: true
+})
+
+export const readRequestBodyWith = (collectionId: string) => ({
+    collectionId, filter: {}, query: { filter: {} }, omitTotalCount: false, group: { by: [], aggregation: [] }, initialFilter: {}, finalFilter: {}, sort: [], paging: { offset: 0, limit: 10 }
+})
+
+export const splitIdToThreeParts = (id: string) => {
+    return [id.slice(0, id.length / 3), id.slice(id.length / 3, id.length / 3 * 2), id.slice(id.length / 3 * 2)]
+}
+
+export const concatToProperty = <T>(obj: T, path: string, value: any): T => {
+    const pathArray = path.split('.')
+    const newObject = { ...obj }
+    let current = newObject
+
+    for (let i = 0; i < pathArray.length - 1; i++) {
+      current = current[pathArray[i]]
+    }
+
+    current[pathArray[pathArray.length - 1]] += value
+    return newObject
+  }
+
+export const resetHooks = (externalDbRouter: ExternalDbRouter) => externalDbRouter.reloadHooks()

apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts

@@ -1,5 +1,7 @@
 import { SystemFields, asWixSchemaHeaders } from '@wix-velo/velo-external-db-commons'
-import { InputField } from '@wix-velo/velo-external-db-types'
+import { InputField, DataOperation, FieldType, CollectionOperation } from '@wix-velo/velo-external-db-types'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
+import { Capabilities, ColumnsCapabilities } from '../types'
 
 export const responseWith = (matcher: any) => expect.objectContaining( { data: matcher } )
 
@@ -40,3 +42,42 @@ const toHaveCollections = (collections: string[]) => expect.objectContaining( {
 const listToHaveCollection = (collectionName: string) => expect.objectContaining( {
     schemas: expect.arrayContaining( [ expect.objectContaining( { id: collectionName } ) ] )
 } )
+
+const collectionCapabilities = (collectionOperations: CollectionOperation[], dataOperations: DataOperation[], fieldTypes: FieldType[]) => ({
+    collectionOperations: expect.arrayContaining(collectionOperations.map(schemaUtils.collectionOperationsToWixDataCollectionOperations)),
+    dataOperations: expect.arrayContaining(dataOperations.map(schemaUtils.dataOperationsToWixDataQueryOperators)),
+    fieldTypes: expect.arrayContaining(fieldTypes.map(schemaUtils.fieldTypeToWixDataEnum)),
+    referenceCapabilities: expect.objectContaining({ supportedNamespaces: [] }),
+    indexing: [],
+    encryption: 0
+})
+
+const filedMatcher = (field: InputField, columnsCapabilities: ColumnsCapabilities) => ({
+    key: field.name,
+    type: schemaUtils.fieldTypeToWixDataEnum(field.type),
+    capabilities: {
+        sortable: columnsCapabilities[field.type].sortable,
+        queryOperators: columnsCapabilities[field.type].columnQueryOperators.map(schemaUtils.queryOperatorsToWixDataQueryOperators)
+    },
+    encrypted: expect.any(Boolean)
+})
+
+const fieldsWith = (fields: InputField[], columnsCapabilities: ColumnsCapabilities) => expect.toIncludeSameMembers(fields.map(f => filedMatcher(f, columnsCapabilities)))
+
+export const collectionResponsesWith = (collectionName: string, fields: InputField[], capabilities: Capabilities) => {
+    const dataOperations = fields.map(f => f.name).includes('_id') ? capabilities.ReadWriteOperations : capabilities.ReadOnlyOperations
+    return {
+        id: collectionName,
+        capabilities: collectionCapabilities(capabilities.CollectionOperations, dataOperations, capabilities.FieldTypes),
+        fields: fieldsWith(fields, capabilities.ColumnsCapabilities),
+    }
+}
+
+export const createCollectionResponseWith = (collectionName: string, fields: InputField[], capabilities: Capabilities) => {
+    const dataOperations = fields.map(f => f.name).includes('_id') ? capabilities.ReadWriteOperations : capabilities.ReadOnlyOperations
+    return {
+        id: collectionName,
+        capabilities: collectionCapabilities(capabilities.CollectionOperations, dataOperations, capabilities.FieldTypes),
+        fields: fieldsWith(fields, capabilities.ColumnsCapabilities),
+    }
+}

apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts

@@ -1,14 +1,34 @@
+import axios from 'axios'
 import { InputField } from '@wix-velo/velo-external-db-types'
+import { streamToArray } from '@wix-velo/test-commons'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
 
-const axios = require('axios').create({
+
+const axiosClient = axios.create({
     baseURL: 'http://localhost:8080'
 })
 
 export const givenCollection = async(name: string, columns: InputField[], auth: any) => {
-    await axios.post('/schemas/create', { collectionName: name }, auth)
-    for (const column of columns) {
-        await axios.post('/schemas/column/add', { collectionName: name, column: column }, auth)
+    const collection = {
+        id: name,
+        fields: columns.map(schemaUtils.InputFieldToWixFormatField)
     }
+    await axiosClient.post('/collections/create', { collection }, { ...auth, responseType: 'stream' })
 }
 
-export const retrieveSchemaFor = async(collectionName: string, auth: any) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth)
+export const deleteAllCollections = async(auth: any) => {
+    const res = await axiosClient.post('/collections/get', { collectionIds: [] }, { ...auth, responseType: 'stream' })
+    const dataRes = await streamToArray(res.data) as any []
+    const collectionIds = dataRes.map(d => d.id)
+
+    for (const collectionId of collectionIds) {
+        await axiosClient.post('/collections/delete', { collectionId }, { ...auth, responseType: 'stream' })
+    }
+
+}
+
+export const retrieveSchemaFor = async(collectionName: string, auth: any) => {
+    const collectionGetStream = await axiosClient.post('/collections/get', { collectionIds: [collectionName] }, { ...auth, responseType: 'stream' })
+    const [collectionGetRes] = await streamToArray(collectionGetStream.data) as any[]
+    return collectionGetRes
+}

apps/velo-external-db/test/drivers/schema_provider_matchers.ts

@@ -1,6 +1,27 @@
-export const hasSameSchemaFieldsLike = (fields: {field: string, [x: string]: any}[]) => expect.arrayContaining( fields.map((f: any) => expect.objectContaining( f ) ))
+import { SystemFields } from '@wix-velo/velo-external-db-commons'
+import { ResponseField } from '@wix-velo/velo-external-db-types'
+import { Capabilities, ColumnsCapabilities } from '../types'
 
-export const collectionWithDefaultFields = () => hasSameSchemaFieldsLike([ { field: '_id', type: 'text' },
-                                                                          { field: '_createdDate', type: 'datetime' },
-                                                                          { field: '_updatedDate', type: 'datetime' },
-                                                                          { field: '_owner', type: 'text' } ])
+export const hasSameSchemaFieldsLike = (fields: ResponseField[]) => expect.arrayContaining(fields.map((f) => expect.objectContaining( f )))
+
+export const toContainDefaultFields = (columnsCapabilities: ColumnsCapabilities) => hasSameSchemaFieldsLike(SystemFields.map(f => ({
+    field: f.name, 
+    type: f.type,
+    capabilities: columnsCapabilities[f.type]
+})))
+
+
+export const collectionToContainFields = (collectionName: string, fields: any[], capabilities: Capabilities) => ({
+    id: collectionName,
+    fields: hasSameSchemaFieldsLike(fields),
+    capabilities: {
+        collectionOperations: capabilities.CollectionOperations,
+        dataOperations: capabilities.ReadWriteOperations,
+        fieldTypes: capabilities.FieldTypes,
+        referenceCapabilities: { supportedNamespaces: [] },
+        indexing: [],
+        encryption: 'notSupported'
+    }
+})
+
+export const toBeDefaultCollectionWith = (collectionName: string, capabilities: any) => collectionToContainFields(collectionName, SystemFields.map(f => ({ field: f.name, type: f.type })), capabilities)

apps/velo-external-db/test/drivers/wix_data_resources.ts

@@ -0,0 +1,17 @@
+import { Server } from 'http'
+import { app as mockServer } from './wix_data_testkit'
+
+let _server: Server
+const PORT = 9001
+
+export const initWixDataEnv = async() => {
+    _server = mockServer.listen(PORT)
+}
+
+export const shutdownWixDataEnv = async() => {
+    _server.close()
+}
+
+export const wixDataBaseUrl = () => {
+    return `http://localhost:${PORT}`
+}

apps/velo-external-db/test/drivers/wix_data_testkit.ts

@@ -0,0 +1,33 @@
+import { authConfig } from '@wix-velo/test-commons'
+import * as express from 'express'
+
+export const app = express()
+
+app.set('case sensitive routing', true)
+
+app.use(express.json())
+
+app.get('/v1/external-databases/:externalDatabaseId/public-keys', (_req, res) => {
+    res.json({
+        publicKeys: [
+            { id: authConfig.kid, publicKeyPem: authConfig.authPublicKey },
+        ]
+    })
+})
+
+app.use((_req, res) => {
+    res.status(404)
+    res.json({ error: 'NOT_FOUND' })
+})
+
+app.use((err, _req, res, next) => {
+    res.status(err.status)
+    res.json({
+        error: {
+            message: err.message,
+            status: err.status,
+            error: err.error
+        }
+    })
+    next()
+})