diff --git a/.eslintrc.json b/.eslintrc.json
index a9c0d4f59..26e435bc5 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -61,7 +61,8 @@
"@typescript-eslint/no-unused-vars": [ "error", { "ignoreRestSiblings": true, "argsIgnorePattern": "^_" } ],
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/ban-ts-comment": "warn",
- "eol-last": [ "error", "always" ]
+ "eol-last": [ "error", "always" ],
+ "@typescript-eslint/no-empty-interface": "off"
}
}
]
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 42ae49fb9..fc8854196 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -39,12 +39,12 @@ jobs:
matrix:
database: [
"postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
- "spanner",
"mysql", "mysql5",
"mssql", "mssql17",
+ spanner,
"mongo", "mongo4",
- "firestore",
"dynamodb",
+ "firestore",
"google-sheets"
]
diff --git a/apps/velo-external-db/src/app.ts b/apps/velo-external-db/src/app.ts
index 70fc0f258..c6e34d822 100644
--- a/apps/velo-external-db/src/app.ts
+++ b/apps/velo-external-db/src/app.ts
@@ -3,12 +3,10 @@ import { create, readCommonConfig } from '@wix-velo/external-db-config'
import { ExternalDbRouter, Hooks } from '@wix-velo/velo-external-db-core'
import { engineConnectorFor } from './storage/factory'
-
-const initConnector = async(hooks?: Hooks) => {
- const { vendor, type: adapterType, hideAppInfo } = readCommonConfig()
-
+const initConnector = async(wixDataBaseUrl?: string, hooks?: Hooks) => {
+ const { vendor, type: adapterType, externalDatabaseId, allowedMetasites, hideAppInfo } = readCommonConfig()
const configReader = create()
- const { authorization, secretKey, ...dbConfig } = await configReader.readConfig()
+ const { authorization, ...dbConfig } = await configReader.readConfig()
const { connector: engineConnector, providers, cleanup } = await engineConnectorFor(adapterType, dbConfig)
@@ -18,11 +16,13 @@ const initConnector = async(hooks?: Hooks) => {
authorization: {
roleConfig: authorization
},
- secretKey,
+ externalDatabaseId,
+ allowedMetasites,
vendor,
adapterType,
commonExtended: true,
- hideAppInfo
+ hideAppInfo,
+ wixDataBaseUrl: wixDataBaseUrl || 'https://www.wixapis.com/wix-data'
},
hooks,
})
@@ -30,11 +30,11 @@ const initConnector = async(hooks?: Hooks) => {
return { externalDbRouter, cleanup: async() => await cleanup(), schemaProvider: providers.schemaProvider }
}
-export const createApp = async() => {
+export const createApp = async(wixDataBaseUrl?: string) => {
const app = express()
- const initConnectorResponse = await initConnector()
+ const initConnectorResponse = await initConnector(wixDataBaseUrl)
app.use(initConnectorResponse.externalDbRouter.router)
const server = app.listen(8080, () => console.log('Connector listening on port 8080'))
- return { server, ...initConnectorResponse, reload: () => initConnector() }
+ return { server, ...initConnectorResponse, reload: () => initConnector(wixDataBaseUrl) }
}
diff --git a/apps/velo-external-db/src/storage/factory.ts b/apps/velo-external-db/src/storage/factory.ts
index 9948269b8..606521b9a 100644
--- a/apps/velo-external-db/src/storage/factory.ts
+++ b/apps/velo-external-db/src/storage/factory.ts
@@ -31,18 +31,18 @@ export const engineConnectorFor = async(_type: string, config: any): Promise<Dat
const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
return await googleSheetFactory(config)
}
- case 'airtable': {
- const { airtableFactory } = require('@wix-velo/external-db-airtable')
- return await airtableFactory(config)
- }
+ // case 'airtable': {
+ // const { airtableFactory } = require('@wix-velo/external-db-airtable')
+ // return await airtableFactory(config)
+ // }
case 'dynamodb': {
const { dynamoDbFactory } = require('@wix-velo/external-db-dynamodb')
return await dynamoDbFactory(config)
}
- case 'bigquery': {
- const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
- return await bigqueryFactory(config)
- }
+ // case 'bigquery': {
+ // const { bigqueryFactory } = require('@wix-velo/external-db-bigquery')
+ // return await bigqueryFactory(config)
+ // }
default: {
const { stubFactory } = require('./stub-db/stub-connector')
return await stubFactory(type, config)
diff --git a/apps/velo-external-db/test/drivers/data_api_rest_test_support.ts b/apps/velo-external-db/test/drivers/data_api_rest_test_support.ts
index ce9e593ba..3b7c10e00 100644
--- a/apps/velo-external-db/test/drivers/data_api_rest_test_support.ts
+++ b/apps/velo-external-db/test/drivers/data_api_rest_test_support.ts
@@ -1,9 +1,70 @@
+import axios from 'axios'
import { Item } from '@wix-velo/velo-external-db-types'
+import { dataSpi } from '@wix-velo/velo-external-db-core'
+import { streamToArray } from '@wix-velo/test-commons'
-const axios = require('axios').create({
+const axiosInstance = axios.create({
baseURL: 'http://localhost:8080'
})
-export const givenItems = async(items: Item[], collectionName: string, auth: any) => await axios.post('/data/insert/bulk', { collectionName: collectionName, items: items }, auth)
+export const insertRequest = (collectionName: string, items: Item[], overwriteExisting: boolean): dataSpi.InsertRequest => ({
+ collectionId: collectionName,
+ items: items,
+ overwriteExisting,
+ options: {
+ consistentRead: false,
+ appOptions: {},
+ }
+})
+
+export const updateRequest = (collectionName: string, items: Item[]): dataSpi.UpdateRequest => ({
+ collectionId: collectionName,
+ items: items,
+ options: {
+ consistentRead: false,
+ appOptions: {},
+ }
+})
+
+export const countRequest = (collectionName: string, filter?: dataSpi.Filter): dataSpi.CountRequest => ({
+ collectionId: collectionName,
+ filter: filter ?? '',
+ options: {
+ consistentRead: false,
+ appOptions: {},
+ },
+})
+
+export const queryRequest = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], filter?: dataSpi.Filter): dataSpi.QueryRequest => ({
+ collectionId: collectionName,
+ query: {
+ filter: filter ?? '',
+ sort: sort,
+ fields: fields,
+ fieldsets: undefined,
+ paging: {
+ limit: 25,
+ offset: 0,
+ },
+ cursorPaging: null
+ },
+ includeReferencedItems: [],
+ options: {
+ consistentRead: false,
+ appOptions: {},
+ },
+ omitTotalCount: false
+})
+
+
+export const queryCollectionAsArray = (collectionName: string, sort: dataSpi.Sorting[], fields: string[], auth: any, filter?: dataSpi.Filter) =>
+ axiosInstance.post('/data/query',
+ queryRequest(collectionName, sort, fields, filter), { responseType: 'stream', transformRequest: auth.transformRequest })
+ .then(response => streamToArray(response.data))
+
+
+export const pagingMetadata = (count: number, total?: number): dataSpi.QueryResponsePart => ({ pagingMetadata: { count: count, offset: 0, total: total, tooManyToCount: false } })
+
-export const expectAllDataIn = async(collectionName: string, auth: any) => (await axios.post('/data/find', { collectionName: collectionName, filter: '', sort: '', skip: 0, limit: 25 }, auth)).data
+export const givenItems = async(items: Item[], collectionName: string, auth: any) =>
+ await axiosInstance.post('/data/insert', insertRequest(collectionName, items, false), { responseType: 'stream', transformRequest: auth.transformRequest })
diff --git a/apps/velo-external-db/test/drivers/hooks_test_support_v3.ts b/apps/velo-external-db/test/drivers/hooks_test_support_v3.ts
new file mode 100644
index 000000000..3e1b39d77
--- /dev/null
+++ b/apps/velo-external-db/test/drivers/hooks_test_support_v3.ts
@@ -0,0 +1,34 @@
+import { ExternalDbRouter } from '@wix-velo/velo-external-db-core'
+import { Item } from '@wix-velo/velo-external-db-types'
+
+
+export const requestBodyWith = (collectionId: string, items: Item[]) => ({
+ ...writeRequestBodyWith(collectionId, items), ...readRequestBodyWith(collectionId)
+})
+
+export const writeRequestBodyWith = (collectionId: string, items: Item[]) => ({
+ collectionId, items, item: items[0], itemId: items[0]?._id, itemIds: items.map((item: { _id: any }) => item._id), overWriteExisting: true
+})
+
+export const readRequestBodyWith = (collectionId: string) => ({
+ collectionId, filter: {}, query: { filter: {} }, omitTotalCount: false, group: { by: [], aggregation: [] }, initialFilter: {}, finalFilter: {}, sort: [], paging: { offset: 0, limit: 10 }
+})
+
+export const splitIdToThreeParts = (id: string) => {
+ return [id.slice(0, id.length / 3), id.slice(id.length / 3, id.length / 3 * 2), id.slice(id.length / 3 * 2)]
+}
+
+export const concatToProperty = <T>(obj: T, path: string, value: any): T => {
+ const pathArray = path.split('.')
+ const newObject = { ...obj }
+ let current = newObject
+
+ for (let i = 0; i < pathArray.length - 1; i++) {
+ current = current[pathArray[i]]
+ }
+
+ current[pathArray[pathArray.length - 1]] += value
+ return newObject
+ }
+
+export const resetHooks = (externalDbRouter: ExternalDbRouter) => externalDbRouter.reloadHooks()
diff --git a/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts b/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts
index c7b6c014e..1e1717ace 100644
--- a/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts
+++ b/apps/velo-external-db/test/drivers/schema_api_rest_matchers.ts
@@ -1,5 +1,7 @@
import { SystemFields, asWixSchemaHeaders } from '@wix-velo/velo-external-db-commons'
-import { InputField } from '@wix-velo/velo-external-db-types'
+import { InputField, DataOperation, FieldType, CollectionOperation } from '@wix-velo/velo-external-db-types'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
+import { Capabilities, ColumnsCapabilities } from '../types'
export const responseWith = (matcher: any) => expect.objectContaining( { data: matcher } )
@@ -40,3 +42,42 @@ const toHaveCollections = (collections: string[]) => expect.objectContaining( {
const listToHaveCollection = (collectionName: string) => expect.objectContaining( {
schemas: expect.arrayContaining( [ expect.objectContaining( { id: collectionName } ) ] )
} )
+
+const collectionCapabilities = (collectionOperations: CollectionOperation[], dataOperations: DataOperation[], fieldTypes: FieldType[]) => ({
+ collectionOperations: expect.arrayContaining(collectionOperations.map(schemaUtils.collectionOperationsToWixDataCollectionOperations)),
+ dataOperations: expect.arrayContaining(dataOperations.map(schemaUtils.dataOperationsToWixDataQueryOperators)),
+ fieldTypes: expect.arrayContaining(fieldTypes.map(schemaUtils.fieldTypeToWixDataEnum)),
+ referenceCapabilities: expect.objectContaining({ supportedNamespaces: [] }),
+ indexing: [],
+ encryption: 0
+})
+
+const filedMatcher = (field: InputField, columnsCapabilities: ColumnsCapabilities) => ({
+ key: field.name,
+ type: schemaUtils.fieldTypeToWixDataEnum(field.type),
+ capabilities: {
+ sortable: columnsCapabilities[field.type].sortable,
+ queryOperators: columnsCapabilities[field.type].columnQueryOperators.map(schemaUtils.queryOperatorsToWixDataQueryOperators)
+ },
+ encrypted: expect.any(Boolean)
+})
+
+const fieldsWith = (fields: InputField[], columnsCapabilities: ColumnsCapabilities) => expect.toIncludeSameMembers(fields.map(f => filedMatcher(f, columnsCapabilities)))
+
+export const collectionResponsesWith = (collectionName: string, fields: InputField[], capabilities: Capabilities) => {
+ const dataOperations = fields.map(f => f.name).includes('_id') ? capabilities.ReadWriteOperations : capabilities.ReadOnlyOperations
+ return {
+ id: collectionName,
+ capabilities: collectionCapabilities(capabilities.CollectionOperations, dataOperations, capabilities.FieldTypes),
+ fields: fieldsWith(fields, capabilities.ColumnsCapabilities),
+ }
+}
+
+export const createCollectionResponseWith = (collectionName: string, fields: InputField[], capabilities: Capabilities) => {
+ const dataOperations = fields.map(f => f.name).includes('_id') ? capabilities.ReadWriteOperations : capabilities.ReadOnlyOperations
+ return {
+ id: collectionName,
+ capabilities: collectionCapabilities(capabilities.CollectionOperations, dataOperations, capabilities.FieldTypes),
+ fields: fieldsWith(fields, capabilities.ColumnsCapabilities),
+ }
+}
diff --git a/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts b/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts
index bf492fc90..558cc26b6 100644
--- a/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts
+++ b/apps/velo-external-db/test/drivers/schema_api_rest_test_support.ts
@@ -1,14 +1,34 @@
+import axios from 'axios'
import { InputField } from '@wix-velo/velo-external-db-types'
+import { streamToArray } from '@wix-velo/test-commons'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
-const axios = require('axios').create({
+
+const axiosClient = axios.create({
baseURL: 'http://localhost:8080'
})
export const givenCollection = async(name: string, columns: InputField[], auth: any) => {
- await axios.post('/schemas/create', { collectionName: name }, auth)
- for (const column of columns) {
- await axios.post('/schemas/column/add', { collectionName: name, column: column }, auth)
+ const collection = {
+ id: name,
+ fields: columns.map(schemaUtils.InputFieldToWixFormatField)
}
+ await axiosClient.post('/collections/create', { collection }, { ...auth, responseType: 'stream' })
}
-export const retrieveSchemaFor = async(collectionName: string, auth: any) => axios.post('/schemas/find', { schemaIds: [collectionName] }, auth)
+export const deleteAllCollections = async(auth: any) => {
+ const res = await axiosClient.post('/collections/get', { collectionIds: [] }, { ...auth, responseType: 'stream' })
+ const dataRes = await streamToArray(res.data) as any []
+ const collectionIds = dataRes.map(d => d.id)
+
+ for (const collectionId of collectionIds) {
+ await axiosClient.post('/collections/delete', { collectionId }, { ...auth, responseType: 'stream' })
+ }
+
+}
+
+export const retrieveSchemaFor = async(collectionName: string, auth: any) => {
+ const collectionGetStream = await axiosClient.post('/collections/get', { collectionIds: [collectionName] }, { ...auth, responseType: 'stream' })
+ const [collectionGetRes] = await streamToArray(collectionGetStream.data) as any[]
+ return collectionGetRes
+}
diff --git a/apps/velo-external-db/test/drivers/schema_provider_matchers.ts b/apps/velo-external-db/test/drivers/schema_provider_matchers.ts
index 6e41f7eed..9b806ac2b 100644
--- a/apps/velo-external-db/test/drivers/schema_provider_matchers.ts
+++ b/apps/velo-external-db/test/drivers/schema_provider_matchers.ts
@@ -1,6 +1,27 @@
-export const hasSameSchemaFieldsLike = (fields: {field: string, [x: string]: any}[]) => expect.arrayContaining( fields.map((f: any) => expect.objectContaining( f ) ))
+import { SystemFields } from '@wix-velo/velo-external-db-commons'
+import { ResponseField } from '@wix-velo/velo-external-db-types'
+import { Capabilities, ColumnsCapabilities } from '../types'
-export const collectionWithDefaultFields = () => hasSameSchemaFieldsLike([ { field: '_id', type: 'text' },
- { field: '_createdDate', type: 'datetime' },
- { field: '_updatedDate', type: 'datetime' },
- { field: '_owner', type: 'text' } ])
+export const hasSameSchemaFieldsLike = (fields: ResponseField[]) => expect.arrayContaining(fields.map((f) => expect.objectContaining( f )))
+
+export const toContainDefaultFields = (columnsCapabilities: ColumnsCapabilities) => hasSameSchemaFieldsLike(SystemFields.map(f => ({
+ field: f.name,
+ type: f.type,
+ capabilities: columnsCapabilities[f.type]
+})))
+
+
+export const collectionToContainFields = (collectionName: string, fields: any[], capabilities: Capabilities) => ({
+ id: collectionName,
+ fields: hasSameSchemaFieldsLike(fields),
+ capabilities: {
+ collectionOperations: capabilities.CollectionOperations,
+ dataOperations: capabilities.ReadWriteOperations,
+ fieldTypes: capabilities.FieldTypes,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: 'notSupported'
+ }
+})
+
+export const toBeDefaultCollectionWith = (collectionName: string, capabilities: any) => collectionToContainFields(collectionName, SystemFields.map(f => ({ field: f.name, type: f.type })), capabilities)
diff --git a/apps/velo-external-db/test/drivers/wix_data_resources.ts b/apps/velo-external-db/test/drivers/wix_data_resources.ts
new file mode 100644
index 000000000..02b136867
--- /dev/null
+++ b/apps/velo-external-db/test/drivers/wix_data_resources.ts
@@ -0,0 +1,17 @@
+import { Server } from 'http'
+import { app as mockServer } from './wix_data_testkit'
+
+let _server: Server
+const PORT = 9001
+
+export const initWixDataEnv = async() => {
+ _server = mockServer.listen(PORT)
+}
+
+export const shutdownWixDataEnv = async() => {
+ _server.close()
+}
+
+export const wixDataBaseUrl = () => {
+ return `http://localhost:${PORT}`
+}
diff --git a/apps/velo-external-db/test/drivers/wix_data_testkit.ts b/apps/velo-external-db/test/drivers/wix_data_testkit.ts
new file mode 100644
index 000000000..ebe170fcc
--- /dev/null
+++ b/apps/velo-external-db/test/drivers/wix_data_testkit.ts
@@ -0,0 +1,33 @@
+import { authConfig } from '@wix-velo/test-commons'
+import * as express from 'express'
+
+export const app = express()
+
+app.set('case sensitive routing', true)
+
+app.use(express.json())
+
+app.get('/v1/external-databases/:externalDatabaseId/public-keys', (_req, res) => {
+ res.json({
+ publicKeys: [
+ { id: authConfig.kid, publicKeyPem: authConfig.authPublicKey },
+ ]
+ })
+})
+
+app.use((_req, res) => {
+ res.status(404)
+ res.json({ error: 'NOT_FOUND' })
+})
+
+app.use((err, _req, res, next) => {
+ res.status(err.status)
+ res.json({
+ error: {
+ message: err.message,
+ status: err.status,
+ error: err.error
+ }
+ })
+ next()
+})
diff --git a/apps/velo-external-db/test/e2e/app.e2e.spec.ts b/apps/velo-external-db/test/e2e/app.e2e.spec.ts
index 30f71a531..e5dcca42f 100644
--- a/apps/velo-external-db/test/e2e/app.e2e.spec.ts
+++ b/apps/velo-external-db/test/e2e/app.e2e.spec.ts
@@ -2,6 +2,7 @@ import { authOwner } from '@wix-velo/external-db-testkit'
import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env } from '../resources/e2e_resources'
import { givenHideAppInfoEnvIsTrue } from '../drivers/app_info_config_test_support'
+import { CollectionCapability } from '@wix-velo/velo-external-db-core'
const axios = require('axios').create({
baseURL: 'http://localhost:8080'
})
@@ -20,7 +21,7 @@ describe(`Velo External DB: ${currentDbImplementationName()}`, () => {
})
test('answer provision with stub response', async() => {
- expect((await axios.post('/provision', { }, authOwner)).data).toEqual(expect.objectContaining({ protocolVersion: 2, vendor: 'azure' }))
+ expect((await axios.post('/provision', { }, authOwner)).data).toEqual(expect.objectContaining({ protocolVersion: 3, vendor: 'azure' }))
})
test('answer app info with stub response', async() => {
@@ -31,6 +32,15 @@ describe(`Velo External DB: ${currentDbImplementationName()}`, () => {
expect(appInfo).not.toContain(value)
})
})
+ test('answer capability', async() => {
+
+ expect((await axios.get('/capabilities', { }, authOwner)).data).toEqual(expect.objectContaining({
+ capabilities: {
+ collection: [CollectionCapability.CREATE]
+ }
+ }))
+ })
+
afterAll(async() => await teardownApp())
diff --git a/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts b/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts
new file mode 100644
index 000000000..5a582be11
--- /dev/null
+++ b/apps/velo-external-db/test/e2e/app_auth.e2e._spec.ts
@@ -0,0 +1,36 @@
+import { Uninitialized, gen } from '@wix-velo/test-commons'
+import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName } from '../resources/e2e_resources'
+
+
+describe(`Velo External DB authorization: ${currentDbImplementationName()}`, () => {
+ beforeAll(async() => {
+ await setupDb()
+
+ await initApp()
+ }, 20000)
+
+ afterAll(async() => {
+ await dbTeardown()
+ }, 20000)
+
+ // each(['data/query', 'data/aggregate', 'data/insert', 'data/insert/bulk', 'data/get', 'data/update',
+ // 'data/update/bulk', 'data/remove', 'data/remove/bulk', 'data/count'])
+ // .test('should throw 401 on a request to %s without the appropriate role', async(api) => {
+ // return expect(() => axios.post(api, { collectionName: ctx.collectionName }, authVisitor)).rejects.toThrow('401')
+ // })
+
+ // test('wrong secretKey will throw an appropriate error with the right format', async() => {
+ // return expect(() => axios.post('/schemas/list', {}, authOwnerWithoutSecretKey)).rejects.toMatchObject(errorResponseWith(401, 'You are not authorized'))
+ // })
+
+
+ const ctx = {
+ collectionName: Uninitialized,
+ }
+
+ beforeEach(async() => {
+ ctx.collectionName = gen.randomCollectionName()
+ })
+
+ afterAll(async() => await teardownApp())
+})
diff --git a/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts
deleted file mode 100644
index fd533b696..000000000
--- a/apps/velo-external-db/test/e2e/app_auth.e2e.spec.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import { Uninitialized, gen } from '@wix-velo/test-commons'
-import { authVisitor, authOwnerWithoutSecretKey, errorResponseWith } from '@wix-velo/external-db-testkit'
-import each from 'jest-each'
-import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName } from '../resources/e2e_resources'
-
-
-
-const axios = require('axios').create({
- baseURL: 'http://localhost:8080'
-})
-
-describe(`Velo External DB authorization: ${currentDbImplementationName()}`, () => {
- beforeAll(async() => {
- await setupDb()
-
- await initApp()
- }, 20000)
-
- afterAll(async() => {
- await dbTeardown()
- }, 20000)
-
- each(['data/find', 'data/aggregate', 'data/insert', 'data/insert/bulk', 'data/get', 'data/update',
- 'data/update/bulk', 'data/remove', 'data/remove/bulk', 'data/count'])
- .test('should throw 401 on a request to %s without the appropriate role', async(api) => {
- return expect(() => axios.post(api, { collectionName: ctx.collectionName }, authVisitor)).rejects.toThrow('401')
- })
-
- test('wrong secretKey will throw an appropriate error with the right format', async() => {
- return expect(() => axios.post('/schemas/list', {}, authOwnerWithoutSecretKey)).rejects.toMatchObject(errorResponseWith(401, 'You are not authorized'))
- })
-
- const ctx = {
- collectionName: Uninitialized,
- }
-
- beforeEach(async() => {
- ctx.collectionName = gen.randomCollectionName()
- })
-
- afterAll(async() => await teardownApp())
-})
diff --git a/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts
index 19d359adf..ad1415bc7 100644
--- a/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts
+++ b/apps/velo-external-db/test/e2e/app_data.e2e.spec.ts
@@ -1,19 +1,21 @@
-import { Uninitialized, gen as genCommon } from '@wix-velo/test-commons'
-import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField } = SchemaOperations
-import { testIfSupportedOperationsIncludes } from '@wix-velo/test-commons'
+import axios from 'axios'
+import each from 'jest-each'
+import * as Chance from 'chance'
+import { Uninitialized, gen as genCommon, testIfSupportedOperationsIncludes, streamToArray } from '@wix-velo/test-commons'
+import { InputField, SchemaOperations, Item } from '@wix-velo/velo-external-db-types'
+import { dataSpi } from '@wix-velo/velo-external-db-core'
+import { authAdmin, authOwner, authVisitor } from '@wix-velo/external-db-testkit'
import * as gen from '../gen'
import * as schema from '../drivers/schema_api_rest_test_support'
-import * as data from '../drivers/data_api_rest_test_support'
import * as matchers from '../drivers/schema_api_rest_matchers'
-import { authAdmin, authOwner, authVisitor } from '@wix-velo/external-db-testkit'
+import * as data from '../drivers/data_api_rest_test_support'
import * as authorization from '../drivers/authorization_test_support'
-import Chance = require('chance')
import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources'
+const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField, QueryNestedFields, NonAtomicBulkInsert, AtomicBulkInsert } = SchemaOperations
const chance = Chance()
-const axios = require('axios').create({
+const axiosInstance = axios.create({
baseURL: 'http://localhost:8080'
})
@@ -31,11 +33,12 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin)
await authorization.givenCollectionWithVisitorReadPolicy(ctx.collectionName)
- await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', sort: [{ fieldName: ctx.column.name }], skip: 0, limit: 25 }, authVisitor) ).resolves.toEqual(
- expect.objectContaining({ data: {
- items: [ ctx.item, ctx.anotherItem ].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1),
- totalCount: 2
- } }))
+
+ const itemsByOrder = [ctx.item, ctx.anotherItem].sort((a, b) => (a[ctx.column.name] > b[ctx.column.name]) ? 1 : -1).map(item => ({ item }))
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [{ fieldName: ctx.column.name, order: dataSpi.SortOrder.ASC }], undefined, authVisitor)).resolves.toEqual(
+ ([...itemsByOrder, data.pagingMetadata(2, 2)])
+ )
})
testIfSupportedOperationsIncludes(supportedOperations, [FilterByEveryField])('find api - filter by date', async() => {
@@ -45,164 +48,228 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
_createdDate: { $gte: ctx.pastVeloDate }
}
- await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: filterByDate, skip: 0, limit: 25 }, authOwner) ).resolves.toEqual(
- expect.objectContaining({ data: {
- items: [ ctx.item ],
- totalCount: 1
- } }))
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner, filterByDate)).resolves.toEqual(
+ expect.toIncludeSameMembers([{ item: ctx.item }, data.pagingMetadata(1, 1)]))
})
testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('find api with projection', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item ], ctx.collectionName, authAdmin)
- await expect( axios.post('/data/find', { collectionName: ctx.collectionName, filter: '', skip: 0, limit: 25, projection: [ctx.column.name] }, authOwner) ).resolves.toEqual(
- expect.objectContaining({ data: {
- items: [ ctx.item ].map(item => ({ [ctx.column.name]: item[ctx.column.name], _id: ctx.item._id })),
- totalCount: 1
- } }))
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], [ctx.column.name], authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([{ item: { [ctx.column.name]: ctx.item[ctx.column.name], _id: ctx.item._id } }, data.pagingMetadata(1, 1)])
+ )
})
//todo: create another test without sort for these implementations
test('insert api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await axios.post('/data/insert', { collectionName: ctx.collectionName, item: ctx.item }, authAdmin)
- await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ctx.item], totalCount: 1 })
+ const response = await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin })
+
+ const expectedItems = ctx.items.map(item => ({ item }))
+
+ await expect(streamToArray(response.data)).resolves.toEqual(expectedItems)
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers(
+ [
+ ...expectedItems,
+ data.pagingMetadata(ctx.items.length, ctx.items.length)
+ ])
+ )
})
- test('bulk insert api', async() => {
+ testIfSupportedOperationsIncludes(supportedOperations, [AtomicBulkInsert])('insert api should fail if item already exists', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin)
- await axios.post('/data/insert/bulk', { collectionName: ctx.collectionName, items: ctx.items }, authAdmin)
+ const response = axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin })
- await expect( data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual( { items: expect.arrayContaining(ctx.items), totalCount: ctx.items.length })
- })
+ const expectedItems = [dataSpi.QueryResponsePart.item(ctx.items[1])]
- testIfSupportedOperationsIncludes(supportedOperations, [ Aggregate ])('aggregate api', async() => {
- await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner)
- await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authAdmin)
-
- await expect( axios.post('/data/aggregate',
- {
- collectionName: ctx.collectionName,
- filter: { _id: { $eq: ctx.numberItem._id } },
- processingStep: {
- _id: {
- field1: '$_id',
- field2: '$_owner',
- },
- myAvg: {
- $avg: `$${ctx.numberColumns[0].name}`
- },
- mySum: {
- $sum: `$${ctx.numberColumns[1].name}`
- }
- },
- postFilteringStep: {
- $and: [
- { myAvg: { $gt: 0 } },
- { mySum: { $gt: 0 } }
- ],
- },
- }, authAdmin) ).resolves.toEqual(matchers.responseWith({ items: [ { _id: ctx.numberItem._id, _owner: ctx.numberItem._owner, myAvg: ctx.numberItem[ctx.numberColumns[0].name], mySum: ctx.numberItem[ctx.numberColumns[1].name] } ],
- totalCount: 0 }))
+ await expect(response).rejects.toThrow('409')
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers(
+ [
+ ...expectedItems,
+ data.pagingMetadata(expectedItems.length, expectedItems.length)
+ ])
+ )
})
- testIfSupportedOperationsIncludes(supportedOperations, [ DeleteImmediately ])('delete one api', async() => {
+ testIfSupportedOperationsIncludes(supportedOperations, [NonAtomicBulkInsert])('insert api should throw 409 error if item already exists and continue inserting the rest', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
+ await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin)
+
+ const response = axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false), { responseType: 'stream', ...authAdmin })
+
+ const expectedItems = ctx.items.map(i => dataSpi.QueryResponsePart.item(i))
+
+ await expect(response).rejects.toThrow('409')
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers(
+ [
+ ...expectedItems,
+ data.pagingMetadata(expectedItems.length, expectedItems.length)
+ ])
+ )
+ })
- await axios.post('/data/remove', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin)
+ test.skip('insert api should succeed if item already exists and overwriteExisting is on', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ await data.givenItems([ ctx.item ], ctx.collectionName, authAdmin)
+
+ const response = await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, [ctx.modifiedItem, ...ctx.items], true), { responseType: 'stream', ...authOwner })
+ const expectedItems = [ctx.modifiedItem, ...ctx.items].map(dataSpi.QueryResponsePart.item)
+
+ await expect(streamToArray(response.data)).resolves.toEqual(expectedItems)
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers(
+ [
+ ...expectedItems,
+ data.pagingMetadata(expectedItems.length, expectedItems.length)
+ ])
+ )
+ })
- await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 })
+ testIfSupportedOperationsIncludes(supportedOperations, [ Aggregate ])('aggregate api', async() => {
+
+ await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner)
+ await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authOwner)
+ const response = await axiosInstance.post('/data/aggregate',
+ {
+ collectionId: ctx.collectionName,
+ initialFilter: { _id: { $eq: ctx.numberItem._id } },
+ group: {
+ by: ['_id', '_owner'], aggregation: [
+ {
+ name: 'myAvg',
+ avg: ctx.numberColumns[0].name
+ },
+ {
+ name: 'mySum',
+ sum: ctx.numberColumns[1].name
+ }
+ ]
+ },
+ finalFilter: {
+ $and: [
+ { myAvg: { $gt: 0 } },
+ { mySum: { $gt: 0 } }
+ ],
+ },
+ }, { responseType: 'stream', ...authOwner })
+
+ await expect(streamToArray(response.data)).resolves.toEqual(
+ expect.toIncludeSameMembers([{ item: {
+ _id: ctx.numberItem._id,
+ _owner: ctx.numberItem._owner,
+ myAvg: ctx.numberItem[ctx.numberColumns[0].name],
+ mySum: ctx.numberItem[ctx.numberColumns[1].name]
+ } },
+ data.pagingMetadata(1, 1)
+ ]))
})
testIfSupportedOperationsIncludes(supportedOperations, [ DeleteImmediately ])('bulk delete api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems(ctx.items, ctx.collectionName, authAdmin)
- await axios.post('/data/remove/bulk', { collectionName: ctx.collectionName, itemIds: ctx.items.map((i: { _id: any }) => i._id) }, authAdmin)
-
- await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ ], totalCount: 0 })
- })
+ const response = await axiosInstance.post('/data/remove', {
+ collectionId: ctx.collectionName, itemIds: ctx.items.map(i => i._id)
+ }, { responseType: 'stream', ...authAdmin })
- test('get by id api', async() => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
+ const expectedItems = ctx.items.map(item => ({ item }))
- await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id }, authAdmin) ).resolves.toEqual(matchers.responseWith({ item: ctx.item }))
+ await expect(streamToArray(response.data)).resolves.toEqual(expect.toIncludeSameMembers(expectedItems))
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual([data.pagingMetadata(0, 0)])
})
- test('get by id api should throw 404 if not found', async() => {
+ test('query by id api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
+ await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
- await expect( axios.post('/data/get', { collectionName: ctx.collectionName, itemId: 'wrong' }, authAdmin) ).rejects.toThrow('404')
+ const filter = {
+ _id: { $eq: ctx.item._id }
+ }
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, undefined, authOwner, filter)).resolves.toEqual(expect.toIncludeSameMembers(
+ [...[dataSpi.QueryResponsePart.item(ctx.item)], data.pagingMetadata(1, 1)])
+ )
})
- testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('get by id api with projection', async() => {
+ test('query by id api should return empty result if not found', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
- await expect(axios.post('/data/get', { collectionName: ctx.collectionName, itemId: ctx.item._id, projection: [ctx.column.name] }, authAdmin)).resolves.toEqual(
- matchers.responseWith({
- item: { [ctx.column.name]: ctx.item[ctx.column.name], _id: ctx.item._id }
- }))
+ const filter = {
+ _id: { $eq: 'wrong' }
+ }
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, undefined, authOwner, filter)).resolves.toEqual(
+ ([data.pagingMetadata(0, 0)])
+ )
})
- testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('update api', async() => {
+ testIfSupportedOperationsIncludes(supportedOperations, [ Projection ])('query by id api with projection', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
- await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.modifiedItem }, authAdmin)
+ const filter = {
+ _id: { $eq: ctx.item._id }
+ }
- await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({ items: [ctx.modifiedItem], totalCount: 1 })
+ await expect(data.queryCollectionAsArray(ctx.collectionName, undefined, [ctx.column.name], authOwner, filter)).resolves.toEqual(expect.toIncludeSameMembers(
+ [dataSpi.QueryResponsePart.item({ [ctx.column.name]: ctx.item[ctx.column.name], _id: ctx.item._id }), data.pagingMetadata(1, 1)])
+ )
})
- testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('bulk update api', async() => {
+ testIfSupportedOperationsIncludes(supportedOperations, [ UpdateImmediately ])('update api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems(ctx.items, ctx.collectionName, authAdmin)
+ const response = await axiosInstance.post('/data/update', data.updateRequest(ctx.collectionName, ctx.modifiedItems), { responseType: 'stream', ...authAdmin })
- await axios.post('/data/update/bulk', { collectionName: ctx.collectionName, items: ctx.modifiedItems }, authAdmin)
+ const expectedItems = ctx.modifiedItems.map(dataSpi.QueryResponsePart.item)
- await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual( { items: expect.arrayContaining(ctx.modifiedItems), totalCount: ctx.modifiedItems.length })
+ await expect(streamToArray(response.data)).resolves.toEqual(expectedItems)
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers(
+ [
+ ...expectedItems,
+ data.pagingMetadata(ctx.modifiedItems.length, ctx.modifiedItems.length)
+ ]))
})
test('count api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin)
-
- await expect( axios.post('/data/count', { collectionName: ctx.collectionName, filter: '' }, authAdmin) ).resolves.toEqual(matchers.responseWith( { totalCount: 2 } ))
+ await expect( axiosInstance.post('/data/count', data.countRequest(ctx.collectionName), authAdmin) ).resolves.toEqual(
+ matchers.responseWith( { totalCount: 2 } ))
})
testIfSupportedOperationsIncludes(supportedOperations, [ Truncate ])('truncate api', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item, ctx.anotherItem], ctx.collectionName, authAdmin)
-
- await axios.post('/data/truncate', { collectionName: ctx.collectionName }, authAdmin)
-
- await expect( data.expectAllDataIn(ctx.collectionName, authAdmin) ).resolves.toEqual({ items: [ ], totalCount: 0 })
+ await axiosInstance.post('/data/truncate', { collectionId: ctx.collectionName }, authAdmin)
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual([data.pagingMetadata(0, 0)])
})
test('insert undefined to number columns should inserted as null', async() => {
await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner)
delete ctx.numberItem[ctx.numberColumns[0].name]
delete ctx.numberItem[ctx.numberColumns[1].name]
+
+ await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, [ctx.numberItem], false), { responseType: 'stream', ...authAdmin })
- await axios.post('/data/insert', { collectionName: ctx.collectionName, item: ctx.numberItem }, authAdmin)
-
-
- await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({
- items: [
- {
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers(
+ [
+ dataSpi.QueryResponsePart.item({
...ctx.numberItem,
[ctx.numberColumns[0].name]: null,
[ctx.numberColumns[1].name]: null,
- }
- ], totalCount: 1
- })
+ }),
+ data.pagingMetadata(1, 1)
+ ]))
})
@@ -212,24 +279,113 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
ctx.numberItem[ctx.numberColumns[0].name] = null
ctx.numberItem[ctx.numberColumns[1].name] = null
- await axios.post('/data/update', { collectionName: ctx.collectionName, item: ctx.numberItem }, authAdmin)
+ await axiosInstance.post('/data/update', data.updateRequest(ctx.collectionName, [ctx.numberItem]), { responseType: 'stream', ...authAdmin })
+
- await expect(data.expectAllDataIn(ctx.collectionName, authAdmin)).resolves.toEqual({
- items: [
- {
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeSameMembers(
+ [
+ dataSpi.QueryResponsePart.item({
...ctx.numberItem,
[ctx.numberColumns[0].name]: null,
[ctx.numberColumns[1].name]: null,
+ }),
+ data.pagingMetadata(1, 1)
+ ]))
+ })
+
+ testIfSupportedOperationsIncludes(supportedOperations, [ QueryNestedFields ])('query on nested fields', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.objectColumn], authOwner)
+ await data.givenItems([ctx.objectItem], ctx.collectionName, authAdmin)
+
+ const filter = {
+ [`${ctx.objectColumn.name}.${ctx.nestedFieldName}`]: { $eq: ctx.objectItem[ctx.objectColumn.name][ctx.nestedFieldName] }
+ }
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner, filter)).resolves.toEqual(expect.toIncludeSameMembers(
+ [ dataSpi.QueryResponsePart.item(ctx.objectItem), data.pagingMetadata(1, 1) ]
+ ))
+ })
+
+ describe('error handling', () => {
+ test.skip('insert api with duplicate _id should fail with WDE0074, 409', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ await data.givenItems([ctx.item], ctx.collectionName, authAdmin)
+ let error
+
+ await axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, [ctx.item], false), authAdmin).catch(e => error = e)
+
+ expect(error).toBeDefined()
+ expect(error.response.status).toEqual(409)
+ expect(error.response.data).toEqual(expect.objectContaining({
+ code: 'WDE0074',
+ data: {
+ itemId: ctx.item._id,
+ collectionId: ctx.collectionName
+ }
+ }))
+ })
+
+ each([
+ ['update', '/data/update', data.updateRequest.bind(null, 'nonExistingCollection', [])],
+ ['count', '/data/count', data.countRequest.bind(null, 'nonExistingCollection')],
+ ['insert', '/data/insert', data.insertRequest.bind(null, 'nonExistingCollection', [], false)],
+ ['query', '/data/query', data.queryRequest.bind(null, 'nonExistingCollection', [], undefined)],
+ ])
+ .test('%s api on non existing collection should fail with WDE0025, 404', async(_, api, request) => {
+ let error
+
+ await axiosInstance.post(api, request(), authAdmin).catch(e => error = e)
+
+ expect(error).toBeDefined()
+ expect(error.response.status).toEqual(404)
+ expect(error.response.data).toEqual(expect.objectContaining({
+ code: 'WDE0025',
+ data: {
+ collectionId: 'nonExistingCollection'
}
- ], totalCount: 1
+ }))
+ })
+
+ test('filter non existing column should fail with WDE0147, 400', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ let error
+
+ await axiosInstance.post('/data/query', data.queryRequest(ctx.collectionName, [], undefined, { nonExistingColumn: { $eq: 'value' } }), authAdmin).catch(e => error = e)
+
+ expect(error).toBeDefined()
+ expect(error.response.status).toEqual(400)
+ expect(error.response.data).toEqual(expect.objectContaining({
+ code: 'WDE0147',
+ data: {
+ collectionId: ctx.collectionName,
+ propertyName: 'nonExistingColumn'
+ }
+ }))
})
})
+ interface Ctx {
+ collectionName: string
+ column: InputField
+ numberColumns: InputField[]
+ objectColumn: InputField
+ item: Item
+ items: Item[]
+ modifiedItem: Item
+ modifiedItems: Item[]
+ anotherItem: Item
+ numberItem: Item
+ anotherNumberItem: Item
+ objectItem: Item
+ nestedFieldName: string
+ pastVeloDate: { $date: string; }
+ }
- const ctx = {
+ const ctx: Ctx = {
collectionName: Uninitialized,
column: Uninitialized,
numberColumns: Uninitialized,
+ objectColumn: Uninitialized,
item: Uninitialized,
items: Uninitialized,
modifiedItem: Uninitialized,
@@ -237,6 +393,8 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
anotherItem: Uninitialized,
numberItem: Uninitialized,
anotherNumberItem: Uninitialized,
+ objectItem: Uninitialized,
+ nestedFieldName: Uninitialized,
pastVeloDate: Uninitialized,
}
@@ -246,6 +404,7 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
ctx.collectionName = gen.randomCollectionName()
ctx.column = gen.randomColumn()
ctx.numberColumns = gen.randomNumberColumns()
+ ctx.objectColumn = gen.randomObjectColumn()
ctx.item = genCommon.randomEntity([ctx.column.name])
ctx.items = Array.from({ length: 10 }, () => genCommon.randomEntity([ctx.column.name]))
ctx.modifiedItems = ctx.items.map((i: any) => ( { ...i, [ctx.column.name]: chance.word() } ) )
@@ -253,6 +412,8 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`, ()
ctx.anotherItem = genCommon.randomEntity([ctx.column.name])
ctx.numberItem = genCommon.randomNumberEntity(ctx.numberColumns)
ctx.anotherNumberItem = genCommon.randomNumberEntity(ctx.numberColumns)
+ ctx.nestedFieldName = chance.word()
+ ctx.objectItem = { ...genCommon.randomEntity(), [ctx.objectColumn.name]: { [ctx.nestedFieldName]: chance.word() } }
ctx.pastVeloDate = genCommon.pastVeloDate()
})
})
diff --git a/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts
index 1e8341b5b..32c4705d8 100644
--- a/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts
+++ b/apps/velo-external-db/test/e2e/app_data_hooks.e2e.spec.ts
@@ -1,14 +1,17 @@
-import each from 'jest-each'
import { authOwner, errorResponseWith } from '@wix-velo/external-db-testkit'
-import { testSupportedOperations } from '@wix-velo/test-commons'
-import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { streamToArray } from '@wix-velo/test-commons'
+import { dataSpi, types as coreTypes, collectionSpi } from '@wix-velo/velo-external-db-core'
+import { DataOperation, InputField, ItemWithId, SchemaOperations } from '@wix-velo/velo-external-db-types'
import { Uninitialized, gen as genCommon } from '@wix-velo/test-commons'
import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, env, supportedOperations } from '../resources/e2e_resources'
import gen = require('../gen')
import schema = require('../drivers/schema_api_rest_test_support')
-import data = require('../drivers/data_api_rest_test_support')
-import hooks = require('../drivers/hooks_test_support')
-const { UpdateImmediately, DeleteImmediately, Aggregate } = SchemaOperations
+import * as data from '../drivers/data_api_rest_test_support'
+import hooks = require('../drivers/hooks_test_support_v3')
+import * as matchers from '../drivers/schema_api_rest_matchers'
+import each from 'jest-each'
+
+const { Aggregate } = SchemaOperations
const axios = require('axios').create({
@@ -26,276 +29,453 @@ describe(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () =>
await dbTeardown()
}, 20000)
-
- describe('After hooks', () => {
- describe('Write Operations', () => {
- each(testSupportedOperations(supportedOperations, [
- ['afterInsert', '/data/insert'],
- ['afterBulkInsert', '/data/insert/bulk'],
- ['afterUpdate', '/data/update', { neededOperations: [UpdateImmediately] }],
- ['afterBulkUpdate', '/data/update/bulk', { neededOperations: [UpdateImmediately] }],
- ['afterRemove', '/data/remove', { neededOperations: [DeleteImmediately] }],
- ['afterBulkRemove', '/data/remove/bulk', { neededOperations: [DeleteImmediately] }]
- ])).test('specific hook %s should overwrite non-specific and change payload', async(hookName: string, api: string) => {
+ describe('Before Hooks', () => {
+ describe('Read Operations', () => {
+ test('before query request - should be able to modify the request, specific hooks should overwrite non-specific', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- if (!['afterInsert', 'afterBulkInsert'].includes(hookName)) {
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
- }
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+
+
+ const [idPart1, idPart2, idPart3] = hooks.splitIdToThreeParts(ctx.item._id)
env.externalDbRouter.reloadHooks({
dataHooks: {
- afterAll: (payload, _requestContext, _serviceContext) => {
- return { ...payload, [hookName]: false, afterAll: true, afterWrite: false }
+ beforeAll: (payload: dataSpi.QueryRequest, _requestContext, _serviceContext) => {
+ return {
+ ...payload, omitTotalCount: true, query: { ...payload.query, filter: { _id: { $eq: idPart1 } } }
+ }
},
- afterWrite: (payload, _requestContext, _serviceContext) => {
- return { ...payload, [hookName]: false, afterWrite: true }
+ beforeRead: (payload: dataSpi.QueryRequest, _requestContext, _serviceContext) => {
+ return {
+ ...hooks.concatToProperty(payload, 'query.filter._id.$eq', idPart2),
+ }
},
- [hookName]: (payload, _requestContext, _serviceContext) => {
- return { ...payload, [hookName]: true }
+ beforeQuery: (payload: dataSpi.QueryRequest, _requestContext, _serviceContext) => {
+ return {
+ ...hooks.concatToProperty(payload, 'query.filter._id.$eq', idPart3),
+ }
}
}
})
- await expect(axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, ctx.items), authOwner)).resolves.toEqual(
- expect.objectContaining({ data: expect.objectContaining({ [hookName]: true, afterAll: true, afterWrite: true }) })
- )
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner, { _id: { $ne: ctx.item._id } })).resolves.toEqual(
+ expect.toIncludeSameMembers([{ item: ctx.item }, data.pagingMetadata(1)]))
})
- })
-
- describe('Read Operations', () => {
- each(testSupportedOperations(supportedOperations, [
- ['afterGetById', '/data/get'],
- ['afterFind', '/data/find'],
- ['afterAggregate', '/data/aggregate', { neededOperations: [Aggregate] }],
- ['afterCount', '/data/count']
- ])).test('specific hook %s should overwrite non-specific and change payload', async(hookName: string, api: string) => {
- if (hooks.skipAggregationIfNotSupported(hookName, supportedOperations))
- return
+ test('before count request - should be able to modify the query, specific hooks should overwrite non-specific', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+
+ const [idPart1, idPart2, idPart3] = hooks.splitIdToThreeParts(ctx.item._id)
env.externalDbRouter.reloadHooks({
dataHooks: {
- afterAll: (payload, _requestContext, _serviceContext) => {
- return { ...payload, afterAll: true, [hookName]: false }
+ beforeAll: (payload: dataSpi.QueryRequest, _requestContext, _serviceContext) => {
+ return {
+ ...payload, filter: { _id: { $eq: idPart1 } }
+ }
},
- afterRead: (payload, _requestContext, _serviceContext) => {
- return { ...payload, afterAll: false, [hookName]: false }
+ beforeRead: (payload: dataSpi.QueryRequest, _requestContext, _serviceContext) => {
+ return {
+ ...hooks.concatToProperty(payload, 'filter._id.$eq', idPart2),
+ }
},
- [hookName]: (payload, _requestContext, _serviceContext) => {
- return { ...payload, [hookName]: true }
+ beforeCount: (payload: dataSpi.CountRequest, _requestContext, _serviceContext): dataSpi.CountRequest => {
+ return {
+ ...hooks.concatToProperty(payload, 'filter._id.$eq', idPart3),
+ }
}
}
})
- await expect(axios.post(api, hooks.readRequestBodyWith(ctx.collectionName, ctx.items), authOwner)).resolves.toEqual(
- expect.objectContaining({ data: expect.objectContaining({ [hookName]: true, afterAll: false }) })
- )
+ await expect(axios.post('/data/count', data.countRequest(ctx.collectionName, { _id: { $ne: ctx.item._id } }), authOwner)).resolves.toEqual(
+ matchers.responseWith({ totalCount: 1 }))
})
- })
- })
-
- describe('Before hooks', () => {
- describe('Write Operations', () => {
- each(testSupportedOperations(supportedOperations, [
- ['beforeInsert', '/data/insert'],
- ['beforeUpdate', '/data/update', { neededOperations: [UpdateImmediately] }],
- ])).test('specific hook %s should overwrite non-specific and change payload', async(hookName: string, api: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column, ctx.beforeAllColumn, ctx.beforeWriteColumn, ctx.beforeHookColumn], authOwner)
- if (hookName !== 'beforeInsert') {
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
- }
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- beforeAll: (payload, _requestContext, _serviceContext) => (
- { ...payload, item: { ...payload.item, beforeAll: true, beforeWrite: false, beforeHook: false } }
- ),
- beforeWrite: (payload, _requestContext, _serviceContext) => (
- { ...payload, item: { ...payload.item, beforeWrite: true, beforeHook: false } }
- ),
- [hookName]: ({ item }, _requestContext, _serviceContext) => ({
- item: { ...item, beforeHook: true }
- })
- }
- })
+ if (supportedOperations.includes(Aggregate)) {
+ test('before aggregate request - should be able to modify group, initialFilter and finalFilter', async() => {
+ await schema.givenCollection(ctx.collectionName, ctx.numberColumns, authOwner)
+ await data.givenItems([ctx.numberItem, ctx.anotherNumberItem], ctx.collectionName, authOwner)
- await expect(axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), authOwner)).resolves.toEqual(
- expect.objectContaining({
- data: {
- item: expect.objectContaining({
- beforeAll: true, beforeWrite: true, beforeHook: true
- })
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ beforeAll: (payload: dataSpi.AggregateRequest, _requestContext, _serviceContext): dataSpi.AggregateRequest => {
+ return {
+ ...payload,
+ group: { ...payload.group, by: [] },
+ initialFilter: { _id: { $eq: ctx.numberItem._id } },
+ }
+ },
+ beforeRead: (payload: dataSpi.AggregateRequest, _requestContext, _serviceContext): dataSpi.AggregateRequest => {
+ return {
+ ...payload,
+ group: { ...payload.group, by: ['_id'] },
+ finalFilter: { myAvg: { $gt: 0 } },
+ }
+ },
+ beforeAggregate: (payload: dataSpi.AggregateRequest, _requestContext, _serviceContext): dataSpi.AggregateRequest => {
+ return {
+ ...payload,
+ group: { ...payload.group, by: ['_id', '_owner'] },
+ }
+ }
}
})
- )
- })
- each(testSupportedOperations(supportedOperations, [
- ['beforeBulkInsert', '/data/insert/bulk'],
- ['beforeBulkUpdate', '/data/update/bulk', { neededOperations: [UpdateImmediately] }],
- ])).test('specific hook %s should overwrite non-specific and change payload', async(hookName: string, api: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column, ctx.beforeAllColumn, ctx.beforeWriteColumn, ctx.beforeHookColumn], authOwner)
- if (hookName !== 'beforeBulkInsert') {
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
- }
+ const response = await axios.post('/data/aggregate',
+ {
+ collectionId: ctx.collectionName,
+ initialFilter: { _id: { $ne: ctx.numberItem._id } },
+ group: {
+ by: ['_id'], aggregation: [
+ {
+ name: 'myAvg',
+ avg: ctx.numberColumns[0].name
+ },
+ {
+ name: 'mySum',
+ sum: ctx.numberColumns[1].name
+ }
+ ]
+ },
+ finalFilter: { myAvg: { $lt: 0 } },
+ }, { responseType: 'stream', ...authOwner })
+
+ await expect(streamToArray(response.data)).resolves.toEqual(
+ expect.toIncludeSameMembers([{
+ item: {
+ _id: ctx.numberItem._id,
+ _owner: ctx.numberItem._owner,
+ myAvg: ctx.numberItem[ctx.numberColumns[0].name],
+ mySum: ctx.numberItem[ctx.numberColumns[1].name]
+ }
+ },
+ data.pagingMetadata(1, 1)
+ ]))
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- beforeAll: (payload, _requestContext, _serviceContext) => (
- { ...payload, items: payload.items.map(item => ({ ...item, beforeAll: true, beforeWrite: false, beforeHook: false })) }
- ),
- beforeWrite: (payload, _requestContext, _serviceContext) => (
- { ...payload, items: payload.items.map(item => ({ ...item, beforeWrite: true, beforeHook: false })) }
- ),
- [hookName]: ({ items }, _requestContext, _serviceContext) => ({
- items: items.map((item: any) => ({ ...item, beforeHook: true }))
- })
- }
})
- await expect(axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, ctx.items), authOwner)).resolves.toEqual(
- expect.objectContaining({
- data: {
- items: ctx.items.map((item: any) => ({
- ...item, beforeAll: true, beforeWrite: true, beforeHook: true
- }))
- }
- })
- )
- })
- each(['beforeAll', 'beforeWrite', 'beforeRemove'])
- .test('hook %s with data/remove/bulk api should throw 400 with the appropriate message if hook throwing', async(hookName: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ }
+ })
+ describe('Write Operations', () => {
+ each([
+ ['insert', 'beforeInsert', '/data/insert'],
+ ['update', 'beforeUpdate', '/data/update'],
+ ])
+ .test('before %s request - should be able to modify the item', async(operation, hookName, api) => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column, ctx.afterAllColumn, ctx.afterWriteColumn, ctx.afterHookColumn], authOwner)
+ if (operation !== 'insert') {
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ }
env.externalDbRouter.reloadHooks({
dataHooks: {
+ beforeAll: (payload, requestContext: coreTypes.RequestContext, _serviceContext) => {
+ if (requestContext.operation !== DataOperation.query) {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterWriteColumn.name]: false,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ }
+ },
+ beforeWrite: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterWriteColumn.name]: true,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
[hookName]: (payload, _requestContext, _serviceContext) => {
- if (payload.itemId === ctx.item._id) {
- throw ('Should not be removed')
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterHookColumn.name]: true,
+ }))
}
}
}
})
- await expect(axios.post('/data/remove', hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), authOwner)).rejects.toMatchObject(
- errorResponseWith(400, 'Should not be removed')
+ await axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), authOwner)
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([{
+ item: {
+ ...ctx.item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterWriteColumn.name]: true,
+ [ctx.afterHookColumn.name]: true,
+ }
+ }, data.pagingMetadata(1, 1)])
)
})
- each(['beforeAll', 'beforeWrite', 'beforeBulkRemove'])
- .test('hook %s with data/remove/bulk api should throw 400 with the appropriate message if hook throwing', async(hookName: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
+ test('before remove request - should be able to modify the item id', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+
+ const [idPart1, idPart2, idPart3] = hooks.splitIdToThreeParts(ctx.item._id)
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- [hookName]: (payload, _requestContext, _serviceContext) => {
- if (payload.itemIds[0] === ctx.items[0]._id) {
- throw ('Should not be removed')
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ beforeAll: (payload: dataSpi.RemoveRequest, requestContext: coreTypes.RequestContext, _serviceContext) => {
+ if (requestContext.operation !== DataOperation.query) {
+ return {
+ ...payload, itemIds: [idPart1]
}
}
+ },
+ beforeWrite: (payload: dataSpi.RemoveRequest, _requestContext, _serviceContext) => {
+ return {
+ ...payload, itemIds: [`${payload.itemIds[0]}${idPart2}`]
+ }
+ },
+ beforeRemove: (payload: dataSpi.RemoveRequest, _requestContext, _serviceContext) => {
+ return {
+ ...payload, itemIds: [`${payload.itemIds[0]}${idPart3}`]
+ }
}
- })
-
- await expect(axios.post('/data/remove/bulk', hooks.writeRequestBodyWith(ctx.collectionName, ctx.items), authOwner)).rejects.toMatchObject(
- errorResponseWith(400, 'Should not be removed')
- )
+ }
})
- })
- describe('Read Operations', () => {
- each(['beforeAll', 'beforeRead', 'beforeFind'])
- .test('%s should able to change filter payload /data/find', async(hookName: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ await axios.post('/data/remove', hooks.writeRequestBodyWith(ctx.collectionName, [ctx.numberItem]), authOwner)
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- [hookName]: (payload, _requestContext, _serviceContext) => {
- return { ...payload, filter: { _id: { $eq: ctx.item._id } } }
- },
- }
- })
-
- const response = await axios.post('/data/find', hooks.findRequestBodyWith(ctx.collectionName, { _id: { $ne: ctx.item._id } }), authOwner)
- expect(response.data.items).toEqual([ctx.item])
- })
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([data.pagingMetadata(0, 0)])
+ )
+ })
- test('beforeFind should be able to change projection payload /data/find', async() => {
+ test('before truncate request - should be able to modify the collection name', async() => {
await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ const [collectionIdPart1, collectionIdPart2, collectionIdPart3] = hooks.splitIdToThreeParts(ctx.collectionName)
+
env.externalDbRouter.reloadHooks({
dataHooks: {
- beforeFind: (payload, _requestContext, _serviceContext) => {
- return { ...payload, projection: ['_id'] }
+ beforeAll: (payload: dataSpi.TruncateRequest, requestContext: coreTypes.RequestContext, _serviceContext) => {
+ if (requestContext.operation !== DataOperation.query) {
+ return { ...payload, collectionId: collectionIdPart1 }
+ }
+ },
+ beforeWrite: (payload: dataSpi.TruncateRequest, _requestContext, _serviceContext) => {
+ return hooks.concatToProperty(payload, 'collectionId', collectionIdPart2)
+ },
+ beforeTruncate: (payload: dataSpi.TruncateRequest, _requestContext, _serviceContext) => {
+ return hooks.concatToProperty(payload, 'collectionId', collectionIdPart3)
}
}
})
- const response = await axios.post('/data/find', hooks.findRequestBodyWith(ctx.collectionName, { _id: { $eq: ctx.item._id } }), authOwner)
- expect(response.data.items).toEqual([{ _id: ctx.item._id }])
+ await axios.post('/data/truncate', hooks.writeRequestBodyWith('wrongCollectionId', []), authOwner)
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([data.pagingMetadata(0, 0)])
+ )
})
- each(['beforeAll', 'beforeRead', 'beforeGetById'])
- .test('%s should able to change payload /data/get', async(hookName: string) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ })
+ })
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- [hookName]: (_payload, _requestContext, _serviceContext) => ({
- itemId: ctx.item._id
- })
+ describe('After Hooks', () => {
+ describe('Read Operations', () => {
+ test('after query request - should be able to modify query response', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column, ctx.afterAllColumn, ctx.afterReadColumn, ctx.afterHookColumn], authOwner)
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ afterAll: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterReadColumn.name]: false,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
+ afterRead: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterReadColumn.name]: true,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
+ afterQuery: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterHookColumn.name]: true,
+ }))
+ }
}
- })
+ }
+ })
+
+ await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([{
+ item: {
+ ...ctx.item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterHookColumn.name]: true,
+ [ctx.afterReadColumn.name]: true,
+ }
+ }, data.pagingMetadata(1, 1)]))
+ })
+
+ test('after count request - should be able to modify count response', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
- const response = await axios.post('/data/get', hooks.getRequestBodyWith(ctx.collectionName, 'wrongId'), authOwner)
- expect(response.data.item).toEqual(ctx.item)
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ afterAll: (payload, _requestContext, _serviceContext) => {
+ return { ...payload, totalCount: payload.totalCount + 2 }
+ },
+ afterRead: (payload, _requestContext, _serviceContext) => {
+ return { ...payload, totalCount: payload.totalCount * 2 }
+ },
+ afterCount: (payload, _requestContext, _serviceContext) => {
+ return { ...payload, totalCount: payload.totalCount - 3 }
+ }
+ }
})
- each(['beforeAll', 'beforeRead', 'beforeCount'])
- .test('%s should able to change payload /data/count', async(hookName: any) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ await expect(axios.post('/data/count', data.countRequest(ctx.collectionName), authOwner)).resolves.toEqual(
+ matchers.responseWith({ totalCount: 3 }))
+ })
+
+ if (supportedOperations.includes(Aggregate)) {
+ test('after aggregate request - should be able to modify response', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.afterAllColumn, ctx.afterReadColumn, ctx.afterHookColumn], authOwner)
+ await data.givenItems([{ ...ctx.item, [ctx.afterAllColumn.name]: false, [ctx.afterReadColumn.name]: false, [ctx.afterHookColumn.name]: false }],
+ ctx.collectionName, authOwner)
env.externalDbRouter.reloadHooks({
dataHooks: {
- [hookName]: (payload, _requestContext, _serviceContext) => {
- return { ...payload, filter: { _id: { $eq: ctx.item._id } } }
+ afterAll: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterReadColumn.name]: false,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
+ afterRead: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterReadColumn.name]: true,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
+ afterAggregate: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterHookColumn.name]: true,
+ }))
+ }
}
}
})
- const response = await axios.post('/data/count', hooks.findRequestBodyWith(ctx.collectionName, { _id: { $ne: ctx.item._id } }), authOwner)
- expect(response.data.totalCount).toEqual(1)
+ const response = await axios.post('/data/aggregate',
+ {
+ collectionId: ctx.collectionName,
+ initialFilter: { _id: { $eq: ctx.item._id } },
+ group: {
+ by: [ctx.afterAllColumn.name, ctx.afterReadColumn.name, ctx.afterHookColumn.name],
+ aggregation: []
+ },
+ finalFilter: {},
+ }, { responseType: 'stream', ...authOwner })
+
+ await expect(streamToArray(response.data)).resolves.toEqual(
+ expect.toIncludeSameMembers([{
+ item: expect.objectContaining({
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterHookColumn.name]: true,
+ [ctx.afterReadColumn.name]: true,
+ })
+ },
+ data.pagingMetadata(1, 1)
+ ]))
+
})
- if (supportedOperations.includes(Aggregate)) {
- each(['beforeAll', 'beforeRead', 'beforeAggregate'])
- .test('%s should able to change payload /data/aggregate', async(hookName: string) => {
- if (hooks.skipAggregationIfNotSupported(hookName, supportedOperations))
- return
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ }
+ })
+ describe('Write Operations', () => {
+ each([
+ ['insert', 'afterInsert', '/data/insert'],
+ ['update', 'afterUpdate', '/data/update'],
+ ['remove', 'afterRemove', '/data/remove'],
+ ]).test('after %s request - should be able to modify response', async(operation, hookName, api) => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column, ctx.afterAllColumn, ctx.afterWriteColumn, ctx.afterHookColumn], authOwner)
+ if (operation !== 'insert') {
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ }
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- [hookName]: (payload, _requestContext, _serviceContext) => {
- return { ...payload, filter: { _id: { $eq: ctx.item._id } } }
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ afterAll: (payload, requestContext: coreTypes.RequestContext, _serviceContext) => {
+ if (requestContext.operation !== DataOperation.query) {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterWriteColumn.name]: false,
+ [ctx.afterHookColumn.name]: false,
+ }))
}
}
- })
+ },
+ afterWrite: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterWriteColumn.name]: true,
+ [ctx.afterHookColumn.name]: false,
+ }))
+ }
+ },
+ [hookName]: (payload, _requestContext, _serviceContext) => {
+ return {
+ ...payload, items: payload.items.map(item => ({
+ ...item,
+ [ctx.afterHookColumn.name]: true,
+ }))
+ }
+ }
+ }
+ })
- const response = await axios.post('/data/aggregate', hooks.aggregateRequestBodyWith(ctx.collectionName, { _id: { $ne: ctx.item._id } } ), authOwner)
- expect(response.data.items).toEqual([{ _id: ctx.item._id }])
- })
- }
+ const response = await axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), { responseType: 'stream', ...authOwner })
+
+ await expect(streamToArray(response.data)).resolves.toEqual(
+ expect.toIncludeSameMembers([{
+ item: {
+ ...ctx.item,
+ [ctx.afterAllColumn.name]: true,
+ [ctx.afterWriteColumn.name]: true,
+ [ctx.afterHookColumn.name]: true,
+ }
+ }]))
+ })
})
})
@@ -316,7 +496,7 @@ describe(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () =>
)
})
- test('If not specified should throw 400 - Error object', async() => {
+ test('If not specified should throw 500 - Error object', async() => {
env.externalDbRouter.reloadHooks({
dataHooks: {
beforeAll: (_payload, _requestContext, _serviceContext) => {
@@ -327,11 +507,11 @@ describe(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () =>
})
await expect(axios.post('/data/remove', hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), authOwner)).rejects.toMatchObject(
- errorResponseWith(400, 'message')
+ errorResponseWith(500, 'message')
)
})
- test('If not specified should throw 400 - string', async() => {
+ test('If not specified should throw 500 - string', async() => {
env.externalDbRouter.reloadHooks({
dataHooks: {
beforeAll: (_payload, _requestContext, _serviceContext) => {
@@ -341,120 +521,116 @@ describe(`Velo External DB Data Hooks: ${currentDbImplementationName()}`, () =>
})
await expect(axios.post('/data/remove', hooks.writeRequestBodyWith(ctx.collectionName, [ctx.item]), authOwner)).rejects.toMatchObject(
- errorResponseWith(400, 'message')
+ errorResponseWith(500, 'message')
)
})
})
+ describe('Custom context, Service context', () => { //skip aggregate if needed!
+ each([
+ ['query', 'Read', 'beforeQuery', 'afterQuery', '/data/query'],
+ ['count', 'Read', 'beforeCount', 'afterCount', '/data/count'],
+ ['insert', 'Write', 'beforeInsert', 'afterInsert', '/data/insert'],
+ ['update', 'Write', 'beforeUpdate', 'afterUpdate', '/data/update'],
+ ['remove', 'Write', 'beforeRemove', 'afterRemove', '/data/remove'],
+ ['truncate', 'Write', 'beforeTruncate', 'afterTruncate', '/data/truncate'],
+ ]).test('%s - should be able to modify custom context from each hook, and use service context', async(operation, operationType, beforeHook, afterHook, api) => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ if (operation !== 'insert') {
+ await data.givenItems([ctx.item], ctx.collectionName, authOwner)
+ }
- describe('Custom Context', () => {
- describe('Read operations', () => {
- each(testSupportedOperations(supportedOperations, [
- ['Get', 'beforeGetById', 'afterGetById', '/data/get'],
- ['Find', 'beforeFind', 'afterFind', '/data/find'],
- ['Aggregate', 'beforeAggregate', 'afterAggregate', '/data/aggregate', { neededOperations: [Aggregate] }],
- ['Count', 'beforeCount', 'afterCount', '/data/count']
- ])).test('customContext should pass by ref on [%s] ', async(_: any, beforeHook: string, afterHook: string, api: string) => {
- if (hooks.skipAggregationIfNotSupported(beforeHook, supportedOperations))
- return
-
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
+ const beforeOperationHookName = `before${operationType}`
+ const afterOperationHookName = `after${operationType}`
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- beforeAll: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['beforeAll'] = true
- },
- beforeRead: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['beforeRead'] = true
- },
- [beforeHook]: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext[beforeHook] = true
- },
- afterAll: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['afterAll'] = true
- },
- afterRead: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['afterRead'] = true
- },
- [afterHook]: (payload: any, _requestContext: any, _serviceContext: any, customContext: { [x: string]: boolean }) => {
- customContext[afterHook] = true
- return { ...payload, customContext }
- }
+ env.externalDbRouter.reloadHooks({
+ dataHooks: {
+ beforeAll: (_payload, _requestContext, _serviceContext, customContext) => {
+ customContext['beforeAll'] = true
+ },
+ [beforeOperationHookName]: (_payload, _requestContext, _serviceContext, customContext) => {
+ customContext['beforeOperation'] = true
+ },
+ [beforeHook]: (_payload, _requestContext, _serviceContext, customContext) => {
+ customContext['beforeHook'] = true
+ },
+ afterAll: (_payload, _requestContext, _serviceContext, customContext) => {
+ customContext['afterAll'] = true
+ },
+ [afterOperationHookName]: (_payload, _requestContext, _serviceContext, customContext) => {
+ customContext['afterOperation'] = true
+ },
+ [afterHook]: async(payload, _requestContext, serviceContext: coreTypes.ServiceContext, customContext) => {
+ customContext['afterHook'] = true
+
+ if (customContext['beforeAll'] && customContext['beforeOperation'] &&
+ customContext['beforeHook'] && customContext['afterAll'] &&
+ customContext['afterOperation'] && customContext['afterHook']) {
+
+ await serviceContext.schemaService.create(ctx.newCollection)
+ await serviceContext.dataService.insert(ctx.newCollection.id, ctx.newItem)
+ }
}
- })
- const response = await axios.post(api, hooks.readRequestBodyWith(ctx.collectionName, ctx.items), authOwner)
- expect(response.data.customContext).toEqual({
- beforeAll: true, beforeRead: true, [beforeHook]: true, afterAll: true, afterRead: true, [afterHook]: true
- })
- })
- })
-
- describe('Write operations', () => {
- each(testSupportedOperations(supportedOperations, [
- ['Insert', 'beforeInsert', 'afterInsert', '/data/insert'],
- ['Bulk Insert', 'beforeBulkInsert', 'afterBulkInsert', '/data/insert/bulk'],
- ['Update', 'beforeUpdate', 'afterUpdate', '/data/update', { neededOperations: [UpdateImmediately] }],
- ['Bulk Update', 'beforeBulkUpdate', 'afterBulkUpdate', '/data/update/bulk', { neededOperations: [UpdateImmediately] }],
- ['Remove', 'beforeRemove', 'afterRemove', '/data/remove', { neededOperations: [DeleteImmediately] }],
- ['Bulk Remove', 'beforeBulkRemove', 'afterBulkRemove', '/data/remove/bulk', { neededOperations: [DeleteImmediately] }]
- ])).test('customContext should pass by ref on [%s] ', async(_: any, beforeHook: string | number, afterHook: string, api: any) => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- if (!['afterInsert', 'afterBulkInsert'].includes(afterHook)) {
- await data.givenItems(ctx.items, ctx.collectionName, authOwner)
}
- env.externalDbRouter.reloadHooks({
- dataHooks: {
- beforeAll: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['beforeAll'] = true
- },
- beforeWrite: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['beforeWrite'] = true
- },
- [beforeHook]: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext[beforeHook] = true
- },
- afterAll: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['afterAll'] = true
- },
- afterWrite: (_payload, _requestContext, _serviceContext, customContext) => {
- customContext['afterWrite'] = true
- },
- [afterHook]: (payload, _requestContext, _serviceContext, customContext) => {
- customContext[afterHook] = true
- return { ...payload, customContext }
- }
- }
- })
- const response = await axios.post(api, hooks.writeRequestBodyWith(ctx.collectionName, ctx.items), authOwner)
- expect(response.data.customContext).toEqual({
- beforeAll: true, beforeWrite: true, [beforeHook]: true, afterAll: true, afterWrite: true, [afterHook]: true
- })
})
+
+ await axios.post(api, hooks.requestBodyWith(ctx.collectionName, [ctx.item]), { responseType: 'stream', ...authOwner })
+
+ hooks.resetHooks(env.externalDbRouter)
+
+ await expect(data.queryCollectionAsArray(ctx.newCollection.id, [], undefined, authOwner)).resolves.toEqual(
+ expect.toIncludeSameMembers([{ item: ctx.newItem }, data.pagingMetadata(1, 1)]))
})
})
- const ctx = {
+ interface Ctx {
+ collectionName: string
+ column: InputField
+ item: ItemWithId
+ items: ItemWithId[]
+ numberItem: ItemWithId
+ anotherNumberItem: ItemWithId
+ afterAllColumn: InputField
+ afterReadColumn: InputField
+ afterWriteColumn: InputField
+ afterHookColumn: InputField
+ numberColumns: InputField[]
+ newCollection: collectionSpi.Collection
+ newItem: ItemWithId
+ }
+
+ const ctx: Ctx = {
collectionName: Uninitialized,
column: Uninitialized,
item: Uninitialized,
items: Uninitialized,
- beforeAllColumn: Uninitialized,
- beforeReadColumn: Uninitialized,
- beforeWriteColumn: Uninitialized,
- beforeHookColumn: Uninitialized,
+ numberItem: Uninitialized,
+ anotherNumberItem: Uninitialized,
+ afterAllColumn: Uninitialized,
+ afterReadColumn: Uninitialized,
+ afterWriteColumn: Uninitialized,
+ afterHookColumn: Uninitialized,
+ numberColumns: Uninitialized,
+ newCollection: Uninitialized,
+ newItem: Uninitialized
}
beforeEach(async() => {
ctx.collectionName = gen.randomCollectionName()
+ ctx.newCollection = gen.randomCollection()
ctx.column = gen.randomColumn()
- ctx.beforeAllColumn = { name: 'beforeAll', type: 'boolean' }
- ctx.beforeWriteColumn = { name: 'beforeWrite', type: 'boolean' }
- ctx.beforeReadColumn = { name: 'beforeRead', type: 'boolean' }
- ctx.beforeHookColumn = { name: 'beforeHook', type: 'boolean' }
- ctx.item = genCommon.randomEntity([ctx.column.name])
- ctx.items = Array.from({ length: 10 }, () => genCommon.randomEntity([ctx.column.name]))
+ ctx.afterAllColumn = { name: 'afterAll', type: 'boolean' }
+ ctx.afterWriteColumn = { name: 'afterWrite', type: 'boolean' }
+ ctx.afterReadColumn = { name: 'afterRead', type: 'boolean' }
+ ctx.afterHookColumn = { name: 'afterHook', type: 'boolean' }
+ ctx.item = genCommon.randomEntity([ctx.column.name]) as ItemWithId
+ ctx.items = Array.from({ length: 10 }, () => genCommon.randomEntity([ctx.column.name])) as ItemWithId[]
+
+ ctx.newItem = genCommon.randomEntity([]) as ItemWithId
+ ctx.numberColumns = gen.randomNumberColumns()
+ ctx.numberItem = genCommon.randomNumberEntity(ctx.numberColumns) as ItemWithId
+ ctx.anotherNumberItem = genCommon.randomNumberEntity(ctx.numberColumns) as ItemWithId
+
hooks.resetHooks(env.externalDbRouter)
})
diff --git a/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts
index 975ef72ca..994312cbe 100644
--- a/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts
+++ b/apps/velo-external-db/test/e2e/app_schema.e2e.spec.ts
@@ -1,19 +1,22 @@
+import { SystemFields } from '@wix-velo/velo-external-db-commons'
import { Uninitialized, gen as genCommon, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons'
-import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const { RemoveColumn } = SchemaOperations
+import { InputField, SchemaOperations } from '@wix-velo/velo-external-db-types'
+const { RemoveColumn, ChangeColumnType } = SchemaOperations
import * as schema from '../drivers/schema_api_rest_test_support'
import * as matchers from '../drivers/schema_api_rest_matchers'
+import { schemaUtils } from '@wix-velo/velo-external-db-core'
import { authOwner } from '@wix-velo/external-db-testkit'
import * as gen from '../gen'
import Chance = require('chance')
-import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources'
+import axios from 'axios'
+import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations, env } from '../resources/e2e_resources'
const chance = Chance()
-const axios = require('axios').create({
+const axiosClient = axios.create({
baseURL: 'http://localhost:8080'
})
-describe(`Velo External DB Schema REST API: ${currentDbImplementationName()}`, () => {
+describe(`Schema REST API: ${currentDbImplementationName()}`, () => {
beforeAll(async() => {
await setupDb()
@@ -23,47 +26,97 @@ describe(`Velo External DB Schema REST API: ${currentDbImplementationName()}`,
afterAll(async() => {
await dbTeardown()
}, 20000)
+
+ describe('Velo External DB Collections REST API', () => {
+ beforeEach(async() => {
+ await schema.deleteAllCollections(authOwner)
+ })
- test('list', async() => {
- await expect( axios.post('/schemas/list', {}, authOwner) ).resolves.toEqual( matchers.collectionResponseWithNoCollections() )
- })
+ test('collection get', async() => {
+ await schema.givenCollection(ctx.collectionName, [], authOwner)
- test('list headers', async() => {
- await schema.givenCollection(ctx.collectionName, [], authOwner)
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields], env.capabilities))
+ })
- await expect( axios.post('/schemas/list/headers', {}, authOwner) ).resolves.toEqual( matchers.collectionResponseWithCollections([ctx.collectionName]) )
- })
+ test('collection create - collection without fields', async() => {
+ const collection = {
+ id: ctx.collectionName,
+ fields: []
+ }
+ await axiosClient.post('/collections/create', { collection }, { ...authOwner, responseType: 'stream' })
- test('create', async() => {
- await axios.post('/schemas/create', { collectionName: ctx.collectionName }, authOwner)
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponseWith(ctx.collectionName, [...SystemFields], env.capabilities))
+ })
- await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) )
- })
+ test('collection create - collection with fields', async() => {
+ const collection = {
+ id: ctx.collectionName,
+ fields: [ctx.column].map(schemaUtils.InputFieldToWixFormatField)
+ }
- test('find', async() => {
- await schema.givenCollection(ctx.collectionName, [], authOwner)
+ await axiosClient.post('/collections/create', { collection }, { ...authOwner, responseType: 'stream' })
- await expect( axios.post('/schemas/find', { schemaIds: [ctx.collectionName] }, authOwner)).resolves.toEqual( matchers.collectionResponseWithDefaultFieldsFor(ctx.collectionName) )
- })
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponseWith(ctx.collectionName, [...SystemFields, ctx.column], env.capabilities))
+ })
- test('add column', async() => {
- await schema.givenCollection(ctx.collectionName, [], authOwner)
+ test('collection update - add column', async() => {
+ await schema.givenCollection(ctx.collectionName, [], authOwner)
- await axios.post('/schemas/column/add', { collectionName: ctx.collectionName, column: ctx.column }, authOwner)
+ const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner)
- await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.toEqual( matchers.collectionResponseHasField( ctx.column ) )
- })
+ collection.fields.push(schemaUtils.InputFieldToWixFormatField(ctx.column))
+
+ await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' })
+
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields, ctx.column], env.capabilities))
+ })
+
+ testIfSupportedOperationsIncludes(supportedOperations, [ RemoveColumn ])('collection update - remove column', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
- testIfSupportedOperationsIncludes(supportedOperations, [ RemoveColumn ])('remove column', async() => {
- await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner)
- await axios.post('/schemas/column/remove', { collectionName: ctx.collectionName, columnName: ctx.column.name }, authOwner)
+ const systemFieldsNames = SystemFields.map(f => f.name)
+ collection.fields = collection.fields.filter((f: any) => systemFieldsNames.includes(f.key))
- await expect( schema.retrieveSchemaFor(ctx.collectionName, authOwner) ).resolves.not.toEqual( matchers.collectionResponseHasField( ctx.column ) )
+ await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' })
+
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.collectionResponsesWith(ctx.collectionName, [...SystemFields], env.capabilities))
+ })
+
+ testIfSupportedOperationsIncludes(supportedOperations, [ ChangeColumnType ])('collection update - change column type', async() => {
+ await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+ const collection: any = await schema.retrieveSchemaFor(ctx.collectionName, authOwner)
+
+ const columnIndex = collection.fields.findIndex((f: any) => f.key === ctx.column.name)
+ collection.fields[columnIndex].type = schemaUtils.fieldTypeToWixDataEnum('number')
+
+ await axiosClient.post('/collections/update', { collection }, { ...authOwner, responseType: 'stream' })
+
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).resolves.toEqual(matchers.createCollectionResponseWith(ctx.collectionName, [...SystemFields, { name: ctx.column.name, type: 'number' }], env.capabilities))
+ })
+
+ test('collection delete', async() => {
+ await schema.givenCollection(ctx.collectionName, [], authOwner)
+ await axiosClient.post('/collections/delete', { collectionId: ctx.collectionName }, { ...authOwner, responseType: 'stream' })
+ await expect(schema.retrieveSchemaFor(ctx.collectionName, authOwner)).rejects.toThrow('404')
+ })
})
+ interface Ctx {
+ collectionName: string
+ column: InputField
+ numberColumns: InputField[],
+ item: { [x: string]: any }
+ items: { [x: string]: any}[]
+ modifiedItem: { [x: string]: any }
+ modifiedItems: { [x: string]: any }
+ anotherItem: { [x: string]: any }
+ numberItem: { [x: string]: any }
+ anotherNumberItem: { [x: string]: any }
+ }
- const ctx = {
+ const ctx: Ctx = {
collectionName: Uninitialized,
column: Uninitialized,
numberColumns: Uninitialized,
diff --git a/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts b/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts
index ae371394a..20b1498f2 100644
--- a/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts
+++ b/apps/velo-external-db/test/e2e/app_schema_hooks.e2e.spec.ts
@@ -17,7 +17,8 @@ const axios = require('axios').create({
baseURL: 'http://localhost:8080'
})
-describe(`Velo External DB Schema Hooks: ${currentDbImplementationName()}`, () => {
+// eslint-disable-next-line jest/no-disabled-tests
+describe.skip(`Velo External DB Schema Hooks: ${currentDbImplementationName()}`, () => {
beforeAll(async() => {
await setupDb()
diff --git a/apps/velo-external-db/test/env/env.db.setup.js b/apps/velo-external-db/test/env/env.db.setup.js
index 22b402c47..54371acce 100644
--- a/apps/velo-external-db/test/env/env.db.setup.js
+++ b/apps/velo-external-db/test/env/env.db.setup.js
@@ -10,9 +10,9 @@ const { testResources: firestore } = require ('@wix-velo/external-db-firestore')
const { testResources: mssql } = require ('@wix-velo/external-db-mssql')
const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets')
-const { testResources: airtable } = require('@wix-velo/external-db-airtable')
+// const { testResources: airtable } = require('@wix-velo/external-db-airtable')
const { testResources: dynamoDb } = require('@wix-velo/external-db-dynamodb')
-const { testResources: bigquery } = require('@wix-velo/external-db-bigquery')
+// const { testResources: bigquery } = require('@wix-velo/external-db-bigquery')
const { sleep } = require('@wix-velo/test-commons')
const ci = require('./ci_utils')
@@ -46,17 +46,17 @@ const initEnv = async(testEngine) => {
await googleSheet.initEnv()
break
- case 'airtable':
- await airtable.initEnv()
- break
+ // case 'airtable':
+ // await airtable.initEnv()
+ // break
case 'dynamodb':
await dynamoDb.initEnv()
break
- case 'bigquery':
- await bigquery.initEnv()
- break
+ // case 'bigquery':
+ // await bigquery.initEnv()
+ // break
}
}
@@ -94,9 +94,9 @@ const cleanup = async(testEngine) => {
await dynamoDb.cleanup()
break
- case 'bigquery':
- await bigquery.cleanup()
- break
+ // case 'bigquery':
+ // await bigquery.cleanup()
+ // break
}
}
diff --git a/apps/velo-external-db/test/env/env.db.teardown.js b/apps/velo-external-db/test/env/env.db.teardown.js
index 63a4e09c7..ec86f1cfa 100644
--- a/apps/velo-external-db/test/env/env.db.teardown.js
+++ b/apps/velo-external-db/test/env/env.db.teardown.js
@@ -5,9 +5,9 @@ const { testResources: firestore } = require ('@wix-velo/external-db-firestore')
const { testResources: mssql } = require ('@wix-velo/external-db-mssql')
const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets')
-const { testResources: airtable } = require('@wix-velo/external-db-airtable')
+// const { testResources: airtable } = require('@wix-velo/external-db-airtable')
const { testResources: dynamo } = require('@wix-velo/external-db-dynamodb')
-const { testResources: bigquery } = require('@wix-velo/external-db-bigquery')
+// const { testResources: bigquery } = require('@wix-velo/external-db-bigquery')
const ci = require('./ci_utils')
@@ -37,9 +37,9 @@ const shutdownEnv = async(testEngine) => {
await googleSheet.shutdownEnv()
break
- case 'airtable':
- await airtable.shutdownEnv()
- break
+ // case 'airtable':
+ // await airtable.shutdownEnv()
+ // break
case 'dynamodb':
await dynamo.shutdownEnv()
@@ -49,9 +49,9 @@ const shutdownEnv = async(testEngine) => {
await mongo.shutdownEnv()
break
- case 'bigquery':
- await bigquery.shutdownEnv()
- break
+ // case 'bigquery':
+ // await bigquery.shutdownEnv()
+ // break
}
}
diff --git a/apps/velo-external-db/test/gen.ts b/apps/velo-external-db/test/gen.ts
index b6429d074..2800717fe 100644
--- a/apps/velo-external-db/test/gen.ts
+++ b/apps/velo-external-db/test/gen.ts
@@ -1,5 +1,6 @@
import { SystemFields } from '@wix-velo/velo-external-db-commons'
+import { collectionSpi } from '@wix-velo/velo-external-db-core'
import { InputField } from '@wix-velo/velo-external-db-types'
import * as Chance from 'chance'
@@ -73,12 +74,12 @@ export const randomObjectDbEntity = (columns: InputField[]) => {
return entity
}
-export const randomNumberColumns = () => {
+export const randomNumberColumns = (): InputField[] => {
return [ { name: chance.word(), type: 'number', subtype: 'int', isPrimary: false },
{ name: chance.word(), type: 'number', subtype: 'decimal', precision: '10,2', isPrimary: false } ]
}
-export const randomColumn = () => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } )
+export const randomColumn = (): InputField => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } )
export const randomObjectColumn = () => ( { name: chance.word(), type: 'object' } )
@@ -105,3 +106,10 @@ export const randomMatchesValueWithDashes = () => {
}
return arr.join('-')
}
+
+export const randomCollection = (): collectionSpi.Collection => {
+ return {
+ id: randomCollectionName(),
+ fields: [],
+ }
+}
diff --git a/apps/velo-external-db/test/resources/e2e_resources.ts b/apps/velo-external-db/test/resources/e2e_resources.ts
index b8b78df40..26030f173 100644
--- a/apps/velo-external-db/test/resources/e2e_resources.ts
+++ b/apps/velo-external-db/test/resources/e2e_resources.ts
@@ -13,56 +13,49 @@ import { testResources as bigquery } from '@wix-velo/external-db-bigquery'
import { E2EResources } from '@wix-velo/external-db-testkit'
import { Uninitialized } from '@wix-velo/test-commons'
-import { ExternalDbRouter } from '@wix-velo/velo-external-db-core'
-import { Server } from 'http'
-import { ConnectionCleanUp, ISchemaProvider } from '@wix-velo/velo-external-db-types'
-interface App {
- server: Server;
- schemaProvider: ISchemaProvider;
- cleanup: ConnectionCleanUp;
- started: boolean;
- reload: (hooks?: any) => Promise<{
- externalDbRouter: ExternalDbRouter;
- }>;
- externalDbRouter: ExternalDbRouter;
-}
+import { initWixDataEnv, shutdownWixDataEnv, wixDataBaseUrl } from '../drivers/wix_data_resources'
+import { E2E_ENV } from '../types'
-type Internals = () => App
-export let env:{
- app: App,
- externalDbRouter: ExternalDbRouter,
- internals: Internals,
- enviormentVariables: Record<string, string>
-} = {
+export let env: E2E_ENV = {
app: Uninitialized,
internals: Uninitialized,
externalDbRouter: Uninitialized,
- enviormentVariables: Uninitialized
+ capabilities: Uninitialized,
+ enviormentVariables: Uninitialized,
}
+const createAppWithWixDataBaseUrl = createApp.bind(null, wixDataBaseUrl())
+
const testSuits = {
- mysql: new E2EResources(mysql, createApp),
- postgres: new E2EResources(postgres, createApp),
- spanner: new E2EResources(spanner, createApp),
- firestore: new E2EResources(firestore, createApp),
- mssql: new E2EResources(mssql, createApp),
- mongo: new E2EResources(mongo, createApp),
- 'google-sheet': new E2EResources(googleSheet, createApp),
+ mysql: new E2EResources(mysql, createAppWithWixDataBaseUrl),
+ postgres: new E2EResources(postgres, createAppWithWixDataBaseUrl),
+ spanner: new E2EResources(spanner, createAppWithWixDataBaseUrl),
+ firestore: new E2EResources(firestore, createAppWithWixDataBaseUrl),
+ mssql: new E2EResources(mssql, createAppWithWixDataBaseUrl),
+ mongo: new E2EResources(mongo, createAppWithWixDataBaseUrl),
+ 'google-sheet': new E2EResources(googleSheet, createAppWithWixDataBaseUrl),
airtable: new E2EResources(airtable, createApp),
- dynamodb: new E2EResources(dynamo, createApp),
+ dynamodb: new E2EResources(dynamo, createAppWithWixDataBaseUrl),
bigquery: new E2EResources(bigquery, createApp),
}
export const testedSuit = () => testSuits[process.env.TEST_ENGINE]
export const supportedOperations = testedSuit().supportedOperations
-export const setupDb = () => testedSuit().setUpDb()
+export const setupDb = async() => {
+ await initWixDataEnv()
+ await testedSuit().setUpDb()
+}
export const currentDbImplementationName = () => testedSuit().currentDbImplementationName
export const initApp = async() => {
env = await testedSuit().initApp()
+ env.capabilities = testedSuit().implementation.capabilities
env.enviormentVariables = testedSuit().implementation.enviormentVariables
}
-export const teardownApp = async() => testedSuit().teardownApp()
+export const teardownApp = async() => {
+ await testedSuit().teardownApp()
+ await shutdownWixDataEnv()
+}
export const dbTeardown = async() => testedSuit().dbTeardown()
diff --git a/apps/velo-external-db/test/resources/provider_resources.ts b/apps/velo-external-db/test/resources/provider_resources.ts
index 977246e8b..8442065d4 100644
--- a/apps/velo-external-db/test/resources/provider_resources.ts
+++ b/apps/velo-external-db/test/resources/provider_resources.ts
@@ -21,21 +21,14 @@ import * as bigquery from '@wix-velo/external-db-bigquery'
import * as googleSheet from '@wix-velo/external-db-google-sheets'
-import { AnyFixMe, ConnectionCleanUp, IDataProvider, ISchemaProvider } from '@wix-velo/velo-external-db-types'
+import { ProviderResourcesEnv } from '../types'
-// const googleSheet = require('@wix-velo/external-db-google-sheets')
-// const googleSheetTestEnv = require('./engines/google_sheets_resources')
-
-export const env: {
- dataProvider: IDataProvider
- schemaProvider: ISchemaProvider
- cleanup: ConnectionCleanUp
- driver: AnyFixMe
-} = {
+export const env: ProviderResourcesEnv = {
dataProvider: Uninitialized,
schemaProvider: Uninitialized,
cleanup: Uninitialized,
driver: Uninitialized,
+ capabilities: Uninitialized,
}
const dbInit = async(impl: any) => {
@@ -48,6 +41,7 @@ const dbInit = async(impl: any) => {
env.dataProvider = new impl.DataProvider(pool, driver.filterParser)
env.schemaProvider = new impl.SchemaProvider(pool, testResources.schemaProviderTestVariables?.() )
env.driver = driver
+ env.capabilities = impl.testResources.capabilities
env.cleanup = cleanup
}
@@ -56,6 +50,7 @@ export const dbTeardown = async() => {
env.dataProvider = Uninitialized
env.schemaProvider = Uninitialized
env.driver = Uninitialized
+ env.capabilities = Uninitialized
}
const postgresTestEnvInit = async() => await dbInit(postgres)
@@ -70,16 +65,16 @@ const bigqueryTestEnvInit = async() => await dbInit(bigquery)
const googleSheetTestEnvInit = async() => await dbInit(googleSheet)
const testSuits = {
- mysql: suiteDef('MySql', mysqlTestEnvInit, mysql.testResources.supportedOperations),
- postgres: suiteDef('Postgres', postgresTestEnvInit, postgres.testResources.supportedOperations),
- spanner: suiteDef('Spanner', spannerTestEnvInit, spanner.testResources.supportedOperations),
- firestore: suiteDef('Firestore', firestoreTestEnvInit, firestore.testResources.supportedOperations),
- mssql: suiteDef('Sql Server', mssqlTestEnvInit, mssql.testResources.supportedOperations),
- mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources.supportedOperations),
+ mysql: suiteDef('MySql', mysqlTestEnvInit, mysql.testResources),
+ postgres: suiteDef('Postgres', postgresTestEnvInit, postgres.testResources),
+ spanner: suiteDef('Spanner', spannerTestEnvInit, spanner.testResources),
+ firestore: suiteDef('Firestore', firestoreTestEnvInit, firestore.testResources),
+ mssql: suiteDef('Sql Server', mssqlTestEnvInit, mssql.testResources),
+ mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources),
airtable: suiteDef('Airtable', airTableTestEnvInit, airtable.testResources.supportedOperations),
- dynamodb: suiteDef('DynamoDb', dynamoTestEnvInit, dynamo.testResources.supportedOperations),
+ dynamodb: suiteDef('DynamoDb', dynamoTestEnvInit, dynamo.testResources),
bigquery: suiteDef('BigQuery', bigqueryTestEnvInit, bigquery.testResources.supportedOperations),
- 'google-sheet': suiteDef('Google-Sheet', googleSheetTestEnvInit, googleSheet.supportedOperations),
+ 'google-sheet': suiteDef('Google-Sheet', googleSheetTestEnvInit, googleSheet.testResources),
}
const testedSuit = () => testSuits[process.env.TEST_ENGINE]
diff --git a/apps/velo-external-db/test/resources/test_suite_definition.ts b/apps/velo-external-db/test/resources/test_suite_definition.ts
index 45275b36a..09c7598e0 100644
--- a/apps/velo-external-db/test/resources/test_suite_definition.ts
+++ b/apps/velo-external-db/test/resources/test_suite_definition.ts
@@ -1 +1,6 @@
-export const suiteDef = (name: string, setup: any, supportedOperations: any) => ( { name, setup, supportedOperations } )
+export const suiteDef = (name: string, setup: any, testResources: any) => ({
+ name,
+ setup,
+ supportedOperations: testResources.supportedOperations,
+ capabilities: testResources.capabilities
+ })
diff --git a/apps/velo-external-db/test/storage/data_provider.spec.ts b/apps/velo-external-db/test/storage/data_provider.spec.ts
index 06a820f86..7bd11a1c0 100644
--- a/apps/velo-external-db/test/storage/data_provider.spec.ts
+++ b/apps/velo-external-db/test/storage/data_provider.spec.ts
@@ -220,8 +220,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => {
env.driver.stubEmptyFilterFor(ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1)
+ env.driver.stubEmptyOrderByFor(ctx.sort)
- await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] },
+ await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] },
{ _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] }
]))
})
@@ -232,8 +233,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => {
env.driver.stubEmptyFilterFor(ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 1)
+ env.driver.stubEmptyOrderByFor(ctx.sort)
- await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] },
+ await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual(expect.arrayContaining([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] },
{ _id: ctx.anotherNumberEntity._id, [ctx.aliasColumns[0]]: ctx.anotherNumberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.anotherNumberEntity[ctx.numericColumns[1].name] }
]))
})
@@ -244,8 +246,9 @@ describe(`Data API: ${currentDbImplementationName()}`, () => {
env.driver.givenFilterByIdWith(ctx.numberEntity._id, ctx.filter)
env.driver.givenAggregateQueryWith(ctx.aggregation.processingStep, ctx.numericColumns, ctx.aliasColumns, ['_id'], ctx.aggregation.postFilteringStep, 2)
+ env.driver.stubEmptyOrderByFor(ctx.sort)
- await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }])
+ await expect( env.dataProvider.aggregate(ctx.numericCollectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit) ).resolves.toEqual([{ _id: ctx.numberEntity._id, [ctx.aliasColumns[0]]: ctx.numberEntity[ctx.numericColumns[0].name], [ctx.aliasColumns[1]]: ctx.numberEntity[ctx.numericColumns[1].name] }])
})
diff --git a/apps/velo-external-db/test/storage/schema_provider.spec.ts b/apps/velo-external-db/test/storage/schema_provider.spec.ts
index 1177dc726..8406a401a 100644
--- a/apps/velo-external-db/test/storage/schema_provider.spec.ts
+++ b/apps/velo-external-db/test/storage/schema_provider.spec.ts
@@ -3,7 +3,7 @@ import { errors, SystemFields } from '@wix-velo/velo-external-db-commons'
import { SchemaOperations } from '@wix-velo/velo-external-db-types'
import { Uninitialized, gen, testIfSupportedOperationsIncludes } from '@wix-velo/test-commons'
import { env, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/provider_resources'
-import { collectionWithDefaultFields, hasSameSchemaFieldsLike } from '../drivers/schema_provider_matchers'
+import { toContainDefaultFields, collectionToContainFields, toBeDefaultCollectionWith, hasSameSchemaFieldsLike } from '../drivers/schema_provider_matchers'
const chance = new Chance()
const { CollectionDoesNotExists, FieldAlreadyExists, CannotModifySystemField, FieldDoesNotExist } = errors
const { RemoveColumn } = SchemaOperations
@@ -39,11 +39,11 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => {
await expect( env.schemaProvider.list() ).resolves.toEqual(expect.arrayContaining([
expect.objectContaining({
id: ctx.collectionName,
- fields: collectionWithDefaultFields()
+ fields: toContainDefaultFields(env.capabilities.ColumnsCapabilities)
}),
expect.objectContaining({
id: ctx.anotherCollectionName,
- fields: collectionWithDefaultFields()
+ fields: toContainDefaultFields(env.capabilities.ColumnsCapabilities)
})
]))
})
@@ -51,7 +51,7 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => {
test('create collection with default columns', async() => {
await env.schemaProvider.create(ctx.collectionName)
- await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields())
+ await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName, env.capabilities))
})
test('drop collection', async() => {
@@ -65,13 +65,13 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => {
test('collection name and variables are case sensitive', async() => {
await env.schemaProvider.create(ctx.collectionName.toUpperCase())
- await expect( env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) ).resolves.toEqual(collectionWithDefaultFields())
+ await expect( env.schemaProvider.describeCollection(ctx.collectionName.toUpperCase()) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName.toUpperCase(), env.capabilities))
})
test('retrieve collection data by collection name', async() => {
await env.schemaProvider.create(ctx.collectionName)
- await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionWithDefaultFields())
+ await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(toBeDefaultCollectionWith(ctx.collectionName, env.capabilities))
})
test('create collection twice will do nothing', async() => {
@@ -87,7 +87,7 @@ describe(`Schema API: ${currentDbImplementationName()}`, () => {
test('add column on a an existing collection', async() => {
await env.schemaProvider.create(ctx.collectionName, [])
await env.schemaProvider.addColumn(ctx.collectionName, { name: ctx.columnName, type: 'datetime', subtype: 'timestamp' })
- await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual( hasSameSchemaFieldsLike([{ field: ctx.columnName }]))
+ await expect( env.schemaProvider.describeCollection(ctx.collectionName) ).resolves.toEqual(collectionToContainFields(ctx.collectionName, [{ field: ctx.columnName }], env.capabilities))
})
test('add duplicate column will fail', async() => {
diff --git a/apps/velo-external-db/test/types.ts b/apps/velo-external-db/test/types.ts
new file mode 100644
index 000000000..8239037a5
--- /dev/null
+++ b/apps/velo-external-db/test/types.ts
@@ -0,0 +1,54 @@
+
+import { Server } from 'http'
+import { ExternalDbRouter } from '@wix-velo/velo-external-db-core'
+import {
+ ConnectionCleanUp,
+ ISchemaProvider,
+ IDataProvider,
+ DataOperation,
+ FieldType,
+ CollectionOperation,
+ AnyFixMe
+} from '@wix-velo/velo-external-db-types'
+
+
+export interface ColumnsCapabilities {
+ [columnTypeName: string]: { sortable: boolean, columnQueryOperators: string[]}
+}
+
+export interface Capabilities {
+ ReadWriteOperations: DataOperation[]
+ ReadOnlyOperations: DataOperation[]
+ FieldTypes: FieldType[]
+ CollectionOperations: CollectionOperation[]
+ ColumnsCapabilities: ColumnsCapabilities
+}
+
+export interface App {
+ server: Server
+ schemaProvider: ISchemaProvider
+ cleanup: ConnectionCleanUp
+ started: boolean
+ reload: (hooks?: any) => Promise<{ externalDbRouter: ExternalDbRouter }>
+ externalDbRouter: ExternalDbRouter
+}
+
+type Internals = () => App
+
+export interface E2E_ENV {
+ app: App
+ externalDbRouter: ExternalDbRouter
+ internals: Internals
+ capabilities: Capabilities,
+ enviormentVariables: { [key: string]: string }
+}
+
+export interface ProviderResourcesEnv {
+ dataProvider: IDataProvider
+ schemaProvider: ISchemaProvider
+ cleanup: ConnectionCleanUp
+ driver: AnyFixMe
+ capabilities: Capabilities
+}
+
+
diff --git a/libs/external-db-config/src/readers/aws_config_reader.ts b/libs/external-db-config/src/readers/aws_config_reader.ts
index 988acfb1b..413eacc53 100644
--- a/libs/external-db-config/src/readers/aws_config_reader.ts
+++ b/libs/external-db-config/src/readers/aws_config_reader.ts
@@ -13,8 +13,8 @@ export class AwsConfigReader implements IConfigReader {
async readConfig() {
const { config } = await this.readExternalAndLocalConfig()
- const { host, username, password, DB, SECRET_KEY, DB_PORT } = config
- return { host: host, user: username, password: password, db: DB, secretKey: SECRET_KEY, port: DB_PORT }
+ const { host, username, password, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, DB_PORT } = config
+ return { host: host, user: username, password: password, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, port: DB_PORT }
}
async readExternalConfig() {
@@ -29,8 +29,8 @@ export class AwsConfigReader implements IConfigReader {
async readExternalAndLocalConfig() {
const { externalConfig, secretMangerError }: {[key: string]: any} = await this.readExternalConfig()
- const { host, username, password, DB, SECRET_KEY, HOST, PASSWORD, USER, DB_PORT }: {[key: string]: string} = { ...process.env, ...externalConfig }
- const config = { host: host || HOST, username: username || USER, password: password || PASSWORD, DB, SECRET_KEY, DB_PORT }
+ const { host, username, password, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, HOST, PASSWORD, USER, DB_PORT }: {[key: string]: string} = { ...process.env, ...externalConfig }
+ const config = { host: host || HOST, username: username || USER, password: password || PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, DB_PORT }
return { config, secretMangerError }
}
}
@@ -46,15 +46,15 @@ export class AwsDynamoConfigReader implements IConfigReader {
async readConfig() {
const { config } = await this.readExternalAndLocalConfig()
if (process.env['NODE_ENV'] === 'test') {
- return { region: this.region, secretKey: config.SECRET_KEY, endpoint: process.env['ENDPOINT_URL'] }
+ return { region: this.region, externalDatabaseId: config.EXTERNAL_DATABASE_ID, endpoint: process.env['ENDPOINT_URL'] }
}
- return { region: this.region, secretKey: config.SECRET_KEY }
+ return { region: this.region, externalDatabaseId: config.EXTERNAL_DATABASE_ID, allowedMetasites: config.ALLOWED_METASITES }
}
async readExternalAndLocalConfig() {
const { externalConfig, secretMangerError }: {[key: string]: any} = await this.readExternalConfig()
- const { SECRET_KEY = undefined } = { ...process.env, ...externalConfig }
- const config = { SECRET_KEY }
+ const { EXTERNAL_DATABASE_ID = undefined, ALLOWED_METASITES = undefined } = { ...process.env, ...externalConfig }
+ const config = { EXTERNAL_DATABASE_ID, ALLOWED_METASITES }
return { config, secretMangerError: secretMangerError }
}
@@ -90,8 +90,8 @@ export class AwsMongoConfigReader implements IConfigReader {
async readExternalAndLocalConfig() {
const { externalConfig, secretMangerError } :{[key: string]: any} = await this.readExternalConfig()
- const { SECRET_KEY, URI }: {SECRET_KEY: string, URI: string} = { ...process.env, ...externalConfig }
- const config = { SECRET_KEY, URI }
+ const { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI }: {EXTERNAL_DATABASE_ID: string, ALLOWED_METASITES: string, URI: string} = { ...process.env, ...externalConfig }
+ const config = { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI }
return { config, secretMangerError: secretMangerError }
}
@@ -99,8 +99,8 @@ export class AwsMongoConfigReader implements IConfigReader {
async readConfig() {
const { config } = await this.readExternalAndLocalConfig()
- const { SECRET_KEY, URI } = config
+ const { EXTERNAL_DATABASE_ID, ALLOWED_METASITES, URI } = config
- return { secretKey: SECRET_KEY, connectionUri: URI }
+ return { externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, connectionUri: URI }
}
}
diff --git a/libs/external-db-config/src/readers/azure_config_reader.ts b/libs/external-db-config/src/readers/azure_config_reader.ts
index 317e4fafb..4218b417b 100644
--- a/libs/external-db-config/src/readers/azure_config_reader.ts
+++ b/libs/external-db-config/src/readers/azure_config_reader.ts
@@ -5,7 +5,7 @@ export class AzureConfigReader implements IConfigReader {
}
async readConfig() {
- const { HOST, USER, PASSWORD, DB, SECRET_KEY, UNSECURED_ENV, DB_PORT } = process.env
- return { host: HOST, user: USER, password: PASSWORD, db: DB, secretKey: SECRET_KEY, unsecuredEnv: UNSECURED_ENV, port: DB_PORT }
+ const { HOST, USER, PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, UNSECURED_ENV, DB_PORT } = process.env
+ return { host: HOST, user: USER, password: PASSWORD, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, unsecuredEnv: UNSECURED_ENV, port: DB_PORT }
}
}
diff --git a/libs/external-db-config/src/readers/common_config_reader.ts b/libs/external-db-config/src/readers/common_config_reader.ts
index 9fa5982a1..fa0f476a2 100644
--- a/libs/external-db-config/src/readers/common_config_reader.ts
+++ b/libs/external-db-config/src/readers/common_config_reader.ts
@@ -4,7 +4,7 @@ export default class CommonConfigReader implements IConfigReader {
constructor() { }
readConfig() {
- const { CLOUD_VENDOR, TYPE, REGION, SECRET_NAME, HIDE_APP_INFO } = process.env
- return { vendor: CLOUD_VENDOR, type: TYPE, region: REGION, secretId: SECRET_NAME, hideAppInfo: HIDE_APP_INFO ? HIDE_APP_INFO === 'true' : undefined }
+ const { CLOUD_VENDOR, TYPE, REGION, SECRET_NAME, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, HIDE_APP_INFO } = process.env
+ return { vendor: CLOUD_VENDOR, type: TYPE, region: REGION, secretId: SECRET_NAME, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, hideAppInfo: HIDE_APP_INFO ? HIDE_APP_INFO === 'true' : undefined }
}
}
diff --git a/libs/external-db-config/src/readers/gcp_config_reader.ts b/libs/external-db-config/src/readers/gcp_config_reader.ts
index 0eb985182..c7d489de8 100644
--- a/libs/external-db-config/src/readers/gcp_config_reader.ts
+++ b/libs/external-db-config/src/readers/gcp_config_reader.ts
@@ -5,8 +5,8 @@ export class GcpConfigReader implements IConfigReader {
}
async readConfig() {
- const { CLOUD_SQL_CONNECTION_NAME, USER, PASSWORD, DB, SECRET_KEY, DB_PORT } = process.env
- return { cloudSqlConnectionName: CLOUD_SQL_CONNECTION_NAME, user: USER, password: PASSWORD, db: DB, secretKey: SECRET_KEY, port: DB_PORT }
+ const { CLOUD_SQL_CONNECTION_NAME, USER, PASSWORD, DB, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, DB_PORT } = process.env
+ return { cloudSqlConnectionName: CLOUD_SQL_CONNECTION_NAME, user: USER, password: PASSWORD, db: DB, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, port: DB_PORT }
}
}
@@ -16,8 +16,8 @@ export class GcpSpannerConfigReader implements IConfigReader {
}
async readConfig() {
- const { PROJECT_ID, INSTANCE_ID, DATABASE_ID, SECRET_KEY } = process.env
- return { projectId: PROJECT_ID, instanceId: INSTANCE_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY }
+ const { PROJECT_ID, INSTANCE_ID, DATABASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env
+ return { projectId: PROJECT_ID, instanceId: INSTANCE_ID, databaseId: DATABASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES }
}
@@ -27,8 +27,8 @@ export class GcpFirestoreConfigReader implements IConfigReader {
constructor() { }
async readConfig() {
- const { PROJECT_ID, SECRET_KEY } = process.env
- return { projectId: PROJECT_ID, secretKey: SECRET_KEY }
+ const { PROJECT_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env
+ return { projectId: PROJECT_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES }
}
@@ -38,8 +38,8 @@ export class GcpGoogleSheetsConfigReader implements IConfigReader {
constructor() { }
async readConfig() {
- const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, SECRET_KEY } = process.env
- return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, secretKey: SECRET_KEY }
+ const { CLIENT_EMAIL, SHEET_ID, API_PRIVATE_KEY, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env
+ return { clientEmail: CLIENT_EMAIL, apiPrivateKey: API_PRIVATE_KEY, sheetId: SHEET_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES }
}
}
@@ -48,8 +48,8 @@ export class GcpMongoConfigReader implements IConfigReader {
constructor() { }
async readConfig() {
- const { URI, SECRET_KEY } = process.env
- return { connectionUri: URI, secretKey: SECRET_KEY }
+ const { URI, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env
+ return { connectionUri: URI, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES }
}
}
@@ -57,8 +57,8 @@ export class GcpAirtableConfigReader implements IConfigReader {
constructor() { }
async readConfig() {
- const { AIRTABLE_API_KEY, META_API_KEY, BASE_ID, SECRET_KEY, BASE_URL } = process.env
- return { apiPrivateKey: AIRTABLE_API_KEY, metaApiKey: META_API_KEY, baseId: BASE_ID, secretKey: SECRET_KEY, baseUrl: BASE_URL }
+ const { AIRTABLE_API_KEY, META_API_KEY, BASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES, BASE_URL } = process.env
+ return { apiPrivateKey: AIRTABLE_API_KEY, metaApiKey: META_API_KEY, baseId: BASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES, baseUrl: BASE_URL }
}
}
@@ -67,7 +67,7 @@ export class GcpBigQueryConfigReader implements IConfigReader {
}
async readConfig() {
- const { PROJECT_ID, DATABASE_ID, SECRET_KEY } = process.env
- return { projectId: PROJECT_ID, databaseId: DATABASE_ID, secretKey: SECRET_KEY }
+ const { PROJECT_ID, DATABASE_ID, EXTERNAL_DATABASE_ID, ALLOWED_METASITES } = process.env
+ return { projectId: PROJECT_ID, databaseId: DATABASE_ID, externalDatabaseId: EXTERNAL_DATABASE_ID, allowedMetasites: ALLOWED_METASITES }
}
}
diff --git a/libs/external-db-config/src/service/config_validator.spec.ts b/libs/external-db-config/src/service/config_validator.spec.ts
index d59ad6856..2d36fd2fd 100644
--- a/libs/external-db-config/src/service/config_validator.spec.ts
+++ b/libs/external-db-config/src/service/config_validator.spec.ts
@@ -10,7 +10,7 @@ describe('Config Reader Client', () => {
test('read config will retrieve config from secret provider and validate retrieved data', async() => {
driver.givenConfig(ctx.config)
- driver.givenCommonConfig(ctx.secretKey)
+ driver.givenCommonConfig(ctx.externalDatabaseId, ctx.allowedMetasites)
driver.givenAuthorizationConfig(ctx.authorizationConfig)
expect( env.configValidator.readConfig() ).toEqual(matchers.configResponseFor(ctx.config, ctx.authorizationConfig))
@@ -85,7 +85,8 @@ describe('Config Reader Client', () => {
configStatus: Uninitialized,
missingProperties: Uninitialized,
moreMissingProperties: Uninitialized,
- secretKey: Uninitialized,
+ externalDatabaseId: Uninitialized,
+ allowedMetasites: Uninitialized,
authorizationConfig: Uninitialized,
}
@@ -102,7 +103,8 @@ describe('Config Reader Client', () => {
ctx.configStatus = gen.randomConfig()
ctx.missingProperties = Array.from({ length: 5 }, () => chance.word())
ctx.moreMissingProperties = Array.from({ length: 5 }, () => chance.word())
- ctx.secretKey = chance.guid()
+ ctx.externalDatabaseId = chance.guid()
+ ctx.allowedMetasites = chance.guid()
env.configValidator = new ConfigValidator(driver.configValidator, driver.authorizationConfigValidator, driver.commonConfigValidator)
})
})
diff --git a/libs/external-db-config/src/validators/common_config_validator.spec.ts b/libs/external-db-config/src/validators/common_config_validator.spec.ts
index ec61b8494..42ea22149 100644
--- a/libs/external-db-config/src/validators/common_config_validator.spec.ts
+++ b/libs/external-db-config/src/validators/common_config_validator.spec.ts
@@ -11,9 +11,9 @@ describe('MySqlConfigValidator', () => {
expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: [] })
})
- test('not extended common config validator will return if secretKey is missing', () => {
+ test('not extended common config validator will return if externalDatabaseId or allowedMetasites are missing', () => {
env.CommonConfigValidator = new CommonConfigValidator({})
- expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: ['secretKey'] })
+ expect(env.CommonConfigValidator.validate()).toEqual({ missingRequiredSecretsKeys: ['externalDatabaseId', 'allowedMetasites'] })
})
each(
diff --git a/libs/external-db-config/src/validators/common_config_validator.ts b/libs/external-db-config/src/validators/common_config_validator.ts
index c82ec8f06..d7ef19835 100644
--- a/libs/external-db-config/src/validators/common_config_validator.ts
+++ b/libs/external-db-config/src/validators/common_config_validator.ts
@@ -22,7 +22,7 @@ export class CommonConfigValidator {
validateBasic() {
return {
- missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['secretKey'])
+ missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['externalDatabaseId', 'allowedMetasites'])
}
}
@@ -32,7 +32,7 @@ export class CommonConfigValidator {
return {
validType,
validVendor,
- missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['type', 'vendor', 'secretKey'])
+ missingRequiredSecretsKeys: checkRequiredKeys(this.config, ['type', 'vendor', 'externalDatabaseId', 'allowedMetasites'])
}
}
}
diff --git a/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts b/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts
index ee8138bbd..533249adb 100644
--- a/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/aws_mongo_config_test_support.ts
@@ -17,8 +17,11 @@ export const defineValidConfig = (config: MongoConfig) => {
if (config.connectionUri) {
awsConfig['URI'] = config.connectionUri
}
- if (config.secretKey) {
- awsConfig['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ awsConfig['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ awsConfig['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
awsConfig['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -30,8 +33,11 @@ const defineLocalEnvs = (config: MongoConfig) => {
if (config.connectionUri) {
process.env['URI'] = config.connectionUri
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -42,7 +48,8 @@ export const defineInvalidConfig = () => defineValidConfig({})
export const validConfig = () => ({
connectionUri: chance.word(),
- secretKey: chance.word()
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word()
})
export const defineSplittedConfig = (config: MongoConfig) => {
@@ -56,8 +63,8 @@ export const validConfigWithAuthorization = () => ({
authorization: validAuthorizationConfig.collectionPermissions
})
-export const ExpectedProperties = ['URI', 'SECRET_KEY', 'PERMISSIONS']
-export const RequiredProperties = ['URI', 'SECRET_KEY']
+export const ExpectedProperties = ['URI', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'PERMISSIONS']
+export const RequiredProperties = ['URI', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES']
export const reset = () => {
mockedAwsSdk.reset()
diff --git a/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts
index 84f091a30..02f87ea69 100644
--- a/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/aws_mysql_config_test_support.ts
@@ -26,8 +26,11 @@ export const defineValidConfig = (config: MySqlConfig) => {
if (config.db) {
awsConfig['DB'] = config.db
}
- if (config.secretKey) {
- awsConfig['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ awsConfig['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ awsConfig['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
awsConfig['PERMISSIONS'] = JSON.stringify(config.authorization)
@@ -48,8 +51,11 @@ const defineLocalEnvs = (config: MySqlConfig) => {
if (config.db) {
process.env['DB'] = config.db
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify(config.authorization)
@@ -70,7 +76,8 @@ export const validConfig = (): MySqlConfig => ({
user: chance.word(),
password: chance.word(),
db: chance.word(),
- secretKey: chance.word(),
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word(),
})
export const validConfigWithAuthorization = (): MySqlConfig => ({
@@ -89,8 +96,8 @@ export const validConfigWithAuthConfig = () => ({
}
})
-export const ExpectedProperties = ['host', 'username', 'password', 'DB', 'SECRET_KEY', 'PERMISSIONS']
-export const RequiredProperties = ['host', 'username', 'password', 'DB', 'SECRET_KEY']
+export const ExpectedProperties = ['host', 'username', 'password', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'PERMISSIONS']
+export const RequiredProperties = ['host', 'username', 'password', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES']
export const reset = () => {
mockedAwsSdk.reset()
diff --git a/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts
index 80e94b845..d2ef52a80 100644
--- a/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/azure_mysql_config_test_support.ts
@@ -17,8 +17,11 @@ export const defineValidConfig = (config: MySqlConfig) => {
if (config.db) {
process.env['DB'] = config.db
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -39,7 +42,8 @@ export const validConfig = (): MySqlConfig => ({
user: chance.word(),
password: chance.word(),
db: chance.word(),
- secretKey: chance.word(),
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word(),
})
export const validConfigWithAuthorization = (): MySqlConfig => ({
@@ -58,7 +62,7 @@ export const validConfigWithAuthConfig = () => ({
export const defineInvalidConfig = () => defineValidConfig({})
-export const ExpectedProperties = ['HOST', 'USER', 'PASSWORD', 'DB', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
+export const ExpectedProperties = ['HOST', 'USER', 'PASSWORD', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
export const reset = () => ExpectedProperties.forEach(p => delete process.env[p])
diff --git a/libs/external-db-config/test/drivers/external_db_config_test_support.ts b/libs/external-db-config/test/drivers/external_db_config_test_support.ts
index 1717ed094..20da29647 100644
--- a/libs/external-db-config/test/drivers/external_db_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/external_db_config_test_support.ts
@@ -27,9 +27,9 @@ export const givenValidConfig = () =>
when(configValidator.validate).calledWith()
.mockReturnValue({ missingRequiredSecretsKeys: [] })
-export const givenCommonConfig = (secretKey: any) =>
+export const givenCommonConfig = (externalDatabaseId: any, allowedMetasites: any) =>
when(commonConfigValidator.readConfig).calledWith()
- .mockReturnValue({ secretKey })
+ .mockReturnValue({ externalDatabaseId, allowedMetasites })
export const givenValidCommonConfig = () =>
when(commonConfigValidator.validate).calledWith()
diff --git a/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts
index c21cff85d..97998f70f 100644
--- a/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/gcp_firestore_config_test_support.ts
@@ -8,8 +8,11 @@ export const defineValidConfig = (config: FiresStoreConfig) => {
if (config.projectId) {
process.env['PROJECT_ID'] = config.projectId
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -27,7 +30,8 @@ export const defineValidConfig = (config: FiresStoreConfig) => {
export const validConfig = (): FiresStoreConfig => ({
projectId: chance.word(),
- secretKey: chance.word(),
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word(),
})
export const validConfigWithAuthorization = () => ({
@@ -46,7 +50,7 @@ export const validConfigWithAuthConfig = () => ({
export const defineInvalidConfig = () => defineValidConfig({})
-export const ExpectedProperties = ['PROJECT_ID', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
+export const ExpectedProperties = ['PROJECT_ID', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
export const reset = () => ExpectedProperties.forEach(p => delete process.env[p])
diff --git a/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts
index d582c6f96..bcdca9889 100644
--- a/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/gcp_mysql_config_test_support.ts
@@ -17,8 +17,11 @@ export const defineValidConfig = (config: MySqlConfig) => {
if (config.db) {
process.env['DB'] = config.db
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -39,7 +42,8 @@ export const validConfig = (): MySqlConfig => ({
user: chance.word(),
password: chance.word(),
db: chance.word(),
- secretKey: chance.word(),
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word(),
})
export const validConfigWithAuthorization = () => ({
@@ -56,7 +60,7 @@ export const validConfigWithAuthConfig = () => ({
}
})
-export const ExpectedProperties = ['CLOUD_SQL_CONNECTION_NAME', 'USER', 'PASSWORD', 'DB', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
+export const ExpectedProperties = ['CLOUD_SQL_CONNECTION_NAME', 'USER', 'PASSWORD', 'DB', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
export const defineInvalidConfig = () => defineValidConfig({})
diff --git a/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts b/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts
index 110a41d70..0b41c36d2 100644
--- a/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts
+++ b/libs/external-db-config/test/drivers/gcp_spanner_config_test_support.ts
@@ -14,8 +14,11 @@ export const defineValidConfig = (config: SpannerConfig) => {
if (config.databaseId) {
process.env['DATABASE_ID'] = config.databaseId
}
- if (config.secretKey) {
- process.env['SECRET_KEY'] = config.secretKey
+ if (config.externalDatabaseId) {
+ process.env['EXTERNAL_DATABASE_ID'] = config.externalDatabaseId
+ }
+ if (config.allowedMetasites) {
+ process.env['ALLOWED_METASITES'] = config.allowedMetasites
}
if (config.authorization) {
process.env['PERMISSIONS'] = JSON.stringify( config.authorization )
@@ -35,7 +38,8 @@ export const validConfig = (): SpannerConfig => ({
projectId: chance.word(),
instanceId: chance.word(),
databaseId: chance.word(),
- secretKey: chance.word(),
+ externalDatabaseId: chance.word(),
+ allowedMetasites: chance.word(),
})
export const validConfigWithAuthorization = (): SpannerConfig => ({
@@ -56,7 +60,7 @@ export const validConfigWithAuthConfig = () => ({
export const defineInvalidConfig = () => defineValidConfig({})
-export const ExpectedProperties = ['PROJECT_ID', 'INSTANCE_ID', 'DATABASE_ID', 'SECRET_KEY', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
+export const ExpectedProperties = ['PROJECT_ID', 'INSTANCE_ID', 'DATABASE_ID', 'EXTERNAL_DATABASE_ID', 'ALLOWED_METASITES', 'callbackUrl', 'clientId', 'clientSecret', 'PERMISSIONS']
export const reset = () => ExpectedProperties.forEach(p => delete process.env[p])
diff --git a/libs/external-db-config/test/gen.ts b/libs/external-db-config/test/gen.ts
index 93418a488..ec2c0ca62 100644
--- a/libs/external-db-config/test/gen.ts
+++ b/libs/external-db-config/test/gen.ts
@@ -10,11 +10,13 @@ export const randomConfig = () => ({
})
export const randomCommonConfig = () => ({
- secretKey: chance.guid(),
+ externalDatabaseId: chance.guid(),
+ allowedMetasites: chance.guid(),
})
export const randomExtendedCommonConfig = () => ({
- secretKey: chance.guid(),
+ externalDatabaseId: chance.guid(),
+ allowedMetasites: chance.guid(),
vendor: chance.pickone(supportedVendors),
type: chance.pickone(supportedDBs),
})
diff --git a/libs/external-db-config/test/test_types.ts b/libs/external-db-config/test/test_types.ts
index cf2fad9b6..3cdfedc77 100644
--- a/libs/external-db-config/test/test_types.ts
+++ b/libs/external-db-config/test/test_types.ts
@@ -1,13 +1,15 @@
export interface MongoConfig {
connectionUri?: string
- secretKey?: string
+ externalDatabaseId?: string
+ allowedMetasites?: string
authorization?: any
}
export interface MongoAwsConfig {
URI?: string
- SECRET_KEY?: string
+ EXTERNAL_DATABASE_ID?: string
+ ALLOWED_METASITES?: string
PERMISSIONS?: string
}
@@ -17,7 +19,8 @@ export interface MySqlConfig {
user?: string
password?: string
db?: string
- secretKey?: string
+ externalDatabaseId?: string
+ allowedMetasites?: string
authorization?: any
auth?: any
}
@@ -27,7 +30,8 @@ export interface AwsMysqlConfig {
username?: string
password?: string
DB?: string
- SECRET_KEY?: string
+ EXTERNAL_DATABASE_ID?: string
+ ALLOWED_METASITES?: string
PERMISSIONS?: string
}
@@ -36,11 +40,14 @@ export interface CommonConfig {
vendor?: string
secretKey?: string
hideAppInfo?: boolean
+ externalDatabaseId?: string
+ allowedMetasites?: string
}
export interface FiresStoreConfig {
projectId?: string
- secretKey?: string
+ externalDatabaseId?: string
+ allowedMetasites?: string
authorization?: any
auth?: any
}
@@ -49,7 +56,8 @@ export interface SpannerConfig {
projectId?: string
instanceId?: string
databaseId?: string
- secretKey?: string
+ externalDatabaseId?: string
+ allowedMetasites?: string
authorization?: any
auth?: any
}
diff --git a/libs/external-db-config/test/test_utils.ts b/libs/external-db-config/test/test_utils.ts
index a50ed9ee4..b411917db 100644
--- a/libs/external-db-config/test/test_utils.ts
+++ b/libs/external-db-config/test/test_utils.ts
@@ -23,4 +23,4 @@ export const splitConfig = (config: {[key: string]: any}) => {
return { firstPart, secondPart }
}
-export const extendedCommonConfigRequiredProperties = ['secretKey', 'vendor', 'type']
+export const extendedCommonConfigRequiredProperties = ['externalDatabaseId', 'allowedMetasites', 'vendor', 'type']
diff --git a/libs/external-db-dynamodb/src/dynamo_capabilities.ts b/libs/external-db-dynamodb/src/dynamo_capabilities.ts
new file mode 100644
index 000000000..7ebc407d9
--- /dev/null
+++ b/libs/external-db-dynamodb/src/dynamo_capabilities.ts
@@ -0,0 +1,20 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { eq, ne, string_contains, string_begins, gt, gte, lt, lte, include } = AdapterOperators
+const UnsupportedCapabilities = [DataOperation.insertReferences, DataOperation.removeReferences, DataOperation.queryReferenced]
+
+
+export const ReadWriteOperations = Object.values(DataOperation).filter(op => !UnsupportedCapabilities.includes(op))
+
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, include, gt, gte, lt, lte] },
+ url: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, include, gt, gte, lt, lte] },
+ number: { sortable: false, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: false, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, include, gt, gte, lt, lte] },
+ datetime: { sortable: false, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-dynamodb/src/dynamo_data_provider.ts b/libs/external-db-dynamodb/src/dynamo_data_provider.ts
index cf67f64bb..5a723879a 100644
--- a/libs/external-db-dynamodb/src/dynamo_data_provider.ts
+++ b/libs/external-db-dynamodb/src/dynamo_data_provider.ts
@@ -5,6 +5,7 @@ import { DynamoDB } from '@aws-sdk/client-dynamodb'
import FilterParser from './sql_filter_transformer'
import { IDataProvider, AdapterFilter as Filter, Item } from '@wix-velo/velo-external-db-types'
import * as dynamoRequests from './dynamo_data_requests_utils'
+import { translateErrorCodes } from './sql_exception_translator'
export default class DataProvider implements IDataProvider {
filterParser: FilterParser
@@ -40,10 +41,13 @@ export default class DataProvider implements IDataProvider {
return Count || 0
}
- async insert(collectionName: string, items: Item[]): Promise<number> {
+ async insert(collectionName: string, items: Item[], _fields?: any[], upsert = false): Promise<number> {
validateTable(collectionName)
await this.docClient
- .batchWrite(dynamoRequests.batchPutItemsCommand(collectionName, items.map(patchDateTime)))
+ .transactWrite({
+ TransactItems: items.map((item: Item) => dynamoRequests.insertSingleItemCommand(collectionName, patchDateTime(item), upsert))
+ }).catch(e => translateErrorCodes(e, collectionName, { items }))
+
return items.length
}
diff --git a/libs/external-db-dynamodb/src/dynamo_data_requests_utils.ts b/libs/external-db-dynamodb/src/dynamo_data_requests_utils.ts
index 742f12958..f016b8b4b 100644
--- a/libs/external-db-dynamodb/src/dynamo_data_requests_utils.ts
+++ b/libs/external-db-dynamodb/src/dynamo_data_requests_utils.ts
@@ -1,4 +1,5 @@
-import { updateFieldsFor } from '@wix-velo/velo-external-db-commons'
+import { BatchWriteCommandInput } from '@aws-sdk/lib-dynamodb'
+import { updateFieldsFor } from '@wix-velo/velo-external-db-commons'
import { Item } from '@wix-velo/velo-external-db-types'
import { isEmptyObject } from './dynamo_utils'
import { DynamoParsedFilter } from './types'
@@ -9,7 +10,7 @@ export const findCommand = (collectionName: string, filter: DynamoParsedFilter,
delete filter.ProjectionExpression
}
return {
- TableName: collectionName,
+ TableName: collectionName,
...filter,
Limit: limit
}
@@ -17,7 +18,7 @@ export const findCommand = (collectionName: string, filter: DynamoParsedFilter,
export const countCommand = (collectionName: string, filter: DynamoParsedFilter) => {
return {
- TableName: collectionName,
+ TableName: collectionName,
...filter,
Select: 'COUNT'
}
@@ -28,19 +29,9 @@ export const getAllIdsCommand = (collectionName: string) => ({
AttributesToGet: ['_id']
})
-export const batchPutItemsCommand = (collectionName: string, items: Item[]) => ({
- RequestItems: {
- [collectionName]: items.map(putSingleItemCommand)
- }
-})
-export const putSingleItemCommand = (item: Item) => ({
- PutRequest: {
- Item: item
- }
-})
-export const batchDeleteItemsCommand = (collectionName: string, itemIds: string[]) => ({
+export const batchDeleteItemsCommand = (collectionName: string, itemIds: string[]): BatchWriteCommandInput => ({
RequestItems: {
[collectionName]: itemIds.map(deleteSingleItemCommand)
}
@@ -54,7 +45,24 @@ export const deleteSingleItemCommand = (id: string) => ({
}
})
-export const updateSingleItemCommand = (collectionName: string, item: Item) => {
+export const insertSingleItemCommand = (collectionName: string, item: Item, upsert: boolean) => {
+ const upsertCondition = upsert ? {} : {
+ ConditionExpression: 'attribute_not_exists(#_id)',
+ ExpressionAttributeNames: {
+ '#_id': '_id'
+ }
+ }
+
+ return {
+ Put: {
+ TableName: collectionName,
+ Item: item,
+ ...upsertCondition
+ }
+ }
+}
+
+export const updateSingleItemCommand = (collectionName: string, item: Item) => {
const updateFields = updateFieldsFor(item)
const updateExpression = `SET ${updateFields.map(f => `#${f} = :${f}`).join(', ')}`
const expressionAttributeNames = updateFields.reduce((pv, cv) => ({ ...pv, [`#${cv}`]: cv }), {})
diff --git a/libs/external-db-dynamodb/src/dynamo_schema_provider.ts b/libs/external-db-dynamodb/src/dynamo_schema_provider.ts
index c00041122..1b5c675b1 100644
--- a/libs/external-db-dynamodb/src/dynamo_schema_provider.ts
+++ b/libs/external-db-dynamodb/src/dynamo_schema_provider.ts
@@ -1,12 +1,15 @@
-import { SystemTable, validateTable, reformatFields } from './dynamo_utils'
+import { SystemTable, validateTable } from './dynamo_utils'
import { translateErrorCodes } from './sql_exception_translator'
-import { SystemFields, validateSystemFields, errors } from '@wix-velo/velo-external-db-commons'
+import { SystemFields, validateSystemFields, errors, EmptyCapabilities } from '@wix-velo/velo-external-db-commons'
import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb'
import * as dynamoRequests from './dynamo_schema_requests_utils'
import { DynamoDB } from '@aws-sdk/client-dynamodb'
-import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
+import { CollectionCapabilities, Encryption, InputField, ISchemaProvider, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
+import { CollectionOperations, ColumnsCapabilities, FieldTypes, ReadWriteOperations } from './dynamo_capabilities'
+import { supportedOperations } from './supported_operations'
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = errors
+
export default class SchemaProvider implements ISchemaProvider {
client: DynamoDB
docClient: DynamoDBDocument
@@ -23,7 +26,8 @@ export default class SchemaProvider implements ISchemaProvider {
return Items ? Items.map((table: { [x:string]: any, tableName?: any, fields?: any }) => ({
id: table.tableName,
- fields: [...SystemFields, ...table.fields].map(reformatFields)
+ fields: [...SystemFields, ...table.fields].map(this.appendAdditionalRowDetails),
+ capabilities: this.collectionCapabilities()
})) : []
}
@@ -36,9 +40,7 @@ export default class SchemaProvider implements ISchemaProvider {
}
supportedOperations(): SchemaOperations[] {
- const { List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately } = SchemaOperations
-
- return [ List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately ]
+ return supportedOperations
}
async create(collectionName: string, columns: InputField[]): Promise<void> {
@@ -71,7 +73,7 @@ export default class SchemaProvider implements ISchemaProvider {
await validateSystemFields(column.name)
const { fields } = await this.collectionDataFor(collectionName)
- if (fields.find((f: { name: any }) => f.name === column.name)) {
+ if (fields.find((f) => f.name === column.name)) {
throw new FieldAlreadyExists('Collection already has a field with the same name')
}
@@ -86,21 +88,41 @@ export default class SchemaProvider implements ISchemaProvider {
const { fields } = await this.collectionDataFor(collectionName)
- if (!fields.some((f: { name: any }) => f.name === columnName)) {
- throw new FieldDoesNotExist('Collection does not contain a field with this name')
+ if (!fields.some((f) => f.name === columnName)) {
+ throw new FieldDoesNotExist('Collection does not contain a field with this name', collectionName, columnName)
}
await this.docClient
- .update(dynamoRequests.removeColumnExpression(collectionName, fields.filter((f: { name: any }) => f.name !== columnName)))
+ .update(dynamoRequests.updateColumnsExpression(collectionName, fields.filter((f: { name: any }) => f.name !== columnName)))
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
+ async describeCollection(collectionName: string): Promise<Table> {
await this.ensureSystemTableExists()
validateTable(collectionName)
const collection = await this.collectionDataFor(collectionName)
- return [...SystemFields, ...collection.fields].map( reformatFields )
+ return {
+ id: collectionName,
+ fields: [...SystemFields, ...collection.fields].map(this.appendAdditionalRowDetails),
+ capabilities: this.collectionCapabilities()
+ }
+ }
+
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
+ await this.ensureSystemTableExists()
+ validateTable(collectionName)
+ await validateSystemFields(column.name)
+
+ const { fields } = await this.collectionDataFor(collectionName)
+
+ if (!fields.some((f) => f.name === column.name)) {
+ throw new FieldDoesNotExist('Collection does not contain a field with this name', collectionName, column.name)
+ }
+
+ await this.docClient
+ .update(dynamoRequests.updateColumnsExpression(collectionName, fields.map((f) => f.name === column.name ? column : f)))
+
}
async ensureSystemTableExists() {
@@ -124,13 +146,13 @@ export default class SchemaProvider implements ISchemaProvider {
.delete(dynamoRequests.deleteTableFromSystemTableExpression(collectionName))
}
- async collectionDataFor(collectionName: string, toReturn?: boolean | undefined): Promise<any> {
+ async collectionDataFor(collectionName: string, toReturn?: boolean | undefined) {
validateTable(collectionName)
const { Item } = await this.docClient
.get(dynamoRequests.getCollectionFromSystemTableExpression(collectionName))
- if (!Item && !toReturn ) throw new CollectionDoesNotExists('Collection does not exists')
- return Item
+ if (!Item && !toReturn ) throw new CollectionDoesNotExists('Collection does not exists', collectionName)
+ return Item as { tableName: string, fields: { name: string, type: string, subtype?: string }[] }
}
async systemTableExists() {
@@ -139,4 +161,24 @@ export default class SchemaProvider implements ISchemaProvider {
.then(() => true)
.catch(() => false)
}
+
+
+ private appendAdditionalRowDetails(row: {name: string, type: string}) {
+ return {
+ field: row.name,
+ type: row.type,
+ capabilities: ColumnsCapabilities[row.type as keyof typeof ColumnsCapabilities] ?? EmptyCapabilities
+ }
+ }
+
+ private collectionCapabilities(): CollectionCapabilities {
+ return {
+ dataOperations: ReadWriteOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
+ }
+ }
}
diff --git a/libs/external-db-dynamodb/src/dynamo_schema_requests_utils.ts b/libs/external-db-dynamodb/src/dynamo_schema_requests_utils.ts
index eb8c2b200..c8ed62a54 100644
--- a/libs/external-db-dynamodb/src/dynamo_schema_requests_utils.ts
+++ b/libs/external-db-dynamodb/src/dynamo_schema_requests_utils.ts
@@ -1,7 +1,8 @@
+import { InputField } from '@wix-velo/velo-external-db-types'
import { SystemTable } from './dynamo_utils'
-export const removeColumnExpression = (collectionName: any, columns: any) => ({
+export const updateColumnsExpression = (collectionName: any, columns: any) => ({
TableName: SystemTable,
Key: {
tableName: collectionName
@@ -31,6 +32,22 @@ export const addColumnExpression = (collectionName: any, column: any) => ({
ReturnValues: 'UPDATED_NEW'
})
+export const changeColumnTypeExpression = (collectionName: string, column: InputField) => ({
+ TableName: SystemTable,
+ Key: {
+ tableName: collectionName
+ },
+ UpdateExpression: 'SET #attrName = list_append(list_append(:attrValue1, list_remove(#attrName, :attrValue2)), :attrValue3)',
+ ExpressionAttributeNames: {
+ '#attrName': 'fields'
+ },
+ ExpressionAttributeValues: {
+ ':attrValue1': [column],
+ ':attrValue2': column.name,
+ ':attrValue3': [column]
+ },
+})
+
export const createTableExpression = (collectionName: any) => ({
TableName: collectionName,
KeySchema: [{ AttributeName: '_id', KeyType: 'HASH' }],
diff --git a/libs/external-db-dynamodb/src/dynamo_utils.ts b/libs/external-db-dynamodb/src/dynamo_utils.ts
index 19250f299..62f7d81ba 100644
--- a/libs/external-db-dynamodb/src/dynamo_utils.ts
+++ b/libs/external-db-dynamodb/src/dynamo_utils.ts
@@ -1,5 +1,4 @@
import { errors } from '@wix-velo/velo-external-db-commons'
-import { InputField, ResponseField } from '@wix-velo/velo-external-db-types'
import { Counter } from './sql_filter_transformer'
const { InvalidQuery } = errors
@@ -28,14 +27,6 @@ export const patchFixDates = (record: { [x: string]: any }) => {
return fixedRecord
}
-
-export const reformatFields = (field: InputField): ResponseField => {
- return {
- field: field.name,
- type: field.type,
- }
-}
-
export const patchCollectionKeys = () => (['_id'])
export const canQuery = (filterExpr: { ExpressionAttributeNames: { [s: string]: unknown } | ArrayLike<unknown> }, collectionKeys: unknown[]) => {
@@ -47,5 +38,5 @@ export const canQuery = (filterExpr: { ExpressionAttributeNames: { [s: string]:
export const isEmptyObject = (obj: Record<string, unknown>) => Object.keys(obj).length === 0
-export const fieldNameWithCounter = (fieldName: string, counter: Counter) => `#${fieldName}${counter.nameCounter++}`
-export const attributeValueNameWithCounter = (fieldName: any, counter: Counter) => `:${fieldName}${counter.valueCounter++}`
+export const fieldNameWithCounter = (fieldName: string, counter: Counter) => `#${fieldName.split('.').join('.#').split('.').map(s => s.concat(`${counter.nameCounter++}`)).join('.')}`
+export const attributeValueNameWithCounter = (fieldName: any, counter: Counter) => `:${fieldName.split('.')[0]}${counter.valueCounter++}`
diff --git a/libs/external-db-dynamodb/src/sql_exception_translator.ts b/libs/external-db-dynamodb/src/sql_exception_translator.ts
index 309ab8bbb..0898a2dcc 100644
--- a/libs/external-db-dynamodb/src/sql_exception_translator.ts
+++ b/libs/external-db-dynamodb/src/sql_exception_translator.ts
@@ -1,14 +1,20 @@
import { errors } from '@wix-velo/velo-external-db-commons'
-const { CollectionDoesNotExists, DbConnectionError } = errors
+import { Item } from '@wix-velo/velo-external-db-types'
+const { CollectionDoesNotExists, DbConnectionError, ItemAlreadyExists } = errors
-export const notThrowingTranslateErrorCodes = (err: any) => {
+export const notThrowingTranslateErrorCodes = (err: any, collectionName?: string, metaData?: { items?: Item[] }) => {
switch (err.name) {
case 'ResourceNotFoundException':
- return new CollectionDoesNotExists('Collection does not exists')
+ return new CollectionDoesNotExists('Collection does not exists', collectionName)
case 'CredentialsProviderError':
return new DbConnectionError('AWS_SECRET_ACCESS_KEY or AWS_ACCESS_KEY_ID are missing')
case 'InvalidSignatureException':
return new DbConnectionError('AWS_SECRET_ACCESS_KEY or AWS_ACCESS_KEY_ID are invalid')
+ case 'TransactionCanceledException':
+ if (err.message.includes('ConditionalCheckFailed')) {
+ const itemId = metaData?.items?.[err.CancellationReasons.findIndex((reason: any) => reason.Code === 'ConditionalCheckFailed')]._id
+ return new ItemAlreadyExists('Item already exists', collectionName, itemId)
+ }
}
switch (err.message) {
@@ -21,7 +27,7 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
}
}
-export const translateErrorCodes = (err: any) => {
- throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName?: string, metaData?: {items?: Item[]}) => {
+ throw notThrowingTranslateErrorCodes(err, collectionName, metaData)
}
diff --git a/libs/external-db-dynamodb/src/sql_filter_transformer.spec.ts b/libs/external-db-dynamodb/src/sql_filter_transformer.spec.ts
index ef9d9f317..87a495e13 100644
--- a/libs/external-db-dynamodb/src/sql_filter_transformer.spec.ts
+++ b/libs/external-db-dynamodb/src/sql_filter_transformer.spec.ts
@@ -205,6 +205,29 @@ describe('Sql Parser', () => {
})
})
+ describe('handle queries on nested fields', () => {
+ test('correctly transform nested field query', () => {
+ const operator = ctx.filterWithoutInclude.operator
+ const filter = {
+ operator,
+ fieldName: `${ctx.fieldName}.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}`,
+ value: ctx.filterWithoutInclude.value
+ }
+
+ expect( env.filterParser.parseFilter(filter) ).toEqual([{
+ filterExpr: {
+ FilterExpression: `#${ctx.fieldName}0.#${ctx.nestedFieldName}1.#${ctx.anotherNestedFieldName}2 ${env.filterParser.adapterOperatorToDynamoOperator(operator)} :${ctx.fieldName}0`,
+ ExpressionAttributeNames: {
+ [`#${ctx.fieldName}0`]: ctx.fieldName,
+ [`#${ctx.nestedFieldName}1`]: ctx.nestedFieldName,
+ [`#${ctx.anotherNestedFieldName}2`]: ctx.anotherNestedFieldName
+ },
+ ExpressionAttributeValues: { [`:${ctx.fieldName}0`]: ctx.filterWithoutInclude.value }
+ }
+ }])
+ })
+ })
+
describe('handle multi field operator', () => {
each([
and, or
@@ -276,9 +299,13 @@ describe('Sql Parser', () => {
fieldListValue: Uninitialized,
anotherFieldName: Uninitialized,
moreFieldName: Uninitialized,
+ nestedFieldName: Uninitialized,
+ anotherNestedFieldName: Uninitialized,
filter: Uninitialized,
idFilterNotEqual: Uninitialized,
anotherFilter: Uninitialized,
+ filterWithoutInclude: Uninitialized,
+
}
@@ -292,6 +319,8 @@ describe('Sql Parser', () => {
ctx.fieldName = chance.word()
ctx.anotherFieldName = chance.word()
ctx.moreFieldName = chance.word()
+ ctx.nestedFieldName = chance.word()
+ ctx.anotherNestedFieldName = chance.word()
ctx.fieldValue = chance.word()
ctx.fieldListValue = [chance.word(), chance.word(), chance.word(), chance.word(), chance.word()]
@@ -299,6 +328,7 @@ describe('Sql Parser', () => {
ctx.filter = gen.randomWrappedFilter()
ctx.idFilterNotEqual = idFilter({ withoutEqual: true })
ctx.anotherFilter = gen.randomWrappedFilter()
+ ctx.filterWithoutInclude = gen.randomDomainFilterWithoutInclude()
})
beforeAll(function() {
diff --git a/libs/external-db-dynamodb/src/sql_filter_transformer.ts b/libs/external-db-dynamodb/src/sql_filter_transformer.ts
index 2b5a597f8..a27db62ad 100644
--- a/libs/external-db-dynamodb/src/sql_filter_transformer.ts
+++ b/libs/external-db-dynamodb/src/sql_filter_transformer.ts
@@ -65,7 +65,23 @@ export default class FilterParser {
}
const expressionAttributeName = attributeNameWithCounter(fieldName, counter)
-
+
+ if (this.isNestedField(fieldName)) {
+ const expressionAttributeValue = attributeValueNameWithCounter(fieldName, counter)
+ return [{
+ filterExpr: {
+ FilterExpression: `${expressionAttributeName} ${this.adapterOperatorToDynamoOperator(operator)} ${expressionAttributeValue}`,
+ ExpressionAttributeNames: expressionAttributeName.split('.').reduce((pV, cV) => ({
+ ...pV,
+ [cV]: cV.slice(1, cV.length - 1)
+ }), {}),
+ ExpressionAttributeValues: {
+ [expressionAttributeValue]: this.valueForOperator(value, operator)
+ }
+ }
+ }]
+ }
+
if (this.isSingleFieldOperator(operator)) {
const expressionAttributeValue = attributeValueNameWithCounter(fieldName, counter)
return [{
@@ -80,7 +96,7 @@ export default class FilterParser {
}
}]
}
-
+
if (this.isSingleFieldStringOperator(operator)) {
const expressionAttributeValue = attributeValueNameWithCounter(fieldName, counter)
return [{
@@ -141,6 +157,10 @@ export default class FilterParser {
return value
}
+ isNestedField(fieldName: string) {
+ return fieldName.includes('.')
+ }
+
adapterOperatorToDynamoOperator(operator: any) {
switch (operator) {
case eq:
diff --git a/libs/external-db-dynamodb/src/supported_operations.ts b/libs/external-db-dynamodb/src/supported_operations.ts
index 9e6e490bc..532050b56 100644
--- a/libs/external-db-dynamodb/src/supported_operations.ts
+++ b/libs/external-db-dynamodb/src/supported_operations.ts
@@ -1,4 +1,16 @@
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const { List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, Projection, StartWithCaseSensitive, NotOperator, FindObject, IncludeOperator, FilterByEveryField } = SchemaOperations
-export const supportedOperations = [ List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, Projection, StartWithCaseSensitive, NotOperator, FindObject, IncludeOperator, FilterByEveryField ]
+const notSupportedOperations = [
+ SchemaOperations.FindWithSort,
+ SchemaOperations.Aggregate,
+ SchemaOperations.StartWithCaseInsensitive,
+ SchemaOperations.FindObject,
+ SchemaOperations.IncludeOperator,
+ SchemaOperations.Matches,
+ SchemaOperations.NonAtomicBulkInsert
+]
+
+
+
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-dynamodb/tests/e2e-testkit/dynamodb_resources.ts b/libs/external-db-dynamodb/tests/e2e-testkit/dynamodb_resources.ts
index f928c331a..4e4c51fdd 100644
--- a/libs/external-db-dynamodb/tests/e2e-testkit/dynamodb_resources.ts
+++ b/libs/external-db-dynamodb/tests/e2e-testkit/dynamodb_resources.ts
@@ -1,6 +1,7 @@
import * as compose from 'docker-compose'
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/dynamo_capabilities'
export const connection = async() => {
const { connection, schemaProvider, cleanup } = init(connectionConfig(), accessOptions())
diff --git a/libs/external-db-firestore/src/firestore_capabilities.ts b/libs/external-db-firestore/src/firestore_capabilities.ts
new file mode 100644
index 000000000..40bee8729
--- /dev/null
+++ b/libs/external-db-firestore/src/firestore_capabilities.ts
@@ -0,0 +1,21 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, string_begins, gt, gte, lt, lte, include } = AdapterOperators
+const UnsupportedCapabilities = [DataOperation.insertReferences, DataOperation.removeReferences, DataOperation.queryReferenced]
+
+
+export const ReadWriteOperations = Object.values(DataOperation).filter(op => !UnsupportedCapabilities.includes(op))
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, string_begins, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, string_begins, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [eq, string_begins, include, gt, gte, lt, lte] },
+ datetime: { sortable: true, columnQueryOperators: [eq, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-firestore/src/firestore_data_provider.ts b/libs/external-db-firestore/src/firestore_data_provider.ts
index e12812908..f261b790b 100644
--- a/libs/external-db-firestore/src/firestore_data_provider.ts
+++ b/libs/external-db-firestore/src/firestore_data_provider.ts
@@ -1,7 +1,15 @@
import { Firestore, WriteBatch, Query, DocumentData } from '@google-cloud/firestore'
-import { AdapterAggregation, AdapterFilter, IDataProvider, Item, AdapterFilter as Filter } from '@wix-velo/velo-external-db-types'
+import {
+ AdapterAggregation,
+ AdapterFilter,
+ IDataProvider,
+ Item,
+ AdapterFilter as Filter,
+ ResponseField
+} from '@wix-velo/velo-external-db-types'
import FilterParser from './sql_filter_transformer'
import { asEntity } from './firestore_utils'
+import { translateErrorCodes } from './sql_exception_translator'
export default class DataProvider implements IDataProvider {
database: Firestore
@@ -25,7 +33,7 @@ export default class DataProvider implements IDataProvider {
const projectedCollectionRef = projection ? collectionRef2.select(...projection) : collectionRef2
- const docs = (await projectedCollectionRef.limit(limit).offset(skip).get()).docs
+ const docs = (await projectedCollectionRef.limit(limit).offset(skip).get().catch(translateErrorCodes)).docs
return docs.map((doc) => asEntity(doc))
}
@@ -35,18 +43,25 @@ export default class DataProvider implements IDataProvider {
const collectionRef = filterOperations.reduce((c: Query<DocumentData>, { fieldName, opStr, value }) => c.where(fieldName, opStr, value), this.database.collection(collectionName))
- return (await collectionRef.get()).size
+ return (await collectionRef.get().catch(translateErrorCodes)).size
}
- async insert(collectionName: string, items: Item[]): Promise<number> {
- const batch = items.reduce((b, i) => b.set(this.database.doc(`${collectionName}/${i._id}`), i), this.database.batch())
- return (await batch.commit()).length
+ async insert(collectionName: string, items: Item[], _fields?: ResponseField[], upsert?: boolean): Promise<number> {
+
+ const batch = items.reduce((b, i) =>
+ upsert
+ ? b.set(this.database.doc(`${collectionName}/${i._id}`), i)
+ : b.create(this.database.doc(`${collectionName}/${i._id}`), i)
+ , this.database.batch()
+ )
+
+ return (await batch.commit().catch(translateErrorCodes)).length
}
async update(collectionName: any, items: any[]): Promise<number> {
const batch = items.reduce((b: { update: (arg0: FirebaseFirestore.DocumentReference<FirebaseFirestore.DocumentData>, arg1: any) => any }, i: { _id: any }) => b.update(this.database.doc(`${collectionName}/${i._id}`), i), this.database.batch())
- return (await batch.commit()).length
+ return (await batch.commit().catch(translateErrorCodes)).length
}
async delete(collectionName: string, itemIds: any[]) {
diff --git a/libs/external-db-firestore/src/firestore_schema_provider.ts b/libs/external-db-firestore/src/firestore_schema_provider.ts
index e8300106d..5ac59fa01 100644
--- a/libs/external-db-firestore/src/firestore_schema_provider.ts
+++ b/libs/external-db-firestore/src/firestore_schema_provider.ts
@@ -1,7 +1,20 @@
import { Firestore } from '@google-cloud/firestore'
-import { SystemFields, validateSystemFields, errors } from '@wix-velo/velo-external-db-commons'
-import { InputField, ISchemaProvider, ResponseField, Table, SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { SystemFields, validateSystemFields, errors, EmptyCapabilities } from '@wix-velo/velo-external-db-commons'
+import {
+ InputField,
+ ISchemaProvider,
+ Table,
+ SchemaOperations,
+ CollectionCapabilities, Encryption
+} from '@wix-velo/velo-external-db-types'
import { table } from './types'
+import {
+ CollectionOperations,
+ FieldTypes,
+ ReadWriteOperations,
+ ColumnsCapabilities
+} from './firestore_capabilities'
+
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = errors
const SystemTable = '_descriptor'
@@ -15,17 +28,20 @@ export default class SchemaProvider implements ISchemaProvider {
return {
field: field.name,
type: field.type,
+ capabilities: this.fieldCapabilities(field)
}
}
async list(): Promise<Table[]> {
const l = await this.database.collection(SystemTable).get()
const tables: {[x:string]: table[]} = l.docs.reduce((o, d) => ({ ...o, [d.id]: d.data() }), {})
+
return Object.entries(tables)
- .map(([collectionName, rs]: [string, any]) => ({
- id: collectionName,
- fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) )
- }))
+ .map(([collectionName, rs]: [string, any]) => ({
+ id: collectionName,
+ fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) ),
+ capabilities: this.collectionCapabilities()
+ }))
}
async listHeaders() {
@@ -61,9 +77,9 @@ export default class SchemaProvider implements ISchemaProvider {
const collection = await collectionRef.get()
if (!collection.exists) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
}
- const { fields } = collection.data() as any
+ const { fields } = collection.data() as { id: string, fields: { type: string, subtype: string, name: string}[]}
if (fields.find((f: { name: string }) => f.name === column.name)) {
throw new FieldAlreadyExists('Collection already has a field with the same name')
@@ -81,7 +97,7 @@ export default class SchemaProvider implements ISchemaProvider {
const collection = await collectionRef.get()
if (!collection.exists) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
}
const { fields } = collection.data() as any
@@ -94,23 +110,57 @@ export default class SchemaProvider implements ISchemaProvider {
})
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
+ const collectionRef = this.database.collection(SystemTable).doc(collectionName)
+ const collection = await collectionRef.get()
+
+ if (!collection.exists) {
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
+ }
+
+ const { fields } = collection.data() as any
+
+ await collectionRef.update({
+ fields: [...fields, column]
+ })
+ }
+
+ async describeCollection(collectionName: string): Promise<Table> {
const collection = await this.database.collection(SystemTable)
- .doc(collectionName)
- .get()
+ .doc(collectionName)
+ .get()
if (!collection.exists) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
}
const { fields } = collection.data() as any
- return [...SystemFields, ...fields].map(this.reformatFields.bind(this))
+ return {
+ id: collectionName,
+ fields: [...SystemFields, ...fields].map( this.reformatFields.bind(this) ),
+ capabilities: this.collectionCapabilities()
+ }
}
async drop(collectionName: string) {
// todo: drop collection https://firebase.google.com/docs/firestore/manage-data/delete-data
await this.database.collection(SystemTable).doc(collectionName).delete()
}
+
+ private fieldCapabilities(field: InputField) {
+ return ColumnsCapabilities[field.type as keyof typeof ColumnsCapabilities] ?? EmptyCapabilities
+ }
+
+ private collectionCapabilities(): CollectionCapabilities {
+ return {
+ dataOperations: ReadWriteOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
+ }
+ }
}
diff --git a/libs/external-db-firestore/src/sql_exception_translator.ts b/libs/external-db-firestore/src/sql_exception_translator.ts
index ad88df3c2..4fc139ddf 100644
--- a/libs/external-db-firestore/src/sql_exception_translator.ts
+++ b/libs/external-db-firestore/src/sql_exception_translator.ts
@@ -1,9 +1,25 @@
import { errors } from '@wix-velo/velo-external-db-commons'
-const { DbConnectionError, UnrecognizedError } = errors
+const { DbConnectionError, UnrecognizedError, ItemAlreadyExists } = errors
+const extractValue = (details: string, valueName: string) => extractValueFromErrorMessage(details, new RegExp(`${valueName}:\\s*"([^"]+)"`))
+
+const extractValueFromErrorMessage = (msg: string, regex: RegExp) => {
+ try {
+ const match = msg.match(regex)
+ const value = (match && match[1])
+ return value || ''
+ } catch(e) {
+ return ''
+ }
+}
export const notThrowingTranslateErrorCodes = (err: any) => {
+ const collectionName = extractValue(err.details, 'type')
+ const itemId = extractValue(err.details, 'name')
+
switch (err.code) {
+ case 6:
+ return new ItemAlreadyExists(`Item already exists: ${err.details}`, collectionName, itemId)
case 7:
return new DbConnectionError(`Permission denied - Cloud Firestore API has not been enabled: ${err.details}`)
case 16:
diff --git a/libs/external-db-firestore/src/supported_operations.ts b/libs/external-db-firestore/src/supported_operations.ts
index 18780c1e0..0ba90029b 100644
--- a/libs/external-db-firestore/src/supported_operations.ts
+++ b/libs/external-db-firestore/src/supported_operations.ts
@@ -1,5 +1,5 @@
import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const { List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, StartWithCaseSensitive, FindObject, IncludeOperator, FilterByEveryField } = SchemaOperations
+const { List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, StartWithCaseSensitive, FindObject, IncludeOperator, FilterByEveryField, QueryNestedFields } = SchemaOperations
-export const supportedOperations = [ List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, StartWithCaseSensitive, FindObject, IncludeOperator, FilterByEveryField ]
+export const supportedOperations = [ List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, BulkDelete, Truncate, DeleteImmediately, UpdateImmediately, StartWithCaseSensitive, FindObject, IncludeOperator, FilterByEveryField, QueryNestedFields ]
diff --git a/libs/external-db-firestore/tests/e2e-testkit/firestore_resources.ts b/libs/external-db-firestore/tests/e2e-testkit/firestore_resources.ts
index 2d49b0d97..7ee09b1cd 100644
--- a/libs/external-db-firestore/tests/e2e-testkit/firestore_resources.ts
+++ b/libs/external-db-firestore/tests/e2e-testkit/firestore_resources.ts
@@ -3,6 +3,7 @@ import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/firestore_capabilities'
const setEmulatorOn = () => process.env['FIRESTORE_EMULATOR_HOST'] = 'localhost:8082'
diff --git a/libs/external-db-google-sheets/src/google_sheet_capabilities.ts b/libs/external-db-google-sheets/src/google_sheet_capabilities.ts
new file mode 100644
index 000000000..f7d938576
--- /dev/null
+++ b/libs/external-db-google-sheets/src/google_sheet_capabilities.ts
@@ -0,0 +1,24 @@
+import {
+ DataOperation,
+ FieldType,
+} from '@wix-velo/velo-external-db-types'
+
+export const ColumnsCapabilities = {
+ text: { sortable: false, columnQueryOperators: [] },
+ url: { sortable: false, columnQueryOperators: [] },
+ number: { sortable: false, columnQueryOperators: [] },
+ boolean: { sortable: false, columnQueryOperators: [] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [] },
+ datetime: { sortable: false, columnQueryOperators: [] },
+}
+
+export const ReadWriteOperations = [
+ DataOperation.insert,
+ DataOperation.update,
+ DataOperation.remove,
+ DataOperation.truncate,
+]
+export const ReadOnlyOperations = []
+export const FieldTypes = [ FieldType.text ]
+export const CollectionOperations = []
diff --git a/libs/external-db-google-sheets/src/google_sheet_exception_translator.ts b/libs/external-db-google-sheets/src/google_sheet_exception_translator.ts
index b51949acf..473f88af5 100644
--- a/libs/external-db-google-sheets/src/google_sheet_exception_translator.ts
+++ b/libs/external-db-google-sheets/src/google_sheet_exception_translator.ts
@@ -17,7 +17,7 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
case '400':
return new DbConnectionError('Client email is invalid')
default :
- return new UnrecognizedError(`${err.message}`)
+ return new DbConnectionError(`${err.message}`)
}
}
diff --git a/libs/external-db-google-sheets/src/google_sheet_schema_provider.ts b/libs/external-db-google-sheets/src/google_sheet_schema_provider.ts
index b01dd4bbd..cc4286ac3 100644
--- a/libs/external-db-google-sheets/src/google_sheet_schema_provider.ts
+++ b/libs/external-db-google-sheets/src/google_sheet_schema_provider.ts
@@ -1,9 +1,10 @@
import { GoogleSpreadsheet, GoogleSpreadsheetWorksheet } from 'google-spreadsheet'
-import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { CollectionCapabilities, Encryption, SchemaOperations } from '@wix-velo/velo-external-db-types'
import { SystemFields, validateSystemFields, parseTableData, errors } from '@wix-velo/velo-external-db-commons'
-import { ISchemaProvider, ResponseField, InputField, Table } from '@wix-velo/velo-external-db-types'
+import { ISchemaProvider, InputField, Table } from '@wix-velo/velo-external-db-types'
import { translateErrorCodes } from './google_sheet_exception_translator'
import { describeSheetHeaders, headersFrom, sheetFor } from './google_sheet_utils'
+import { CollectionOperations, FieldTypes, ReadOnlyOperations, ReadWriteOperations, ColumnsCapabilities } from './google_sheet_capabilities'
export default class SchemaProvider implements ISchemaProvider {
doc: GoogleSpreadsheet
@@ -25,7 +26,8 @@ export default class SchemaProvider implements ISchemaProvider {
return Object.entries(parsedSheetsHeadersData)
.map(([collectionName, rs]) => ({
id: collectionName,
- fields: rs.map(this.translateDbTypes.bind(this))
+ fields: rs.map(this.translateDbTypes.bind(this)),
+ capabilities: this.collectionCapabilities(rs.map(r => r.field))
}))
}
@@ -48,9 +50,14 @@ export default class SchemaProvider implements ISchemaProvider {
}
}
- async describeCollection(collectionName: string) {
+ async describeCollection(collectionName: string): Promise<Table> {
const sheet = await sheetFor(collectionName, this.doc)
- return await describeSheetHeaders(sheet)
+ const fields = await describeSheetHeaders(sheet)
+ return {
+ id: collectionName,
+ fields,
+ capabilities: this.collectionCapabilities(fields.map(f => f.field))
+ }
}
async addColumn(collectionName: string, column: InputField) {
@@ -74,11 +81,27 @@ export default class SchemaProvider implements ISchemaProvider {
await sheet.delete()
}
- translateDbTypes(row: ResponseField) {
+ translateDbTypes(row: { field: string, type: string }) {
return {
field: row.field,
- type: row.type
+ type: row.type,
+ capabilities: ColumnsCapabilities[row.type as keyof typeof ColumnsCapabilities]
+ }
+ }
+
+ private collectionCapabilities(fieldNames: string[]): CollectionCapabilities {
+ return {
+ dataOperations: fieldNames.includes('_id') ? ReadWriteOperations : ReadOnlyOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
}
}
+
+ async changeColumnType(_collectionName: string, _column: InputField): Promise<void> {
+ throw new Error('Method not implemented.')
+ }
}
diff --git a/libs/external-db-google-sheets/src/google_sheet_utils.ts b/libs/external-db-google-sheets/src/google_sheet_utils.ts
index d356decbc..5fa5c3f10 100644
--- a/libs/external-db-google-sheets/src/google_sheet_utils.ts
+++ b/libs/external-db-google-sheets/src/google_sheet_utils.ts
@@ -27,7 +27,7 @@ export const sheetFor = async(sheetTitle: string, doc: GoogleSpreadsheet) => {
}
if (!doc.sheetsByTitle[sheetTitle]) {
- throw new errors.CollectionDoesNotExists('Collection does not exists')
+ throw new errors.CollectionDoesNotExists('Collection does not exists', sheetTitle)
}
return doc.sheetsByTitle[sheetTitle]
diff --git a/libs/external-db-google-sheets/tests/e2e-testkit/google_sheets_resources.ts b/libs/external-db-google-sheets/tests/e2e-testkit/google_sheets_resources.ts
index eed65f2e5..b14635b9a 100644
--- a/libs/external-db-google-sheets/tests/e2e-testkit/google_sheets_resources.ts
+++ b/libs/external-db-google-sheets/tests/e2e-testkit/google_sheets_resources.ts
@@ -10,6 +10,8 @@ let _server: Server
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/google_sheet_capabilities'
+
export const connection = async() => {
const googleSheetsConfig = {
sheetId: SHEET_ID,
diff --git a/libs/external-db-mongo/src/exception_translator.ts b/libs/external-db-mongo/src/exception_translator.ts
index 7b65001d6..c35d8c59b 100644
--- a/libs/external-db-mongo/src/exception_translator.ts
+++ b/libs/external-db-mongo/src/exception_translator.ts
@@ -1,15 +1,17 @@
import { errors } from '@wix-velo/velo-external-db-commons'
const { ItemAlreadyExists } = errors
-const notThrowingTranslateErrorCodes = (err: any) => {
+const extractItemIdFromError = (err: any) => err.message.split('"')[1]
+
+const notThrowingTranslateErrorCodes = (err: any, collectionName: string) => {
switch (err.code) {
- case 11000:
- return new ItemAlreadyExists(`Item already exists: ${err.message}`)
+ case 11000:
+ return new ItemAlreadyExists(`Item already exists: ${err.message}`, collectionName, extractItemIdFromError(err))
default:
return new Error (`default ${err.message}`)
}
}
-export const translateErrorCodes = (err: any) => {
- throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName: string) => {
+ throw notThrowingTranslateErrorCodes(err, collectionName)
}
diff --git a/libs/external-db-mongo/src/mongo_capabilities.ts b/libs/external-db-mongo/src/mongo_capabilities.ts
new file mode 100644
index 000000000..104d6f5ce
--- /dev/null
+++ b/libs/external-db-mongo/src/mongo_capabilities.ts
@@ -0,0 +1,19 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+
+export const ReadWriteOperations = Object.values(DataOperation)
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [] },
+ datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-mongo/src/mongo_data_provider.ts b/libs/external-db-mongo/src/mongo_data_provider.ts
index b99859c52..4a5abaa3a 100644
--- a/libs/external-db-mongo/src/mongo_data_provider.ts
+++ b/libs/external-db-mongo/src/mongo_data_provider.ts
@@ -1,6 +1,6 @@
import { translateErrorCodes } from './exception_translator'
-import { unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils'
-import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types'
+import { insertExpressionFor, isEmptyObject, unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils'
+import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort, } from '@wix-velo/velo-external-db-types'
import FilterParser from './sql_filter_transformer'
import { MongoClient } from 'mongodb'
@@ -34,14 +34,14 @@ export default class DataProvider implements IDataProvider {
.count(filterExpr)
}
- async insert(collectionName: string, items: Item[] ): Promise<number> {
+ async insert(collectionName: string, items: Item[], _fields: any[], upsert = false): Promise<number> {
validateTable(collectionName)
- const result = await this.client.db()
- .collection(collectionName)
- //@ts-ignore - Type 'string' is not assignable to type 'ObjectId', objectId Can be a 24 character hex string, 12 byte binary Buffer, or a number. and we cant assume that on the _id input
- .insertMany(items)
- .catch(translateErrorCodes)
- return result.insertedCount
+ const { insertedCount, upsertedCount } = await this.client.db()
+ .collection(collectionName)
+ .bulkWrite(insertExpressionFor(items, upsert), { ordered: false })
+ .catch(e => translateErrorCodes(e, collectionName))
+
+ return insertedCount + upsertedCount
}
async update(collectionName: string, items: Item[]): Promise<number> {
@@ -49,7 +49,7 @@ export default class DataProvider implements IDataProvider {
const result = await this.client.db()
.collection(collectionName)
.bulkWrite( updateExpressionFor(items) )
- return result.nModified
+ return result.nModified
}
async delete(collectionName: string, itemIds: string[]): Promise<number> {
@@ -67,17 +67,26 @@ export default class DataProvider implements IDataProvider {
.deleteMany({})
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise<Item[]> {
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise<Item[]> {
validateTable(collectionName)
+ const additionalAggregationStages = []
const { fieldsStatement, havingFilter } = this.filterParser.parseAggregation(aggregation)
const { filterExpr } = this.filterParser.transform(filter)
+ const sortExpr = this.filterParser.orderAggregationBy(sort)
+
+ !isEmptyObject(sortExpr.$sort)? additionalAggregationStages.push(sortExpr) : null
+ skip? additionalAggregationStages.push({ $skip: skip }) : null
+ limit? additionalAggregationStages.push({ $limit: limit }) : null
+
const result = await this.client.db()
- .collection(collectionName)
- .aggregate( [ { $match: filterExpr },
- fieldsStatement,
- havingFilter
- ] )
- .toArray()
+ .collection(collectionName)
+ .aggregate([
+ { $match: filterExpr },
+ fieldsStatement,
+ havingFilter,
+ ...additionalAggregationStages
+ ])
+ .toArray()
return result.map( unpackIdFieldForItem )
}
diff --git a/libs/external-db-mongo/src/mongo_schema_provider.ts b/libs/external-db-mongo/src/mongo_schema_provider.ts
index 26d2aa9f9..3f7b899cf 100644
--- a/libs/external-db-mongo/src/mongo_schema_provider.ts
+++ b/libs/external-db-mongo/src/mongo_schema_provider.ts
@@ -1,33 +1,49 @@
-import { SystemFields, validateSystemFields, AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
-import { InputField, ResponseField, ISchemaProvider, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
-const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = require('@wix-velo/velo-external-db-commons').errors
-import { validateTable, SystemTable } from './mongo_utils'
+import { MongoClient } from 'mongodb'
+import { SystemFields, validateSystemFields, AllSchemaOperations, EmptyCapabilities, errors } from '@wix-velo/velo-external-db-commons'
+import { InputField, ResponseField, ISchemaProvider, SchemaOperations, Table, CollectionCapabilities, Encryption } from '@wix-velo/velo-external-db-types'
+import { validateTable, SystemTable, updateExpressionFor, CollectionObject } from './mongo_utils'
+import { CollectionOperations, FieldTypes, ReadWriteOperations, ColumnsCapabilities } from './mongo_capabilities'
+const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist } = errors
+
export default class SchemaProvider implements ISchemaProvider {
- client: any
+ client: MongoClient
constructor(client: any) {
this.client = client
}
- reformatFields(field: InputField ) {
+ reformatFields(field: {name: string, type: string}): ResponseField {
return {
field: field.name,
type: field.type,
+ capabilities: ColumnsCapabilities[field.type as keyof typeof ColumnsCapabilities] ?? EmptyCapabilities
}
}
+ private collectionCapabilities(): CollectionCapabilities {
+ return {
+ dataOperations: ReadWriteOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ encryption: Encryption.notSupported,
+ indexing: [],
+ referenceCapabilities: { supportedNamespaces: [] }
+ }
+ }
+
async list(): Promise<Table[]> {
await this.ensureSystemTableExists()
const resp = await this.client.db()
.collection(SystemTable)
- .find({})
+ .find<CollectionObject>({})
const l = await resp.toArray()
const tables = l.reduce((o: any, d: { _id: string; fields: any }) => ({ ...o, [d._id]: { fields: d.fields } }), {})
return Object.entries(tables)
.map(([collectionName, rs]: [string, any]) => ({
id: collectionName,
- fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) )
+ fields: [...SystemFields, ...rs.fields].map( this.reformatFields.bind(this) ),
+ capabilities: this.collectionCapabilities()
}))
}
@@ -37,7 +53,7 @@ export default class SchemaProvider implements ISchemaProvider {
const resp = await this.client.db()
.collection(SystemTable)
- .find({})
+ .find<CollectionObject>({})
const data = await resp.toArray()
return data.map((rs: { _id: string }) => rs._id)
}
@@ -52,7 +68,7 @@ export default class SchemaProvider implements ISchemaProvider {
if (!collection) {
await this.client.db()
.collection(SystemTable)
- .insertOne( { _id: collectionName, fields: columns || [] })
+ .insertOne({ _id: collectionName as any, fields: columns || [] })
await this.client.db()
.createCollection(collectionName)
}
@@ -98,14 +114,33 @@ export default class SchemaProvider implements ISchemaProvider {
{ $pull: { fields: { name: { $eq: columnName } } } } )
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
- validateTable(collectionName)
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
const collection = await this.collectionDataFor(collectionName)
+
if (!collection) {
throw new CollectionDoesNotExists('Collection does not exists')
}
+
+ await this.client.db()
+ .collection(SystemTable)
+ .bulkWrite(updateExpressionFor([{
+ _id: collection._id,
+ fields: [...collection.fields.filter((f: InputField) => f.name !== column.name), column]
+ }]))
- return [...SystemFields, ...collection.fields].map( this.reformatFields.bind(this) )
+ }
+
+ async describeCollection(collectionName: string): Promise<Table> {
+ validateTable(collectionName)
+ const collection = await this.collectionDataFor(collectionName)
+ if (!collection) {
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
+ }
+ return {
+ id: collectionName,
+ fields: [...SystemFields, ...collection.fields].map( this.reformatFields.bind(this) ),
+ capabilities: this.collectionCapabilities()
+ }
}
async drop(collectionName: string): Promise<void> {
@@ -120,11 +155,11 @@ export default class SchemaProvider implements ISchemaProvider {
}
}
- async collectionDataFor(collectionName: string): Promise<any> { //fixme: any
+ async collectionDataFor(collectionName: string) {
validateTable(collectionName)
return await this.client.db()
.collection(SystemTable)
- .findOne({ _id: collectionName })
+ .findOne<CollectionObject>({ _id: collectionName })
}
async ensureSystemTableExists(): Promise<void> {
diff --git a/libs/external-db-mongo/src/mongo_utils.spec.ts b/libs/external-db-mongo/src/mongo_utils.spec.ts
index 8b31003c0..6ff585318 100644
--- a/libs/external-db-mongo/src/mongo_utils.spec.ts
+++ b/libs/external-db-mongo/src/mongo_utils.spec.ts
@@ -1,5 +1,5 @@
const { InvalidQuery } = require('@wix-velo/velo-external-db-commons').errors
-import { unpackIdFieldForItem, validateTable } from './mongo_utils'
+import { unpackIdFieldForItem, validateTable, insertExpressionFor, isEmptyObject } from './mongo_utils'
describe('Mongo Utils', () => {
describe('unpackIdFieldForItem', () => {
@@ -48,4 +48,28 @@ describe('Mongo Utils', () => {
expect(() => validateTable('someTable')).not.toThrow()
})
})
+
+ describe('insertExpressionFor', () => {
+ test('insertExpressionFor with upsert set to false will return insert expression', () => {
+ expect(insertExpressionFor([{ _id: 'itemId' }], false)[0]).toEqual({ insertOne: { document: { _id: 'itemId' } } })
+ })
+ test('insertExpressionFor with upsert set to true will return update expression', () => {
+ expect(insertExpressionFor([{ _id: 'itemId' }], true)[0]).toEqual({
+ updateOne: {
+ filter: { _id: 'itemId' },
+ update: { $set: { _id: 'itemId' } },
+ upsert: true
+ }
+ })
+ })
+ })
+
+ describe('isEmptyObject', () => {
+ test('isEmptyObject will return true for empty object', () => {
+ expect(isEmptyObject({})).toBe(true)
+ expect(isEmptyObject({ a: {} }.a)).toBe(true)
+ }
+ )
+
+ })
})
diff --git a/libs/external-db-mongo/src/mongo_utils.ts b/libs/external-db-mongo/src/mongo_utils.ts
index 3ef8d2a1a..c970c911c 100644
--- a/libs/external-db-mongo/src/mongo_utils.ts
+++ b/libs/external-db-mongo/src/mongo_utils.ts
@@ -35,14 +35,28 @@ export const isConnected = (client: { topology: { isConnected: () => any } }) =>
return client && client.topology && client.topology.isConnected()
}
-const updateExpressionForItem = (item: { _id: any }) => ({
+const insertExpressionForItem = (item: { _id: any }) => ({
+ insertOne: {
+ document: { ...item, _id: item._id as any }
+ }
+})
+
+const updateExpressionForItem = (item: { _id: any }, upsert: boolean) => ({
updateOne: {
filter: { _id: item._id },
- update: { $set: { ...item } }
+ update: { $set: { ...item } },
+ upsert
}
})
-export const updateExpressionFor = (items: any[]) => items.map(updateExpressionForItem)
+export const insertExpressionFor = (items: any[], upsert: boolean) => {
+ return upsert?
+ items.map(i => updateExpressionForItem(i, upsert)):
+ items.map(i => insertExpressionForItem(i))
+}
+
+
+export const updateExpressionFor = (items: any[], upsert = false) => items.map(i => updateExpressionForItem(i, upsert))
export const unpackIdFieldForItem = (item: { [x: string]: any, _id?: any }) => {
if (isObject(item._id)) {
@@ -56,3 +70,10 @@ export const unpackIdFieldForItem = (item: { [x: string]: any, _id?: any }) => {
export const EmptySort = {
sortExpr: { sort: [] },
}
+
+export interface CollectionObject {
+ _id: string,
+ fields: { name: string, type: string, subtype?: string }[]
+}
+
+export const isEmptyObject = (obj: any) => Object.keys(obj).length === 0 && obj.constructor === Object
diff --git a/libs/external-db-mongo/src/sql_filter_transformer.spec.ts b/libs/external-db-mongo/src/sql_filter_transformer.spec.ts
index 28af5067a..e90566334 100644
--- a/libs/external-db-mongo/src/sql_filter_transformer.spec.ts
+++ b/libs/external-db-mongo/src/sql_filter_transformer.spec.ts
@@ -1,5 +1,5 @@
import each from 'jest-each'
-import Chance = require('chance')
+import * as Chance from 'chance'
import { AdapterOperators, errors } from '@wix-velo/velo-external-db-commons'
import { AdapterFunctions } from '@wix-velo/velo-external-db-types'
import { Uninitialized, gen } from '@wix-velo/test-commons'
@@ -416,6 +416,28 @@ describe('Sql Parser', () => {
havingFilter: { $match: {} },
})
})
+
+ test('orderAggregationBy', () => {
+ expect(env.filterParser.orderAggregationBy([
+ { fieldName: ctx.fieldName, direction: ctx.direction },
+ ])).toEqual({
+ $sort: {
+ [ctx.fieldName]: ctx.direction === 'asc' ? 1 : -1
+ }
+ })
+
+ expect(env.filterParser.orderAggregationBy([
+ { fieldName: ctx.fieldName, direction: ctx.direction },
+ { fieldName: ctx.anotherFieldName, direction: ctx.anotherDirection },
+ ])).toEqual({
+ $sort: {
+ [ctx.fieldName]: ctx.direction === 'asc' ? 1 : -1,
+ [ctx.anotherFieldName]: ctx.anotherDirection === 'asc' ? 1 : -1
+ }
+ })
+
+
+ })
})
})
@@ -423,16 +445,18 @@ describe('Sql Parser', () => {
})
interface Context {
- fieldName: any
- fieldValue: any
- anotherValue: any
- moreValue: any
- fieldListValue: any
- anotherFieldName: any
- moreFieldName: any
- filter: any
- anotherFilter: any
- offset: any
+ fieldName: string
+ fieldValue: string
+ anotherValue: string
+ moreValue: string
+ fieldListValue: string[]
+ anotherFieldName: string
+ moreFieldName: string
+ filter: { fieldName: string; operator: string; value: string | string[] }
+ anotherFilter: { fieldName: string; operator: string; value: string | string[]; }
+ offset: number
+ direction: 'asc' | 'desc'
+ anotherDirection: 'asc' | 'desc'
}
const ctx : Context = {
@@ -446,6 +470,8 @@ describe('Sql Parser', () => {
filter: Uninitialized,
anotherFilter: Uninitialized,
offset: Uninitialized,
+ direction: Uninitialized,
+ anotherDirection: Uninitialized,
}
interface Enviorment {
@@ -470,6 +496,9 @@ describe('Sql Parser', () => {
ctx.anotherFilter = gen.randomWrappedFilter()
ctx.offset = chance.natural({ min: 2, max: 20 })
+
+ ctx.direction = chance.pickone(['asc', 'desc'])
+ ctx.anotherDirection = chance.pickone(['asc', 'desc'])
})
beforeAll(function() {
diff --git a/libs/external-db-mongo/src/sql_filter_transformer.ts b/libs/external-db-mongo/src/sql_filter_transformer.ts
index eebcbd7ae..308867257 100644
--- a/libs/external-db-mongo/src/sql_filter_transformer.ts
+++ b/libs/external-db-mongo/src/sql_filter_transformer.ts
@@ -141,6 +141,15 @@ export default class FilterParser {
}
}
+ orderAggregationBy(sort: Sort[]) {
+ return {
+ $sort: sort.reduce((acc, s) => {
+ const direction = s.direction === 'asc'? 1 : -1
+ return { ...acc, [s.fieldName]: direction }
+ }, {})
+ }
+ }
+
parseSort({ fieldName, direction }: Sort): { expr: MongoFieldSort } | [] {
if (typeof fieldName !== 'string') {
return []
diff --git a/libs/external-db-mongo/src/supported_operations.ts b/libs/external-db-mongo/src/supported_operations.ts
index 642b18976..c266ba31d 100644
--- a/libs/external-db-mongo/src/supported_operations.ts
+++ b/libs/external-db-mongo/src/supported_operations.ts
@@ -1 +1,5 @@
-export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons'
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+const notSupportedOperations = [SchemaOperations.AtomicBulkInsert]
+
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts b/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts
index e20d595b7..0a40fdb48 100644
--- a/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts
+++ b/libs/external-db-mongo/tests/drivers/sql_filter_transformer_test_support.ts
@@ -7,6 +7,7 @@ export const filterParser = {
parseFilter: jest.fn(),
orderBy: jest.fn(),
parseAggregation: jest.fn(),
+ orderAggregationBy: jest.fn(),
selectFieldsFor: jest.fn()
}
@@ -23,6 +24,8 @@ export const stubEmptyFilterFor = (filter: any) => {
export const stubEmptyOrderByFor = (sort: any) => {
when(filterParser.orderBy).calledWith(sort)
.mockReturnValue(EmptySort)
+ when(filterParser.orderAggregationBy).calledWith(sort)
+ .mockReturnValue({ $sort: {} })
}
export const givenOrderByFor = (column: any, sort: any) => {
@@ -97,6 +100,7 @@ export const givenIncludeFilterForIdColumn = (filter: any, value: any) =>
export const reset = () => {
filterParser.transform.mockClear()
filterParser.orderBy.mockClear()
+ filterParser.orderAggregationBy.mockClear()
filterParser.parseAggregation.mockClear()
filterParser.parseFilter.mockClear()
filterParser.selectFieldsFor.mockClear()
diff --git a/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts b/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts
index d68464377..f1d17243e 100644
--- a/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts
+++ b/libs/external-db-mongo/tests/e2e-testkit/mongo_resources.ts
@@ -2,6 +2,8 @@ import * as compose from 'docker-compose'
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/mongo_capabilities'
+
export const connection = async() => {
const { connection, schemaProvider, cleanup } = await init({ connectionUri: 'mongodb://root:pass@localhost/testdb' })
diff --git a/libs/external-db-mssql/src/mssql_capabilities.ts b/libs/external-db-mssql/src/mssql_capabilities.ts
new file mode 100644
index 000000000..64b42731e
--- /dev/null
+++ b/libs/external-db-mssql/src/mssql_capabilities.ts
@@ -0,0 +1,24 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import {
+ CollectionOperation,
+ DataOperation,
+ FieldType,
+} from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+}
+
+export const ReadWriteOperations = Object.values(DataOperation)
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
diff --git a/libs/external-db-mssql/src/mssql_data_provider.ts b/libs/external-db-mssql/src/mssql_data_provider.ts
index fbf9c0bd2..be597dc1c 100644
--- a/libs/external-db-mssql/src/mssql_data_provider.ts
+++ b/libs/external-db-mssql/src/mssql_data_provider.ts
@@ -2,7 +2,7 @@ import { escapeId, validateLiteral, escape, patchFieldName, escapeTable } from '
import { updateFieldsFor } from '@wix-velo/velo-external-db-commons'
import { translateErrorCodes } from './sql_exception_translator'
import { ConnectionPool as MSSQLPool } from 'mssql'
-import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types'
+import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort } from '@wix-velo/velo-external-db-types'
import FilterParser from './sql_filter_transformer'
export default class DataProvider implements IDataProvider {
@@ -20,32 +20,39 @@ export default class DataProvider implements IDataProvider {
const projectionExpr = this.filterParser.selectFieldsFor(projection)
const sql = `SELECT ${projectionExpr} FROM ${escapeTable(collectionName)} ${filterExpr} ${sortExpr} ${pagingQueryStr}`
- return await this.query(sql, parameters)
+ return await this.query(sql, parameters, collectionName)
}
async count(collectionName: string, filter: Filter): Promise<number> {
const { filterExpr, parameters } = this.filterParser.transform(filter)
const sql = `SELECT COUNT(*) as num FROM ${escapeTable(collectionName)} ${filterExpr}`
- const rs = await this.query(sql, parameters)
+ const rs = await this.query(sql, parameters, collectionName)
return rs[0]['num']
}
- patch(item: Item) {
- return Object.entries(item).reduce((o, [k, v]) => ( { ...o, [patchFieldName(k)]: v } ), {})
+ patch(item: Item, i?: number) {
+ return Object.entries(item).reduce((o, [k, v]) => ( { ...o, [patchFieldName(k, i)]: v } ), {})
}
- async insert(collectionName: string, items: any[], fields: any[]): Promise<number> {
+ async insert(collectionName: string, items: any[], fields: any[], upsert?: boolean): Promise<number> {
const fieldsNames = fields.map((f: { field: any }) => f.field)
- const rss = await Promise.all(items.map((item: any) => this.insertSingle(collectionName, item, fieldsNames)))
-
- return rss.reduce((s, rs) => s + rs, 0)
- }
+ let sql
+ if (upsert) {
+ sql = `MERGE ${escapeTable(collectionName)} as target`
+ +` USING (VALUES ${items.map((item: any, i: any) => `(${Object.keys(item).map((key: string) => validateLiteral(key, i) ).join(', ')})`).join(', ')}) as source`
+ +` (${fieldsNames.map( escapeId ).join(', ')}) ON target._id = source._id`
+ +' WHEN NOT MATCHED '
+ +` THEN INSERT (${fieldsNames.map( escapeId ).join(', ')}) VALUES (${fieldsNames.map((f) => `source.${f}` ).join(', ')})`
+ +' WHEN MATCHED'
+ +` THEN UPDATE SET ${fieldsNames.map((f) => `${escapeId(f)} = source.${f}`).join(', ')};`
+ }
+ else {
+ sql = `INSERT INTO ${escapeTable(collectionName)} (${fieldsNames.map( escapeId ).join(', ')}) VALUES ${items.map((item: any, i: any) => `(${Object.keys(item).map((key: string) => validateLiteral(key, i) ).join(', ')})`).join(', ')}`
+ }
- insertSingle(collectionName: string, item: Item, fieldsNames: string[]): Promise<number> {
- const sql = `INSERT INTO ${escapeTable(collectionName)} (${fieldsNames.map( escapeId ).join(', ')}) VALUES (${Object.keys(item).map( validateLiteral ).join(', ')})`
- return this.query(sql, this.patch(item), true)
+ return await this.query(sql, items.reduce((p: any, t: any, i: any) => ( { ...p, ...this.patch(t, i) } ), {}), collectionName, true)
}
async update(collectionName: string, items: Item[]): Promise<number> {
@@ -57,39 +64,41 @@ export default class DataProvider implements IDataProvider {
const updateFields = updateFieldsFor(item)
const sql = `UPDATE ${escapeTable(collectionName)} SET ${updateFields.map(f => `${escapeId(f)} = ${validateLiteral(f)}`).join(', ')} WHERE _id = ${validateLiteral('_id')}`
- return await this.query(sql, this.patch(item), true)
+ return await this.query(sql, this.patch(item), collectionName, true)
}
async delete(collectionName: string, itemIds: string[]): Promise<number> {
const sql = `DELETE FROM ${escapeTable(collectionName)} WHERE _id IN (${itemIds.map((t: any, i: any) => validateLiteral(`_id${i}`)).join(', ')})`
- const rs = await this.query(sql, itemIds.reduce((p: any, t: any, i: any) => ( { ...p, [patchFieldName(`_id${i}`)]: t } ), {}), true)
- .catch( translateErrorCodes )
+ const rs = await this.query(sql, itemIds.reduce((p: any, t: any, i: any) => ( { ...p, [patchFieldName(`_id${i}`)]: t } ), {}), collectionName, true)
+ .catch(e => translateErrorCodes(e, collectionName) )
return rs
}
async truncate(collectionName: string): Promise<void> {
- await this.sql.query(`TRUNCATE TABLE ${escapeTable(collectionName)}`).catch( translateErrorCodes )
+ await this.sql.query(`TRUNCATE TABLE ${escapeTable(collectionName)}`).catch(e => translateErrorCodes(e, collectionName))
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise<Item[]> {
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise<Item[]> {
const { filterExpr: whereFilterExpr, parameters: whereParameters } = this.filterParser.transform(filter)
const { fieldsStatement, groupByColumns, havingFilter, parameters } = this.filterParser.parseAggregation(aggregation)
+ const { sortExpr } = this.filterParser.orderBy(sort)
+ const pagingQueryStr = this.pagingQueryFor(skip, limit)
- const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter}`
+ const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter} ${sortExpr} ${pagingQueryStr}`
- return await this.query(sql, { ...whereParameters, ...parameters })
+ return await this.query(sql, { ...whereParameters, ...parameters }, collectionName)
}
- async query(sql: string, parameters: any, op?: false): Promise<Item[]>
- async query(sql: string, parameters: any, op?: true): Promise<number>
- async query(sql: string, parameters: any, op?: boolean| undefined): Promise<Item[] | number> {
+ async query(sql: string, parameters: any, collectionName: string, op?: false): Promise<Item[]>
+ async query(sql: string, parameters: any, collectionName: string, op?: true): Promise<number>
+ async query(sql: string, parameters: any, collectionName: string, op?: boolean| undefined): Promise<Item[] | number> {
const request = Object.entries(parameters)
.reduce((r, [k, v]) => r.input(k, v),
this.sql.request())
const rs = await request.query(sql)
- .catch( translateErrorCodes )
+ .catch(e => translateErrorCodes(e, collectionName) )
if (op) {
return rs.rowsAffected[0]
@@ -109,5 +118,7 @@ export default class DataProvider implements IDataProvider {
return `${offsetSql} ${limitSql}`.trim()
}
-
+ translateErrorCodes(collectionName: string, e: any) {
+ return translateErrorCodes(e, collectionName)
+ }
}
diff --git a/libs/external-db-mssql/src/mssql_schema_provider.ts b/libs/external-db-mssql/src/mssql_schema_provider.ts
index f1fc1bc13..882e7e610 100644
--- a/libs/external-db-mssql/src/mssql_schema_provider.ts
+++ b/libs/external-db-mssql/src/mssql_schema_provider.ts
@@ -1,11 +1,12 @@
+import { ConnectionPool as MSSQLPool } from 'mssql'
+import { CollectionCapabilities, FieldAttributes, InputField, ISchemaProvider, ResponseField, SchemaOperations, Table, Encryption } from '@wix-velo/velo-external-db-types'
+import { SystemFields, validateSystemFields, parseTableData, EmptyCapabilities } from '@wix-velo/velo-external-db-commons'
+import { errors } from '@wix-velo/velo-external-db-commons'
import { translateErrorCodes, notThrowingTranslateErrorCodes } from './sql_exception_translator'
import SchemaColumnTranslator from './sql_schema_translator'
import { escapeId, escapeTable } from './mssql_utils'
-import { SystemFields, validateSystemFields, parseTableData } from '@wix-velo/velo-external-db-commons'
import { supportedOperations } from './supported_operations'
-import { ConnectionPool as MSSQLPool } from 'mssql'
-import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
-import { errors } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperations, ColumnsCapabilities, FieldTypes, ReadOnlyOperations, ReadWriteOperations } from './mssql_capabilities'
const { CollectionDoesNotExists, CollectionAlreadyExists } = errors
export default class SchemaProvider implements ISchemaProvider {
@@ -29,7 +30,8 @@ export default class SchemaProvider implements ISchemaProvider {
return Object.entries(tables)
.map(([collectionName, rs]) => ({
id: collectionName,
- fields: rs.map( this.translateDbTypes.bind(this) )
+ fields: rs.map(this.appendAdditionalRowDetails.bind(this)),
+ capabilities: this.collectionCapabilities(rs.map(r => r.field))
}))
}
@@ -74,21 +76,47 @@ export default class SchemaProvider implements ISchemaProvider {
.catch( translateErrorCodes )
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
+ async describeCollection(collectionName: string): Promise<Table> {
const rs = await this.sql.request()
.input('db', this.dbName)
.input('tableName', collectionName)
.query('SELECT TABLE_NAME as table_name, COLUMN_NAME as field, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = @db AND TABLE_NAME = @tableName')
if (rs.recordset.length === 0) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
+ }
+ const fields = rs.recordset.map(this.appendAdditionalRowDetails.bind(this))
+
+ return {
+ id: collectionName,
+ fields: fields as ResponseField[],
+ capabilities: this.collectionCapabilities(fields.map((f: ResponseField) => f.field))
}
+ }
+
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
+ await validateSystemFields(column.name)
+ await this.sql.query(`ALTER TABLE ${escapeTable(collectionName)} ALTER COLUMN ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
+ .catch(translateErrorCodes)
+ }
- return rs.recordset.map( this.translateDbTypes.bind(this) )
+ private appendAdditionalRowDetails(row: { field: string} & FieldAttributes): ResponseField {
+ const type = this.sqlSchemaTranslator.translateType(row.type) as keyof typeof ColumnsCapabilities
+ return {
+ ...row,
+ type: this.sqlSchemaTranslator.translateType(row.type),
+ capabilities: ColumnsCapabilities[type] ?? EmptyCapabilities
+ }
}
- translateDbTypes(row: ResponseField) {
- row.type = this.sqlSchemaTranslator.translateType(row.type)
- return row
+ private collectionCapabilities(fieldNames: string[]): CollectionCapabilities {
+ return {
+ dataOperations: fieldNames.includes('_id') ? ReadWriteOperations : ReadOnlyOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported,
+ }
}
}
diff --git a/libs/external-db-mssql/src/mysql_utils.spec.ts b/libs/external-db-mssql/src/mssql_utils.spec.ts
similarity index 100%
rename from libs/external-db-mssql/src/mysql_utils.spec.ts
rename to libs/external-db-mssql/src/mssql_utils.spec.ts
diff --git a/libs/external-db-mssql/src/mssql_utils.ts b/libs/external-db-mssql/src/mssql_utils.ts
index 222bbc5ef..76d9e2a87 100644
--- a/libs/external-db-mssql/src/mssql_utils.ts
+++ b/libs/external-db-mssql/src/mssql_utils.ts
@@ -25,8 +25,8 @@ export const escapeTable = (s: string) => {
return escapeId(s)
}
-export const patchFieldName = (s: any) => `x${SqlString.escape(s).substring(1).slice(0, -1)}`
-export const validateLiteral = (s: any) => `@${patchFieldName(s)}`
+export const patchFieldName = (s: any, i?: number) => i ? `x${SqlString.escape(s).substring(1).slice(0, -1)}${i}` : SqlString.escape(s).substring(1).slice(0, -1)
+export const validateLiteral = (s: any, i?: number) => `@${patchFieldName(s, i)}`
export const validateLiteralWithCounter = (s: any, counter: Counter) => validateLiteral(`${s}${counter.valueCounter++}`)
diff --git a/libs/external-db-mssql/src/sql_exception_translator.ts b/libs/external-db-mssql/src/sql_exception_translator.ts
index 7437ac8a1..dd2dc679b 100644
--- a/libs/external-db-mssql/src/sql_exception_translator.ts
+++ b/libs/external-db-mssql/src/sql_exception_translator.ts
@@ -1,19 +1,19 @@
import { errors } from '@wix-velo/velo-external-db-commons'
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist, CollectionAlreadyExists, DbConnectionError, ItemAlreadyExists } = errors
-export const notThrowingTranslateErrorCodes = (err: any) => {
+export const notThrowingTranslateErrorCodes = (err: any, collectionName?: string) => {
if (err.number) {
switch (err.number) {
case 4902:
- return new CollectionDoesNotExists('Collection does not exists')
+ return new CollectionDoesNotExists('Collection does not exists', collectionName)
case 2705:
- return new FieldAlreadyExists('Collection already has a field with the same name')
+ return new FieldAlreadyExists('Collection already has a field with the same name', collectionName, extractColumnName(err.message))
case 2627:
- return new ItemAlreadyExists(`Item already exists: ${err.message}`)
+ return new ItemAlreadyExists(`Item already exists: ${err.message}`, collectionName, extractDuplicateKey(err.message))
case 4924:
- return new FieldDoesNotExist('Collection does not contain a field with this name')
+ return new FieldDoesNotExist('Collection does not contain a field with this name', collectionName)
case 2714:
- return new CollectionAlreadyExists('Collection already exists')
+ return new CollectionAlreadyExists('Collection already exists', collectionName)
default:
return new Error(`default ${err.message}`)
}
@@ -30,6 +30,27 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
}
}
-export const translateErrorCodes = (err: any) => {
- throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName?: string) => {
+ throw notThrowingTranslateErrorCodes(err, collectionName)
}
+
+
+
+const extractDuplicateKey = (errorMessage: string) => {
+ const regex = /The duplicate key value is \((.*)\)/i
+ const match = errorMessage.match(regex)
+ if (match) {
+ return match[1]
+ }
+ return ''
+ }
+
+const extractColumnName = (errorMessage: string) => {
+ const regex = /Column name '(\w+)'/i
+ const match = errorMessage.match(regex)
+ if (match) {
+ return match[1]
+ }
+ return ''
+ }
+
diff --git a/libs/external-db-mssql/src/sql_schema_translator.spec.ts b/libs/external-db-mssql/src/sql_schema_translator.spec.ts
index b672f1f21..0791c9d91 100644
--- a/libs/external-db-mssql/src/sql_schema_translator.spec.ts
+++ b/libs/external-db-mssql/src/sql_schema_translator.spec.ts
@@ -19,15 +19,15 @@ describe('Sql Schema Column Translator', () => {
})
test('decimal float', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(15, 2)`)
})
test('decimal float with precision', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float', precision: '7, 3' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(7,3)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float', precision: '7, 3' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(7,3)`)
})
test('decimal double', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} REAL(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} REAL(15, 2)`)
})
test('decimal double with precision', () => {
@@ -35,7 +35,7 @@ describe('Sql Schema Column Translator', () => {
})
test('decimal generic', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(15, 2)`)
})
test('decimal generic with precision', () => {
diff --git a/libs/external-db-mssql/src/sql_schema_translator.ts b/libs/external-db-mssql/src/sql_schema_translator.ts
index df094af2b..34c457a0a 100644
--- a/libs/external-db-mssql/src/sql_schema_translator.ts
+++ b/libs/external-db-mssql/src/sql_schema_translator.ts
@@ -61,13 +61,11 @@ export default class SchemaColumnTranslator {
case 'number_bigint':
return 'BIGINT'
-
- case 'number_float':
- return `FLOAT${this.parsePrecision(precision)}`
-
+
case 'number_double':
return `REAL${this.parsePrecision(precision)}`
+ case 'number_float':
case 'number_decimal':
return `DECIMAL${this.parsePrecision(precision)}`
@@ -107,7 +105,7 @@ export default class SchemaColumnTranslator {
const parsed = precision.split(',').map((s: string) => s.trim()).map((s: string) => parseInt(s))
return `(${parsed.join(',')})`
} catch (e) {
- return '(5,2)'
+ return '(15, 2)'
}
}
diff --git a/libs/external-db-mssql/src/supported_operations.ts b/libs/external-db-mssql/src/supported_operations.ts
index 20133804e..b1c9e223b 100644
--- a/libs/external-db-mssql/src/supported_operations.ts
+++ b/libs/external-db-mssql/src/supported_operations.ts
@@ -1,5 +1,5 @@
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
import { SchemaOperations } from '@wix-velo/velo-external-db-types'
-const { List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, FindWithSort, Aggregate, BulkDelete,
- Truncate, UpdateImmediately, DeleteImmediately, StartWithCaseSensitive, StartWithCaseInsensitive, Projection, NotOperator, Matches, IncludeOperator, FilterByEveryField } = SchemaOperations
+const notSupportedOperations = [SchemaOperations.QueryNestedFields, SchemaOperations.FindObject, SchemaOperations.NonAtomicBulkInsert]
-export const supportedOperations = [ List, ListHeaders, Create, Drop, AddColumn, RemoveColumn, Describe, FindWithSort, Aggregate, BulkDelete, Truncate, UpdateImmediately, DeleteImmediately, StartWithCaseSensitive, StartWithCaseInsensitive, Projection, NotOperator, Matches, IncludeOperator, FilterByEveryField ]
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-mssql/tests/e2e-testkit/mssql_resources.ts b/libs/external-db-mssql/tests/e2e-testkit/mssql_resources.ts
index 289628231..fa6e51933 100644
--- a/libs/external-db-mssql/tests/e2e-testkit/mssql_resources.ts
+++ b/libs/external-db-mssql/tests/e2e-testkit/mssql_resources.ts
@@ -2,6 +2,8 @@ import * as compose from 'docker-compose'
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/mssql_capabilities'
+
const testEnvConfig = {
host: 'localhost',
user: 'sa',
diff --git a/libs/external-db-mysql/src/mysql_capabilities.ts b/libs/external-db-mysql/src/mysql_capabilities.ts
new file mode 100644
index 000000000..7b22111e9
--- /dev/null
+++ b/libs/external-db-mysql/src/mysql_capabilities.ts
@@ -0,0 +1,21 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+const UnsupportedCapabilities = [DataOperation.insertReferences, DataOperation.removeReferences, DataOperation.queryReferenced]
+
+
+export const ReadWriteOperations = Object.values(DataOperation).filter(op => !UnsupportedCapabilities.includes(op))
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-mysql/src/mysql_data_provider.ts b/libs/external-db-mysql/src/mysql_data_provider.ts
index 603f950f5..23e384536 100644
--- a/libs/external-db-mysql/src/mysql_data_provider.ts
+++ b/libs/external-db-mysql/src/mysql_data_provider.ts
@@ -4,7 +4,7 @@ import { promisify } from 'util'
import { asParamArrays, updateFieldsFor } from '@wix-velo/velo-external-db-commons'
import { translateErrorCodes } from './sql_exception_translator'
import { wildCardWith } from './mysql_utils'
-import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types'
+import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort } from '@wix-velo/velo-external-db-types'
import { IMySqlFilterParser } from './sql_filter_transformer'
import { MySqlQuery } from './types'
@@ -26,7 +26,7 @@ export default class DataProvider implements IDataProvider {
const sql = `SELECT ${projectionExpr} FROM ${escapeTable(collectionName)} ${filterExpr} ${sortExpr} LIMIT ?, ?`
const resultset = await this.query(sql, [...parameters, skip, limit])
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
return resultset
}
@@ -34,17 +34,18 @@ export default class DataProvider implements IDataProvider {
const { filterExpr, parameters } = this.filterParser.transform(filter)
const sql = `SELECT COUNT(*) AS num FROM ${escapeTable(collectionName)} ${filterExpr}`
const resultset = await this.query(sql, parameters)
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
return resultset[0]['num']
}
- async insert(collectionName: string, items: Item[], fields: any[]): Promise<number> {
+ async insert(collectionName: string, items: Item[], fields: any[], upsert?: boolean): Promise<number> {
const escapedFieldsNames = fields.map( (f: { field: any }) => escapeId(f.field)).join(', ')
- const sql = `INSERT INTO ${escapeTable(collectionName)} (${escapedFieldsNames}) VALUES ?`
+ const op = upsert ? 'REPLACE' : 'INSERT'
+ const sql = `${op} INTO ${escapeTable(collectionName)} (${escapedFieldsNames}) VALUES ?`
const data = items.map((item: Item) => asParamArrays( patchItem(item) ) )
const resultset = await this.query(sql, [data])
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
return resultset.affectedRows
}
@@ -57,7 +58,7 @@ export default class DataProvider implements IDataProvider {
// @ts-ignore
const resultset = await this.query(queries, [].concat(...updatables))
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
return Array.isArray(resultset) ? resultset.reduce((s, r) => s + r.changedRows, 0) : resultset.changedRows
}
@@ -65,21 +66,22 @@ export default class DataProvider implements IDataProvider {
async delete(collectionName: string, itemIds: string[]): Promise<number> {
const sql = `DELETE FROM ${escapeTable(collectionName)} WHERE _id IN (${wildCardWith(itemIds.length, '?')})`
const rs = await this.query(sql, itemIds)
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
return rs.affectedRows
}
async truncate(collectionName: string): Promise<void> {
- await this.query(`TRUNCATE ${escapeTable(collectionName)}`).catch( translateErrorCodes )
+ await this.query(`TRUNCATE ${escapeTable(collectionName)}`).catch( err => translateErrorCodes(err, collectionName) )
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise<Item[]> {
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise<Item[]> {
const { filterExpr: whereFilterExpr, parameters: whereParameters } = this.filterParser.transform(filter)
const { fieldsStatement, groupByColumns, havingFilter, parameters } = this.filterParser.parseAggregation(aggregation)
+ const { sortExpr } = this.filterParser.orderBy(sort)
- const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter}`
- const resultset = await this.query(sql, [...whereParameters, ...parameters])
- .catch( translateErrorCodes )
+ const sql = `SELECT ${fieldsStatement} FROM ${escapeTable(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeId ).join(', ')} ${havingFilter} ${sortExpr} LIMIT ?, ?`
+ const resultset = await this.query(sql, [...whereParameters, ...parameters, skip, limit])
+ .catch( err => translateErrorCodes(err, collectionName) )
return resultset
}
}
diff --git a/libs/external-db-mysql/src/mysql_operations.ts b/libs/external-db-mysql/src/mysql_operations.ts
index ffb378fa0..34cc19064 100644
--- a/libs/external-db-mysql/src/mysql_operations.ts
+++ b/libs/external-db-mysql/src/mysql_operations.ts
@@ -13,6 +13,6 @@ export default class DatabaseOperations implements IDatabaseOperations {
async validateConnection() {
return await this.query('SELECT 1').then(() => { return { valid: true } })
- .catch((e: any) => { return { valid: false, error: notThrowingTranslateErrorCodes(e) } })
+ .catch((e: any) => { return { valid: false, error: notThrowingTranslateErrorCodes(e, '') } })
}
}
diff --git a/libs/external-db-mysql/src/mysql_schema_provider.ts b/libs/external-db-mysql/src/mysql_schema_provider.ts
index 48438db9c..e867749ef 100644
--- a/libs/external-db-mysql/src/mysql_schema_provider.ts
+++ b/libs/external-db-mysql/src/mysql_schema_provider.ts
@@ -1,11 +1,12 @@
+import { Pool as MySqlPool } from 'mysql'
import { promisify } from 'util'
+import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations, EmptyCapabilities } from '@wix-velo/velo-external-db-commons'
+import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table, CollectionCapabilities, Encryption } from '@wix-velo/velo-external-db-types'
import { translateErrorCodes } from './sql_exception_translator'
import SchemaColumnTranslator, { IMySqlSchemaColumnTranslator } from './sql_schema_translator'
import { escapeId, escapeTable } from './mysql_utils'
-import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
-import { Pool as MySqlPool } from 'mysql'
import { MySqlQuery } from './types'
-import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
+import { CollectionOperations, FieldTypes, ReadOnlyOperations, ReadWriteOperations, ColumnsCapabilities } from './mysql_capabilities'
export default class SchemaProvider implements ISchemaProvider {
pool: MySqlPool
@@ -22,11 +23,13 @@ export default class SchemaProvider implements ISchemaProvider {
async list(): Promise<Table[]> {
const currentDb = this.pool.config.connectionConfig.database
const data = await this.query('SELECT TABLE_NAME as table_name, COLUMN_NAME as field, DATA_TYPE as type FROM information_schema.columns WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME, ORDINAL_POSITION', currentDb)
- const tables: {[x:string]: {table_name: string, field: string, type: string}[]} = parseTableData( data )
+ const tables: {[x:string]: { field: string, type: string}[]} = parseTableData( data )
+
return Object.entries(tables)
.map(([collectionName, rs]) => ({
id: collectionName,
- fields: rs.map( this.translateDbTypes.bind(this) )
+ fields: rs.map(this.appendAdditionalRowDetails.bind(this)),
+ capabilities: this.collectionCapabilities(rs.map(r => r.field))
} ))
}
@@ -47,35 +50,65 @@ export default class SchemaProvider implements ISchemaProvider {
await this.query(`CREATE TABLE IF NOT EXISTS ${escapeTable(collectionName)} (${dbColumnsSql}, PRIMARY KEY (${primaryKeySql}))`,
[...(columns || []).map((c: { name: any }) => c.name)])
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
}
async drop(collectionName: string): Promise<void> {
await this.query(`DROP TABLE IF EXISTS ${escapeTable(collectionName)}`)
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
}
async addColumn(collectionName: string, column: InputField): Promise<void> {
await validateSystemFields(column.name)
await this.query(`ALTER TABLE ${escapeTable(collectionName)} ADD ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
+ }
+
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
+ await validateSystemFields(column.name)
+ await this.query(`ALTER TABLE ${escapeTable(collectionName)} MODIFY ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
+ .catch( err => translateErrorCodes(err, collectionName) )
}
async removeColumn(collectionName: string, columnName: string): Promise<void> {
await validateSystemFields(columnName)
return await this.query(`ALTER TABLE ${escapeTable(collectionName)} DROP COLUMN ${escapeId(columnName)}`)
- .catch( translateErrorCodes )
+ .catch( err => translateErrorCodes(err, collectionName) )
+ }
+
+ async describeCollection(collectionName: string): Promise<Table> {
+ interface describeTableResponse {
+ Field: string,
+ Type: string,
+ }
+
+ const res: describeTableResponse[] = await this.query(`DESCRIBE ${escapeTable(collectionName)}`)
+ .catch( err => translateErrorCodes(err, collectionName) )
+ const fields = res.map(r => ({ field: r.Field, type: r.Type })).map(this.appendAdditionalRowDetails.bind(this))
+ return {
+ id: collectionName,
+ fields: fields as ResponseField[],
+ capabilities: this.collectionCapabilities(res.map(f => f.Field))
+ }
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
- const res = await this.query(`DESCRIBE ${escapeTable(collectionName)}`)
- .catch( translateErrorCodes )
- return res.map((r: { Field: string; Type: string }) => ({ field: r.Field, type: r.Type }))
- .map( this.translateDbTypes.bind(this) )
+ private appendAdditionalRowDetails(row: {field: string, type: string}) : ResponseField {
+ const type = this.sqlSchemaTranslator.translateType(row.type) as keyof typeof ColumnsCapabilities
+ return {
+ field: row.field,
+ type,
+ capabilities: ColumnsCapabilities[type] ?? EmptyCapabilities
+ }
}
- translateDbTypes(row: ResponseField): ResponseField {
- row.type = this.sqlSchemaTranslator.translateType(row.type)
- return row
+ private collectionCapabilities(fieldNames: string[]): CollectionCapabilities {
+ return {
+ dataOperations: fieldNames.includes('_id') ? ReadWriteOperations : ReadOnlyOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
+ }
}
}
diff --git a/libs/external-db-mysql/src/mysql_utils.spec.ts b/libs/external-db-mysql/src/mysql_utils.spec.ts
index a863d6cee..60b49dc55 100644
--- a/libs/external-db-mysql/src/mysql_utils.spec.ts
+++ b/libs/external-db-mysql/src/mysql_utils.spec.ts
@@ -1,6 +1,7 @@
-import { escapeTable, escapeId } from './mysql_utils'
+import { escapeTable, escapeId, } from './mysql_utils'
import { errors } from '@wix-velo/velo-external-db-commons'
const { InvalidQuery } = errors
+// const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
describe('Mysql Utils', () => {
test('escape collection id will not allow dots', () => {
@@ -10,4 +11,5 @@ describe('Mysql Utils', () => {
test('escape collection id', () => {
expect( escapeTable('some_table_name') ).toEqual(escapeId('some_table_name'))
})
+
})
diff --git a/libs/external-db-mysql/src/sql_exception_translator.ts b/libs/external-db-mysql/src/sql_exception_translator.ts
index 7bd1b63f4..fa50b4eb4 100644
--- a/libs/external-db-mysql/src/sql_exception_translator.ts
+++ b/libs/external-db-mysql/src/sql_exception_translator.ts
@@ -1,14 +1,28 @@
import { errors } from '@wix-velo/velo-external-db-commons'
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist, DbConnectionError, ItemAlreadyExists, UnrecognizedError } = errors
-export const notThrowingTranslateErrorCodes = (err: any) => {
+const extractDuplicatedColumnName = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Duplicate column name '(.*)'/)
+const extractDuplicatedItem = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Duplicate entry '(.*)' for key .*/)
+const extractUnknownColumn = (error: any) => extractValueFromErrorMessage(error.sqlMessage, /Unknown column '(.*)' in 'field list'/)
+
+const extractValueFromErrorMessage = (msg: string, regex: RegExp) => {
+ try {
+ const match = msg.match(regex)
+ const value = (match && match[1])
+ return value || ''
+ } catch(e) {
+ return ''
+ }
+}
+
+export const notThrowingTranslateErrorCodes = (err: any, collectionName: string) => {
switch (err.code) {
case 'ER_CANT_DROP_FIELD_OR_KEY':
- return new FieldDoesNotExist('Collection does not contain a field with this name')
+ return new FieldDoesNotExist('Collection does not contain a field with this name', collectionName, extractUnknownColumn(err))
case 'ER_DUP_FIELDNAME':
- return new FieldAlreadyExists('Collection already has a field with the same name')
+ return new FieldAlreadyExists('Collection already has a field with the same name', collectionName, extractDuplicatedColumnName(err))
case 'ER_NO_SUCH_TABLE':
- return new CollectionDoesNotExists('Collection does not exists')
+ return new CollectionDoesNotExists('Collection does not exists', collectionName)
case 'ER_DBACCESS_DENIED_ERROR':
case 'ER_BAD_DB_ERROR':
return new DbConnectionError(`Database does not exists or you don't have access to it, sql message: ${err.sqlMessage}`)
@@ -18,13 +32,13 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
case 'ENOTFOUND':
return new DbConnectionError(`Access to database denied - host is unavailable, sql message: ${err.sqlMessage} `)
case 'ER_DUP_ENTRY':
- return new ItemAlreadyExists(`Item already exists: ${err.sqlMessage}`)
+ return new ItemAlreadyExists(`Item already exists: ${err.sqlMessage}`, collectionName, extractDuplicatedItem(err))
default :
console.error(err)
return new UnrecognizedError(`${err.code} ${err.sqlMessage}`)
}
}
-export const translateErrorCodes = (err: any) => {
- throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName: string) => {
+ throw notThrowingTranslateErrorCodes(err, collectionName)
}
diff --git a/libs/external-db-mysql/src/sql_filter_transformer.spec.ts b/libs/external-db-mysql/src/sql_filter_transformer.spec.ts
index 46c8ba464..976fe170a 100644
--- a/libs/external-db-mysql/src/sql_filter_transformer.spec.ts
+++ b/libs/external-db-mysql/src/sql_filter_transformer.spec.ts
@@ -270,7 +270,24 @@ describe('Sql Parser', () => {
}])
})
})
+
+ describe('handle queries on nested fields', () => {
+ test('correctly transform nested field query', () => {
+ const operator = ctx.filterWithoutInclude.operator
+ const filter = {
+ operator,
+ fieldName: `${ctx.fieldName}.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}`,
+ value: ctx.filterWithoutInclude.value
+ }
+
+ expect( env.filterParser.parseFilter(filter) ).toEqual([{
+ filterExpr: `${escapeId(ctx.fieldName)} ->> '$.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}' ${env.filterParser.adapterOperatorToMySqlOperator(operator, ctx.filterWithoutInclude.value)} ?`,
+ parameters: [ctx.filterWithoutInclude.value].flat()
+ }])
+ })
+ })
})
+
describe('handle multi field operator', () => {
each([
and, or
@@ -418,7 +435,10 @@ describe('Sql Parser', () => {
filter: Uninitialized,
anotherFilter: Uninitialized,
anotherValue: Uninitialized,
- moreValue: Uninitialized
+ moreValue: Uninitialized,
+ nestedFieldName: Uninitialized,
+ anotherNestedFieldName: Uninitialized,
+ filterWithoutInclude: Uninitialized,
}
const env = {
@@ -429,6 +449,8 @@ describe('Sql Parser', () => {
ctx.fieldName = chance.word()
ctx.anotherFieldName = chance.word()
ctx.moreFieldName = chance.word()
+ ctx.nestedFieldName = chance.word()
+ ctx.anotherNestedFieldName = chance.word()
ctx.fieldValue = chance.word()
ctx.anotherValue = chance.word()
@@ -437,6 +459,7 @@ describe('Sql Parser', () => {
ctx.filter = gen.randomWrappedFilter()
ctx.anotherFilter = gen.randomWrappedFilter()
+ ctx.filterWithoutInclude = gen.randomDomainFilterWithoutInclude()
})
beforeAll(function() {
diff --git a/libs/external-db-mysql/src/sql_filter_transformer.ts b/libs/external-db-mysql/src/sql_filter_transformer.ts
index f10abae21..3c6e61564 100644
--- a/libs/external-db-mysql/src/sql_filter_transformer.ts
+++ b/libs/external-db-mysql/src/sql_filter_transformer.ts
@@ -53,6 +53,15 @@ export default class FilterParser implements IMySqlFilterParser {
}]
}
+ if (this.isNestedField(fieldName)) {
+ const [nestedFieldName, ...nestedFieldPath] = fieldName.split('.')
+
+ return [{
+ filterExpr: `${escapeId(nestedFieldName)} ->> '$.${nestedFieldPath.join('.')}' ${this.adapterOperatorToMySqlOperator(operator, value)} ${this.valueForOperator(value, operator)}`.trim(),
+ parameters: !isNull(value) ? [].concat( this.patchTrueFalseValue(value) ) : []
+ }]
+ }
+
if (this.isSingleFieldOperator(operator)) {
return [{
filterExpr: `${escapeId(fieldName)} ${this.adapterOperatorToMySqlOperator(operator, value)} ${this.valueForOperator(value, operator)}`.trim(),
@@ -94,6 +103,10 @@ export default class FilterParser implements IMySqlFilterParser {
return [string_contains, string_begins, string_ends].includes(operator)
}
+ isNestedField(fieldName: string) {
+ return fieldName.includes('.')
+ }
+
valueForOperator(value: string | any[], operator: any) {
if (operator === include) {
if (isNull(value) || value.length === 0) {
diff --git a/libs/external-db-mysql/src/sql_schema_translator.spec.ts b/libs/external-db-mysql/src/sql_schema_translator.spec.ts
index 9182dcb37..20ea2e16b 100644
--- a/libs/external-db-mysql/src/sql_schema_translator.spec.ts
+++ b/libs/external-db-mysql/src/sql_schema_translator.spec.ts
@@ -19,7 +19,7 @@ describe('Sql Schema Column Translator', () => {
})
test('decimal float', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'float' }) ).toEqual(`${escapeId(ctx.fieldName)} FLOAT(15,2)`)
})
test('decimal float with precision', () => {
@@ -27,7 +27,7 @@ describe('Sql Schema Column Translator', () => {
})
test('decimal double', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} DOUBLE(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'double' }) ).toEqual(`${escapeId(ctx.fieldName)} DOUBLE(15,2)`)
})
test('decimal double with precision', () => {
@@ -35,7 +35,7 @@ describe('Sql Schema Column Translator', () => {
})
test('decimal generic', () => {
- expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(5,2)`)
+ expect( env.schemaTranslator.columnToDbColumnSql({ name: ctx.fieldName, type: 'number', subtype: 'decimal' }) ).toEqual(`${escapeId(ctx.fieldName)} DECIMAL(15,2)`)
})
test('decimal generic with precision', () => {
diff --git a/libs/external-db-mysql/src/sql_schema_translator.ts b/libs/external-db-mysql/src/sql_schema_translator.ts
index 8fe629aff..f9d1dbca8 100644
--- a/libs/external-db-mysql/src/sql_schema_translator.ts
+++ b/libs/external-db-mysql/src/sql_schema_translator.ts
@@ -125,7 +125,7 @@ export default class SchemaColumnTranslator implements IMySqlSchemaColumnTransla
const parsed = precision.split(',').map((s: string) => s.trim()).map((s: string) => parseInt(s))
return `(${parsed.join(',')})`
} catch (e) {
- return '(5,2)'
+ return '(15,2)'
}
}
diff --git a/libs/external-db-mysql/src/supported_operations.ts b/libs/external-db-mysql/src/supported_operations.ts
index 642b18976..a2dc49fa4 100644
--- a/libs/external-db-mysql/src/supported_operations.ts
+++ b/libs/external-db-mysql/src/supported_operations.ts
@@ -1 +1,5 @@
-export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons'
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+const notSupportedOperations = [SchemaOperations.NonAtomicBulkInsert]
+
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts b/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts
index f22203680..4bc7d4b2e 100644
--- a/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts
+++ b/libs/external-db-mysql/tests/e2e-testkit/mysql_resources.ts
@@ -3,6 +3,7 @@ import { waitUntil } from 'async-wait-until'
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/mysql_capabilities'
export const connection = () => {
const { connection, schemaProvider, cleanup } = init({ host: 'localhost', user: 'test-user', password: 'password', db: 'test-db' }, { connectionLimit: 1, queueLimit: 0 })
diff --git a/libs/external-db-postgres/src/postgres_capabilities.ts b/libs/external-db-postgres/src/postgres_capabilities.ts
new file mode 100644
index 000000000..04a414516
--- /dev/null
+++ b/libs/external-db-postgres/src/postgres_capabilities.ts
@@ -0,0 +1,19 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+
+export const ReadWriteOperations = Object.values(DataOperation)
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-postgres/src/postgres_data_provider.ts b/libs/external-db-postgres/src/postgres_data_provider.ts
index 5a62a0e40..64a78bac4 100644
--- a/libs/external-db-postgres/src/postgres_data_provider.ts
+++ b/libs/external-db-postgres/src/postgres_data_provider.ts
@@ -1,5 +1,5 @@
import { Pool } from 'pg'
-import { escapeIdentifier, prepareStatementVariables } from './postgres_utils'
+import { escapeIdentifier, prepareStatementVariables, prepareStatementVariablesForBulkInsert } from './postgres_utils'
import { asParamArrays, patchDateTime, updateFieldsFor } from '@wix-velo/velo-external-db-commons'
import { translateErrorCodes } from './sql_exception_translator'
import { IDataProvider, AdapterFilter as Filter, Sort, Item, AdapterAggregation as Aggregation, ResponseField } from '@wix-velo/velo-external-db-types'
@@ -18,7 +18,8 @@ export default class DataProvider implements IDataProvider {
const { filterExpr, parameters, offset } = this.filterParser.transform(filter)
const { sortExpr } = this.filterParser.orderBy(sort)
const projectionExpr = this.filterParser.selectFieldsFor(projection)
- const resultSet = await this.pool.query(`SELECT ${projectionExpr} FROM ${escapeIdentifier(collectionName)} ${filterExpr} ${sortExpr} OFFSET $${offset} LIMIT $${offset + 1}`, [...parameters, skip, limit])
+ const sql = `SELECT ${projectionExpr} FROM ${escapeIdentifier(collectionName)} ${filterExpr} ${sortExpr} OFFSET $${offset} LIMIT $${offset + 1}`
+ const resultSet = await this.pool.query(sql, [...parameters, skip, limit])
.catch( translateErrorCodes )
return resultSet.rows
}
@@ -30,16 +31,15 @@ export default class DataProvider implements IDataProvider {
return parseInt(resultSet.rows[0]['num'], 10)
}
- async insert(collectionName: string, items: Item[], fields: ResponseField[]) {
+ async insert(collectionName: string, items: Item[], fields: ResponseField[], upsert?: boolean) {
+ const itemsAsParams = items.map((item: Item) => asParamArrays( patchDateTime(item) ))
const escapedFieldsNames = fields.map( (f: { field: string }) => escapeIdentifier(f.field)).join(', ')
- const res = await Promise.all(
- items.map(async(item: { [x: string]: any }) => {
- const data = asParamArrays( patchDateTime(item) )
- const res = await this.pool.query(`INSERT INTO ${escapeIdentifier(collectionName)} (${escapedFieldsNames}) VALUES (${prepareStatementVariables(fields.length)})`, data)
- .catch( translateErrorCodes )
- return res.rowCount
- } ) )
- return res.reduce((sum, i) => i + sum, 0)
+ const upsertAddon = upsert ? ` ON CONFLICT (_id) DO UPDATE SET ${fields.map(f => `${escapeIdentifier(f.field)} = EXCLUDED.${escapeIdentifier(f.field)}`).join(', ')}` : ''
+ const query = `INSERT INTO ${escapeIdentifier(collectionName)} (${escapedFieldsNames}) VALUES ${prepareStatementVariablesForBulkInsert(items.length, fields.length)}${upsertAddon}`
+
+ await this.pool.query(query, itemsAsParams.flat()).catch( translateErrorCodes )
+
+ return items.length
}
async update(collectionName: string, items: Item[]) {
@@ -66,12 +66,14 @@ export default class DataProvider implements IDataProvider {
await this.pool.query(`TRUNCATE ${escapeIdentifier(collectionName)}`).catch( translateErrorCodes )
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise<Item[]> {
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise<Item[]> {
+
const { filterExpr: whereFilterExpr, parameters: whereParameters, offset } = this.filterParser.transform(filter)
- const { fieldsStatement, groupByColumns, havingFilter: filterExpr, parameters: havingParameters } = this.filterParser.parseAggregation(aggregation, offset)
+ const { fieldsStatement, groupByColumns, havingFilter: filterExpr, parameters: havingParameters, offset: offsetAfterAggregation } = this.filterParser.parseAggregation(aggregation, offset)
+ const { sortExpr } = this.filterParser.orderBy(sort)
- const sql = `SELECT ${fieldsStatement} FROM ${escapeIdentifier(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeIdentifier ).join(', ')} ${filterExpr}`
- const rs = await this.pool.query(sql, [...whereParameters, ...havingParameters])
+ const sql = `SELECT ${fieldsStatement} FROM ${escapeIdentifier(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map( escapeIdentifier ).join(', ')} ${filterExpr} ${sortExpr} OFFSET $${offsetAfterAggregation} LIMIT $${offsetAfterAggregation+1}`
+ const rs = await this.pool.query(sql, [...whereParameters, ...havingParameters, skip, limit])
.catch( translateErrorCodes )
return rs.rows
}
diff --git a/libs/external-db-postgres/src/postgres_schema_provider.ts b/libs/external-db-postgres/src/postgres_schema_provider.ts
index 0184b43c7..1fbfea418 100644
--- a/libs/external-db-postgres/src/postgres_schema_provider.ts
+++ b/libs/external-db-postgres/src/postgres_schema_provider.ts
@@ -1,9 +1,25 @@
import { Pool } from 'pg'
-import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations, errors } from '@wix-velo/velo-external-db-commons'
+import {
+ SystemFields,
+ validateSystemFields,
+ parseTableData,
+ AllSchemaOperations,
+ errors,
+ EmptyCapabilities
+} from '@wix-velo/velo-external-db-commons'
import { translateErrorCodes } from './sql_exception_translator'
import SchemaColumnTranslator from './sql_schema_translator'
import { escapeIdentifier } from './postgres_utils'
-import { InputField, ISchemaProvider, ResponseField, Table } from '@wix-velo/velo-external-db-types'
+import {
+ CollectionCapabilities,
+ Encryption,
+ InputField,
+ ISchemaProvider,
+ ResponseField,
+ Table
+} from '@wix-velo/velo-external-db-types'
+import { CollectionOperations, FieldTypes, ReadOnlyOperations, ReadWriteOperations, ColumnsCapabilities } from './postgres_capabilities'
+
const { CollectionDoesNotExists } = errors
export default class SchemaProvider implements ISchemaProvider {
@@ -21,7 +37,8 @@ export default class SchemaProvider implements ISchemaProvider {
return Object.entries(tables)
.map(([collectionName, rs]) => ({
id: collectionName,
- fields: rs.map( this.translateDbTypes.bind(this) )
+ fields: rs.map( this.appendAdditionalRowDetails.bind(this) ),
+ capabilities: this.collectionCapabilities(rs.map(r => r.field))
}))
}
@@ -57,6 +74,13 @@ export default class SchemaProvider implements ISchemaProvider {
.catch( translateErrorCodes )
}
+ async changeColumnType(collectionName: string, column: InputField): Promise<void> {
+ await validateSystemFields(column.name)
+ const query = `ALTER TABLE ${escapeIdentifier(collectionName)} ALTER COLUMN ${escapeIdentifier(column.name)} TYPE ${this.sqlSchemaTranslator.dbTypeFor(column)} USING (${escapeIdentifier(column.name)}::${this.sqlSchemaTranslator.dbTypeFor(column)})`
+ await this.pool.query(query)
+ .catch( err => translateErrorCodes(err) )
+ }
+
async removeColumn(collectionName: string, columnName: string) {
await validateSystemFields(columnName)
@@ -65,13 +89,39 @@ export default class SchemaProvider implements ISchemaProvider {
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
+ async describeCollection(collectionName: string): Promise<Table> {
const res = await this.pool.query('SELECT table_name, column_name AS field, data_type, udt_name AS type, character_maximum_length FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 ORDER BY table_name', ['public', collectionName])
.catch( translateErrorCodes )
if (res.rows.length === 0) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
+ }
+
+ const fields = res.rows.map(r => ({ field: r.field, type: r.type })).map(r => this.appendAdditionalRowDetails(r))
+ return {
+ id: collectionName,
+ fields: fields,
+ capabilities: this.collectionCapabilities(res.rows.map(r => r.field))
+ }
+ }
+
+ private collectionCapabilities(fieldNames: string[]): CollectionCapabilities {
+ return {
+ dataOperations: fieldNames.includes('_id') ? ReadWriteOperations : ReadOnlyOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
+ }
+ }
+
+ private appendAdditionalRowDetails(row: ResponseField) {
+ const type = this.sqlSchemaTranslator.translateType(row.type) as keyof typeof ColumnsCapabilities
+ return {
+ ...row,
+ type: this.sqlSchemaTranslator.translateType(row.type),
+ capabilities: ColumnsCapabilities[type] ?? EmptyCapabilities
}
- return res.rows.map( this.translateDbTypes.bind(this) )
}
translateDbTypes(row: ResponseField) {
diff --git a/libs/external-db-postgres/src/postgres_utils.spec.ts b/libs/external-db-postgres/src/postgres_utils.spec.ts
new file mode 100644
index 000000000..19df3a5a2
--- /dev/null
+++ b/libs/external-db-postgres/src/postgres_utils.spec.ts
@@ -0,0 +1,27 @@
+
+
+import { prepareStatementVariablesForBulkInsert } from './postgres_utils'
+
+describe('Postgres utils', () => {
+ describe('Prepare statement variables for BulkInsert', () => {
+ test('creates bulk insert statement for 2,2', () => {
+ const expected = '($1,$2),($3,$4)'
+ const result = prepareStatementVariablesForBulkInsert(2, 2)
+
+ expect(result).toEqual(expected)
+ })
+ test('creates bulk insert statement for 10,1', () => {
+ const expected = '($1),($2),($3),($4),($5),($6),($7),($8),($9),($10)'
+ const result = prepareStatementVariablesForBulkInsert(10, 1)
+
+ expect(result).toEqual(expected)
+ })
+ test('creates bulk insert statement for 1,10', () => {
+ const expected = '($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)'
+ const result = prepareStatementVariablesForBulkInsert(1, 10)
+
+ expect(result).toEqual(expected)
+ })
+ })
+
+})
diff --git a/libs/external-db-postgres/src/postgres_utils.ts b/libs/external-db-postgres/src/postgres_utils.ts
index 340fd3684..a13d96cec 100644
--- a/libs/external-db-postgres/src/postgres_utils.ts
+++ b/libs/external-db-postgres/src/postgres_utils.ts
@@ -1,5 +1,6 @@
// Ported from PostgreSQL 9.2.4 source code in src/interfaces/libpq/fe-exec.c
+
export const escapeIdentifier = (str: string) => str === '*' ? '*' : `"${(str || '').replace(/"/g, '""')}"`
export const prepareStatementVariables = (n: number) => {
@@ -7,3 +8,15 @@ export const prepareStatementVariables = (n: number) => {
.map(i => `$${i}`)
.join(', ')
}
+
+export const prepareStatementVariablesForBulkInsert = (rowsCount: number, columnsCount: number) => {
+ const segments = []
+ for(let row=0; row < rowsCount; row++) {
+ const segment = []
+ for(let col=0; col < columnsCount; col++) {
+ segment.push(`$${col+1 + row * columnsCount}`)
+ }
+ segments.push('(' + segment.join(',') + ')')
+ }
+ return segments.join(',')
+}
diff --git a/libs/external-db-postgres/src/sql_exception_translator.ts b/libs/external-db-postgres/src/sql_exception_translator.ts
index c9dfb0848..e7776246c 100644
--- a/libs/external-db-postgres/src/sql_exception_translator.ts
+++ b/libs/external-db-postgres/src/sql_exception_translator.ts
@@ -2,6 +2,18 @@ import { errors } from '@wix-velo/velo-external-db-commons'
import { IBaseHttpError } from '@wix-velo/velo-external-db-types'
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist, DbConnectionError, ItemAlreadyExists, UnrecognizedError } = errors
+const extractDuplicatedItem = (error: any) => extractValueFromErrorMessage(error.detail, /Key \(_id\)=\((.*)\) already exists\./)
+
+const extractValueFromErrorMessage = (msg: string, regex: RegExp) => {
+ try {
+ const match = msg.match(regex)
+ const value = (match && match[1])
+ return value || ''
+ } catch(e) {
+ return ''
+ }
+}
+
export const notThrowingTranslateErrorCodes = (err: any): IBaseHttpError => {
switch (err.code) {
case '42703':
@@ -9,9 +21,9 @@ export const notThrowingTranslateErrorCodes = (err: any): IBaseHttpError => {
case '42701':
return new FieldAlreadyExists('Collection already has a field with the same name')
case '23505':
- return new ItemAlreadyExists(`Item already exists: ${err.message}`)
+ return new ItemAlreadyExists(`Item already exists: ${err.message}`, err.table, extractDuplicatedItem(err))
case '42P01':
- return new CollectionDoesNotExists('Collection does not exists')
+ return new CollectionDoesNotExists('Collection does not exists', err.table)
case '28P01':
return new DbConnectionError(`Access to database denied - probably wrong credentials,sql message: ${err.message}`)
case '3D000':
diff --git a/libs/external-db-postgres/src/sql_filter_transformer.spec.ts b/libs/external-db-postgres/src/sql_filter_transformer.spec.ts
index fb9e8c9a9..2a6486d26 100644
--- a/libs/external-db-postgres/src/sql_filter_transformer.spec.ts
+++ b/libs/external-db-postgres/src/sql_filter_transformer.spec.ts
@@ -280,6 +280,26 @@ describe('Sql Parser', () => {
})
})
+ describe('handle queries on nested fields', () => {
+ test('correctly transform nested field query', () => {
+ const operator = ctx.filterWithoutInclude.operator
+ const filter = {
+ operator,
+ fieldName: `${ctx.fieldName}.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}`,
+ value: ctx.filterWithoutInclude.value
+ }
+
+ const parsedFilter = env.filterParser.parseFilter(filter, ctx.offset)
+
+ expect( parsedFilter ).toEqual([{
+ filterExpr: `${escapeIdentifier(ctx.fieldName)} ->> '${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}' ${env.filterParser.adapterOperatorToMySqlOperator(operator, ctx.filterWithoutInclude.value)} $${ctx.offset}`,
+ parameters: [ctx.filterWithoutInclude.value].flat(),
+ filterColumns: [],
+ offset: ctx.offset + 1,
+ }])
+ })
+ })
+
describe('handle multi field operator', () => {
each([
and, or
@@ -342,7 +362,8 @@ describe('Sql Parser', () => {
fieldsStatement: escapeIdentifier(ctx.fieldName),
groupByColumns: [ctx.fieldName],
havingFilter: '',
- parameters: []
+ parameters: [],
+ offset: 1,
})
})
@@ -359,6 +380,7 @@ describe('Sql Parser', () => {
groupByColumns: [ctx.fieldName, ctx.anotherFieldName],
havingFilter: '',
parameters: [],
+ offset: 1,
})
})
@@ -380,6 +402,7 @@ describe('Sql Parser', () => {
groupByColumns: [ctx.fieldName],
havingFilter: `HAVING AVG(${escapeIdentifier(ctx.anotherFieldName)}) > $${ctx.offset}`,
parameters: [ctx.fieldValue],
+ offset: ctx.offset + 1,
})
})
@@ -401,6 +424,7 @@ describe('Sql Parser', () => {
groupByColumns: [ctx.fieldName],
havingFilter: '',
parameters: [],
+ offset: 1,
})
})
@@ -417,6 +441,7 @@ describe('Sql Parser', () => {
groupByColumns: [ctx.fieldName],
havingFilter: '',
parameters: [],
+ offset: 1,
})
})
})
@@ -427,6 +452,8 @@ describe('Sql Parser', () => {
const ctx = {
fieldName: Uninitialized,
+ nestedFieldName: Uninitialized,
+ anotherNestedFieldName: Uninitialized,
fieldValue: Uninitialized,
anotherValue: Uninitialized,
moreValue: Uninitialized,
@@ -446,6 +473,8 @@ describe('Sql Parser', () => {
beforeEach(() => {
ctx.fieldName = chance.word()
+ ctx.nestedFieldName = chance.word()
+ ctx.anotherNestedFieldName = chance.word()
ctx.anotherFieldName = chance.word()
ctx.moreFieldName = chance.word()
@@ -457,6 +486,7 @@ describe('Sql Parser', () => {
ctx.filter = gen.randomWrappedFilter()
ctx.anotherFilter = gen.randomWrappedFilter()
+ ctx.filterWithoutInclude = gen.randomDomainFilterWithoutInclude()
ctx.offset = chance.natural({ min: 2, max: 20 })
})
diff --git a/libs/external-db-postgres/src/sql_filter_transformer.ts b/libs/external-db-postgres/src/sql_filter_transformer.ts
index a9663bbaa..36a42319f 100644
--- a/libs/external-db-postgres/src/sql_filter_transformer.ts
+++ b/libs/external-db-postgres/src/sql_filter_transformer.ts
@@ -55,13 +55,15 @@ export default class FilterParser {
const havingFilter = this.parseFilter(aggregation.postFilter, offset, aliasToFunction)
- const { filterExpr, parameters } = this.extractFilterExprAndParams(havingFilter)
+ const { filterExpr, parameters, offset: offsetAfterAggregation } = this.extractFilterExprAndParams(havingFilter, offset)
+
return {
fieldsStatement: filterColumnsStr.join(', '),
groupByColumns,
havingFilter: filterExpr,
parameters: parameters,
+ offset: offsetAfterAggregation
}
}
@@ -77,10 +79,10 @@ export default class FilterParser {
return { filterColumnsStr, aliasToFunction }
}
- extractFilterExprAndParams(havingFilter: any[]) {
- return havingFilter.map(({ filterExpr, parameters }) => ({ filterExpr: filterExpr !== '' ? `HAVING ${filterExpr}` : '',
- parameters: parameters }))
- .concat(EmptyFilter)[0]
+ extractFilterExprAndParams(havingFilter: any[], offset: number) {
+ return havingFilter.map(({ filterExpr, parameters, offset }) => ({ filterExpr: filterExpr !== '' ? `HAVING ${filterExpr}` : '',
+ parameters: parameters, offset }))
+ .concat({ ...EmptyFilter, offset: offset ?? 1 })[0]
}
parseFilter(filter: Filter, offset: number, inlineFields: { [key: string]: any }) : ParsedFilter[] {
@@ -119,6 +121,17 @@ export default class FilterParser {
}]
}
+ if (this.isNestedField(fieldName)) {
+ const [nestedFieldName, ...nestedFieldPath] = fieldName.split('.')
+ const params = this.valueForOperator(value, operator, offset)
+ return [{
+ filterExpr: `${escapeIdentifier(nestedFieldName)} ->> '${nestedFieldPath.join('.')}' ${this.adapterOperatorToMySqlOperator(operator, value)} ${params.sql}`.trim(),
+ parameters: !isNull(value) ? [].concat( this.patchTrueFalseValue(value) ) : [],
+ offset: params.offset,
+ filterColumns: [],
+ }]
+ }
+
if (this.isSingleFieldOperator(operator)) {
const params = this.valueForOperator(value, operator, offset)
@@ -153,6 +166,10 @@ export default class FilterParser {
return []
}
+ isNestedField(fieldName: string) {
+ return fieldName.includes('.')
+ }
+
valueForStringOperator(operator: string, value: any) {
switch (operator) {
case string_contains:
diff --git a/libs/external-db-postgres/src/supported_operations.ts b/libs/external-db-postgres/src/supported_operations.ts
index 642b18976..a2dc49fa4 100644
--- a/libs/external-db-postgres/src/supported_operations.ts
+++ b/libs/external-db-postgres/src/supported_operations.ts
@@ -1 +1,5 @@
-export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons'
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+const notSupportedOperations = [SchemaOperations.NonAtomicBulkInsert]
+
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-postgres/tests/e2e-testkit/postgres_resources.ts b/libs/external-db-postgres/tests/e2e-testkit/postgres_resources.ts
index bda850ef1..bb85417e3 100644
--- a/libs/external-db-postgres/tests/e2e-testkit/postgres_resources.ts
+++ b/libs/external-db-postgres/tests/e2e-testkit/postgres_resources.ts
@@ -1,6 +1,7 @@
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
import * as compose from 'docker-compose'
+export * as capabilities from '../../src/postgres_capabilities'
export const connection = () => {
const { connection, schemaProvider, cleanup } = init({ host: 'localhost', user: 'test-user', password: 'password', db: 'test-db' }, { max: 1 })
diff --git a/libs/external-db-spanner/src/spanner_capabilities.ts b/libs/external-db-spanner/src/spanner_capabilities.ts
new file mode 100644
index 000000000..7b22111e9
--- /dev/null
+++ b/libs/external-db-spanner/src/spanner_capabilities.ts
@@ -0,0 +1,21 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+const UnsupportedCapabilities = [DataOperation.insertReferences, DataOperation.removeReferences, DataOperation.queryReferenced]
+
+
+export const ReadWriteOperations = Object.values(DataOperation).filter(op => !UnsupportedCapabilities.includes(op))
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+ text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+ boolean: { sortable: true, columnQueryOperators: [eq] },
+ image: { sortable: false, columnQueryOperators: [] },
+ object: { sortable: false, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+ datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}
diff --git a/libs/external-db-spanner/src/spanner_data_provider.ts b/libs/external-db-spanner/src/spanner_data_provider.ts
index 0f5a928c8..712218afa 100644
--- a/libs/external-db-spanner/src/spanner_data_provider.ts
+++ b/libs/external-db-spanner/src/spanner_data_provider.ts
@@ -1,6 +1,6 @@
import { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, patchFloat, extractFloatFields } from './spanner_utils'
import { translateErrorCodes } from './sql_exception_translator'
-import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types'
+import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort } from '@wix-velo/velo-external-db-types'
import { Database as SpannerDb } from '@google-cloud/spanner'
import FilterParser from './sql_filter_transformer'
@@ -45,13 +45,14 @@ export default class DataProvider implements IDataProvider {
return objs[0]['num']
}
- async insert(collectionName: string, items: Item[], fields: any): Promise <number> {
+ async insert(collectionName: string, items: Item[], fields: any, upsert = false): Promise <number> {
const floatFields = extractFloatFields(fields)
- await this.database.table(collectionName)
- .insert(
- (items.map((item: any) => patchFloat(item, floatFields)))
- .map(this.asDBEntity.bind(this))
- ).catch(translateErrorCodes)
+
+ const preparedItems = items.map((item: any) => patchFloat(item, floatFields)).map(this.asDBEntity.bind(this))
+
+ upsert ? await this.database.table(collectionName).upsert(preparedItems).catch((err) => translateErrorCodes(err, collectionName)) :
+ await this.database.table(collectionName).insert(preparedItems).catch((err) => translateErrorCodes(err, collectionName))
+
return items.length
}
@@ -86,7 +87,7 @@ export default class DataProvider implements IDataProvider {
.update(
(items.map((item: any) => patchFloat(item, floatFields)))
.map(this.asDBEntity.bind(this))
- )
+ ).catch((err) => translateErrorCodes(err, collectionName))
return items.length
}
@@ -112,13 +113,14 @@ export default class DataProvider implements IDataProvider {
await this.delete(collectionName, itemIds)
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise <Item[]> {
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: any, limit: any,): Promise <Item[]> {
const { filterExpr: whereFilterExpr, parameters: whereParameters } = this.filterParser.transform(filter)
+ const { sortExpr } = this.filterParser.orderBy(sort)
const { fieldsStatement, groupByColumns, havingFilter, parameters } = this.filterParser.parseAggregation(aggregation)
const query = {
- sql: `SELECT ${fieldsStatement} FROM ${escapeId(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map(column => escapeId(column)).join(', ')} ${havingFilter}`,
- params: { ...whereParameters, ...parameters },
+ sql: `SELECT ${fieldsStatement} FROM ${escapeId(collectionName)} ${whereFilterExpr} GROUP BY ${groupByColumns.map(column => escapeId(column)).join(', ')} ${havingFilter} ${sortExpr} LIMIT @limit OFFSET @skip`,
+ params: { ...whereParameters, ...parameters, skip, limit },
}
const [rows] = await this.database.run(query)
diff --git a/libs/external-db-spanner/src/spanner_schema_provider.ts b/libs/external-db-spanner/src/spanner_schema_provider.ts
index 22bccc486..078ea7f1b 100644
--- a/libs/external-db-spanner/src/spanner_schema_provider.ts
+++ b/libs/external-db-spanner/src/spanner_schema_provider.ts
@@ -1,10 +1,11 @@
-import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { SystemFields, validateSystemFields, parseTableData, AllSchemaOperations, EmptyCapabilities } from '@wix-velo/velo-external-db-commons'
import { errors } from '@wix-velo/velo-external-db-commons'
import SchemaColumnTranslator from './sql_schema_translator'
import { notThrowingTranslateErrorCodes } from './sql_exception_translator'
import { recordSetToObj, escapeId, patchFieldName, unpatchFieldName, escapeFieldId } from './spanner_utils'
import { Database as SpannerDb } from '@google-cloud/spanner'
-import { InputField, ISchemaProvider, ResponseField, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
+import { CollectionCapabilities, Encryption, InputField, ISchemaProvider, SchemaOperations, Table } from '@wix-velo/velo-external-db-types'
+import { CollectionOperations, ColumnsCapabilities, FieldTypes, ReadOnlyOperations, ReadWriteOperations } from './spanner_capabilities'
const { CollectionDoesNotExists, CollectionAlreadyExists } = errors
export default class SchemaProvider implements ISchemaProvider {
@@ -18,7 +19,7 @@ export default class SchemaProvider implements ISchemaProvider {
async list(): Promise<Table[]> {
const query = {
- sql: 'SELECT table_name, COLUMN_NAME, SPANNER_TYPE FROM information_schema.columns WHERE table_catalog = @tableCatalog and table_schema = @tableSchema',
+ sql: 'SELECT table_name, COLUMN_NAME as field, SPANNER_TYPE as type FROM information_schema.columns WHERE table_catalog = @tableCatalog and table_schema = @tableSchema',
params: {
tableSchema: '',
tableCatalog: '',
@@ -26,14 +27,15 @@ export default class SchemaProvider implements ISchemaProvider {
}
const [rows] = await this.database.run(query)
- const res = recordSetToObj(rows)
+ const res = recordSetToObj(rows) as { table_name: string, field: string, type: string }[]
- const tables: {[x:string]: {table_name: string, field: string, type: string}[]} = parseTableData(res)
+ const tables = parseTableData(res)
return Object.entries(tables)
.map(([collectionName, rs]) => ({
id: collectionName,
- fields: rs.map( this.reformatFields.bind(this) )
+ fields: rs.map( this.appendAdditionalFieldDetails.bind(this) ),
+ capabilities: this.collectionCapabilities(rs.map(r => r.field))
}))
}
@@ -60,24 +62,24 @@ export default class SchemaProvider implements ISchemaProvider {
.join(', ')
const primaryKeySql = SystemFields.filter(f => f.isPrimary).map(f => escapeFieldId(f.name)).join(', ')
- await this.updateSchema(`CREATE TABLE ${escapeId(collectionName)} (${dbColumnsSql}) PRIMARY KEY (${primaryKeySql})`, CollectionAlreadyExists)
+ await this.updateSchema(`CREATE TABLE ${escapeId(collectionName)} (${dbColumnsSql}) PRIMARY KEY (${primaryKeySql})`, collectionName, CollectionAlreadyExists)
}
async addColumn(collectionName: string, column: InputField): Promise<void> {
await validateSystemFields(column.name)
- await this.updateSchema(`ALTER TABLE ${escapeId(collectionName)} ADD COLUMN ${this.sqlSchemaTranslator.columnToDbColumnSql(column)}`)
+ await this.updateSchema(`ALTER TABLE ${escapeId(collectionName)} ADD COLUMN ${this.sqlSchemaTranslator.columnToDbColumnSql(column)}`, collectionName)
}
async removeColumn(collectionName: string, columnName: string): Promise<void> {
await validateSystemFields(columnName)
- await this.updateSchema(`ALTER TABLE ${escapeId(collectionName)} DROP COLUMN ${escapeId(columnName)}`)
+ await this.updateSchema(`ALTER TABLE ${escapeId(collectionName)} DROP COLUMN ${escapeId(columnName)}`, collectionName)
}
- async describeCollection(collectionName: string): Promise<ResponseField[]> {
+ async describeCollection(collectionName: string): Promise<Table> {
const query = {
- sql: 'SELECT table_name, COLUMN_NAME, SPANNER_TYPE FROM information_schema.columns WHERE table_catalog = @tableCatalog and table_schema = @tableSchema and table_name = @tableName',
+ sql: 'SELECT table_name, COLUMN_NAME as field, SPANNER_TYPE as type FROM information_schema.columns WHERE table_catalog = @tableCatalog and table_schema = @tableSchema and table_name = @tableName',
params: {
tableSchema: '',
tableCatalog: '',
@@ -89,40 +91,58 @@ export default class SchemaProvider implements ISchemaProvider {
const res = recordSetToObj(rows)
if (res.length === 0) {
- throw new CollectionDoesNotExists('Collection does not exists')
+ throw new CollectionDoesNotExists('Collection does not exists', collectionName)
}
- return res.map( this.reformatFields.bind(this) )
+ return {
+ id: collectionName,
+ fields: res.map( this.appendAdditionalFieldDetails.bind(this) ),
+ capabilities: this.collectionCapabilities(res.map(f => f.field))
+ }
}
async drop(collectionName: string): Promise<void> {
- await this.updateSchema(`DROP TABLE ${escapeId(collectionName)}`)
+ await this.updateSchema(`DROP TABLE ${escapeId(collectionName)}`, collectionName)
}
+ async changeColumnType(collectionName: string, _column: InputField): Promise<void> {
+ throw new errors.UnsupportedSchemaOperation('changeColumnType is not supported', collectionName, 'changeColumnType')
+ }
- async updateSchema(sql: string, catching: any = undefined) {
+ async updateSchema(sql: string, collectionName: string, catching: any = undefined ) {
try {
const [operation] = await this.database.updateSchema([sql])
await operation.promise()
} catch (err) {
- const e = notThrowingTranslateErrorCodes(err)
+ const e = notThrowingTranslateErrorCodes(err, collectionName)
if (!catching || (catching && !(e instanceof catching))) {
throw e
}
}
}
- fixColumn(c: InputField) {
+ private fixColumn(c: InputField) {
return { ...c, name: patchFieldName(c.name) }
}
- reformatFields(r: { [x: string]: string }) {
- const { type, subtype } = this.sqlSchemaTranslator.translateType(r['SPANNER_TYPE'])
+ private appendAdditionalFieldDetails(row: { field: string, type: string }) {
+ const type = this.sqlSchemaTranslator.translateType(row.type).type as keyof typeof ColumnsCapabilities
+ return {
+ field: unpatchFieldName(row.field),
+ ...this.sqlSchemaTranslator.translateType(row.type),
+ capabilities: ColumnsCapabilities[type] ?? EmptyCapabilities
+ }
+ }
+
+ private collectionCapabilities(fieldNames: string[]): CollectionCapabilities {
return {
- field: unpatchFieldName(r['COLUMN_NAME']),
- type,
- subtype
+ dataOperations: fieldNames.map(unpatchFieldName).includes('_id') ? ReadWriteOperations : ReadOnlyOperations,
+ fieldTypes: FieldTypes,
+ collectionOperations: CollectionOperations,
+ referenceCapabilities: { supportedNamespaces: [] },
+ indexing: [],
+ encryption: Encryption.notSupported
}
}
diff --git a/libs/external-db-spanner/src/sql_exception_translator.ts b/libs/external-db-spanner/src/sql_exception_translator.ts
index f8b10f7d6..7165b38d8 100644
--- a/libs/external-db-spanner/src/sql_exception_translator.ts
+++ b/libs/external-db-spanner/src/sql_exception_translator.ts
@@ -1,7 +1,16 @@
import { errors } from '@wix-velo/velo-external-db-commons'
const { CollectionDoesNotExists, FieldAlreadyExists, FieldDoesNotExist, DbConnectionError, CollectionAlreadyExists, ItemAlreadyExists, InvalidQuery, UnrecognizedError } = errors
+const extractId = (msg: string | null) => {
+ msg = msg || ''
+ const regex = /String\("([A-Za-z0-9-]+)"\)/i
+ const match = msg.match(regex)
+ if (match) {
+ return match[1]
+ }
+ return ''
+ }
-export const notThrowingTranslateErrorCodes = (err: any) => {
+export const notThrowingTranslateErrorCodes = (err: any, collectionName?: string) => {
switch (err.code) {
case 9:
if (err.details.includes('column')) {
@@ -11,19 +20,22 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
}
case 5:
if (err.details.includes('Column')) {
- return new FieldDoesNotExist(err.details)
+ return new FieldDoesNotExist(err.details, collectionName)
} else if (err.details.includes('Instance')) {
return new DbConnectionError(`Access to database denied - wrong credentials or host is unavailable, sql message: ${err.details} `)
} else if (err.details.includes('Database')) {
return new DbConnectionError(`Database does not exists or you don't have access to it, sql message: ${err.details}`)
} else if (err.details.includes('Table')) {
- return new CollectionDoesNotExists(err.details)
+ console.log({ details: err.details, collectionName })
+
+ return new CollectionDoesNotExists(err.details, collectionName)
} else {
return new InvalidQuery(`${err.details}`)
}
case 6:
if (err.details.includes('already exists'))
- return new ItemAlreadyExists(`Item already exists: ${err.details}`)
+ return new ItemAlreadyExists(`Item already exists: ${err.details}`, collectionName, extractId(err.details))
+
else
return new InvalidQuery(`${err.details}`)
case 7:
@@ -35,6 +47,6 @@ export const notThrowingTranslateErrorCodes = (err: any) => {
}
}
-export const translateErrorCodes = (err: any) => {
- throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName?: string) => {
+ throw notThrowingTranslateErrorCodes(err, collectionName)
}
diff --git a/libs/external-db-spanner/src/sql_filter_transformer.spec.ts b/libs/external-db-spanner/src/sql_filter_transformer.spec.ts
index c2877f21f..0754a615a 100644
--- a/libs/external-db-spanner/src/sql_filter_transformer.spec.ts
+++ b/libs/external-db-spanner/src/sql_filter_transformer.spec.ts
@@ -301,6 +301,22 @@ describe('Sql Parser', () => {
})
})
+ describe('handle queries on nested fields', () => {
+ test('correctly transform nested field query', () => {
+ const operator = ctx.filterWithoutInclude.operator
+ const filter = {
+ operator,
+ fieldName: `${ctx.fieldName}.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}`,
+ value: ctx.filterWithoutInclude.value
+ }
+
+ expect( env.filterParser.parseFilter(filter) ).toEqual([{
+ filterExpr: `JSON_VALUE(${escapeId(ctx.fieldName)}, '$.${ctx.nestedFieldName}.${ctx.anotherNestedFieldName}') ${env.filterParser.adapterOperatorToMySqlOperator(operator, ctx.filterWithoutInclude.value)} @${ctx.fieldName}0`,
+ parameters: { [`${ctx.fieldName}0`]: ctx.filterWithoutInclude.value }
+ }])
+ })
+ })
+
describe('handle multi field operator', () => {
each([
and, or
@@ -455,6 +471,10 @@ describe('Sql Parser', () => {
filter: Uninitialized,
anotherFilter: Uninitialized,
offset: Uninitialized,
+ filterWithoutInclude: Uninitialized,
+ nestedFieldName: Uninitialized,
+ anotherNestedFieldName: Uninitialized,
+
}
const env: {
@@ -467,6 +487,8 @@ describe('Sql Parser', () => {
ctx.fieldName = chance.word()
ctx.anotherFieldName = chance.word()
ctx.moreFieldName = chance.word()
+ ctx.nestedFieldName = chance.word()
+ ctx.anotherNestedFieldName = chance.word()
ctx.fieldValue = chance.word()
ctx.anotherValue = chance.word()
@@ -478,6 +500,7 @@ describe('Sql Parser', () => {
ctx.offset = chance.natural({ min: 2, max: 20 })
+ ctx.filterWithoutInclude = gen.randomDomainFilterWithoutInclude()
})
beforeAll(function() {
diff --git a/libs/external-db-spanner/src/sql_filter_transformer.ts b/libs/external-db-spanner/src/sql_filter_transformer.ts
index c95b7c219..8c3e63e28 100644
--- a/libs/external-db-spanner/src/sql_filter_transformer.ts
+++ b/libs/external-db-spanner/src/sql_filter_transformer.ts
@@ -116,6 +116,16 @@ export default class FilterParser {
}]
}
+ if (this.isNestedField(fieldName)) {
+ const [nestedFieldName, ...nestedFieldPath] = fieldName.split('.')
+ const literals = this.valueForOperator(nestedFieldName, value, operator, counter).sql
+
+ return [{
+ filterExpr: `JSON_VALUE(${this.inlineVariableIfNeeded(nestedFieldName, inlineFields)}, '$.${nestedFieldPath.join('.')}') ${this.adapterOperatorToMySqlOperator(operator, value)} ${literals}`.trim(),
+ parameters: this.parametersFor(nestedFieldName, value, counter)
+ }]
+ }
+
if (this.isSingleFieldOperator(operator)) {
const literals = this.valueForOperator(fieldName, value, operator, counter).sql
@@ -156,6 +166,10 @@ export default class FilterParser {
return []
}
+ isNestedField(fieldName: string) {
+ return fieldName.includes('.')
+ }
+
parametersFor(name: string, value: any, counter: Counter) {
if (!isNull(value)) {
if (!Array.isArray(value)) {
diff --git a/libs/external-db-spanner/src/supported_operations.ts b/libs/external-db-spanner/src/supported_operations.ts
index b2bafcd12..1a43303df 100644
--- a/libs/external-db-spanner/src/supported_operations.ts
+++ b/libs/external-db-spanner/src/supported_operations.ts
@@ -1 +1,6 @@
-export { AllSchemaOperations as supportedOperations } from '@wix-velo/velo-external-db-commons'
+import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+//change column types - https://cloud.google.com/spanner/docs/schema-updates#supported_schema_updates
+const notSupportedOperations = [SchemaOperations.ChangeColumnType, SchemaOperations.NonAtomicBulkInsert]
+
+export const supportedOperations = AllSchemaOperations.filter(op => !notSupportedOperations.includes(op))
diff --git a/libs/external-db-spanner/tests/e2e-testkit/spanner_resources.ts b/libs/external-db-spanner/tests/e2e-testkit/spanner_resources.ts
index c20530df4..f51399603 100644
--- a/libs/external-db-spanner/tests/e2e-testkit/spanner_resources.ts
+++ b/libs/external-db-spanner/tests/e2e-testkit/spanner_resources.ts
@@ -2,6 +2,8 @@ import * as compose from 'docker-compose'
import init from '../../src/connection_provider'
export { supportedOperations } from '../../src/supported_operations'
+export * as capabilities from '../../src/spanner_capabilities'
+
const setEmulatorOn = () => process.env['SPANNER_EMULATOR_HOST'] = 'localhost:9010'
export const connection = () => {
diff --git a/libs/external-db-testkit/src/lib/auth_test_support.ts b/libs/external-db-testkit/src/lib/auth_test_support.ts
index d086b93e6..0f350bcbf 100644
--- a/libs/external-db-testkit/src/lib/auth_test_support.ts
+++ b/libs/external-db-testkit/src/lib/auth_test_support.ts
@@ -1,40 +1,53 @@
import * as Chance from 'chance'
+import { AxiosRequestHeaders } from 'axios'
+import * as jwt from 'jsonwebtoken'
+import { authConfig } from '@wix-velo/test-commons'
const chance = Chance()
const axios = require('axios').create({
baseURL: 'http://localhost:8080',
})
-const secretKey = chance.word()
+const allowedMetasite = chance.word()
+const externalDatabaseId = chance.word()
export const authInit = () => {
- process.env['SECRET_KEY'] = secretKey
+ process.env['ALLOWED_METASITES'] = allowedMetasite
+ process.env['EXTERNAL_DATABASE_ID'] = externalDatabaseId
}
-const appendSecretKeyToRequest = (dataRaw: string) => {
+const appendRoleToRequest = (role: string) => (dataRaw: string) => {
const data = JSON.parse( dataRaw )
- return JSON.stringify({ ...data, ...{ requestContext: { settings: { secretKey: secretKey } } } })
+ return JSON.stringify({ ...data, ...{ requestContext: { ...data.requestContext, role: role } } })
}
-const appendRoleToRequest = (role: string) => (dataRaw: string) => {
+const appendJWTHeaderToRequest = (dataRaw: string, headers: AxiosRequestHeaders) => {
+ headers['Authorization'] = createJwtHeader()
const data = JSON.parse( dataRaw )
- return JSON.stringify({ ...data, ...{ requestContext: { ...data.requestContext, role: role } } })
+ return JSON.stringify({ ...data } )
+}
+
+const TOKEN_ISSUER = 'wix-data.wix.com'
+
+const createJwtHeader = () => {
+ const token = jwt.sign({ iss: TOKEN_ISSUER, siteId: allowedMetasite, aud: externalDatabaseId }, authConfig.authPrivateKey, { algorithm: 'ES256', keyid: authConfig.kid })
+ return `Bearer ${token}`
}
export const authAdmin = { transformRequest: axios.defaults
.transformRequest
- .concat( appendSecretKeyToRequest, appendRoleToRequest('BACKEND_CODE') ) }
+ .concat( appendJWTHeaderToRequest, appendRoleToRequest('BACKEND_CODE') ) }
export const authOwner = { transformRequest: axios.defaults
.transformRequest
- .concat( appendSecretKeyToRequest, appendRoleToRequest('OWNER' ) ) }
+ .concat( appendJWTHeaderToRequest, appendRoleToRequest('OWNER' ) ) }
export const authVisitor = { transformRequest: axios.defaults
.transformRequest
- .concat( appendSecretKeyToRequest, appendRoleToRequest('VISITOR' ) ) }
+ .concat( appendJWTHeaderToRequest, appendRoleToRequest('VISITOR' ) ) }
-export const authOwnerWithoutSecretKey = { transformRequest: axios.defaults
+export const authOwnerWithoutJwt = { transformRequest: axios.defaults
.transformRequest
.concat( appendRoleToRequest('OWNER' ) ) }
-export const errorResponseWith = (status: any, message: string) => ({ response: { data: { message: expect.stringContaining(message) }, status } })
+export const errorResponseWith = (status: any, message: string) => ({ response: { data: { description: expect.stringContaining(message) }, status } })
diff --git a/libs/test-commons/src/index.ts b/libs/test-commons/src/index.ts
index 20290e545..ff771e288 100644
--- a/libs/test-commons/src/index.ts
+++ b/libs/test-commons/src/index.ts
@@ -1,2 +1,3 @@
export * from './libs/test-commons'
export * as gen from './libs/gen'
+export { authConfig } from './libs/auth-config.json'
diff --git a/libs/test-commons/src/libs/auth-config.json b/libs/test-commons/src/libs/auth-config.json
new file mode 100644
index 000000000..c2e77245e
--- /dev/null
+++ b/libs/test-commons/src/libs/auth-config.json
@@ -0,0 +1,8 @@
+{
+ "authConfig": {
+ "kid": "7968bd02-7c7d-446e-83c5-5c993c20a140",
+ "authPublicKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEdnP+fQMJYtljus9pnpEWT02T0uqF\nUacdoxL19vmQdii4DAj+S0pbJ/owcc7HsPvNwhJvIwFtk/4Cm+OYp7fXSQ==\n-----END PUBLIC KEY-----",
+ "authPrivateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEINoWtnYgw8ZcsZkgWDBxAcJF0ziCI4SOVuK17DrQFCWYoAoGCCqGSM49\nAwEHoUQDQgAEdnP+fQMJYtljus9pnpEWT02T0uqFUacdoxL19vmQdii4DAj+S0pb\nJ/owcc7HsPvNwhJvIwFtk/4Cm+OYp7fXSQ==\n-----END EC PRIVATE KEY-----",
+ "otherAuthPublicKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEC0QSOeblgUZjrKzxsLwJ/gcTFV+/\nTIhuEDxhpNaAnY1AvqFuANfCJ++aCWMjmhp1Fy9BZ6pi/lxVJAF4fpMqtw==\n-----END PUBLIC KEY-----"
+ }
+}
\ No newline at end of file
diff --git a/libs/test-commons/src/libs/gen.ts b/libs/test-commons/src/libs/gen.ts
index 3338d4fed..fa8416ae5 100644
--- a/libs/test-commons/src/libs/gen.ts
+++ b/libs/test-commons/src/libs/gen.ts
@@ -1,5 +1,6 @@
import * as Chance from 'chance'
import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { Item } from '@wix-velo/velo-external-db-types'
const { eq, gt, gte, include, lt, lte, ne, string_begins, string_ends, string_contains } = AdapterOperators
const chance = Chance()
@@ -48,8 +49,8 @@ export const randomCollections = () => randomArrayOf( randomCollectionName )
export const randomFieldName = () => chance.word({ length: 5 })
-export const randomEntity = (columns?: any[]) => {
- const entity : {[x:string]: any} = {
+export const randomEntity = (columns?: string[]) => {
+ const entity : Item = {
_id: chance.guid(),
_createdDate: veloDate(),
_updatedDate: veloDate(),
@@ -65,7 +66,7 @@ export const randomEntity = (columns?: any[]) => {
}
export const randomNumberEntity = (columns: any[]) => {
- const entity : {[x:string]: any} = {
+ const entity : Item = {
_id: chance.guid(),
_createdDate: veloDate(),
_updatedDate: veloDate(),
@@ -94,8 +95,10 @@ export const randomObjectFromArray = (array: any[]) => array[chance.integer({ mi
export const randomAdapterOperator = () => ( chance.pickone([ne, lt, lte, gt, gte, include, eq, string_contains, string_begins, string_ends]) )
-export const randomWrappedFilter = (_fieldName?: string) => {
- const operator = randomAdapterOperator()
+export const randomAdapterOperatorWithoutInclude = () => ( chance.pickone([ne, lt, lte, gt, gte, eq, string_contains, string_begins, string_ends]) )
+
+export const randomWrappedFilter = (_fieldName?: string, _operator?: string) => { // TODO: rename to randomDomainFilter
+ const operator = _operator ?? randomAdapterOperator()
const fieldName = _fieldName ?? chance.word()
const value = operator === AdapterOperators.include ? [chance.word(), chance.word(), chance.word(), chance.word(), chance.word()] : chance.word()
return {
@@ -104,3 +107,7 @@ export const randomWrappedFilter = (_fieldName?: string) => {
value
}
}
+
+export const randomDomainFilterWithoutInclude = (_fieldName?: string) => {
+ return randomWrappedFilter(_fieldName || chance.word(), randomAdapterOperatorWithoutInclude())
+}
diff --git a/libs/test-commons/src/libs/test-commons.ts b/libs/test-commons/src/libs/test-commons.ts
index a8ada374f..96241f488 100644
--- a/libs/test-commons/src/libs/test-commons.ts
+++ b/libs/test-commons/src/libs/test-commons.ts
@@ -11,7 +11,7 @@ export const shouldRunOnlyOn = (impl: string[], current: string) => impl.include
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
//@ts-ignore
-export const testIfSupportedOperationsIncludes = (supportedOperations: SchemaOperations[], operation: string[]): any => operation.every((o: any) => supportedOperations.includes(o)) ? test : test.skip
+export const testIfSupportedOperationsIncludes = (supportedOperations: SchemaOperations[], operation: string[]): any => operation.every((o: any) => supportedOperations.includes(o)) ? test : test.skip
export const testSupportedOperations = (supportedOperations: SchemaOperations[], arrayTable: any[][]): string[][] => {
return arrayTable.filter(i => {
@@ -19,3 +19,21 @@ export const testSupportedOperations = (supportedOperations: SchemaOperations[],
return !isObject(lastItem) || lastItem['neededOperations'].every((i: any) => supportedOperations.includes(i))
})
}
+
+export const streamToArray = async(stream: any) => {
+
+ return new Promise((resolve, reject) => {
+ const arr: any[] = []
+
+ stream.on('data', (data: any) => {
+ arr.push(JSON.parse(data.toString()))
+ })
+
+ stream.on('end', () => {
+ resolve(arr)
+ })
+
+ stream.on('error', (err: Error) => reject(err))
+
+ })
+}
diff --git a/libs/velo-external-db-commons/src/libs/errors.ts b/libs/velo-external-db-commons/src/libs/errors.ts
index 2bbf64425..15836c94d 100644
--- a/libs/velo-external-db-commons/src/libs/errors.ts
+++ b/libs/velo-external-db-commons/src/libs/errors.ts
@@ -1,90 +1,111 @@
class BaseHttpError extends Error {
- status: number
- constructor(message: string, status: number) {
+ constructor(message: string) {
super(message)
- this.status = status
}
}
export class UnauthorizedError extends BaseHttpError {
constructor(message: string) {
- super(message, 401)
+ super(message)
}
}
export class CollectionDoesNotExists extends BaseHttpError {
- constructor(message: string) {
- super(message, 404)
+ collectionName: string
+ constructor(message: string, collectionName?: string) {
+ super(message)
+ this.collectionName = collectionName || ''
}
}
export class CollectionAlreadyExists extends BaseHttpError {
- constructor(message: string) {
- super(message, 400)
+ collectionName: string
+ constructor(message: string, collectionName?: string) {
+ super(message)
+ this.collectionName = collectionName || ''
}
}
export class FieldAlreadyExists extends BaseHttpError {
- constructor(message: string) {
- super(message, 400)
+ collectionName: string
+ fieldName: string
+
+ constructor(message: string, collectionName?: string, fieldName?: string) {
+ super(message)
+ this.collectionName = collectionName || ''
+ this.fieldName = fieldName || ''
}
}
export class ItemAlreadyExists extends BaseHttpError {
- constructor(message: string) {
- super(message, 400)
+ itemId: string
+ collectionName: string
+
+ constructor(message: string, collectionName?: string, itemId?: string) {
+ super(message)
+ this.itemId = itemId || ''
+ this.collectionName = collectionName || ''
}
}
export class FieldDoesNotExist extends BaseHttpError {
- constructor(message: string) {
- super(message, 404)
+ propertyName: string
+ collectionName: string
+ constructor(message: string, collectionName?: string, propertyName?: string) {
+ super(message)
+ this.propertyName = propertyName || ''
+ this.collectionName = collectionName || ''
}
}
export class CannotModifySystemField extends BaseHttpError {
constructor(message: string) {
- super(message, 400)
+ super(message)
}
}
export class InvalidQuery extends BaseHttpError {
constructor(message: string) {
- super(message, 400)
+ super(message)
}
}
export class InvalidRequest extends BaseHttpError {
constructor(message: string) {
- super(message, 400)
+ super(message)
}
}
export class DbConnectionError extends BaseHttpError {
constructor(message: string) {
- super(message, 500)
+ super(message)
}
}
export class ItemNotFound extends BaseHttpError {
constructor(message: string) {
- super(message, 404)
+ super(message)
}
}
-export class UnsupportedOperation extends BaseHttpError {
- constructor(message: string) {
- super(message, 405)
+export class UnsupportedSchemaOperation extends BaseHttpError {
+ collectionName: string
+ operation: string
+
+ constructor(message: string, collectionName?: string, operation?: string) {
+ super(message)
+ this.collectionName = collectionName || ''
+ this.operation = operation || ''
}
}
export class UnsupportedDatabase extends BaseHttpError {
constructor(message: string) {
- super(message, 405)
+ super(message)
}
}
export class UnrecognizedError extends BaseHttpError {
constructor(message: string) {
- super(`Unrecognized Error: ${message}`, 400)
+ super(`Unrecognized Error: ${message}`)
}
}
diff --git a/libs/velo-external-db-commons/src/libs/schema_commons.ts b/libs/velo-external-db-commons/src/libs/schema_commons.ts
index 99e5226f7..f0e82e05c 100644
--- a/libs/velo-external-db-commons/src/libs/schema_commons.ts
+++ b/libs/velo-external-db-commons/src/libs/schema_commons.ts
@@ -23,9 +23,11 @@ export const QueryOperatorsByFieldType = {
url: ['eq', 'ne', 'contains', 'hasSome'],
datetime: ['eq', 'ne', 'gt', 'gte', 'lt', 'lte'],
image: [],
- object: ['eq', 'ne'],
+ object: ['eq', 'ne', 'contains', 'startsWith', 'endsWith', 'hasSome', 'matches', 'gt', 'gte', 'lt', 'lte'],
}
+export const EmptyCapabilities = { sortable: false, columnQueryOperators: [] }
+
const QueryOperationsByFieldType: {[x: string]: any} = {
number: [...QueryOperatorsByFieldType.number, 'urlized'],
text: [...QueryOperatorsByFieldType.text, 'urlized', 'isEmpty', 'isNotEmpty'],
diff --git a/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts b/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts
index fbcac5680..653b3e9d1 100644
--- a/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts
+++ b/libs/velo-external-db-core/src/converters/aggregation_transformer.spec.ts
@@ -6,6 +6,7 @@ import { errors } from '@wix-velo/velo-external-db-commons'
import AggregationTransformer from './aggregation_transformer'
import { EmptyFilter } from './utils'
import * as driver from '../../test/drivers/filter_transformer_test_support'
+import { Group } from '../spi-model/data_source'
const chance = Chance()
const { InvalidQuery } = errors
@@ -17,127 +18,96 @@ describe('Aggregation Transformer', () => {
describe('correctly transform Wix functions to adapter functions', () => {
each([
- '$avg', '$max', '$min', '$sum'
+ 'avg', 'max', 'min', 'sum', 'count'
])
- .test('correctly transform [%s]', (f: string) => {
- const AdapterFunction = f.substring(1)
- expect(env.AggregationTransformer.wixFunctionToAdapterFunction(f)).toEqual((AdapterFunctions as any)[AdapterFunction])
- })
+ .test('correctly transform [%s]', (f: string) => {
+ const AdapterFunction = f as AdapterFunctions
+ expect(env.AggregationTransformer.wixFunctionToAdapterFunction(f)).toEqual(AdapterFunctions[AdapterFunction])
+ })
test('transform unknown function will throw an exception', () => {
- expect( () => env.AggregationTransformer.wixFunctionToAdapterFunction('$wrong')).toThrow(InvalidQuery)
+ expect(() => env.AggregationTransformer.wixFunctionToAdapterFunction('wrong')).toThrow(InvalidQuery)
})
})
test('single id field without function or postFilter', () => {
- env.driver.stubEmptyFilterFor(null)
-
- const processingStep = { _id: `$${ctx.fieldName}` }
- const postFilteringStep = null
+ env.driver.stubEmptyFilterForUndefined()
+
+ const group = { by: [ctx.fieldName], aggregation: [] } as Group
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ expect(env.AggregationTransformer.transform({ group })).toEqual({
projection: [{ name: ctx.fieldName }],
postFilter: EmptyFilter
})
})
test('multiple id fields without function or postFilter', () => {
- env.driver.stubEmptyFilterFor(null)
-
- const processingStep = {
- _id: {
- field1: `$${ctx.fieldName}`,
- field2: `$${ctx.anotherFieldName}`
- }
- }
- const postFilteringStep = null
+ env.driver.stubEmptyFilterForUndefined()
+
+ const group = { by: [ctx.fieldName, ctx.anotherFieldName], aggregation: [] } as Group
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ expect(env.AggregationTransformer.transform({ group })).toEqual({
projection: [
- { name: ctx.fieldName },
- { name: ctx.anotherFieldName }
- ],
+ { name: ctx.fieldName },
+ { name: ctx.anotherFieldName }
+ ],
postFilter: EmptyFilter
})
})
test('single id field with function field and without postFilter', () => {
- env.driver.stubEmptyFilterFor(null)
-
- const processingStep = {
- _id: `$${ctx.fieldName}`,
- [ctx.fieldAlias]: {
- $avg: `$${ctx.anotherFieldName}`
- }
- }
- const postFilteringStep = null
+ env.driver.stubEmptyFilterForUndefined()
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }] } as Group
+
+ expect(env.AggregationTransformer.transform({ group })).toEqual({
projection: [
- { name: ctx.fieldName },
- { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg }
- ],
+ { name: ctx.fieldName },
+ { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg }
+ ],
postFilter: EmptyFilter
})
})
test('single id field with count function and without postFilter', () => {
- env.driver.stubEmptyFilterFor(null)
+ env.driver.stubEmptyFilterForUndefined()
- const processingStep = {
- _id: `$${ctx.fieldName}`,
- [ctx.fieldAlias]: {
- $sum: 1
- }
- }
- const postFilteringStep = null
+ const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, count: 1 }] } as Group
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ expect(env.AggregationTransformer.transform({ group })).toEqual({
projection: [
- { name: ctx.fieldName },
- { alias: ctx.fieldAlias, function: AdapterFunctions.count, name: '*' }
- ],
+ { name: ctx.fieldName },
+ { alias: ctx.fieldAlias, function: AdapterFunctions.count, name: '*' }
+ ],
postFilter: EmptyFilter
})
})
-
+
test('multiple function fields and without postFilter', () => {
- env.driver.stubEmptyFilterFor(null)
-
- const processingStep = {
- _id: `$${ctx.fieldName}`,
- [ctx.fieldAlias]: {
- $avg: `$${ctx.anotherFieldName}`
- },
- [ctx.anotherFieldAlias]: {
- $sum: `$${ctx.moreFieldName}`
- }
- }
- const postFilteringStep = null
+ env.driver.stubEmptyFilterForUndefined()
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ const group = {
+ by: [ctx.fieldName],
+ aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }, { name: ctx.anotherFieldAlias, sum: ctx.moreFieldName }]
+ } as Group
+
+ expect(env.AggregationTransformer.transform({ group })).toEqual({
projection: [
- { name: ctx.fieldName },
- { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg },
- { name: ctx.moreFieldName, alias: ctx.anotherFieldAlias, function: AdapterFunctions.sum }
- ],
+ { name: ctx.fieldName },
+ { name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg },
+ { name: ctx.moreFieldName, alias: ctx.anotherFieldAlias, function: AdapterFunctions.sum }
+ ],
postFilter: EmptyFilter
})
})
test('function and postFilter', () => {
env.driver.givenFilterByIdWith(ctx.id, ctx.filter)
-
- const processingStep = {
- _id: `$${ctx.fieldName}`,
- [ctx.fieldAlias]: {
- $avg: `$${ctx.anotherFieldName}`
- }
- }
- const postFilteringStep = ctx.filter
+ const group = { by: [ctx.fieldName], aggregation: [{ name: ctx.fieldAlias, avg: ctx.anotherFieldName }] } as Group
+ const finalFilter = ctx.filter
- expect(env.AggregationTransformer.transform({ processingStep, postFilteringStep })).toEqual({
+ expect(env.AggregationTransformer.transform({ group, finalFilter })).toEqual({
projection: [
{ name: ctx.fieldName },
{ name: ctx.anotherFieldName, alias: ctx.fieldAlias, function: AdapterFunctions.avg }
diff --git a/libs/velo-external-db-core/src/converters/aggregation_transformer.ts b/libs/velo-external-db-core/src/converters/aggregation_transformer.ts
index bc9de23ac..b38fe0f57 100644
--- a/libs/velo-external-db-core/src/converters/aggregation_transformer.ts
+++ b/libs/velo-external-db-core/src/converters/aggregation_transformer.ts
@@ -1,14 +1,12 @@
-import { isObject } from '@wix-velo/velo-external-db-commons'
-import { AdapterAggregation, AdapterFunctions, FieldProjection, FunctionProjection } from '@wix-velo/velo-external-db-types'
+import { AdapterAggregation, AdapterFunctions } from '@wix-velo/velo-external-db-types'
import { IFilterTransformer } from './filter_transformer'
-import { projectionFieldFor, projectionFunctionFor } from './utils'
+import { projectionFunctionFor } from './utils'
import { errors } from '@wix-velo/velo-external-db-commons'
+import { Aggregation, Group } from '../spi-model/data_source'
const { InvalidQuery } = errors
interface IAggregationTransformer {
- transform(aggregation: any): AdapterAggregation
- extractProjectionFunctions(functionsObj: { [x: string]: { [s: string]: string | number } }): FunctionProjection[]
- extractProjectionFields(fields: { [fieldName: string]: string } | string): FieldProjection[]
+ transform(aggregation: { group: Group, finalFilter?: any }): AdapterAggregation
wixFunctionToAdapterFunction(wixFunction: string): AdapterFunctions
}
@@ -18,13 +16,13 @@ export default class AggregationTransformer implements IAggregationTransformer {
this.filterTransformer = filterTransformer
}
- transform({ processingStep, postFilteringStep }: any): AdapterAggregation {
- const { _id: fields, ...functions } = processingStep
+ transform({ group, finalFilter }: { group: Group, finalFilter?: any }): AdapterAggregation {
+ const { by: fields, aggregation } = group
- const projectionFields = this.extractProjectionFields(fields)
- const projectionFunctions = this.extractProjectionFunctions(functions)
-
- const postFilter = this.filterTransformer.transform(postFilteringStep)
+ const projectionFields = fields.map(f => ({ name: f }))
+ const projectionFunctions = this.aggregationToProjectionFunctions(aggregation)
+
+ const postFilter = this.filterTransformer.transform(finalFilter)
const projection = [...projectionFields, ...projectionFunctions]
@@ -34,48 +32,19 @@ export default class AggregationTransformer implements IAggregationTransformer {
}
}
- extractProjectionFunctions(functionsObj: { [x: string]: { [s: string]: string | number } }) {
- const projectionFunctions: { name: any; alias: any; function: any }[] = []
- Object.keys(functionsObj)
- .forEach(fieldAlias => {
- Object.entries(functionsObj[fieldAlias])
- .forEach(([func, field]) => {
- projectionFunctions.push(projectionFunctionFor(field, fieldAlias, this.wixFunctionToAdapterFunction(func)))
- })
- })
-
- return projectionFunctions
- }
-
- extractProjectionFields(fields: { [fieldName: string]: string } | string) {
- const projectionFields = []
-
- if (isObject(fields)) {
- projectionFields.push(...Object.values(fields).map(f => projectionFieldFor(f)) )
- } else {
- projectionFields.push(projectionFieldFor(fields))
- }
-
- return projectionFields
+ aggregationToProjectionFunctions(aggregations: Aggregation[]) {
+ return aggregations.map(aggregation => {
+ const { name: fieldAlias, ...rest } = aggregation
+ const [func, field] = Object.entries(rest)[0]
+ return projectionFunctionFor(field, fieldAlias, this.wixFunctionToAdapterFunction(func))
+ })
}
wixFunctionToAdapterFunction(func: string): AdapterFunctions {
- return this.wixFunctionToAdapterFunctionString(func) as AdapterFunctions
- }
-
- private wixFunctionToAdapterFunctionString(func: string): string {
- switch (func) {
- case '$avg':
- return AdapterFunctions.avg
- case '$max':
- return AdapterFunctions.max
- case '$min':
- return AdapterFunctions.min
- case '$sum':
- return AdapterFunctions.sum
-
- default:
- throw new InvalidQuery(`Unrecognized function ${func}`)
+ if (Object.values(AdapterFunctions).includes(func as any)) {
+ return AdapterFunctions[func as AdapterFunctions] as AdapterFunctions
}
+
+ throw new InvalidQuery(`Unrecognized function ${func}`)
}
}
diff --git a/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts b/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts
index a9bdf63f2..253b08732 100644
--- a/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts
+++ b/libs/velo-external-db-core/src/converters/filter_transformer.spec.ts
@@ -144,6 +144,42 @@ describe('Filter Transformer', () => {
value: [env.FilterTransformer.transform(ctx.filter)]
})
})
+ }),
+
+ describe('transform sort', () => {
+ test('should handle wrong sort', () => {
+ expect(env.FilterTransformer.transformSort('')).toEqual([])
+ expect(env.FilterTransformer.transformSort(undefined)).toEqual([])
+ expect(env.FilterTransformer.transformSort(null)).toEqual([])
+ })
+
+ test('transform empty sort', () => {
+ expect(env.FilterTransformer.transformSort([])).toEqual([])
+ })
+
+ test('transform sort', () => {
+ const sort = [
+ { fieldName: ctx.fieldName, order: 'ASC' },
+ ]
+ expect(env.FilterTransformer.transformSort(sort)).toEqual([{
+ fieldName: ctx.fieldName,
+ direction: 'asc'
+ }])
+ })
+
+ test('transform sort with multiple fields', () => {
+ const sort = [
+ { fieldName: ctx.fieldName, order: 'ASC' },
+ { fieldName: ctx.anotherFieldName, order: 'DESC' },
+ ]
+ expect(env.FilterTransformer.transformSort(sort)).toEqual([{
+ fieldName: ctx.fieldName,
+ direction: 'asc'
+ }, {
+ fieldName: ctx.anotherFieldName,
+ direction: 'desc'
+ }])
+ })
})
interface Enviorment {
@@ -158,6 +194,7 @@ describe('Filter Transformer', () => {
filter: Uninitialized,
anotherFilter: Uninitialized,
fieldName: Uninitialized,
+ anotherFieldName: Uninitialized,
fieldValue: Uninitialized,
operator: Uninitialized,
fieldListValue: Uninitialized,
@@ -168,6 +205,7 @@ describe('Filter Transformer', () => {
ctx.filter = gen.randomFilter()
ctx.anotherFilter = gen.randomFilter()
ctx.fieldName = chance.word()
+ ctx.anotherFieldName = chance.word()
ctx.fieldValue = chance.word()
ctx.operator = gen.randomOperator() as WixDataMultiFieldOperators | WixDataSingleFieldOperators
ctx.fieldListValue = [chance.word(), chance.word(), chance.word(), chance.word(), chance.word()]
diff --git a/libs/velo-external-db-core/src/converters/filter_transformer.ts b/libs/velo-external-db-core/src/converters/filter_transformer.ts
index 2d8b1017c..608dbfb49 100644
--- a/libs/velo-external-db-core/src/converters/filter_transformer.ts
+++ b/libs/velo-external-db-core/src/converters/filter_transformer.ts
@@ -1,7 +1,8 @@
import { AdapterOperators, isObject, patchVeloDateValue } from '@wix-velo/velo-external-db-commons'
import { EmptyFilter } from './utils'
import { errors } from '@wix-velo/velo-external-db-commons'
-import { AdapterFilter, AdapterOperator, WixDataFilter, WixDataMultiFieldOperators, } from '@wix-velo/velo-external-db-types'
+import { AdapterFilter, AdapterOperator, Sort, WixDataFilter, WixDataMultiFieldOperators, } from '@wix-velo/velo-external-db-types'
+import { Sorting } from '../spi-model/data_source'
const { InvalidQuery } = errors
export interface IFilterTransformer {
@@ -41,6 +42,19 @@ export default class FilterTransformer implements IFilterTransformer {
}
}
+ transformSort(sort: any): Sort[] {
+ if (!this.isSortArray(sort)) {
+ return []
+ }
+
+ return (sort as Sorting[]).map(sorting => {
+ return {
+ fieldName: sorting.fieldName,
+ direction: sorting.order.toLowerCase() as 'asc' | 'desc'
+ }
+ })
+ }
+
isMultipleFieldOperator(filter: WixDataFilter) {
return (<any>Object).values(WixDataMultiFieldOperators).includes(Object.keys(filter)[0])
}
@@ -90,4 +104,17 @@ export default class FilterTransformer implements IFilterTransformer {
return (!filter || !isObject(filter) || Object.keys(filter)[0] === undefined)
}
+ isSortArray(sort: any): boolean {
+
+ if (!Array.isArray(sort)) {
+ return false
+ }
+ return sort.every((s: any) => {
+ return this.isSortObject(s)
+ })
+ }
+
+ isSortObject(sort:any): boolean {
+ return sort.fieldName && sort.order
+ }
}
diff --git a/libs/velo-external-db-core/src/converters/query_validator.spec.ts b/libs/velo-external-db-core/src/converters/query_validator.spec.ts
index fe341e9a7..38d3c6044 100644
--- a/libs/velo-external-db-core/src/converters/query_validator.spec.ts
+++ b/libs/velo-external-db-core/src/converters/query_validator.spec.ts
@@ -6,7 +6,7 @@ import { queryAdapterOperatorsFor } from './query_validator_utils'
import QueryValidator from './query_validator'
import Chance = require('chance')
const chance = Chance()
-const { InvalidQuery } = errors
+const { FieldDoesNotExist, InvalidQuery } = errors
describe('Query Validator', () => {
beforeAll(() => {
@@ -24,14 +24,14 @@ describe('Query Validator', () => {
})
- test('will throw InvalidQuery if filter fields doesn\'t exist', () => {
+ test('will throw FieldDoesNotExist if filter fields doesn\'t exist', () => {
const filter = {
fieldName: 'wrong',
operator: ctx.validOperatorForType,
value: ctx.value
}
- expect ( () => env.queryValidator.validateFilter([{ field: ctx.fieldName, type: ctx.type }], filter)).toThrow(InvalidQuery)
+ expect ( () => env.queryValidator.validateFilter([{ field: ctx.fieldName, type: ctx.type }], filter)).toThrow(FieldDoesNotExist)
})
test('will not throw if use allowed operator for type', () => {
@@ -68,7 +68,7 @@ describe('Query Validator', () => {
})
test('should throw Invalid if _id fields doesn\'t exist', () => {
- expect ( () => env.queryValidator.validateGetById([{ field: ctx.fieldName, type: ctx.type }], '0')).toThrow(InvalidQuery)
+ expect ( () => env.queryValidator.validateGetById([{ field: ctx.fieldName, type: ctx.type }], '0')).toThrow(FieldDoesNotExist)
})
})
@@ -114,7 +114,7 @@ describe('Query Validator', () => {
expect ( () => env.queryValidator.validateAggregation([{ field: ctx.fieldName, type: ctx.type }], aggregation)).not.toThrow()
})
- test('will throw Invalid Query with filter on non exist field', () => {
+ test('will throw FieldDoesNotExist with filter on non exist field', () => {
const aggregation = {
projection: [{ name: ctx.fieldName, alias: ctx.anotherFieldName }],
postFilter: {
@@ -123,15 +123,15 @@ describe('Query Validator', () => {
value: ctx.value
}
}
- expect ( () => env.queryValidator.validateAggregation([{ field: ctx.fieldName, type: ctx.type }], aggregation)).toThrow(InvalidQuery)
+ expect ( () => env.queryValidator.validateAggregation([{ field: ctx.fieldName, type: ctx.type }], aggregation)).toThrow(FieldDoesNotExist)
})
- test('will throw Invalid Query with projection with non exist field', () => {
+ test('will throw FieldDoesNotExist with projection with non exist field', () => {
const aggregation = {
projection: [{ name: 'wrong' }],
postFilter: EmptyFilter
}
- expect ( () => env.queryValidator.validateAggregation([{ field: ctx.fieldName, type: ctx.type }], aggregation)).toThrow(InvalidQuery)
+ expect ( () => env.queryValidator.validateAggregation([{ field: ctx.fieldName, type: ctx.type }], aggregation)).toThrow(FieldDoesNotExist)
})
})
@@ -142,9 +142,9 @@ describe('Query Validator', () => {
expect ( () => env.queryValidator.validateProjection([{ field: ctx.fieldName, type: ctx.type }], projection)).not.toThrow()
})
- test('will throw Invalid Query if projection fields doesn\'t exist', () => {
+ test('will throw FieldDoesNotExist if projection fields doesn\'t exist', () => {
const projection = ['wrong']
- expect ( () => env.queryValidator.validateProjection([{ field: ctx.fieldName, type: ctx.type }], projection)).toThrow(InvalidQuery)
+ expect ( () => env.queryValidator.validateProjection([{ field: ctx.fieldName, type: ctx.type }], projection)).toThrow(FieldDoesNotExist)
})
})
diff --git a/libs/velo-external-db-core/src/converters/query_validator.ts b/libs/velo-external-db-core/src/converters/query_validator.ts
index e5874af96..fe6ab2f3c 100644
--- a/libs/velo-external-db-core/src/converters/query_validator.ts
+++ b/libs/velo-external-db-core/src/converters/query_validator.ts
@@ -7,11 +7,11 @@ export default class QueryValidator {
constructor() {
}
- validateFilter(fields: ResponseField[], filter: AdapterFilter ) {
+ validateFilter(fields: ResponseField[], filter: AdapterFilter, collectionName?: string) {
const filterFieldsAndOpsObj = extractFieldsAndOperators(filter)
const filterFields = filterFieldsAndOpsObj.map((f: { name: string }) => f.name)
const fieldNames = fields.map((f: ResponseField) => f.field)
- this.validateFieldsExists(fieldNames, filterFields)
+ this.validateFieldsExists(fieldNames, filterFields, collectionName)
this.validateOperators(fields, filterFieldsAndOpsObj)
}
@@ -33,11 +33,11 @@ export default class QueryValidator {
this.validateFieldsExists(fieldNames, projectionFields)
}
- validateFieldsExists(allFields: string | any[], queryFields: any[]) {
+ validateFieldsExists(allFields: string | any[], queryFields: any[], collectionName?: string) {
const nonExistentFields = queryFields.filter((field: any) => !allFields.includes(field))
if (nonExistentFields.length) {
- throw new InvalidQuery(`fields ${nonExistentFields.join(', ')} don't exist`)
+ throw new errors.FieldDoesNotExist(`fields [${nonExistentFields.join(', ')}] don't exist`, collectionName, nonExistentFields[0])
}
}
diff --git a/libs/velo-external-db-core/src/converters/query_validator_utils.spec.ts b/libs/velo-external-db-core/src/converters/query_validator_utils.spec.ts
index accd47f89..7bc9b3ae2 100644
--- a/libs/velo-external-db-core/src/converters/query_validator_utils.spec.ts
+++ b/libs/velo-external-db-core/src/converters/query_validator_utils.spec.ts
@@ -39,6 +39,15 @@ describe('Query Validator utils spec', () => {
})
).toEqual([{ name: ctx.fieldName, operator: ctx.operator }, { name: ctx.anotherFieldName, operator: ctx.anotherOperator }])
})
+
+ test('correctly extract fields and operators with nested field filter', () => {
+ expect(extractFieldsAndOperators({
+ fieldName: `${ctx.fieldName}.whatEver.nested`,
+ operator: ctx.operator,
+ value: ctx.value
+ })
+ ).toEqual([{ name: ctx.fieldName, operator: ctx.operator }])
+ })
})
describe ('queryAdapterOperatorsFor', () => {
diff --git a/libs/velo-external-db-core/src/converters/query_validator_utils.ts b/libs/velo-external-db-core/src/converters/query_validator_utils.ts
index cfe1ab44e..74d8eda32 100644
--- a/libs/velo-external-db-core/src/converters/query_validator_utils.ts
+++ b/libs/velo-external-db-core/src/converters/query_validator_utils.ts
@@ -9,7 +9,7 @@ export const queryAdapterOperatorsFor = (type: string) => ( (QueryOperatorsByFie
export const extractFieldsAndOperators = (_filter: AdapterFilter): { name: string, operator: AdapterOperator }[] => {
if (_filter === EmptyFilter) return []
const filter = _filter as NotEmptyAdapterFilter
- if (filter.fieldName) return [{ name: filter.fieldName, operator: filter.operator as AdapterOperator }]
+ if (filter.fieldName) return [{ name: filter.fieldName.split('.')[0], operator: filter.operator as AdapterOperator }]
return filter.value.map((filter: any) => extractFieldsAndOperators(filter)).flat()
}
diff --git a/libs/velo-external-db-core/src/converters/utils.ts b/libs/velo-external-db-core/src/converters/utils.ts
index 141313312..18fdf0afa 100644
--- a/libs/velo-external-db-core/src/converters/utils.ts
+++ b/libs/velo-external-db-core/src/converters/utils.ts
@@ -8,10 +8,8 @@ export const projectionFieldFor = (fieldName: any, fieldAlias?: string) => {
}
export const projectionFunctionFor = (fieldName: string | number, fieldAlias: any, func: any) => {
- if (isCountFunc(func, fieldName))
+ if (func === AdapterFunctions.count)
return { alias: fieldAlias, function: AdapterFunctions.count, name: '*' }
- const name = (fieldName as string).substring(1)
- return { name, alias: fieldAlias || name, function: func }
+
+ return { name: fieldName as string, alias: fieldAlias || fieldName as string, function: func }
}
-
-const isCountFunc = (func: any, value: any ) => (func === AdapterFunctions.sum && value === 1)
diff --git a/libs/velo-external-db-core/src/data_hooks_utils.spec.ts b/libs/velo-external-db-core/src/data_hooks_utils.spec.ts
index ce7641887..23b2f7a85 100644
--- a/libs/velo-external-db-core/src/data_hooks_utils.spec.ts
+++ b/libs/velo-external-db-core/src/data_hooks_utils.spec.ts
@@ -2,31 +2,35 @@ import each from 'jest-each'
import * as Chance from 'chance'
import { Uninitialized } from '@wix-velo/test-commons'
import { randomBodyWith } from '../test/gen'
-import { DataHooksForAction, DataOperations, dataPayloadFor, DataActions } from './data_hooks_utils'
+import { DataHooksForAction, dataPayloadFor, DataActions } from './data_hooks_utils'
+import { DataOperation } from '@wix-velo/velo-external-db-types'
+
+const { query: Query, insert: Insert, update: Update, remove: Remove, count: Count, aggregate: Aggregate } = DataOperation
+
const chance = Chance()
describe('Hooks Utils', () => {
describe('Hooks For Action', () => {
describe('Before Read', () => {
- each([DataActions.BeforeFind, DataActions.BeforeAggregate, DataActions.BeforeCount, DataActions.BeforeGetById])
+ each([DataActions.BeforeQuery, DataActions.BeforeAggregate, DataActions.BeforeCount])
.test('Hooks action for %s should return appropriate array', (action) => {
expect(DataHooksForAction[action]).toEqual(['beforeAll', 'beforeRead', action])
})
})
describe('After Read', () => {
- each([DataActions.AfterFind, DataActions.AfterAggregate, DataActions.AfterCount, DataActions.AfterGetById])
+ each([DataActions.AfterQuery, DataActions.AfterAggregate, DataActions.AfterCount])
.test('Hooks action for %s should return appropriate array', (action) => {
expect(DataHooksForAction[action]).toEqual(['afterAll', 'afterRead', action])
})
})
describe('Before Write', () => {
- each([DataActions.BeforeInsert, DataActions.BeforeBulkInsert, DataActions.BeforeUpdate, DataActions.BeforeBulkUpdate, DataActions.BeforeRemove, DataActions.BeforeBulkRemove])
+ each([DataActions.BeforeInsert, DataActions.BeforeUpdate, DataActions.BeforeRemove, DataActions.BeforeTruncate])
.test('Hooks action for %s should return appropriate array', (action) => {
expect(DataHooksForAction[action]).toEqual(['beforeAll', 'beforeWrite', action])
})
})
describe('After Write', () => {
- each([DataActions.AfterInsert, DataActions.AfterBulkInsert, DataActions.AfterUpdate, DataActions.AfterBulkUpdate, DataActions.AfterRemove, DataActions.AfterBulkRemove])
+ each([DataActions.AfterInsert, DataActions.AfterUpdate, DataActions.AfterRemove, DataActions.AfterTruncate])
.test('Hooks action for %s should return appropriate array', (action) => {
expect(DataHooksForAction[action]).toEqual(['afterAll', 'afterWrite', action])
})
@@ -34,75 +38,120 @@ describe('Hooks Utils', () => {
})
describe('Payload For', () => {
- test('Payload for Find should return query object', () => {
- expect(dataPayloadFor(DataOperations.Find, randomBodyWith({ filter: ctx.filter, skip: ctx.skip, limit: ctx.limit, sort: ctx.sort, projection: ctx.projection }))).toEqual({
- filter: ctx.filter,
- skip: ctx.skip,
- limit: ctx.limit,
- sort: ctx.sort,
- projection: ctx.projection,
- })
- })
- test('Payload for Insert should return item', () => {
- expect(dataPayloadFor(DataOperations.Insert, randomBodyWith({ item: ctx.item }))).toEqual({ item: ctx.item })
- })
- test('Payload for BulkInsert should return items', () => {
- expect(dataPayloadFor(DataOperations.BulkInsert, randomBodyWith({ items: ctx.items }))).toEqual({ items: ctx.items })
- })
- test('Payload for Update should return item', () => {
- expect(dataPayloadFor(DataOperations.Update, randomBodyWith({ item: ctx.item }))).toEqual({ item: ctx.item })
- })
- test('Payload for BulkUpdate should return items', () => {
- expect(dataPayloadFor(DataOperations.BulkUpdate, randomBodyWith({ items: ctx.items }))).toEqual({ items: ctx.items })
+ test('Payload for Find should return query request object', () => {
+ expect(dataPayloadFor(Query, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ query: ctx.query,
+ includeReferencedItems: ctx.includeReferencedItems,
+ omitTotalCount: ctx.omitTotalCount,
+ options: ctx.options
+ })
})
- test('Payload for Remove should return item id', () => {
- expect(dataPayloadFor(DataOperations.Remove, randomBodyWith({ itemId: ctx.itemId }))).toEqual({ itemId: ctx.itemId })
+
+ test('Payload for Insert should return insert request object', () => {
+ expect(dataPayloadFor(Insert, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ items: ctx.items,
+ overwriteExisting: ctx.overwriteExisting,
+ options: ctx.options
+ })
})
- test('Payload for BulkRemove should return item ids', () => {
- expect(dataPayloadFor(DataOperations.BulkRemove, randomBodyWith({ itemIds: ctx.itemIds }))).toEqual({ itemIds: ctx.itemIds })
+
+ test('Payload for Update should return update request object', () => {
+ expect(dataPayloadFor(Update, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ items: ctx.items,
+ options: ctx.options
+ })
})
- test('Payload for Count should return filter', () => {
- expect(dataPayloadFor(DataOperations.Count, randomBodyWith({ filter: ctx.filter }))).toEqual({ filter: ctx.filter })
+
+ test('Payload for Remove should return remove request object', () => {
+ expect(dataPayloadFor(Remove, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ itemIds: ctx.itemIds,
+ options: ctx.options
+ })
})
- test('Payload for Get should return item id and projection', () => {
- expect(dataPayloadFor(DataOperations.Get, randomBodyWith({ itemId: ctx.itemId, projection: ctx.projection }))).toEqual({ itemId: ctx.itemId, projection: ctx.projection })
+
+ test('Payload for Count should return count request object', () => {
+ expect(dataPayloadFor(Count, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ filter: ctx.filter,
+ options: ctx.options
+ })
})
- test('Payload for Aggregate should return Aggregation query', () => {
- expect(dataPayloadFor(DataOperations.Aggregate, randomBodyWith({ filter: ctx.filter, processingStep: ctx.processingStep, postFilteringStep: ctx.postFilteringStep })))
- .toEqual(
- {
- filter: ctx.filter,
- processingStep: ctx.processingStep,
- postFilteringStep: ctx.postFilteringStep
- }
- )
+
+ test('Payload for Aggregate should return aggregate request object', () => {
+ expect(dataPayloadFor(Aggregate, ctx.bodyWithAllProps))
+ .toEqual({
+ collectionId: ctx.collectionId,
+ namespace: ctx.namespace,
+ initialFilter: ctx.initialFilter,
+ distinct: ctx.distinct,
+ group: ctx.group,
+ finalFilter: ctx.finalFilter,
+ sort: ctx.sort,
+ paging: ctx.paging,
+ cursorPaging: ctx.cursorPaging,
+ options: ctx.options,
+ omitTotalCount: ctx.omitTotalCount
+ })
})
})
const ctx = {
filter: Uninitialized,
- limit: Uninitialized,
- skip: Uninitialized,
sort: Uninitialized,
- projection: Uninitialized,
- item: Uninitialized,
items: Uninitialized,
- itemId: Uninitialized,
itemIds: Uninitialized,
- processingStep: Uninitialized,
- postFilteringStep: Uninitialized
+ group: Uninitialized,
+ finalFilter: Uninitialized,
+ distinct: Uninitialized,
+ paging: Uninitialized,
+ collectionId: Uninitialized,
+ namespace: Uninitialized,
+ query: Uninitialized,
+ includeReferencedItems: Uninitialized,
+ omitTotalCount: Uninitialized,
+ options: Uninitialized,
+ overwriteExisting: Uninitialized,
+ bodyWithAllProps: Uninitialized,
+ cursorPaging: Uninitialized,
+ initialFilter: Uninitialized
}
beforeEach(() => {
ctx.filter = chance.word()
- ctx.limit = chance.word()
- ctx.skip = chance.word()
ctx.sort = chance.word()
- ctx.projection = chance.word()
- ctx.item = chance.word()
ctx.items = chance.word()
- ctx.itemId = chance.word()
ctx.itemIds = chance.word()
+ ctx.group = chance.word()
+ ctx.finalFilter = chance.word()
+ ctx.distinct = chance.word()
+ ctx.paging = chance.word()
+ ctx.collectionId = chance.word()
+ ctx.namespace = chance.word()
+ ctx.query = chance.word()
+ ctx.includeReferencedItems = chance.word()
+ ctx.omitTotalCount = chance.word()
+ ctx.options = chance.word()
+ ctx.overwriteExisting = chance.word()
+ ctx.cursorPaging = chance.word()
+ ctx.initialFilter = chance.word()
+ ctx.bodyWithAllProps = randomBodyWith({
+ ...ctx
+ })
+
})
})
diff --git a/libs/velo-external-db-core/src/data_hooks_utils.ts b/libs/velo-external-db-core/src/data_hooks_utils.ts
index 04969ca48..e1ee904c3 100644
--- a/libs/velo-external-db-core/src/data_hooks_utils.ts
+++ b/libs/velo-external-db-core/src/data_hooks_utils.ts
@@ -1,109 +1,115 @@
-import { Item, WixDataFilter } from '@wix-velo/velo-external-db-types'
-import { AggregationQuery, FindQuery, RequestContext } from './types'
-
+import { DataOperation } from '@wix-velo/velo-external-db-types'
+import { AggregateRequest, CountRequest, InsertRequest, QueryRequest } from './spi-model/data_source'
+import { RequestContext } from './types'
export const DataHooksForAction: { [key: string]: string[] } = {
- beforeFind: ['beforeAll', 'beforeRead', 'beforeFind'],
- afterFind: ['afterAll', 'afterRead', 'afterFind'],
+ beforeQuery: ['beforeAll', 'beforeRead', 'beforeQuery'],
+ afterQuery: ['afterAll', 'afterRead', 'afterQuery'],
+ beforeCount: ['beforeAll', 'beforeRead', 'beforeCount'],
+ afterCount: ['afterAll', 'afterRead', 'afterCount'],
+ beforeAggregate: ['beforeAll', 'beforeRead', 'beforeAggregate'],
+ afterAggregate: ['afterAll', 'afterRead', 'afterAggregate'],
beforeInsert: ['beforeAll', 'beforeWrite', 'beforeInsert'],
afterInsert: ['afterAll', 'afterWrite', 'afterInsert'],
- beforeBulkInsert: ['beforeAll', 'beforeWrite', 'beforeBulkInsert'],
- afterBulkInsert: ['afterAll', 'afterWrite', 'afterBulkInsert'],
beforeUpdate: ['beforeAll', 'beforeWrite', 'beforeUpdate'],
afterUpdate: ['afterAll', 'afterWrite', 'afterUpdate'],
- beforeBulkUpdate: ['beforeAll', 'beforeWrite', 'beforeBulkUpdate'],
- afterBulkUpdate: ['afterAll', 'afterWrite', 'afterBulkUpdate'],
beforeRemove: ['beforeAll', 'beforeWrite', 'beforeRemove'],
afterRemove: ['afterAll', 'afterWrite', 'afterRemove'],
- beforeBulkRemove: ['beforeAll', 'beforeWrite', 'beforeBulkRemove'],
- afterBulkRemove: ['afterAll', 'afterWrite', 'afterBulkRemove'],
- beforeAggregate: ['beforeAll', 'beforeRead', 'beforeAggregate'],
- afterAggregate: ['afterAll', 'afterRead', 'afterAggregate'],
- beforeCount: ['beforeAll', 'beforeRead', 'beforeCount'],
- afterCount: ['afterAll', 'afterRead', 'afterCount'],
- beforeGetById: ['beforeAll', 'beforeRead', 'beforeGetById'],
- afterGetById: ['afterAll', 'afterRead', 'afterGetById'],
+ beforeTruncate: ['beforeAll', 'beforeWrite', 'beforeTruncate'],
+ afterTruncate: ['afterAll', 'afterWrite', 'afterTruncate'],
}
-
-export enum DataOperations {
- Find = 'find',
- Insert = 'insert',
- BulkInsert = 'bulkInsert',
- Update = 'update',
- BulkUpdate = 'bulkUpdate',
- Remove = 'remove',
- BulkRemove = 'bulkRemove',
- Aggregate = 'aggregate',
- Count = 'count',
- Get = 'getById',
+export enum DataActions {
+ BeforeQuery = 'beforeQuery',
+ AfterQuery = 'afterQuery',
+ BeforeCount = 'beforeCount',
+ AfterCount = 'afterCount',
+ BeforeAggregate = 'beforeAggregate',
+ AfterAggregate = 'afterAggregate',
+ BeforeInsert = 'beforeInsert',
+ AfterInsert = 'afterInsert',
+ BeforeUpdate = 'beforeUpdate',
+ AfterUpdate = 'afterUpdate',
+ BeforeRemove = 'beforeRemove',
+ AfterRemove = 'afterRemove',
+ BeforeTruncate = 'beforeTruncate',
+ AfterTruncate = 'afterTruncate',
}
-export const DataActions = {
- BeforeFind: 'beforeFind',
- AfterFind: 'afterFind',
- BeforeInsert: 'beforeInsert',
- AfterInsert: 'afterInsert',
- BeforeBulkInsert: 'beforeBulkInsert',
- AfterBulkInsert: 'afterBulkInsert',
- BeforeUpdate: 'beforeUpdate',
- AfterUpdate: 'afterUpdate',
- BeforeBulkUpdate: 'beforeBulkUpdate',
- AfterBulkUpdate: 'afterBulkUpdate',
- BeforeRemove: 'beforeRemove',
- AfterRemove: 'afterRemove',
- BeforeBulkRemove: 'beforeBulkRemove',
- AfterBulkRemove: 'afterBulkRemove',
- BeforeAggregate: 'beforeAggregate',
- AfterAggregate: 'afterAggregate',
- BeforeCount: 'beforeCount',
- AfterCount: 'afterCount',
- BeforeGetById: 'beforeGetById',
- AfterGetById: 'afterGetById',
- BeforeAll: 'beforeAll',
- AfterAll: 'afterAll',
- BeforeRead: 'beforeRead',
- AfterRead: 'afterRead',
- BeforeWrite: 'beforeWrite',
- AfterWrite: 'afterWrite'
-}
-export const dataPayloadFor = (operation: DataOperations, body: any) => {
+export const dataPayloadFor = (operation: DataOperation, body: any) => {
switch (operation) {
- case DataOperations.Find:
+ case DataOperation.query:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ query: body.query,
+ includeReferencedItems: body.includeReferencedItems, // not supported
+ omitTotalCount: body.omitTotalCount,
+ options: body.options // not supported
+ } as QueryRequest
+ case DataOperation.count:
return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
filter: body.filter,
- limit: body.limit,
- skip: body.skip,
+ options: body.options // not supported
+ } as CountRequest
+ case DataOperation.aggregate:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ initialFilter: body.initialFilter,
+ distinct: body.distinct, // not supported
+ group: body.group,
+ finalFilter: body.finalFilter,
sort: body.sort,
- projection: body.projection
- } as FindQuery
- case DataOperations.Insert:
- case DataOperations.Update:
- return { item: body.item as Item }
- case DataOperations.BulkInsert:
- case DataOperations.BulkUpdate:
- return { items: body.items as Item[] }
- case DataOperations.Get:
- return { itemId: body.itemId, projection: body.projection }
- case DataOperations.Remove:
- return { itemId: body.itemId as string }
- case DataOperations.BulkRemove:
- return { itemIds: body.itemIds as string[] }
- case DataOperations.Aggregate:
+ paging: body.paging,
+ options: body.options, // not supported
+ omitTotalCount: body.omitTotalCount,
+ cursorPaging: body.cursorPaging, // not supported
+ } as AggregateRequest
+
+ case DataOperation.insert:
return {
- filter: body.filter,
- processingStep: body.processingStep,
- postFilteringStep: body.postFilteringStep
- } as AggregationQuery
- case DataOperations.Count:
- return { filter: body.filter as WixDataFilter }
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ items: body.items,
+ overwriteExisting: body.overwriteExisting,
+ options: body.options, // not supported
+ } as InsertRequest
+ case DataOperation.update:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ items: body.items,
+ options: body.options, // not supported
+ }
+ case DataOperation.remove:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ itemIds: body.itemIds,
+ options: body.options, // not supported
+ }
+ case DataOperation.truncate:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ options: body.options, // not supported
+ }
+ default:
+ return {
+ collectionId: body.collectionId,
+ namespace: body.namespace, // not supported
+ options: body.options, // not supported
+ }
}
}
export const requestContextFor = (operation: any, body: any): RequestContext => ({
operation,
- collectionName: body.collectionName,
+ collectionId: body.collectionId,
instanceId: body.requestContext.instanceId,
memberId: body.requestContext.memberId,
role: body.requestContext.role,
diff --git a/libs/velo-external-db-core/src/index.ts b/libs/velo-external-db-core/src/index.ts
index d0d1caca8..c70b8b8ae 100644
--- a/libs/velo-external-db-core/src/index.ts
+++ b/libs/velo-external-db-core/src/index.ts
@@ -16,6 +16,8 @@ import { RoleAuthorizationService } from '@wix-velo/external-db-security'
import { ConfigValidator, AuthorizationConfigValidator, CommonConfigValidator } from '@wix-velo/external-db-config'
import { ConnectionCleanUp } from '@wix-velo/velo-external-db-types'
import { Router } from 'express'
+import { CollectionCapability } from './spi-model/capabilities'
+import { decodeBase64 } from './utils/base64_utils'
export class ExternalDbRouter {
connector: DbConnector
@@ -36,7 +38,7 @@ export class ExternalDbRouter {
constructor({ connector, config, hooks }: { connector: DbConnector, config: ExternalDbRouterConfig, hooks: {schemaHooks?: SchemaHooks, dataHooks?: DataHooks}}) {
this.isInitialized(connector)
this.connector = connector
- this.configValidator = new ConfigValidator(connector.configValidator, new AuthorizationConfigValidator(config.authorization), new CommonConfigValidator({ secretKey: config.secretKey, vendor: config.vendor, type: config.adapterType }, config.commonExtended))
+ this.configValidator = new ConfigValidator(connector.configValidator, new AuthorizationConfigValidator(config.authorization), new CommonConfigValidator({ externalDatabaseId: config.externalDatabaseId, allowedMetasites: config.allowedMetasites, vendor: config.vendor, type: config.adapterType }, config.commonExtended))
this.config = config
this.operationService = new OperationService(connector.databaseOperations)
this.schemaInformation = new CacheableSchemaInformation(connector.schemaProvider)
@@ -66,4 +68,8 @@ export class ExternalDbRouter {
}
}
-export { DataService, SchemaService, OperationService, CacheableSchemaInformation, FilterTransformer, AggregationTransformer, QueryValidator, SchemaAwareDataService, ItemTransformer, Hooks, ServiceContext }
+export * as types from './types'
+export * as dataSpi from './spi-model/data_source'
+export * as collectionSpi from './spi-model/collection'
+export * as schemaUtils from '../src/utils/schema_utils'
+export { DataService, SchemaService, OperationService, CacheableSchemaInformation, FilterTransformer, AggregationTransformer, QueryValidator, SchemaAwareDataService, ItemTransformer, Hooks, ServiceContext, CollectionCapability, decodeBase64 }
diff --git a/libs/velo-external-db-core/src/router.ts b/libs/velo-external-db-core/src/router.ts
index 6fc49d8b1..5d13c0f74 100644
--- a/libs/velo-external-db-core/src/router.ts
+++ b/libs/velo-external-db-core/src/router.ts
@@ -1,35 +1,38 @@
import * as path from 'path'
import * as BPromise from 'bluebird'
import * as express from 'express'
+import type { Response } from 'express'
import * as compression from 'compression'
import { errorMiddleware } from './web/error-middleware'
import { appInfoFor } from './health/app_info'
import { errors } from '@wix-velo/velo-external-db-commons'
-import { extractRole } from './web/auth-role-middleware'
+
import { config } from './roles-config.json'
-import { secretKeyAuthMiddleware } from './web/auth-middleware'
import { authRoleMiddleware } from './web/auth-role-middleware'
import { unless, includes } from './web/middleware-support'
import { getAppInfoPage } from './utils/router_utils'
-import { DataHooksForAction, DataOperations, dataPayloadFor, DataActions, requestContextFor } from './data_hooks_utils'
-import { SchemaHooksForAction, SchemaOperations, schemaPayloadFor, SchemaActions } from './schema_hooks_utils'
+import { requestContextFor, DataActions, dataPayloadFor, DataHooksForAction } from './data_hooks_utils'
+// import { SchemaHooksForAction } from './schema_hooks_utils'
import SchemaService from './service/schema'
import OperationService from './service/operation'
-import { AnyFixMe } from '@wix-velo/velo-external-db-types'
+import { AnyFixMe, DataOperation, Item } from '@wix-velo/velo-external-db-types'
import SchemaAwareDataService from './service/schema_aware_data'
import FilterTransformer from './converters/filter_transformer'
import AggregationTransformer from './converters/aggregation_transformer'
import { RoleAuthorizationService } from '@wix-velo/external-db-security'
import { DataHooks, Hooks, RequestContext, SchemaHooks, ServiceContext } from './types'
import { ConfigValidator } from '@wix-velo/external-db-config'
+import { JwtAuthenticator } from './web/jwt-auth-middleware'
+import * as dataSource from './spi-model/data_source'
+import * as capabilities from './spi-model/capabilities'
+import { WixDataFacade } from './web/wix_data_facade'
-const { InvalidRequest, ItemNotFound } = errors
-const { Find: FIND, Insert: INSERT, BulkInsert: BULK_INSERT, Update: UPDATE, BulkUpdate: BULK_UPDATE, Remove: REMOVE, BulkRemove: BULK_REMOVE, Aggregate: AGGREGATE, Count: COUNT, Get: GET } = DataOperations
+const { query: Query, count: Count, aggregate: Aggregate, insert: Insert, update: Update, remove: Remove, truncate: Truncate } = DataOperation
-let schemaService: SchemaService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: { secretKey?: any; type?: any; vendor?: any, hideAppInfo?: boolean }, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, roleAuthorizationService: RoleAuthorizationService, dataHooks: DataHooks, schemaHooks: SchemaHooks
+let schemaService: SchemaService, operationService: OperationService, externalDbConfigClient: ConfigValidator, schemaAwareDataService: SchemaAwareDataService, cfg: { externalDatabaseId: string, allowedMetasites: string, type?: any; vendor?: any, wixDataBaseUrl: string, hideAppInfo?: boolean }, filterTransformer: FilterTransformer, aggregationTransformer: AggregationTransformer, dataHooks: DataHooks //roleAuthorizationService: RoleAuthorizationService, schemaHooks: SchemaHooks,
export const initServices = (_schemaAwareDataService: SchemaAwareDataService, _schemaService: SchemaService, _operationService: OperationService,
- _externalDbConfigClient: ConfigValidator, _cfg: { secretKey?: string, type?: string, vendor?: string, hideAppInfo?: boolean },
+ _externalDbConfigClient: ConfigValidator, _cfg: { externalDatabaseId: string, allowedMetasites: string, type?: string, vendor?: string, wixDataBaseUrl: string, hideAppInfo?: boolean },
_filterTransformer: FilterTransformer, _aggregationTransformer: AggregationTransformer,
_roleAuthorizationService: RoleAuthorizationService, _hooks: Hooks) => {
schemaService = _schemaService
@@ -39,9 +42,9 @@ export const initServices = (_schemaAwareDataService: SchemaAwareDataService, _s
schemaAwareDataService = _schemaAwareDataService
filterTransformer = _filterTransformer
aggregationTransformer = _aggregationTransformer
- roleAuthorizationService = _roleAuthorizationService
+ // roleAuthorizationService = _roleAuthorizationService
dataHooks = _hooks?.dataHooks || {}
- schemaHooks = _hooks?.schemaHooks || {}
+ // schemaHooks = _hooks?.schemaHooks || {}
}
const serviceContext = (): ServiceContext => ({
@@ -56,11 +59,6 @@ const executeDataHooksFor = async(action: string, payload: AnyFixMe, requestCont
}, payload)
}
-const executeSchemaHooksFor = async(action: string, payload: any, requestContext: RequestContext, customContext: any) => {
- return BPromise.reduce(SchemaHooksForAction[action], async(lastHookResult: any, hookName: string) => {
- return await executeHook(schemaHooks, hookName, lastHookResult, requestContext, customContext)
- }, payload)
-}
const executeHook = async(hooks: DataHooks | SchemaHooks, _actionName: string, payload: AnyFixMe, requestContext: RequestContext, customContext: any) => {
const actionName = _actionName as keyof typeof hooks
@@ -71,7 +69,7 @@ const executeHook = async(hooks: DataHooks | SchemaHooks, _actionName: string, p
return payloadAfterHook || payload
} catch (e: any) {
if (e.status) throw e
- throw new InvalidRequest(e.message || e)
+ throw new errors.UnrecognizedError(e.message || e)
}
}
return payload
@@ -82,10 +80,26 @@ export const createRouter = () => {
router.use(express.json())
router.use(compression())
router.use('/assets', express.static(path.join(__dirname, 'assets')))
- router.use(unless(['/', '/provision', '/favicon.ico'], secretKeyAuthMiddleware({ secretKey: cfg.secretKey })))
+ const jwtAuthenticator = new JwtAuthenticator(cfg.externalDatabaseId, cfg.allowedMetasites, new WixDataFacade(cfg.wixDataBaseUrl))
+ router.use(unless(['/', '/info', '/capabilities', '/favicon.ico'], jwtAuthenticator.authorizeJwt()))
config.forEach(({ pathPrefix, roles }) => router.use(includes([pathPrefix], authRoleMiddleware({ roles }))))
+ const streamCollection = (collection: any[], res: Response) => {
+ res.contentType('application/x-ndjson')
+ collection.forEach(item => {
+ res.write(JSON.stringify(item))
+ })
+ res.end()
+ }
+
+ const getItemsOneByOne = (collectionName: string, itemIds: string[]): Promise<any[]> => {
+ const idEqExpression = itemIds.map(itemId => ({ _id: { $eq: itemId } }))
+ return Promise.all(
+ idEqExpression.map(eqExp => schemaAwareDataService.find(collectionName, filterTransformer.transform(eqExp), undefined, 0, 1).then(r => r.items[0]))
+ )
+ }
+
// *************** INFO **********************
router.get('/', async(req, res) => {
const { hideAppInfo } = cfg
@@ -95,117 +109,109 @@ export const createRouter = () => {
res.send(appInfoPage)
})
+ router.get('/capabilities', async(req, res) => {
+ const capabilitiesResponse = {
+ capabilities: {
+ collection: [capabilities.CollectionCapability.CREATE]
+ } as capabilities.Capabilities
+ } as capabilities.GetCapabilitiesResponse
+
+ res.json(capabilitiesResponse)
+ })
+
router.post('/provision', async(req, res) => {
const { type, vendor } = cfg
- res.json({ type, vendor, protocolVersion: 2 })
+ res.json({ type, vendor, protocolVersion: 3, adapterVersion: 'v1' })
+ })
+
+ router.get('/info', async(req, res) => {
+ const { externalDatabaseId } = cfg
+ res.json({ dataSourceId: externalDatabaseId })
})
// *************** Data API **********************
- router.post('/data/find', async(req, res, next) => {
+ router.post('/data/query', async(req, res, next) => {
try {
- const { collectionName } = req.body
const customContext = {}
- const { filter, sort, skip, limit, projection } = await executeDataHooksFor(DataActions.BeforeFind, dataPayloadFor(FIND, req.body), requestContextFor(FIND, req.body), customContext)
+ const { collectionId, query, omitTotalCount } = await executeDataHooksFor(DataActions.BeforeQuery, dataPayloadFor(Query, req.body), requestContextFor(Query, req.body), customContext) as dataSource.QueryRequest
+
+ const offset = query.paging ? query.paging.offset : 0
+ const limit = query.paging ? query.paging.limit : 50
+
+ const data = await schemaAwareDataService.find(
+ collectionId,
+ filterTransformer.transform(query.filter),
+ filterTransformer.transformSort(query.sort),
+ offset,
+ limit,
+ query.fields,
+ omitTotalCount
+ )
- await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.find(collectionName, filterTransformer.transform(filter), sort, skip, limit, projection)
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterQuery, data, requestContextFor(Query, req.body), customContext)
+ const responseParts = dataAfterAction.items.map(dataSource.QueryResponsePart.item)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterFind, data, requestContextFor(FIND, req.body), customContext)
- res.json(dataAfterAction)
+ const metadata = dataSource.QueryResponsePart.pagingMetadata(responseParts.length, offset, dataAfterAction.totalCount)
+
+ streamCollection([...responseParts, ...[metadata]], res)
} catch (e) {
next(e)
}
})
- router.post('/data/aggregate', async(req, res, next) => {
+ router.post('/data/count', async(req, res, next) => {
try {
- const { collectionName } = req.body
const customContext = {}
- const { filter, processingStep, postFilteringStep } = await executeDataHooksFor(DataActions.BeforeAggregate, dataPayloadFor(AGGREGATE, req.body), requestContextFor(AGGREGATE, req.body), customContext)
- await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.aggregate(collectionName, filterTransformer.transform(filter), aggregationTransformer.transform({ processingStep, postFilteringStep }))
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterAggregate, data, requestContextFor(AGGREGATE, req.body), customContext)
- res.json(dataAfterAction)
- } catch (e) {
- next(e)
- }
- })
+ const { collectionId, filter } = await executeDataHooksFor(DataActions.BeforeCount, dataPayloadFor(Count, req.body), requestContextFor(Count, req.body), customContext) as dataSource.CountRequest
+ const data = await schemaAwareDataService.count(
+ collectionId,
+ filterTransformer.transform(filter),
+ )
- router.post('/data/insert', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { item } = await executeDataHooksFor(DataActions.BeforeInsert, dataPayloadFor(INSERT, req.body), requestContextFor(INSERT, req.body), customContext)
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.insert(collectionName, item)
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterCount, data, requestContextFor(Count, req.body), customContext)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterInsert, data, requestContextFor(INSERT, req.body), customContext)
- res.json(dataAfterAction)
+ const response = {
+ totalCount: dataAfterAction.totalCount
+ } as dataSource.CountResponse
+
+ res.json(response)
} catch (e) {
next(e)
}
})
- router.post('/data/insert/bulk', async(req, res, next) => {
+ router.post('/data/insert', async(req, res, next) => {
try {
- const { collectionName } = req.body
const customContext = {}
- const { items } = await executeDataHooksFor(DataActions.BeforeBulkInsert, dataPayloadFor(BULK_INSERT, req.body), requestContextFor(BULK_INSERT, req.body), customContext)
+ const { collectionId, items, overwriteExisting } = await executeDataHooksFor(DataActions.BeforeInsert, dataPayloadFor(Insert, req.body), requestContextFor(Insert, req.body), customContext) as dataSource.InsertRequest
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.bulkInsert(collectionName, items)
+ const data = overwriteExisting ?
+ await schemaAwareDataService.bulkUpsert(collectionId, items) :
+ await schemaAwareDataService.bulkInsert(collectionId, items)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkInsert, data, requestContextFor(BULK_INSERT, req.body), customContext)
- res.json(dataAfterAction)
- } catch (e) {
- next(e)
- }
- })
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterInsert, data, requestContextFor(Insert, req.body), customContext)
+ const responseParts = dataAfterAction.items.map(dataSource.InsertResponsePart.item)
- router.post('/data/get', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { itemId, projection } = await executeDataHooksFor(DataActions.BeforeGetById, dataPayloadFor(GET, req.body), requestContextFor(GET, req.body), customContext)
- await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.getById(collectionName, itemId, projection)
-
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterGetById, data, requestContextFor(GET, req.body), customContext)
- if (!dataAfterAction.item) {
- throw new ItemNotFound('Item not found')
- }
- res.json(dataAfterAction)
+ streamCollection(responseParts, res)
} catch (e) {
next(e)
}
})
router.post('/data/update', async(req, res, next) => {
+
try {
- const { collectionName } = req.body
const customContext = {}
- const { item } = await executeDataHooksFor(DataActions.BeforeUpdate, dataPayloadFor(UPDATE, req.body), requestContextFor(UPDATE, req.body), customContext)
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.update(collectionName, item)
+ const { collectionId, items } = await executeDataHooksFor(DataActions.BeforeUpdate, dataPayloadFor(Update, req.body), requestContextFor(Update, req.body), customContext) as dataSource.UpdateRequest
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterUpdate, data, requestContextFor(UPDATE, req.body), customContext)
- res.json(dataAfterAction)
- } catch (e) {
- next(e)
- }
- })
+ const data = await schemaAwareDataService.bulkUpdate(collectionId, items)
- router.post('/data/update/bulk', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { items } = await executeDataHooksFor(DataActions.BeforeBulkUpdate, dataPayloadFor(BULK_UPDATE, req.body), requestContextFor(BULK_UPDATE, req.body), customContext)
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.bulkUpdate(collectionName, items)
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterUpdate, data, requestContextFor(Update, req.body), customContext)
+
+ const responseParts = dataAfterAction.items.map(dataSource.UpdateResponsePart.item)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkUpdate, data, requestContextFor(BULK_UPDATE, req.body), customContext)
- res.json(dataAfterAction)
+ streamCollection(responseParts, res)
} catch (e) {
next(e)
}
@@ -213,151 +219,104 @@ export const createRouter = () => {
router.post('/data/remove', async(req, res, next) => {
try {
- const { collectionName } = req.body
const customContext = {}
- const { itemId } = await executeDataHooksFor(DataActions.BeforeRemove, dataPayloadFor(REMOVE, req.body), requestContextFor(REMOVE, req.body), customContext)
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.delete(collectionName, itemId)
+ const { collectionId, itemIds } = await executeDataHooksFor(DataActions.BeforeRemove, dataPayloadFor(Remove, req.body), requestContextFor(Remove, req.body), customContext) as dataSource.RemoveRequest
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterRemove, data, requestContextFor(REMOVE, req.body), customContext)
- res.json(dataAfterAction)
- } catch (e) {
- next(e)
- }
- })
+ const objectsBeforeRemove = await getItemsOneByOne(collectionId, itemIds)
- router.post('/data/remove/bulk', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { itemIds } = await executeDataHooksFor(DataActions.BeforeBulkRemove, dataPayloadFor(BULK_REMOVE, req.body), requestContextFor(BULK_REMOVE, req.body), customContext)
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.bulkDelete(collectionName, itemIds)
+ await schemaAwareDataService.bulkDelete(collectionId, itemIds)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterBulkRemove, data, requestContextFor(BULK_REMOVE, req.body), customContext)
- res.json(dataAfterAction)
- } catch (e) {
- next(e)
- }
- })
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterRemove, { items: objectsBeforeRemove }, requestContextFor(Remove, req.body), customContext)
- router.post('/data/count', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { filter } = await executeDataHooksFor(DataActions.BeforeCount, dataPayloadFor(COUNT, req.body), requestContextFor(COUNT, req.body), customContext)
- await roleAuthorizationService.authorizeRead(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.count(collectionName, filterTransformer.transform(filter))
+ const responseParts = dataAfterAction.items.map(dataSource.RemoveResponsePart.item)
- const dataAfterAction = await executeDataHooksFor(DataActions.AfterCount, data, requestContextFor(COUNT, req.body), customContext)
- res.json(dataAfterAction)
+ streamCollection(responseParts, res)
} catch (e) {
next(e)
}
})
- router.post('/data/truncate', async(req, res, next) => {
+ router.post('/data/aggregate', async(req, res, next) => {
try {
- const { collectionName } = req.body
- await roleAuthorizationService.authorizeWrite(collectionName, extractRole(req.body))
- const data = await schemaAwareDataService.truncate(collectionName)
- res.json(data)
- } catch (e) {
- next(e)
- }
- })
- // ***********************************************
+ const customContext = {}
+ const { collectionId, initialFilter, group, finalFilter, sort, paging } = await executeDataHooksFor(DataActions.BeforeAggregate, dataPayloadFor(Aggregate, req.body), requestContextFor(Aggregate, req.body), customContext) as dataSource.AggregateRequest
+ const offset = paging ? paging.offset : 0
+ const limit = paging ? paging.limit : 50
- // *************** Schema API **********************
- router.post('/schemas/list', async(req, res, next) => {
- try {
- const customContext = {}
- await executeSchemaHooksFor(SchemaActions.BeforeList, schemaPayloadFor(SchemaOperations.List, req.body), requestContextFor(SchemaOperations.List, req.body), customContext)
+ const data = await schemaAwareDataService.aggregate(collectionId, filterTransformer.transform(initialFilter), aggregationTransformer.transform({ group, finalFilter }), filterTransformer.transformSort(sort), offset, limit)
+
+ const dataAfterAction = await executeDataHooksFor(DataActions.AfterAggregate, data, requestContextFor(Aggregate, req.body), customContext)
- const data = await schemaService.list()
+ const responseParts = dataAfterAction.items.map(dataSource.AggregateResponsePart.item)
+ const metadata = dataSource.AggregateResponsePart.pagingMetadata((dataAfterAction.items as Item[]).length, offset, data.totalCount)
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterList, data, requestContextFor(SchemaOperations.List, req.body), customContext)
- res.json(dataAfterAction)
+ streamCollection([...responseParts, ...[metadata]], res)
} catch (e) {
next(e)
}
})
- router.post('/schemas/list/headers', async(req, res, next) => {
+ router.post('/data/truncate', async(req, res, next) => {
try {
const customContext = {}
- await executeSchemaHooksFor(SchemaActions.BeforeListHeaders, schemaPayloadFor(SchemaOperations.ListHeaders, req.body), requestContextFor(SchemaOperations.ListHeaders, req.body), customContext)
- const data = await schemaService.listHeaders()
-
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterListHeaders, data, requestContextFor(SchemaOperations.ListHeaders, req.body), customContext)
- res.json(dataAfterAction)
+ const { collectionId } = await executeDataHooksFor(DataActions.BeforeTruncate, dataPayloadFor(Truncate, req.body), requestContextFor(Truncate, req.body), customContext) as dataSource.TruncateRequest
+ await schemaAwareDataService.truncate(collectionId)
+ await executeDataHooksFor(DataActions.AfterTruncate, {}, requestContextFor(Truncate, req.body), customContext)
+ res.json({} as dataSource.TruncateResponse)
} catch (e) {
next(e)
}
})
+ // ***********************************************
+
+ // *************** Collections API **********************
+
+ router.post('/collections/get', async(req, res, next) => {
- router.post('/schemas/find', async(req, res, next) => {
+ const { collectionIds } = req.body
try {
- const customContext = {}
- const { schemaIds } = await executeSchemaHooksFor(SchemaActions.BeforeFind, schemaPayloadFor(SchemaOperations.Find, req.body), requestContextFor(SchemaOperations.Find, req.body), customContext)
-
- if (schemaIds && schemaIds.length > 10) {
- throw new InvalidRequest('Too many schemas requested')
- }
- const data = await schemaService.find(schemaIds)
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterFind, data, requestContextFor(SchemaOperations.Find, req.body), customContext)
- res.json(dataAfterAction)
+ const data = await schemaService.list(collectionIds)
+ streamCollection(data.collection, res)
} catch (e) {
next(e)
}
})
- router.post('/schemas/create', async(req, res, next) => {
- try {
- const customContext = {}
- const { collectionName } = await executeSchemaHooksFor(SchemaActions.BeforeCreate, schemaPayloadFor(SchemaOperations.Create, req.body), requestContextFor(SchemaOperations.Create, req.body), customContext)
- const data = await schemaService.create(collectionName)
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterCreate, data, requestContextFor(SchemaOperations.Create, req.body), customContext)
+ router.post('/collections/create', async(req, res, next) => {
+ const { collection } = req.body
- res.json(dataAfterAction)
+ try {
+ const data = await schemaService.create(collection)
+ streamCollection([data.collection], res)
} catch (e) {
next(e)
}
})
- router.post('/schemas/column/add', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { column } = await executeSchemaHooksFor(SchemaActions.BeforeColumnAdd, schemaPayloadFor(SchemaOperations.ColumnAdd, req.body), requestContextFor(SchemaOperations.ColumnAdd, req.body), customContext)
-
- const data = await schemaService.addColumn(collectionName, column)
-
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterColumnAdd, data, requestContextFor(SchemaOperations.ColumnAdd, req.body), customContext)
+ router.post('/collections/update', async(req, res, next) => {
+ const { collection } = req.body
- res.json(dataAfterAction)
+ try {
+ const data = await schemaService.update(collection)
+ streamCollection([data.collection], res)
} catch (e) {
next(e)
}
})
- router.post('/schemas/column/remove', async(req, res, next) => {
- try {
- const { collectionName } = req.body
- const customContext = {}
- const { columnName } = await executeSchemaHooksFor(SchemaActions.BeforeColumnRemove, schemaPayloadFor(SchemaOperations.ColumnRemove, req.body), requestContextFor(SchemaOperations.ColumnRemove, req.body), customContext)
-
- const data = await schemaService.removeColumn(collectionName, columnName)
+ router.post('/collections/delete', async(req, res, next) => {
+ const { collectionId } = req.body
- const dataAfterAction = await executeSchemaHooksFor(SchemaActions.AfterColumnRemove, data, requestContextFor(SchemaOperations.ColumnRemove, req.body), customContext)
- res.json(dataAfterAction)
+ try {
+ const data = await schemaService.delete(collectionId)
+ streamCollection([data.collection], res)
} catch (e) {
next(e)
}
})
- // ***********************************************
+
router.use(errorMiddleware)
diff --git a/libs/velo-external-db-core/src/service/data.spec.ts b/libs/velo-external-db-core/src/service/data.spec.ts
index 4e8fbfafc..f498497a2 100644
--- a/libs/velo-external-db-core/src/service/data.spec.ts
+++ b/libs/velo-external-db-core/src/service/data.spec.ts
@@ -95,9 +95,10 @@ describe('Data Service', () => {
})
test('aggregate api', async() => {
- driver.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation)
+ driver.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)
+ driver.givenCountResult(ctx.total, ctx.collectionName, ctx.filter)
- return expect(env.dataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation)).resolves.toEqual({ items: ctx.entities, totalCount: 0 })
+ return expect(env.dataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({ items: ctx.entities, totalCount: ctx.total })
})
diff --git a/libs/velo-external-db-core/src/service/data.ts b/libs/velo-external-db-core/src/service/data.ts
index 05b8956a7..aa634437c 100644
--- a/libs/velo-external-db-core/src/service/data.ts
+++ b/libs/velo-external-db-core/src/service/data.ts
@@ -1,4 +1,4 @@
-import { AdapterAggregation as Aggregation, AdapterFilter as Filter, IDataProvider, Item, ResponseField } from '@wix-velo/velo-external-db-types'
+import { AdapterAggregation as Aggregation, AdapterFilter as Filter, IDataProvider, Item, ResponseField, Sort } from '@wix-velo/velo-external-db-types'
import { asWixData } from '../converters/data_utils'
import { getByIdFilterFor } from '../utils/data_utils'
@@ -9,13 +9,14 @@ export default class DataService {
this.storage = storage
}
- async find(collectionName: string, _filter: Filter, sort: any, skip: any, limit: any, projection: any) {
+ async find(collectionName: string, _filter: Filter, sort: any, skip: any, limit: any, projection: any, omitTotalCount?: boolean): Promise<{items: any[], totalCount?: number}> {
const items = this.storage.find(collectionName, _filter, sort, skip, limit, projection)
- const totalCount = this.storage.count(collectionName, _filter)
+ const totalCount = omitTotalCount? undefined : this.storage.count(collectionName, _filter)
+
return {
items: (await items).map(asWixData),
totalCount: await totalCount
- }
+ }
}
async getById(collectionName: string, itemId: string, projection: any) {
@@ -34,6 +35,11 @@ export default class DataService {
return { item: asWixData(resp.items[0]) }
}
+ async bulkUpsert(collectionName: string, items: Item[], fields?: ResponseField[]) {
+ await this.storage.insert(collectionName, items, fields, true)
+ return { items: items.map( asWixData ) }
+ }
+
async bulkInsert(collectionName: string, items: Item[], fields?: ResponseField[]) {
await this.storage.insert(collectionName, items, fields)
return { items: items.map( asWixData ) }
@@ -63,11 +69,14 @@ export default class DataService {
return this.storage.truncate(collectionName)
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation) {
+
+ // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ?
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort?: Sort[], skip?: number, limit?: number) {
+ const totalCount = this.storage.count(collectionName, filter)
return {
- items: ((await this.storage.aggregate?.(collectionName, filter, aggregation)) || [])
+ items: ((await this.storage.aggregate?.(collectionName, filter, aggregation, sort, skip, limit)) || [])
.map( asWixData ),
- totalCount: 0
+ totalCount: await totalCount
}
}
}
diff --git a/libs/velo-external-db-core/src/service/schema.spec.ts b/libs/velo-external-db-core/src/service/schema.spec.ts
index f0dc7991d..0b6fe6670 100644
--- a/libs/velo-external-db-core/src/service/schema.spec.ts
+++ b/libs/velo-external-db-core/src/service/schema.spec.ts
@@ -1,90 +1,210 @@
import * as Chance from 'chance'
-import SchemaService from './schema'
-import { AllSchemaOperations, errors } from '@wix-velo/velo-external-db-commons'
import { Uninitialized } from '@wix-velo/test-commons'
+import { errors } from '@wix-velo/velo-external-db-commons'
+import SchemaService from './schema'
import * as driver from '../../test/drivers/schema_provider_test_support'
import * as schema from '../../test/drivers/schema_information_test_support'
import * as matchers from '../../test/drivers/schema_matchers'
import * as gen from '../../test/gen'
-const { schemasListFor, schemaHeadersListFor, schemasWithReadOnlyCapabilitiesFor } = matchers
+import {
+ fieldTypeToWixDataEnum,
+ compareColumnsInDbAndRequest,
+ InputFieldsToWixFormatFields,
+ InputFieldToWixFormatField,
+} from '../utils/schema_utils'
+import {
+ Table,
+ InputField
+ } from '@wix-velo/velo-external-db-types'
+
+const { collectionsListFor } = matchers
const chance = Chance()
describe('Schema Service', () => {
+ describe('Collection new SPI', () => {
+ test('retrieve all collections from provider', async() => {
+ driver.givenAllSchemaOperations()
+ driver.givenColumnCapabilities()
+ driver.givenListResult(ctx.dbsWithIdColumn)
+
+
+ await expect( env.schemaService.list([]) ).resolves.toEqual(collectionsListFor(ctx.dbsWithIdColumn))
+ })
+
+ test('create new collection without fields', async() => {
+ driver.givenAllSchemaOperations()
+ driver.expectCreateOf(ctx.collectionName)
+ schema.expectSchemaRefresh()
+
+ await expect(env.schemaService.create({ id: ctx.collectionName, fields: [] })).resolves.toEqual({
+ collection: { id: ctx.collectionName, fields: [] }
+ })
+ })
+
+ test('create new collection with fields', async() => {
+ const fields = [{
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum(ctx.column.type),
+ }]
+ driver.givenAllSchemaOperations()
+ schema.expectSchemaRefresh()
+ driver.expectCreateWithFieldsOf(ctx.collectionName, fields)
+
+ await expect(env.schemaService.create({ id: ctx.collectionName, fields })).resolves.toEqual({
+ collection: { id: ctx.collectionName, fields }
+ })
+ })
+
+ test('update collection - add new columns', async() => {
+ const newFields = [{
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum(ctx.column.type),
+ }]
+
+ driver.givenAllSchemaOperations()
+ schema.expectSchemaRefresh()
+ driver.givenFindResults([ { id: ctx.collectionName, fields: [] } ])
+
+ await env.schemaService.update({ id: ctx.collectionName, fields: newFields })
+
+
+ expect(driver.schemaProvider.addColumn).toBeCalledTimes(1)
+ expect(driver.schemaProvider.addColumn).toBeCalledWith(ctx.collectionName, {
+ name: ctx.column.name,
+ type: ctx.column.type,
+ subtype: ctx.column.subtype
+ })
+ expect(driver.schemaProvider.removeColumn).not.toBeCalled()
+ expect(driver.schemaProvider.changeColumnType).not.toBeCalled()
+ })
+
+ test('update collection - add new column to non empty collection', async() => {
+ const currentFields = [{
+ field: ctx.column.name,
+ type: ctx.column.type
+ }]
+ const wantedFields = InputFieldsToWixFormatFields([ ctx.column, ctx.anotherColumn ])
+
+ driver.givenAllSchemaOperations()
+ schema.expectSchemaRefresh()
+ driver.givenFindResults([ {
+ id: ctx.collectionName,
+ fields: currentFields
+ }])
+
+ await env.schemaService.update({ id: ctx.collectionName, fields: wantedFields })
+
+ const { columnsToAdd } = compareColumnsInDbAndRequest(currentFields, wantedFields)
+
+ columnsToAdd.forEach(c => expect(driver.schemaProvider.addColumn).toBeCalledWith(ctx.collectionName, c))
+ expect(driver.schemaProvider.removeColumn).not.toBeCalled()
+ expect(driver.schemaProvider.changeColumnType).not.toBeCalled()
+ })
+
+ test('update collection - remove column', async() => {
+ const currentFields = [{
+ field: ctx.column.name,
+ type: ctx.column.type
+ }]
+
+ driver.givenAllSchemaOperations()
+ schema.expectSchemaRefresh()
+ driver.givenFindResults([ {
+ id: ctx.collectionName,
+ fields: currentFields
+ }])
+
+ const { columnsToRemove } = compareColumnsInDbAndRequest(currentFields, [])
+
+ await env.schemaService.update({ id: ctx.collectionName, fields: [] })
+
+ columnsToRemove.forEach(c => expect(driver.schemaProvider.removeColumn).toBeCalledWith(ctx.collectionName, c))
+ expect(driver.schemaProvider.addColumn).not.toBeCalled()
+ expect(driver.schemaProvider.changeColumnType).not.toBeCalled()
+
+ })
+
+ test('update collection - change column type', async() => {
+ const currentField = {
+ field: ctx.column.name,
+ type: 'text'
+ }
+
+ const changedColumnType = {
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum('number')
+ }
+
+ driver.givenAllSchemaOperations()
+ schema.expectSchemaRefresh()
+ driver.givenFindResults([ {
+ id: ctx.collectionName,
+ fields: [currentField]
+ }])
+
+ const { columnsToChangeType } = compareColumnsInDbAndRequest([currentField], [changedColumnType])
+
+ await env.schemaService.update({ id: ctx.collectionName, fields: [changedColumnType] })
+
+ columnsToChangeType.forEach(c => expect(driver.schemaProvider.changeColumnType).toBeCalledWith(ctx.collectionName, c))
+ expect(driver.schemaProvider.addColumn).not.toBeCalled()
+ expect(driver.schemaProvider.removeColumn).not.toBeCalled()
+
+ })
+
+ // TODO: create a test for the case
+ // test('collections without _id column will have read-only capabilities', async() => {})
+
+ test('run unsupported operations should throw', async() => {
+ schema.expectSchemaRefresh()
+ driver.givenAdapterSupportedOperationsWith(ctx.invalidOperations)
+ const field = InputFieldToWixFormatField({
+ name: ctx.column.name,
+ type: 'text'
+ })
+ const changedTypeField = InputFieldToWixFormatField({
+ name: ctx.column.name,
+ type: 'number'
+ })
+
+ driver.givenFindResults([ { id: ctx.collectionName, fields: [] } ])
- test('retrieve all collections from provider', async() => {
- driver.givenAllSchemaOperations()
- driver.givenListResult(ctx.dbsWithIdColumn)
-
- await expect( env.schemaService.list() ).resolves.toEqual( schemasListFor(ctx.dbsWithIdColumn, AllSchemaOperations) )
- })
-
- test('retrieve short list of all collections from provider', async() => {
- driver.givenListHeadersResult(ctx.collections)
-
-
- await expect( env.schemaService.listHeaders() ).resolves.toEqual( schemaHeadersListFor(ctx.collections) )
- })
-
- test('retrieve collections by ids from provider', async() => {
- driver.givenAllSchemaOperations()
- schema.givenSchemaFieldsResultFor(ctx.dbsWithIdColumn)
-
- await expect( env.schemaService.find(ctx.dbsWithIdColumn.map((db: { id: any }) => db.id)) ).resolves.toEqual( schemasListFor(ctx.dbsWithIdColumn, AllSchemaOperations) )
- })
-
- test('create collection name', async() => {
- driver.givenAllSchemaOperations()
- driver.expectCreateOf(ctx.collectionName)
- schema.expectSchemaRefresh()
-
- await expect(env.schemaService.create(ctx.collectionName)).resolves.toEqual({})
- })
-
- test('add column for collection name', async() => {
- driver.givenAllSchemaOperations()
- driver.expectCreateColumnOf(ctx.column, ctx.collectionName)
- schema.expectSchemaRefresh()
-
- await expect(env.schemaService.addColumn(ctx.collectionName, ctx.column)).resolves.toEqual({})
- })
-
- test('remove column from collection name', async() => {
- driver.givenAllSchemaOperations()
- driver.expectRemoveColumnOf(ctx.column, ctx.collectionName)
- schema.expectSchemaRefresh()
-
- await expect(env.schemaService.removeColumn(ctx.collectionName, ctx.column.name)).resolves.toEqual({})
- })
-
- test('collections without _id column will have read-only capabilities', async() => {
- driver.givenAllSchemaOperations()
- driver.givenListResult(ctx.dbsWithoutIdColumn)
+ await expect(env.schemaService.update({ id: ctx.collectionName, fields: [field] })).rejects.toThrow(errors.UnsupportedSchemaOperation)
- await expect( env.schemaService.list() ).resolves.toEqual( schemasWithReadOnlyCapabilitiesFor(ctx.dbsWithoutIdColumn) )
- })
+ driver.givenFindResults([ { id: ctx.collectionName, fields: [{ field: ctx.column.name, type: 'text' }] }])
- test('run unsupported operations should throw', async() => {
- driver.givenAdapterSupportedOperationsWith(ctx.invalidOperations)
+ await expect(env.schemaService.update({ id: ctx.collectionName, fields: [] })).rejects.toThrow(errors.UnsupportedSchemaOperation)
+ await expect(env.schemaService.update({ id: ctx.collectionName, fields: [changedTypeField] })).rejects.toThrow(errors.UnsupportedSchemaOperation)
+ })
- await expect(env.schemaService.create(ctx.collectionName)).rejects.toThrow(errors.UnsupportedOperation)
- await expect(env.schemaService.addColumn(ctx.collectionName, ctx.column)).rejects.toThrow(errors.UnsupportedOperation)
- await expect(env.schemaService.removeColumn(ctx.collectionName, ctx.column.name)).rejects.toThrow(errors.UnsupportedOperation)
})
- const ctx = {
+ interface Ctx {
+ dbsWithoutIdColumn: Table[],
+ dbsWithIdColumn: Table[],
+ collections: string[],
+ collectionName: string,
+ column: InputField,
+ anotherColumn: InputField,
+ invalidOperations: string[],
+ }
+
+
+ const ctx: Ctx = {
dbsWithoutIdColumn: Uninitialized,
dbsWithIdColumn: Uninitialized,
collections: Uninitialized,
collectionName: Uninitialized,
column: Uninitialized,
+ anotherColumn: Uninitialized,
invalidOperations: Uninitialized,
}
- interface Enviorment {
+ interface Environment {
schemaService: SchemaService
}
- const env: Enviorment = {
+ const env: Environment = {
schemaService: Uninitialized,
}
@@ -98,6 +218,7 @@ describe('Schema Service', () => {
ctx.collections = gen.randomCollections()
ctx.collectionName = gen.randomCollectionName()
ctx.column = gen.randomColumn()
+ ctx.anotherColumn = gen.randomColumn()
ctx.invalidOperations = [chance.word(), chance.word()]
diff --git a/libs/velo-external-db-core/src/service/schema.ts b/libs/velo-external-db-core/src/service/schema.ts
index 4ee358bbf..77ed84a35 100644
--- a/libs/velo-external-db-core/src/service/schema.ts
+++ b/libs/velo-external-db-core/src/service/schema.ts
@@ -1,7 +1,24 @@
-import { asWixSchema, asWixSchemaHeaders, allowedOperationsFor, appendQueryOperatorsTo, errors } from '@wix-velo/velo-external-db-commons'
-import { InputField, ISchemaProvider, Table, SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { errors } from '@wix-velo/velo-external-db-commons'
+import { ISchemaProvider,
+ SchemaOperations,
+ ResponseField,
+ CollectionCapabilities,
+ Table
+} from '@wix-velo/velo-external-db-types'
+import * as collectionSpi from '../spi-model/collection'
import CacheableSchemaInformation from './schema_information'
-const { Create, AddColumn, RemoveColumn } = SchemaOperations
+import {
+ queriesToWixDataQueryOperators,
+ fieldTypeToWixDataEnum,
+ WixFormatFieldsToInputFields,
+ responseFieldToWixFormat,
+ compareColumnsInDbAndRequest,
+ dataOperationsToWixDataQueryOperators,
+ collectionOperationsToWixDataCollectionOperations,
+} from '../utils/schema_utils'
+
+
+const { Create, AddColumn, RemoveColumn, ChangeColumnType } = SchemaOperations
export default class SchemaService {
storage: ISchemaProvider
@@ -11,62 +28,109 @@ export default class SchemaService {
this.schemaInformation = schemaInformation
}
- async list() {
- const dbs = await this.storage.list()
- const dbsWithAllowedOperations = this.appendAllowedOperationsTo(dbs)
-
- return { schemas: dbsWithAllowedOperations.map( asWixSchema ) }
+ async list(collectionIds: string[]): Promise<collectionSpi.ListCollectionsResponsePart> {
+ const collections = (!collectionIds || collectionIds.length === 0) ?
+ await this.storage.list() :
+ await Promise.all(collectionIds.map((collectionName: string) => this.schemaInformation.schemaFor(collectionName)))
+
+ return {
+ collection: collections.map(this.formatCollection.bind(this))
+ }
}
- async listHeaders() {
- const collections = await this.storage.listHeaders()
- return { schemas: collections.map((collection) => asWixSchemaHeaders(collection)) }
+ async create(collection: collectionSpi.Collection): Promise<collectionSpi.CreateCollectionResponse> {
+ await this.storage.create(collection.id, WixFormatFieldsToInputFields(collection.fields))
+ await this.schemaInformation.refresh()
+ return { collection }
}
- async find(collectionNames: string[]) {
- const dbs: Table[] = await Promise.all(collectionNames.map(async(collectionName: string) => ({ id: collectionName, fields: await this.schemaInformation.schemaFieldsFor(collectionName) })))
- const dbsWithAllowedOperations = this.appendAllowedOperationsTo(dbs)
+ async update(collection: collectionSpi.Collection): Promise<collectionSpi.UpdateCollectionResponse> {
+ await this.validateOperation(collection.id, Create)
+
+ // remove in the end of development
+ if (!this.storage.changeColumnType) {
+ throw new Error('Your storage does not support the new collection capabilities API')
+ }
- return { schemas: dbsWithAllowedOperations.map( asWixSchema ) }
- }
+ const collectionColumnsInRequest = collection.fields
+ const { fields: collectionColumnsInDb } = await this.storage.describeCollection(collection.id) as Table
+
+ const {
+ columnsToAdd,
+ columnsToRemove,
+ columnsToChangeType
+ } = compareColumnsInDbAndRequest(collectionColumnsInDb, collectionColumnsInRequest)
- async create(collectionName: string) {
- await this.validateOperation(Create)
- await this.storage.create(collectionName)
- await this.schemaInformation.refresh()
- return {}
- }
+ // Adding columns
+ if (columnsToAdd.length > 0) {
+ await this.validateOperation(collection.id, AddColumn)
+ }
+ await Promise.all(columnsToAdd.map(async(field) => await this.storage.addColumn(collection.id, field)))
+
+ // Removing columns
+ if (columnsToRemove.length > 0) {
+ await this.validateOperation(collection.id, RemoveColumn)
+ }
+ await Promise.all(columnsToRemove.map(async(fieldName) => await this.storage.removeColumn(collection.id, fieldName)))
- async addColumn(collectionName: string, column: InputField) {
- await this.validateOperation(AddColumn)
- await this.storage.addColumn(collectionName, column)
- await this.schemaInformation.refresh()
- return {}
- }
+ // Changing columns type
+ if (columnsToChangeType.length > 0) {
+ await this.validateOperation(collection.id, ChangeColumnType)
+ }
+ await Promise.all(columnsToChangeType.map(async(field) => await this.storage.changeColumnType?.(collection.id, field)))
- async removeColumn(collectionName: string, columnName: string) {
- await this.validateOperation(RemoveColumn)
- await this.storage.removeColumn(collectionName, columnName)
await this.schemaInformation.refresh()
- return {}
+
+ return { collection }
}
- appendAllowedOperationsTo(dbs: Table[]) {
- const allowedSchemaOperations = this.storage.supportedOperations()
- return dbs.map((db: Table) => ({
- ...db,
- allowedSchemaOperations,
- allowedOperations: allowedOperationsFor(db),
- fields: appendQueryOperatorsTo(db.fields)
- }))
+ async delete(collectionId: string): Promise<collectionSpi.DeleteCollectionResponse> {
+ const { fields: collectionFields } = await this.storage.describeCollection(collectionId) as Table
+ await this.storage.drop(collectionId)
+ this.schemaInformation.refresh()
+ return { collection: {
+ id: collectionId,
+ fields: responseFieldToWixFormat(collectionFields),
+ } }
}
-
- async validateOperation(operationName: SchemaOperations) {
+ private async validateOperation(collectionName: string, operationName: SchemaOperations) {
const allowedSchemaOperations = this.storage.supportedOperations()
if (!allowedSchemaOperations.includes(operationName))
- throw new errors.UnsupportedOperation(`Your database doesn't support ${operationName} operation`)
+ throw new errors.UnsupportedSchemaOperation(`Your database doesn't support ${operationName} operation`, collectionName, operationName)
+ }
+
+ private formatCollection(collection: Table): collectionSpi.Collection {
+ return {
+ id: collection.id,
+ fields: this.formatFields(collection.fields),
+ capabilities: collection.capabilities? this.formatCollectionCapabilities(collection.capabilities) : undefined
+ }
+ }
+
+ private formatFields(fields: ResponseField[]): collectionSpi.Field[] {
+ return fields.map( field => ({
+ key: field.field,
+ encrypted: false,
+ type: fieldTypeToWixDataEnum(field.type),
+ capabilities: {
+ sortable: field.capabilities? field.capabilities.sortable: undefined,
+ queryOperators: field.capabilities? queriesToWixDataQueryOperators(field.capabilities.columnQueryOperators): undefined
+ }
+ }))
+ }
+
+ private formatCollectionCapabilities(capabilities: CollectionCapabilities): collectionSpi.CollectionCapabilities {
+ return {
+ dataOperations: capabilities.dataOperations.map(dataOperationsToWixDataQueryOperators),
+ fieldTypes: capabilities.fieldTypes.map(fieldTypeToWixDataEnum),
+ collectionOperations: capabilities.collectionOperations.map(collectionOperationsToWixDataCollectionOperations),
+ // TODO: create functions that translate between the domains.
+ referenceCapabilities: { supportedNamespaces: capabilities.referenceCapabilities.supportedNamespaces },
+ indexing: [],
+ encryption: collectionSpi.Encryption.notSupported
+ }
}
}
diff --git a/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts b/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts
index c1615d19b..feb5b0023 100644
--- a/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts
+++ b/libs/velo-external-db-core/src/service/schema_aware_data.spec.ts
@@ -15,10 +15,10 @@ describe ('Schema Aware Data Service', () => {
schema.givenDefaultSchemaFor(ctx.collectionName)
queryValidator.givenValidFilterForDefaultFieldsOf(ctx.transformedFilter)
queryValidator.givenValidProjectionForDefaultFieldsOf(SystemFields)
- data.givenListResult(ctx.entities, ctx.totalCount, ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, ctx.defaultFields)
+ data.givenListResult(ctx.entities, ctx.totalCount, ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, ctx.defaultFields, false)
patcher.givenPatchedBooleanFieldsWith(ctx.patchedEntities, ctx.entities)
- return expect(env.schemaAwareDataService.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({
+ return expect(env.schemaAwareDataService.find(ctx.collectionName, ctx.filter, ctx.sort, ctx.skip, ctx.limit, undefined, false)).resolves.toEqual({
items: ctx.patchedEntities,
totalCount: ctx.totalCount
})
@@ -95,9 +95,9 @@ describe ('Schema Aware Data Service', () => {
queryValidator.givenValidFilterForDefaultFieldsOf(ctx.filter)
queryValidator.givenValidAggregationForDefaultFieldsOf(ctx.aggregation)
- data.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation)
+ data.givenAggregateResult(ctx.entities, ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)
- return expect(env.schemaAwareDataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation)).resolves.toEqual({ items: ctx.entities, totalCount: 0 })
+ return expect(env.schemaAwareDataService.aggregate(ctx.collectionName, ctx.filter, ctx.aggregation, ctx.sort, ctx.skip, ctx.limit)).resolves.toEqual({ items: ctx.entities, totalCount: 0 })
})
test('schema with _id - find will trigger find request with projection includes _id even if it is not in the projection', async() => {
diff --git a/libs/velo-external-db-core/src/service/schema_aware_data.ts b/libs/velo-external-db-core/src/service/schema_aware_data.ts
index 4479de8d1..203c18842 100644
--- a/libs/velo-external-db-core/src/service/schema_aware_data.ts
+++ b/libs/velo-external-db-core/src/service/schema_aware_data.ts
@@ -1,4 +1,4 @@
-import { AdapterAggregation as Aggregation, AdapterFilter as Filter, AnyFixMe, Item, ItemWithId, ResponseField } from '@wix-velo/velo-external-db-types'
+import { AdapterAggregation as Aggregation, AdapterFilter as Filter, AnyFixMe, Item, ItemWithId, ResponseField, Sort } from '@wix-velo/velo-external-db-types'
import QueryValidator from '../converters/query_validator'
import DataService from './data'
import CacheableSchemaInformation from './schema_information'
@@ -15,14 +15,14 @@ export default class SchemaAwareDataService {
this.itemTransformer = itemTransformer
}
- async find(collectionName: string, filter: Filter, sort: any, skip: number, limit: number, _projection?: any): Promise<{ items: ItemWithId[], totalCount: number }> {
+ async find(collectionName: string, filter: Filter, sort: any, skip: number, limit: number, _projection?: any, omitTotalCount?: boolean): Promise<{ items: ItemWithId[], totalCount?: number }> {
const fields = await this.schemaInformation.schemaFieldsFor(collectionName)
await this.validateFilter(collectionName, filter, fields)
const projection = await this.projectionFor(collectionName, _projection)
await this.validateProjection(collectionName, projection, fields)
- const { items, totalCount } = await this.dataService.find(collectionName, filter, sort, skip, limit, projection)
- return { items: this.itemTransformer.patchItems(items, fields), totalCount }
+ const { items, totalCount } = await this.dataService.find(collectionName, filter, sort, skip, limit, projection, omitTotalCount)
+ return { items: this.itemTransformer.patchItems(items, fields), totalCount }
}
async getById(collectionName: string, itemId: string, _projection?: any) {
@@ -44,6 +44,12 @@ export default class SchemaAwareDataService {
return await this.dataService.insert(collectionName, prepared[0], fields)
}
+ async bulkUpsert(collectionName: string, items: Item[]) {
+ const fields = await this.schemaInformation.schemaFieldsFor(collectionName)
+ const prepared = await this.prepareItemsForInsert(fields, items)
+ return await this.dataService.bulkUpsert(collectionName, prepared, fields)
+ }
+
async bulkInsert(collectionName: string, items: Item[]) {
const fields = await this.schemaInformation.schemaFieldsFor(collectionName)
const prepared = await this.prepareItemsForInsert(fields, items)
@@ -72,15 +78,16 @@ export default class SchemaAwareDataService {
return await this.dataService.truncate(collectionName)
}
- async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation) {
+ // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ?
+ async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort?: Sort[], skip?: number, limit?: number) {
await this.validateAggregation(collectionName, aggregation)
await this.validateFilter(collectionName, filter)
- return await this.dataService.aggregate(collectionName, filter, aggregation)
+ return await this.dataService.aggregate(collectionName, filter, aggregation, sort, skip, limit)
}
async validateFilter(collectionName: string, filter: Filter, _fields?: ResponseField[]) {
const fields = _fields ?? await this.schemaInformation.schemaFieldsFor(collectionName)
- this.queryValidator.validateFilter(fields, filter)
+ this.queryValidator.validateFilter(fields, filter, collectionName)
}
async validateGetById(collectionName: string, itemId: string) {
diff --git a/libs/velo-external-db-core/src/service/schema_information.ts b/libs/velo-external-db-core/src/service/schema_information.ts
index 5f55848b3..8dd3c5a7f 100644
--- a/libs/velo-external-db-core/src/service/schema_information.ts
+++ b/libs/velo-external-db-core/src/service/schema_information.ts
@@ -1,5 +1,5 @@
import { errors } from '@wix-velo/velo-external-db-commons'
-import { ISchemaProvider, ResponseField } from '@wix-velo/velo-external-db-types'
+import { ISchemaProvider, ResponseField, Table } from '@wix-velo/velo-external-db-types'
const { CollectionDoesNotExists } = errors
import * as NodeCache from 'node-cache'
@@ -14,25 +14,30 @@ export default class CacheableSchemaInformation {
}
async schemaFieldsFor(collectionName: string): Promise<ResponseField[]> {
+ return (await this.schemaFor(collectionName)).fields
+ }
+
+ async schemaFor(collectionName: string): Promise<Table> {
const schema = this.cache.get(collectionName)
if ( !schema ) {
await this.update(collectionName)
- return this.cache.get(collectionName) as ResponseField[]
+ return this.cache.get(collectionName) as Table
}
- return schema as ResponseField[]
+ return schema as Table
}
async update(collectionName: string) {
const collection = await this.schemaProvider.describeCollection(collectionName)
- if (!collection) throw new CollectionDoesNotExists('Collection does not exists')
+ if (!collection) throw new CollectionDoesNotExists('Collection does not exists', collectionName)
this.cache.set(collectionName, collection, FiveMinutes)
}
async refresh() {
+ await this.clear()
const schema = await this.schemaProvider.list()
if (schema && schema.length)
- schema.forEach((collection: { id: any; fields: any }) => {
- this.cache.set(collection.id, collection.fields, FiveMinutes)
+ schema.forEach((collection: Table) => {
+ this.cache.set(collection.id, collection, FiveMinutes)
})
}
diff --git a/libs/velo-external-db-core/src/spi-model/capabilities.ts b/libs/velo-external-db-core/src/spi-model/capabilities.ts
new file mode 100644
index 000000000..09adb1e83
--- /dev/null
+++ b/libs/velo-external-db-core/src/spi-model/capabilities.ts
@@ -0,0 +1,16 @@
+export interface GetCapabilitiesRequest {}
+
+// Global capabilities that datasource supports.
+export interface GetCapabilitiesResponse {
+ capabilities: Capabilities
+}
+
+export interface Capabilities {
+ // Defines which collection operations is supported.
+ collection: CollectionCapability[]
+}
+
+export enum CollectionCapability {
+ // Supports creating new collections.
+ CREATE = 'CREATE'
+}
diff --git a/libs/velo-external-db-core/src/spi-model/collection.ts b/libs/velo-external-db-core/src/spi-model/collection.ts
new file mode 100644
index 000000000..10dd3a285
--- /dev/null
+++ b/libs/velo-external-db-core/src/spi-model/collection.ts
@@ -0,0 +1,162 @@
+export type listCollections = (req: ListCollectionsRequest) => Promise<ListCollectionsResponsePart>
+
+export type createCollection = (req: CreateCollectionRequest) => Promise<CreateCollectionResponse>
+
+export type updateCollection = (req: UpdateCollectionRequest) => Promise<UpdateCollectionResponse>
+
+export type deleteCollection = (req: DeleteCollectionRequest) => Promise<DeleteCollectionResponse>
+export abstract class CollectionService {
+}
+export interface ListCollectionsRequest {
+ collectionIds: string[];
+}
+export interface ListCollectionsResponsePart {
+ collection: Collection[];
+}
+export interface DeleteCollectionRequest {
+ collectionId: string;
+}
+export interface DeleteCollectionResponse {
+ collection: Collection;
+}
+export interface CreateCollectionRequest {
+ collection: Collection;
+}
+export interface CreateCollectionResponse {
+ collection: Collection;
+}
+export interface UpdateCollectionRequest {
+ collection: Collection;
+}
+export interface UpdateCollectionResponse {
+ collection: Collection;
+}
+export interface Collection {
+ id: string;
+ fields: Field[];
+ capabilities?: CollectionCapabilities;
+}
+
+export interface Field {
+ // Identifier of the field.
+ key: string;
+ // Value is encrypted when `true`. Global data source capabilities define where encryption takes place.
+ encrypted?: boolean;
+ // Type of the field.
+ type: FieldType;
+ // Defines what kind of operations this field supports.
+ // Should be set by datasource itself and ignored in request payload.
+ capabilities?: FieldCapabilities;
+ // Additional options for specific field types, should be one of the following
+ singleReferenceOptions?: SingleReferenceOptions;
+ multiReferenceOptions?: MultiReferenceOptions;
+}
+
+export interface SingleReferenceOptions {
+ referencedCollectionId?: string;
+ referencedNamespace?: string;
+}
+export interface MultiReferenceOptions {
+ referencedCollectionId?: string;
+ referencedNamespace?: string;
+ referencingFieldKey?: string;
+}
+
+export interface FieldCapabilities {
+ // Indicates if field can be used to sort items in collection. Defaults to false.
+ sortable?: boolean;
+ // Query operators (e.g. equals, less than) that can be used for this field.
+ queryOperators?: QueryOperator[];
+ singleReferenceOptions?: SingleReferenceOptions;
+ multiReferenceOptions?: MultiReferenceOptions;
+}
+
+export enum QueryOperator {
+ eq = 0,
+ lt = 1,
+ gt = 2,
+ ne = 3,
+ lte = 4,
+ gte = 5,
+ startsWith = 6,
+ endsWith = 7,
+ contains = 8,
+ hasSome = 9,
+ hasAll = 10,
+ exists = 11,
+ urlized = 12,
+}
+export interface SingleReferenceOptions {
+ // `true` when datasource supports `include_referenced_items` in query method natively.
+ includeSupported?: boolean;
+}
+export interface MultiReferenceOptions {
+ // `true` when datasource supports `include_referenced_items` in query method natively.
+ includeSupported?: boolean;
+}
+
+export interface CollectionCapabilities {
+ // Lists data operations supported by collection.
+ dataOperations: DataOperation[];
+ // Supported field types.
+ fieldTypes: FieldType[];
+ // Describes what kind of reference capabilities is supported.
+ referenceCapabilities: ReferenceCapabilities;
+ // Lists what kind of modifications this collection accept.
+ collectionOperations: CollectionOperation[];
+ // Defines which indexing operations is supported.
+ indexing: IndexingCapabilityEnum[];
+ // Defines if/how encryption is supported.
+ encryption: Encryption;
+}
+
+export enum DataOperation {
+ query = 0,
+ count = 1,
+ queryReferenced = 2,
+ aggregate = 3,
+ insert = 4,
+ update = 5,
+ remove = 6,
+ truncate = 7,
+ insertReferences = 8,
+ removeReferences = 9,
+}
+
+export interface ReferenceCapabilities {
+ supportedNamespaces?: string[];
+}
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface CollectionOperationEnum {
+}
+
+export enum CollectionOperation {
+ update = 0,
+ remove = 1,
+}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IndexingCapabilityEnum {
+}
+
+export enum IndexingCapability {
+ list = 0,
+ create = 1,
+ remove = 2,
+}
+
+export enum Encryption {
+ notSupported = 0,
+ wixDataNative = 1,
+ dataSourceNative = 2,
+}
+export enum FieldType {
+ text = 0,
+ number = 1,
+ boolean = 2,
+ datetime = 3,
+ object = 4,
+ longText = 5,
+ singleReference = 6,
+ multiReference = 7,
+}
diff --git a/libs/velo-external-db-core/src/spi-model/data_source.ts b/libs/velo-external-db-core/src/spi-model/data_source.ts
new file mode 100644
index 000000000..e3bd971c5
--- /dev/null
+++ b/libs/velo-external-db-core/src/spi-model/data_source.ts
@@ -0,0 +1,408 @@
+
+export interface QueryRequest {
+ collectionId: string;
+ namespace?: string;
+ query: QueryV2;
+ includeReferencedItems: string[];
+ options: Options;
+ omitTotalCount: boolean;
+}
+
+export interface QueryV2 {
+ filter: Filter;
+ sort?: Sorting[];
+ fields: string[];
+ fieldsets: string[];
+ paging?: Paging;
+ cursorPaging?: CursorPaging;
+}
+
+export type Filter = any;
+
+export interface Sorting {
+ fieldName: string;
+ order: SortOrder;
+}
+
+export interface Paging {
+ limit: number;
+ offset: number;
+}
+
+export interface CursorPaging {
+ limit: number;
+ cursor?: string;
+}
+
+export interface Options {
+ consistentRead: boolean;
+ appOptions: any;
+}
+
+export enum SortOrder {
+ ASC = 'ASC',
+ DESC = 'DESC'
+}
+
+export interface QueryResponsePart {
+ item?: any;
+ pagingMetadata?: PagingMetadataV2;
+}
+
+export class QueryResponsePart {
+ static item(item: any): QueryResponsePart {
+ return {
+ item: item
+ } as QueryResponsePart
+ }
+
+ static pagingMetadata(count?: number, offset?: number, total?: number): QueryResponsePart {
+ return {
+ pagingMetadata: {
+ count, offset, total, tooManyToCount: false
+ } as PagingMetadataV2
+ }
+ }
+}
+
+export interface PagingMetadataV2 {
+ count?: number;
+ // Offset that was requested.
+ offset?: number;
+ // Total number of items that match the query. Returned if offset paging is used and the `tooManyToCount` flag is not set.
+ total?: number;
+ // Flag that indicates the server failed to calculate the `total` field.
+ tooManyToCount?: boolean
+ // Cursors to navigate through the result pages using `next` and `prev`. Returned if cursor paging is used.
+ cursors?: Cursors
+ // Indicates if there are more results after the current page.
+ // If `true`, another page of results can be retrieved.
+ // If `false`, this is the last page.
+ has_next?: boolean
+}
+
+export interface Cursors {
+ next?: string;
+ // Cursor pointing to previous page in the list of results.
+ prev?: string;
+}
+
+export interface CountRequest {
+ // collection name to query
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // query filter https://bo.wix.com/wix-docs/rnd/platformization-guidelines/api-query-language
+ filter?: any;
+ // request options
+ options: Options;
+}
+
+export interface CountResponse {
+ totalCount: number;
+}
+
+export interface QueryReferencedRequest {
+ // collection name of referencing item
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // referencing item IDs
+ // NOTE if empty reads all referenced items
+ itemIds: string[];
+ // Multi-reference to read referenced items
+ referencePropertyName: string;
+ // Paging
+ paging: Paging;
+ cursorPaging: CursorPaging;
+ // Request options
+ options: Options;
+ // subset of properties to return
+ // empty means all, may not be supported
+ fields: string[]
+ // Indicates if total count calculation should be omitted.
+ // Only affects offset pagination, because cursor paging does not return total count.
+ omitTotalCount: boolean;
+}
+
+// Let's consider "Album" collection containing "songs" property which
+// contains references to "Song" collection.
+// When making references request to "Album" collection the following names are used:
+// - "Album" is called "referencing collection"
+// - "Album" items are called "referencing items"
+// - "Song" is called "referenced collection"
+// - "Song" items are called "referenced items"
+export interface ReferencedItem {
+ // Requested collection item that references returned item
+ referencingItemId: string;
+ // Item from referenced collection that is referenced by referencing item
+ referencedItemId: string;
+ // may not be present if can't be resolved (not found or item is in draft state)
+ // if the only requested field is `_id` item will always be present with only field
+ item?: any;
+ }
+
+export interface QueryReferencedResponsePart {
+ // overall result will contain single paging_metadata
+ // and zero or more items
+ item: ReferencedItem;
+ pagingMetadata: PagingMetadataV2;
+}
+
+export interface AggregateRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // filter to apply before aggregation
+ initialFilter?: any
+ // group and aggregate
+ // property name to return unique values of
+ // may unwind array values or not, depending on implementation
+ distinct: string;
+ group: Group;
+ // filter to apply after aggregation
+ finalFilter?: any
+ // sorting
+ sort?: Sorting[]
+ // paging
+ paging?: Paging;
+ cursorPaging?: CursorPaging;
+ // request options
+ options: Options;
+ // Indicates if total count calculation should be omitted.
+ // Only affects offset pagination, because cursor paging does not return total count.
+ omitTotalCount: boolean;
+}
+
+export interface Group {
+ // properties to group by, if empty single group would be created
+ by: string[];
+ // aggregations, resulted group will contain field with given name and aggregation value
+ aggregation: Aggregation[];
+}
+
+export interface Aggregation {
+ // result property name
+ name: string;
+
+ //TODO: should be one of the following
+ // property to calculate average of
+ avg?: string;
+ // property to calculate min of
+ min?: string;
+ // property to calculate max of
+ max?: string;
+ // property to calculate sum of
+ sum?: string;
+ // count items, value is always 1
+ count?: number;
+}
+
+export interface AggregateResponsePart {
+ // query response consists of any number of items plus single paging metadata
+ // Aggregation result item.
+ // In case of group request, it should contain a field for each `group.by` value
+ // and a field for each `aggregation.name`.
+ // For example, grouping
+ // ```
+ // {by: ["foo", "bar"], aggregation: {name: "someCount", calculate: {count: "baz"}}}
+ // ```
+ // could produce an item:
+ // ```
+ // {foo: "xyz", bar: "456", someCount: 123}
+ // ```
+ // When `group.by` and 'aggregation.name' clash, grouping key should be returned.
+ //
+ // In case of distinct request, it should contain single field, for example
+ // ```
+ // {distinct: "foo"}
+ // ```
+ // could produce an item:
+ // ```
+ // {foo: "xyz"}
+ // ```
+ item?: any;
+ pagingMetadata?: PagingMetadataV2;
+}
+
+export class AggregateResponsePart {
+ static item(item: any) {
+ return {
+ item
+ } as AggregateResponsePart
+ }
+
+ static pagingMetadata(count?: number, offset?: number, total?: number): QueryResponsePart {
+ return {
+ pagingMetadata: {
+ count, offset, total, tooManyToCount: false
+ } as PagingMetadataV2
+ }
+ }
+}
+
+export interface InsertRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // Items to insert
+ items: any[];
+ // if true items would be overwritten by _id if present
+ overwriteExisting: boolean
+ // request options
+ options: Options;
+}
+
+export interface InsertResponsePart {
+ item?: any;
+ // error from [errors list](errors.proto)
+ error?: ApplicationError;
+}
+
+export class InsertResponsePart {
+ static item(item: any) {
+ return {
+ item
+ } as InsertResponsePart
+ }
+
+ static error(error: ApplicationError) {
+ return {
+ error
+ } as InsertResponsePart
+ }
+}
+
+export interface UpdateRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // Items to update, must include _id
+ items: any[];
+ // request options
+ options: Options;
+}
+
+export interface UpdateResponsePart {
+ // results in order of request
+ item?: any;
+ // error from [errors list](errors.proto)
+ error?: ApplicationError;
+}
+
+export class UpdateResponsePart {
+ static item(item: any) {
+ return {
+ item
+ } as UpdateResponsePart
+ }
+
+ static error(error: ApplicationError) {
+ return {
+ error
+ } as UpdateResponsePart
+ }
+}
+
+export interface RemoveRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // Items to update, must include _id
+ itemIds: string[];
+ // request options
+ options: Options;
+}
+
+export interface RemoveResponsePart {
+ // results in order of request
+ // results in order of request
+ item?: any;
+ // error from [errors list](errors.proto)
+ error?: ApplicationError;
+}
+
+export class RemoveResponsePart {
+ static item(item: any) {
+ return {
+ item
+ } as RemoveResponsePart
+ }
+
+ static error(error: ApplicationError) {
+ return {
+ error
+ } as RemoveResponsePart
+ }
+}
+
+export interface TruncateRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // request options
+ options: Options;
+}
+
+export interface TruncateResponse {}
+
+export interface InsertReferencesRequest {
+ // collection name
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // multi-reference property to update
+ referencePropertyName: string;
+ // references to insert
+ references: ReferenceId[]
+ // request options
+ options: Options;
+}
+
+export interface InsertReferencesResponsePart {
+ reference: ReferenceId;
+ // error from [errors list](errors.proto)
+ error: ApplicationError;
+
+}
+
+export interface ReferenceId {
+ // Id of item in requested collection
+ referencingItemId: string;
+ // Id of item in referenced collection
+ referencedItemId: string;
+}
+
+export interface RemoveReferencesRequest {
+ collectionId: string;
+ // Optional namespace assigned to collection/installation
+ namespace?: string;
+ // multi-reference property to update
+ referencePropertyName: string;
+ // reference masks to delete
+ referenceMasks: ReferenceMask[];
+ // request options
+ options: Options;
+
+
+}
+
+export interface ReferenceMask {
+ // Referencing item ID or any item if empty
+ referencingItemId?: string;
+ // Referenced item ID or any item if empty
+ referencedItemId?: string;
+}
+
+export interface RemoveReferencesResponse {}
+
+export interface ApplicationError {
+ code: string;
+ description: string;
+ data: any;
+}
diff --git a/libs/velo-external-db-core/src/spi-model/errors.ts b/libs/velo-external-db-core/src/spi-model/errors.ts
new file mode 100644
index 000000000..04815dc43
--- /dev/null
+++ b/libs/velo-external-db-core/src/spi-model/errors.ts
@@ -0,0 +1,456 @@
+export class ErrorMessage {
+ static unknownError(description?: string, status?: number) {
+ return HttpError.create({
+ code: ApiErrors.WDE0054,
+ description
+ } as ErrorMessage, status || HttpStatusCode.INTERNAL)
+ }
+
+ static operationTimeLimitExceeded(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0028,
+ description
+ } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED)
+ }
+
+ static invalidUpdate(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0007,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static operationIsNotSupportedByCollection(collectionName: string, operation: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0119,
+ description,
+ data: {
+ collectionName,
+ operation
+ } as UnsupportedByCollectionDetails
+ } as ErrorMessage, HttpStatusCode.FAILED_PRECONDITION)
+ }
+
+ static operationIsNotSupportedByDataSource(collectionName: string, operation: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0120,
+ description,
+ data: {
+ collectionName,
+ operation
+ } as UnsupportedByCollectionDetails
+ } as ErrorMessage, HttpStatusCode.FAILED_PRECONDITION)
+ }
+
+ static itemAlreadyExists(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0074,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static uniqIndexConflict(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0123,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static documentTooLargeToIndex(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0133,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static dollarPrefixedFieldNameNotAllowed(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0134,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static requestPerMinuteQuotaExceeded(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0014,
+ description
+ } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED)
+ }
+
+ static processingTimeQuotaExceeded(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0122,
+ description
+ } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED)
+ }
+
+ static storageSpaceQuotaExceeded(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0091,
+ description
+ } as ErrorMessage, HttpStatusCode.RESOURCE_EXHAUSTED)
+ }
+
+ static documentIsTooLarge(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0009,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static itemNotFound(itemId: string, collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0073,
+ description,
+ data: {
+ itemId,
+ collectionId: collectionName
+ } as InvalidItemDetails
+ } as ErrorMessage, HttpStatusCode.NOT_FOUND)
+ }
+
+ static collectionNotFound(collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0025,
+ description,
+ data: {
+ collectionId: collectionName
+ } as InvalidCollectionDetails
+ } as ErrorMessage, HttpStatusCode.NOT_FOUND)
+ }
+
+ static collectionDeleted(collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0026,
+ description,
+ data: {
+ collectionId: collectionName
+ } as InvalidCollectionDetails
+ } as ErrorMessage, HttpStatusCode.NOT_FOUND)
+ }
+
+ static propertyDeleted(collectionName: string, propertyName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0024,
+ description,
+ data: {
+ collectionId: collectionName,
+ propertyName
+ } as InvalidPropertyDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static userDoesNotHavePermissionToPerformAction(collectionName: string, operation: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0027,
+ description,
+ data: {
+ collectionName,
+ operation
+ } as PermissionDeniedDetails
+ } as ErrorMessage, HttpStatusCode.PERMISSION_DENIED)
+ }
+
+ static genericRequestValidationError(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0075,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static notAMultiReferenceProperty(collectionName: string, propertyName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0020,
+ description,
+ data: {
+ collectionId: collectionName,
+ propertyName
+ } as InvalidPropertyDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static datasetIsTooLargeToSort(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0092,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static payloadIsToolarge(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0109,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static sortingByMultipleArrayFieldsIsNotSupported(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0121,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static offsetPagingIsNotSupported(description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0082,
+ description
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+
+ static referenceAlreadyExists(collectionName: string, propertyName: string, referencingItemId: string, referencedItemId: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0029,
+ description,
+ data: {
+ collectionName,
+ propertyName,
+ referencingItemId,
+ referencedItemId
+ } as InvalidReferenceDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static unknownErrorWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0112,
+ description,
+ data: {
+ collectionName,
+ itemId,
+ details,
+ } as IndexingFailureDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static duplicateKeyErrorWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0113,
+ description,
+ data: {
+ collectionName,
+ itemId,
+ details,
+ } as IndexingFailureDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static documentTooLargeWhileBuildingCollectionIndex(collectionName: string, itemId?: string, details?: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0114,
+ description,
+ data: {
+ collectionName,
+ itemId,
+ details,
+ } as IndexingFailureDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static collectionAlreadyExists(collectionName: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0104,
+ description,
+ data: {
+ collectionId: collectionName
+ } as InvalidCollectionDetails
+ } as ErrorMessage, HttpStatusCode.ALREADY_EXISTS)
+ }
+
+ static invalidProperty(collectionName: string, propertyName?: string, description?: string) {
+ return HttpError.create({
+ code: ApiErrors.WDE0147,
+ description,
+ data: {
+ collectionId: collectionName,
+ propertyName
+ } as InvalidPropertyDetails
+ } as ErrorMessage, HttpStatusCode.INVALID_ARGUMENT)
+ }
+}
+
+export interface HttpError {
+ message: ErrorMessage,
+ httpCode: HttpStatusCode
+}
+
+export class HttpError {
+ static create(message: ErrorMessage, httpCode: HttpStatusCode) {
+ return {
+ message,
+ httpCode
+ } as HttpError
+ }
+}
+
+export interface ErrorMessage {
+ code: ApiErrors,
+ description?: string,
+ data: object
+}
+
+
+
+
+enum ApiErrors {
+ // Unknown error
+ WDE0054='WDE0054',
+ // Operation time limit exceeded.
+ WDE0028='WDE0028',
+ // Invalid update. Updated object must have a string _id property.
+ WDE0007='WDE0007',
+ // Operation is not supported by collection
+ WDE0119='WDE0119',
+ // Operation is not supported by data source
+ WDE0120='WDE0120',
+ // Item already exists
+ WDE0074='WDE0074',
+ // Unique index conflict
+ WDE0123='WDE0123',
+ // Document too large to index
+ WDE0133='WDE0133',
+ // Dollar-prefixed field name not allowed
+ WDE0134='WDE0134',
+ // Requests per minute quota exceeded
+ WDE0014='WDE0014',
+ // Processing time quota exceeded
+ WDE0122='WDE0122',
+ // Storage space quota exceeded
+ WDE0091='WDE0091',
+ // Document is too large
+ WDE0009='WDE0009',
+ // Item not found
+ WDE0073='WDE0073',
+ // Collection not found
+ WDE0025='WDE0025',
+ // Collection deleted
+ WDE0026='WDE0026',
+ // Property deleted
+ WDE0024='WDE0024',
+ // User doesn't have permissions to perform action
+ WDE0027='WDE0027',
+ // Generic request validation error
+ WDE0075='WDE0075',
+ // Not a multi-reference property
+ WDE0020='WDE0020',
+ // Dataset is too large to sort
+ WDE0092='WDE0092',
+ // Payload is too large
+ WDE0109='WDE0109',
+ // Sorting by multiple array fields is not supported
+ WDE0121='WDE0121',
+ // Offset paging is not supported
+ WDE0082='WDE0082',
+ // Reference already exists
+ WDE0029='WDE0029',
+ // Unknown error while building collection index
+ WDE0112='WDE0112',
+ // Duplicate key error while building collection index
+ WDE0113='WDE0113',
+ // Document too large while building collection index
+ WDE0114='WDE0114',
+ // Collection already exists
+ WDE0104='WDE0104',
+ // Invalid property
+ WDE0147='WDE0147'
+}
+
+enum HttpStatusCode {
+ OK = 200,
+
+ //Default error codes (applicable to all endpoints)
+
+ // 401 - Identity missing (missing, invalid or expired oAuth token,
+ // signed instance or cookies)
+ UNAUTHENTICATED = 401,
+
+ // 403 - Identity does not have the permission needed for this method / resource
+ PERMISSION_DENIED = 403,
+
+ // 400 - Bad Request. The client sent malformed body
+ // or one of the arguments was invalid
+ INVALID_ARGUMENT = 400,
+
+ // 404 - Resource does not exist
+ NOT_FOUND = 404,
+
+ // 500 - Internal Server Error
+ INTERNAL = 500,
+
+ // 503 - Come back later, server is currently unavailable
+ UNAVAILABLE = 503,
+
+ // 429 - The client has sent too many requests
+ // in a given amount of time (rate limit)
+ RESOURCE_EXHAUSTED = 429,
+
+ //Custom error codes - need to be documented
+
+ // 499 - Request cancelled by the client
+ CANCELED = 499,
+
+ // 409 - Can't recreate same resource or concurrency conflict
+ ALREADY_EXISTS = 409,
+
+ // 428 - request cannot be executed in current system state
+ // such as deleting a non-empty folder or paying with no funds
+ FAILED_PRECONDITION = 428
+
+ //DO NOT USE IN WIX
+ // ABORTED = 11; // 409
+ // OUT_OF_RANGE = 12; // 400
+ // DEADLINE_EXEEDED = 13; // 504
+ // DATA_LOSS = 14; // 500
+ // UNIMPLEMENTED = 15; // 501
+ }
+
+
+interface UnsupportedByCollectionDetails {
+ collectionName: string
+ operation: string
+}
+interface InvalidItemDetails {
+ itemId: string
+ collectionId: string
+}
+interface InvalidCollectionDetails {
+ collectionId: string
+}
+interface InvalidPropertyDetails {
+ collectionId: string
+ propertyName: string
+}
+interface PermissionDeniedDetails {
+ collectionName: string
+ operation: string
+}
+interface InvalidReferenceDetails {
+ collectionName: string
+ propertyName: string
+ referencingItemId: string
+ referencedItemId: string
+}
+interface IndexingFailureDetails {
+ collectionName: string
+ itemId?: string
+ details?: string
+}
diff --git a/libs/velo-external-db-core/src/spi-model/filter.ts b/libs/velo-external-db-core/src/spi-model/filter.ts
new file mode 100644
index 000000000..31667c37b
--- /dev/null
+++ b/libs/velo-external-db-core/src/spi-model/filter.ts
@@ -0,0 +1,42 @@
+
+// type PrimitveType = number | string | boolean
+// type PrimitveTypeArray = PrimitveType[]
+
+// interface Filter {
+// root: And
+// }
+
+
+// interface ToAdapterType {
+// toAdapter(): void
+// }
+
+// type Comperator = Eq | Ne | Lt
+
+// interface FieldComperator {
+// [fieldName: string]: Comperator | PrimitveType | PrimitveTypeArray
+// }
+
+// interface Eq {
+// $eq: PrimitveType
+// }
+
+// interface Ne {
+// $ne: PrimitveType
+// }
+
+// interface Lt {
+// $lt: number
+// }
+
+// interface And {
+// $and: Array<FieldComperator | And | Or | Not>
+// }
+
+// interface Or {
+// $or: Array<FieldComperator | And | Or | Not>
+// }
+
+// interface Not {
+// $not: FieldComperator | And | Or | Not
+// }
diff --git a/libs/velo-external-db-core/src/types.ts b/libs/velo-external-db-core/src/types.ts
index a79ff57d1..d5184d28d 100644
--- a/libs/velo-external-db-core/src/types.ts
+++ b/libs/velo-external-db-core/src/types.ts
@@ -1,51 +1,31 @@
-import { AdapterFilter, InputField, Item, Sort, WixDataFilter, AsWixSchema, AsWixSchemaHeaders, RoleConfig } from '@wix-velo/velo-external-db-types'
+import { InputField, Item, Sort, WixDataFilter, AsWixSchema, AsWixSchemaHeaders, RoleConfig, ItemWithId, DataOperation } from '@wix-velo/velo-external-db-types'
import SchemaService from './service/schema'
import SchemaAwareDataService from './service/schema_aware_data'
+import { AggregateRequest, CountRequest, CountResponse, Group, InsertRequest, Paging, QueryRequest, Sorting, Options, QueryV2, UpdateRequest, RemoveRequest, TruncateRequest } from './spi-model/data_source'
-export interface FindQuery {
- filter?: WixDataFilter;
- sort?: Sort;
- skip?: number;
- limit?: number;
-}
-
-export type AggregationQuery = {
- filter?: WixDataFilter,
- processingStep?: WixDataFilter,
- postProcessingStep?: WixDataFilter
-}
-
export interface Payload {
- filter?: WixDataFilter | AdapterFilter
- sort?: Sort;
- skip?: number;
- limit?: number;
- postProcessingStep?: WixDataFilter | AdapterFilter;
- processingStep?: WixDataFilter | AdapterFilter;
- postFilteringStep?: WixDataFilter | AdapterFilter;
- item?: Item;
+ filter?: WixDataFilter
+ sort?: Sort[] | Sorting[];
+ initialFilter?: WixDataFilter;
+ group?: Group;
+ finalFilter?: WixDataFilter
+ paging?: Paging;
items?: Item[];
- itemId?: string;
itemIds?: string[];
+ collectionId: string;
+ options?: Options;
+ omitTotalCount?: boolean;
+ includeReferencedItems?: string[];
+ namespace?: string;
+ query?: QueryV2;
+ overwriteExisting?: boolean;
+ totalCount?: number;
}
-enum ReadOperation {
- GET = 'GET',
- FIND = 'FIND',
-}
-
-enum WriteOperation {
- INSERT = 'INSERT',
- UPDATE = 'UPDATE',
- DELETE = 'DELETE',
-}
-
-type Operation = ReadOperation | WriteOperation;
-
export interface RequestContext {
- operation: Operation;
- collectionName: string;
+ operation: DataOperation // | SchemaOperation
+ collectionId: string;
instanceId?: string;
role?: string;
memberId?: string;
@@ -67,24 +47,20 @@ export interface DataHooks {
afterRead?: Hook<Payload>;
beforeWrite?: Hook<Payload>;
afterWrite?: Hook<Payload>;
- beforeFind?: Hook<FindQuery>
- afterFind?: Hook<{ items: Item[] }>
- beforeInsert?: Hook<{ item: Item }>
- afterInsert?: Hook<{ item: Item }>
- beforeBulkInsert?: Hook<{ items: Item[] }>
- afterBulkInsert?: Hook<{ items: Item[] }>
- beforeUpdate?: Hook<{ item: Item }>
- afterUpdate?: Hook<{ item: Item }>
- beforeBulkUpdate?: Hook<{ items: Item[] }>
- afterBulkUpdate?: Hook<{ items: Item[] }>
- beforeRemove?: Hook<{ itemId: string }>
- afterRemove?: Hook<{ itemId: string }>
- beforeBulkRemove?: Hook<{ itemIds: string[] }>
- afterBulkRemove?: Hook<{ itemIds: string[] }>
- beforeAggregate?: Hook<AggregationQuery>
- afterAggregate?: Hook<{ items: Item[] }>
- beforeCount?: Hook<WixDataFilter>
- afterCount?: Hook<{ totalCount: number }>
+ beforeQuery?: Hook<QueryRequest>
+ afterQuery?: Hook<{ items: ItemWithId[], totalCount?: number }>
+ beforeCount?: Hook<CountRequest>
+ afterCount?: Hook<CountResponse>
+ beforeAggregate?: Hook<AggregateRequest>
+ afterAggregate?: Hook<{ items: ItemWithId[], totalCount?: number }>
+ beforeInsert?: Hook<InsertRequest>
+ afterInsert?: Hook<{ items: Item[] }>
+ beforeUpdate?: Hook<UpdateRequest>
+ afterUpdate?: Hook<{ items: Item[] }>
+ beforeRemove?: Hook<RemoveRequest>
+ afterRemove?: Hook<{ items: ItemWithId[] }>
+ beforeTruncate?: Hook<TruncateRequest>
+ afterTruncate?: Hook<void>
}
export type DataHook = DataHooks[keyof DataHooks];
@@ -111,12 +87,14 @@ export interface SchemaHooks {
}
export interface ExternalDbRouterConfig {
- secretKey: string
+ externalDatabaseId: string
+ allowedMetasites: string
authorization?: { roleConfig: RoleConfig }
vendor?: string
adapterType?: string
commonExtended?: boolean
hideAppInfo?: boolean
+ wixDataBaseUrl: string
}
export type Hooks = {
diff --git a/libs/velo-external-db-core/src/utils/base64_utils.ts b/libs/velo-external-db-core/src/utils/base64_utils.ts
new file mode 100644
index 000000000..eabebfc0d
--- /dev/null
+++ b/libs/velo-external-db-core/src/utils/base64_utils.ts
@@ -0,0 +1,9 @@
+
+export function decodeBase64(data: string): string {
+ const buff = Buffer.from(data, 'base64')
+ return buff.toString('ascii')
+}
+export function encodeBase64(data: string): string {
+ const buff = Buffer.from(data, 'utf-8')
+ return buff.toString('base64')
+}
diff --git a/libs/velo-external-db-core/src/utils/schema_utils.spec.ts b/libs/velo-external-db-core/src/utils/schema_utils.spec.ts
new file mode 100644
index 000000000..ae4c282ef
--- /dev/null
+++ b/libs/velo-external-db-core/src/utils/schema_utils.spec.ts
@@ -0,0 +1,179 @@
+import * as Chance from 'chance'
+import { InputField } from '@wix-velo/velo-external-db-types'
+import { Uninitialized } from '@wix-velo/test-commons'
+import { FieldType as VeloFieldTypeEnum } from '../spi-model/collection'
+import {
+ fieldTypeToWixDataEnum,
+ wixDataEnumToFieldType,
+ subtypeToFieldType,
+ compareColumnsInDbAndRequest,
+ wixFormatFieldToInputFields
+} from './schema_utils'
+const chance = Chance()
+
+
+describe('Schema utils functions', () => {
+ describe('translate our field type to velo field type emun', () => {
+ test('text type', () => {
+ expect(fieldTypeToWixDataEnum('text')).toBe(VeloFieldTypeEnum.text)
+ })
+ test('number type', () => {
+ expect(fieldTypeToWixDataEnum('number')).toBe(VeloFieldTypeEnum.number)
+ })
+ test('boolean type', () => {
+ expect(fieldTypeToWixDataEnum('boolean')).toBe(VeloFieldTypeEnum.boolean)
+ })
+ test('object type', () => {
+ expect(fieldTypeToWixDataEnum('object')).toBe(VeloFieldTypeEnum.object)
+ })
+ test('datetime type', () => {
+ expect(fieldTypeToWixDataEnum('datetime')).toBe(VeloFieldTypeEnum.datetime)
+ })
+
+ test('unsupported type will throw an error', () => {
+ expect(() => fieldTypeToWixDataEnum('unsupported-type')).toThrowError()
+ })
+ })
+
+ describe('translate velo field type emun to our field type', () => {
+ test('text type', () => {
+ expect(wixDataEnumToFieldType(VeloFieldTypeEnum.text)).toBe('text')
+ })
+ test('number type', () => {
+ expect(wixDataEnumToFieldType(VeloFieldTypeEnum.number)).toBe('number')
+ })
+ test('boolean type', () => {
+ expect(wixDataEnumToFieldType(VeloFieldTypeEnum.boolean)).toBe('boolean')
+ })
+ test('object type', () => {
+ expect(wixDataEnumToFieldType(VeloFieldTypeEnum.object)).toBe('object')
+ })
+
+ test('datetime type', () => {
+ expect(wixDataEnumToFieldType(VeloFieldTypeEnum.datetime)).toBe('datetime')
+ })
+
+ test('unsupported type will throw an error', () => {
+ expect(() => wixDataEnumToFieldType(100)).toThrowError()
+ })
+ })
+
+ describe('translate velo field type enum to our sub type', () => {
+ test('text type', () => {
+ expect(subtypeToFieldType(VeloFieldTypeEnum.text)).toBe('string')
+ })
+ test('number type', () => {
+ expect(subtypeToFieldType(VeloFieldTypeEnum.number)).toBe('float')
+ })
+ test('boolean type', () => {
+ expect(subtypeToFieldType(VeloFieldTypeEnum.boolean)).toBe('')
+ })
+ test('object type', () => {
+ expect(subtypeToFieldType(VeloFieldTypeEnum.object)).toBe('')
+ })
+
+ test('datetime type', () => {
+ expect(subtypeToFieldType(VeloFieldTypeEnum.datetime)).toBe('datetime')
+ })
+
+ test('unsupported type will throw an error', () => {
+ expect(() => wixDataEnumToFieldType(100)).toThrowError()
+ })
+ })
+
+ describe('convert wix format fields to our fields', () => {
+ test('convert velo format fields to our fields', () => {
+ expect(wixFormatFieldToInputFields({ key: ctx.columnName, type: fieldTypeToWixDataEnum('text') })).toEqual({
+ name: ctx.columnName,
+ type: 'text',
+ subtype: 'string',
+ })
+ })
+
+ })
+
+ describe('compare columns in db and request function', () => {
+ test('compareColumnsInDbAndRequest function - add columns', async() => {
+ const columnsInDb = [{
+ field: ctx.column.name,
+ type: ctx.column.type
+ }]
+ const columnsInRequest = [{
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum(ctx.column.type),
+ }]
+ const newColumn = {
+ key: ctx.anotherColumn.name,
+ type: fieldTypeToWixDataEnum(ctx.anotherColumn.type)
+ }
+ expect(compareColumnsInDbAndRequest([], []).columnsToAdd).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToAdd).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, []).columnsToAdd).toEqual([])
+ expect(compareColumnsInDbAndRequest([], columnsInRequest).columnsToAdd).toEqual(columnsInRequest.map(wixFormatFieldToInputFields))
+ expect(compareColumnsInDbAndRequest(columnsInDb, [...columnsInRequest, newColumn]).columnsToAdd).toEqual([newColumn].map(wixFormatFieldToInputFields))
+ })
+
+ test('compareColumnsInDbAndRequest function - remove columns', async() => {
+ const columnsInDb = [{
+ field: ctx.column.name,
+ type: ctx.column.type
+ }]
+ const columnsInRequest = [{
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum(ctx.column.type),
+ }]
+ const newColumn = {
+ key: ctx.anotherColumn.name,
+ type: fieldTypeToWixDataEnum(ctx.anotherColumn.type)
+ }
+ expect(compareColumnsInDbAndRequest([], []).columnsToRemove).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToRemove).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, [...columnsInRequest, newColumn]).columnsToRemove).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, []).columnsToRemove).toEqual(columnsInDb.map(f => f.field))
+ expect(compareColumnsInDbAndRequest(columnsInDb, [newColumn]).columnsToRemove).toEqual(columnsInDb.map(f => f.field))
+ })
+
+ test('compareColumnsInDbAndRequest function - change column type', async() => {
+ const columnsInDb = [{
+ field: ctx.column.name,
+ type: 'text'
+ }]
+
+ const columnsInRequest = [{
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum('text'),
+ }]
+
+ const changedColumnType = {
+ key: ctx.column.name,
+ type: fieldTypeToWixDataEnum('number')
+ }
+
+ expect(compareColumnsInDbAndRequest([], []).columnsToChangeType).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, columnsInRequest).columnsToChangeType).toEqual([])
+ expect(compareColumnsInDbAndRequest(columnsInDb, [changedColumnType]).columnsToChangeType).toEqual([changedColumnType].map(wixFormatFieldToInputFields))
+ })
+ })
+
+ interface Ctx {
+ collectionName: string,
+ columnName: string,
+ column: InputField,
+ anotherColumn: InputField,
+ }
+
+ const ctx: Ctx = {
+ collectionName: Uninitialized,
+ columnName: Uninitialized,
+ column: Uninitialized,
+ anotherColumn: Uninitialized,
+ }
+
+ beforeEach(() => {
+ ctx.collectionName = chance.word({ length: 5 })
+ ctx.columnName = chance.word({ length: 5 })
+ ctx.column = ({ name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false })
+ ctx.anotherColumn = ({ name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false })
+ })
+
+})
diff --git a/libs/velo-external-db-core/src/utils/schema_utils.ts b/libs/velo-external-db-core/src/utils/schema_utils.ts
new file mode 100644
index 000000000..a037c4e53
--- /dev/null
+++ b/libs/velo-external-db-core/src/utils/schema_utils.ts
@@ -0,0 +1,202 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { InputField, ResponseField, FieldType, DataOperation, CollectionOperation } from '@wix-velo/velo-external-db-types'
+import * as collectionSpi from '../spi-model/collection'
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+
+export const fieldTypeToWixDataEnum = ( fieldType: string ): collectionSpi.FieldType => {
+ switch (fieldType) {
+ case FieldType.text:
+ return collectionSpi.FieldType.text
+ case FieldType.longText:
+ return collectionSpi.FieldType.longText
+ case FieldType.number:
+ return collectionSpi.FieldType.number
+ case FieldType.boolean:
+ return collectionSpi.FieldType.boolean
+ case FieldType.object:
+ return collectionSpi.FieldType.object
+ case FieldType.datetime:
+ return collectionSpi.FieldType.datetime
+ case FieldType.singleReference:
+ return collectionSpi.FieldType.singleReference
+ case FieldType.multiReference:
+ return collectionSpi.FieldType.multiReference
+
+ default:
+ throw new Error(`${fieldType} - Unsupported field type`)
+ }
+}
+
+export const wixDataEnumToFieldType = (fieldEnum: number): string => {
+ switch (fieldEnum) {
+ case collectionSpi.FieldType.text:
+ case collectionSpi.FieldType.longText:
+ return FieldType.text
+ case collectionSpi.FieldType.number:
+ return FieldType.number
+ case collectionSpi.FieldType.datetime:
+ return FieldType.datetime
+ case collectionSpi.FieldType.boolean:
+ return FieldType.boolean
+ case collectionSpi.FieldType.object:
+ return FieldType.object
+
+ case collectionSpi.FieldType.singleReference:
+ case collectionSpi.FieldType.multiReference:
+ default:
+ // TODO: throw specific error
+ throw new Error(`Unsupported field type: ${fieldEnum}`)
+ }
+}
+
+export const subtypeToFieldType = (fieldEnum: number): string => {
+ switch (fieldEnum) {
+ case collectionSpi.FieldType.text:
+ case collectionSpi.FieldType.longText:
+ return 'string'
+ case collectionSpi.FieldType.number:
+ return 'float'
+ case collectionSpi.FieldType.datetime:
+ return 'datetime'
+ case collectionSpi.FieldType.boolean:
+ return ''
+ case collectionSpi.FieldType.object:
+ return ''
+
+ case collectionSpi.FieldType.singleReference:
+ case collectionSpi.FieldType.multiReference:
+ default:
+ // TODO: throw specific error
+ throw new Error(`There is no subtype for this type: ${fieldEnum}`)
+ }
+
+}
+
+export const queryOperatorsToWixDataQueryOperators = (queryOperator: string): collectionSpi.QueryOperator => {
+ switch (queryOperator) {
+ case eq:
+ return collectionSpi.QueryOperator.eq
+ case lt:
+ return collectionSpi.QueryOperator.lt
+ case gt:
+ return collectionSpi.QueryOperator.gt
+ case ne:
+ return collectionSpi.QueryOperator.ne
+ case lte:
+ return collectionSpi.QueryOperator.lte
+ case gte:
+ return collectionSpi.QueryOperator.gte
+ case string_begins:
+ return collectionSpi.QueryOperator.startsWith
+ case string_ends:
+ return collectionSpi.QueryOperator.endsWith
+ case string_contains:
+ return collectionSpi.QueryOperator.contains
+ case include:
+ return collectionSpi.QueryOperator.hasSome
+ // case 'hasAll':
+ // return QueryOperator.hasAll
+ // case 'exists':
+ // return QueryOperator.exists
+ // case 'urlized':
+ // return QueryOperator.urlized
+ default:
+ throw new Error(`${queryOperator} - Unsupported query operator`)
+ }
+}
+
+export const dataOperationsToWixDataQueryOperators = (dataOperation: DataOperation): collectionSpi.DataOperation => {
+ switch (dataOperation) {
+ case DataOperation.query:
+ return collectionSpi.DataOperation.query
+ case DataOperation.count:
+ return collectionSpi.DataOperation.count
+ case DataOperation.queryReferenced:
+ return collectionSpi.DataOperation.queryReferenced
+ case DataOperation.aggregate:
+ return collectionSpi.DataOperation.aggregate
+ case DataOperation.insert:
+ return collectionSpi.DataOperation.insert
+ case DataOperation.update:
+ return collectionSpi.DataOperation.update
+ case DataOperation.remove:
+ return collectionSpi.DataOperation.remove
+ case DataOperation.truncate:
+ return collectionSpi.DataOperation.truncate
+ case DataOperation.insertReferences:
+ return collectionSpi.DataOperation.insertReferences
+ case DataOperation.removeReferences:
+ return collectionSpi.DataOperation.removeReferences
+
+ default:
+ throw new Error(`${dataOperation} - Unsupported data operation`)
+ }
+}
+
+export const collectionOperationsToWixDataCollectionOperations = (collectionOperations: CollectionOperation): collectionSpi.CollectionOperation => {
+ switch (collectionOperations) {
+ case CollectionOperation.update:
+ return collectionSpi.CollectionOperation.update
+ case CollectionOperation.remove:
+ return collectionSpi.CollectionOperation.remove
+
+ default:
+ throw new Error(`${collectionOperations} - Unsupported collection operation`)
+ }
+}
+
+export const queriesToWixDataQueryOperators = (queryOperators: string[]): collectionSpi.QueryOperator[] => queryOperators.map(queryOperatorsToWixDataQueryOperators)
+
+
+export const responseFieldToWixFormat = (fields: ResponseField[]): collectionSpi.Field[] => {
+ return fields.map(field => {
+ return {
+ key: field.field,
+ type: fieldTypeToWixDataEnum(field.type)
+ }
+ })
+}
+
+export const wixFormatFieldToInputFields = (field: collectionSpi.Field): InputField => ({
+ name: field.key,
+ type: wixDataEnumToFieldType(field.type),
+ subtype: subtypeToFieldType(field.type)
+})
+
+export const InputFieldToWixFormatField = (field: InputField): collectionSpi.Field => ({
+ key: field.name,
+ type: fieldTypeToWixDataEnum(field.type)
+})
+
+export const WixFormatFieldsToInputFields = (fields: collectionSpi.Field[]): InputField[] => fields.map(wixFormatFieldToInputFields)
+
+export const InputFieldsToWixFormatFields = (fields: InputField[]): collectionSpi.Field[] => fields.map(InputFieldToWixFormatField)
+
+export const compareColumnsInDbAndRequest = (
+ columnsInDb: ResponseField[],
+ columnsInRequest: collectionSpi.Field[]
+): {
+ columnsToAdd: InputField[];
+ columnsToRemove: string[];
+ columnsToChangeType: InputField[];
+} => {
+ const collectionColumnsNamesInDb = columnsInDb.map((f) => f.field)
+ const collectionColumnsNamesInRequest = columnsInRequest.map((f) => f.key)
+
+ const columnsToAdd = columnsInRequest.filter((f) => !collectionColumnsNamesInDb.includes(f.key))
+ .map(wixFormatFieldToInputFields)
+ const columnsToRemove = columnsInDb.filter((f) => !collectionColumnsNamesInRequest.includes(f.field))
+ .map((f) => f.field)
+
+ const columnsToChangeType = columnsInRequest.filter((f) => {
+ const fieldInDb = columnsInDb.find((field) => field.field === f.key)
+ return fieldInDb && fieldInDb.type !== wixDataEnumToFieldType(f.type)
+ })
+ .map(wixFormatFieldToInputFields)
+
+ return {
+ columnsToAdd,
+ columnsToRemove,
+ columnsToChangeType,
+ }
+}
diff --git a/libs/velo-external-db-core/src/web/auth-middleware.spec.ts b/libs/velo-external-db-core/src/web/auth-middleware.spec.ts
deleted file mode 100644
index 380e431e3..000000000
--- a/libs/velo-external-db-core/src/web/auth-middleware.spec.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { Uninitialized } from '@wix-velo/test-commons'
-import { secretKeyAuthMiddleware } from './auth-middleware'
-import * as driver from '../../test/drivers/auth_middleware_test_support' //TODO: change driver location
-import { errors } from '@wix-velo/velo-external-db-commons'
-const { UnauthorizedError } = errors
-import * as Chance from 'chance'
-const chance = Chance()
-
-describe('Auth Middleware', () => {
-
- const ctx = {
- secretKey: Uninitialized,
- anotherSecretKey: Uninitialized,
- next: Uninitialized,
- ownerRole: Uninitialized,
- dataPath: Uninitialized,
- }
-
- const env = {
- auth: Uninitialized,
- }
-
- beforeEach(() => {
- ctx.secretKey = chance.word()
- ctx.anotherSecretKey = chance.word()
- ctx.next = jest.fn().mockName('next')
-
- env.auth = secretKeyAuthMiddleware({ secretKey: ctx.secretKey })
- })
-
- test('should throw when request does not contain auth', () => {
- expect( () => env.auth({ body: { } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: {} } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: '' } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { settings: {} } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { settings: '' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: [] } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { role: '', settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { role: [], settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- expect( () => env.auth({ body: { requestContext: { role: {}, settings: 'x' } } }, Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- })
-
- test('should throw when secret key does not match', () => {
- expect( () => env.auth(driver.requestBodyWith(ctx.anotherSecretKey, ctx.ownerRole, ctx.dataPath), Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
- })
-
- test('should call next when secret key matches', () => {
- env.auth(driver.requestBodyWith(ctx.secretKey, ctx.ownerRole, ctx.dataPath), Uninitialized, ctx.next)
-
- expect(ctx.next).toHaveBeenCalled()
- })
-})
diff --git a/libs/velo-external-db-core/src/web/auth-middleware.ts b/libs/velo-external-db-core/src/web/auth-middleware.ts
deleted file mode 100644
index 7d85f663b..000000000
--- a/libs/velo-external-db-core/src/web/auth-middleware.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import { property } from './middleware-support'
-import { errors } from '@wix-velo/velo-external-db-commons'
-import { Request } from 'express'
-const { UnauthorizedError } = errors
-
-
-const extractSecretKey = (body: any) => property('requestContext.settings.secretKey', body)
-
-const authorizeSecretKey = (req: Request, secretKey: string) => {
- if (extractSecretKey(req.body) !== secretKey) {
- throw new UnauthorizedError('You are not authorized')
- }
-}
-
-export const secretKeyAuthMiddleware = ({ secretKey }: {secretKey: string}) => {
- return (req: any, res: any, next: () => void) => {
- authorizeSecretKey(req, secretKey)
- next()
- }
-}
diff --git a/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts b/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts
index 518b2f0c6..22de5e14d 100644
--- a/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts
+++ b/libs/velo-external-db-core/src/web/auth-role-middleware.spec.ts
@@ -13,8 +13,6 @@ describe('Auth Role Middleware', () => {
permittedRole: Uninitialized,
notPermittedRole: Uninitialized,
next: Uninitialized,
- secretKey: Uninitialized,
-
}
const env = {
@@ -41,12 +39,12 @@ describe('Auth Role Middleware', () => {
})
test('should allow request with permitted role on request', () => {
- env.auth(driver.requestBodyWith(ctx.secretKey, ctx.permittedRole), Uninitialized, ctx.next)
+ env.auth(driver.requestBodyWith(ctx.permittedRole), Uninitialized, ctx.next)
expect(ctx.next).toHaveBeenCalled()
})
test('should not allow request with permitted role on request', () => {
- expect( () => env.auth(driver.requestBodyWith(ctx.secretKey, ctx.notPermittedRole), Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
+ expect( () => env.auth(driver.requestBodyWith(ctx.notPermittedRole), Uninitialized, ctx.next) ).toThrow(UnauthorizedError)
})
})
diff --git a/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts b/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts
new file mode 100644
index 000000000..de0c93114
--- /dev/null
+++ b/libs/velo-external-db-core/src/web/domain-to-spi-error-translator.ts
@@ -0,0 +1,29 @@
+import { errors as domainErrors } from '@wix-velo/velo-external-db-commons'
+import { ErrorMessage } from '../spi-model/errors'
+
+export const domainToSpiErrorTranslator = (err: any) => {
+ switch(err.constructor) {
+ case domainErrors.ItemAlreadyExists:
+ const itemAlreadyExists: domainErrors.ItemAlreadyExists = err
+ return ErrorMessage.itemAlreadyExists(itemAlreadyExists.itemId, itemAlreadyExists.collectionName, itemAlreadyExists.message)
+
+ case domainErrors.CollectionDoesNotExists:
+ const collectionDoesNotExists: domainErrors.CollectionDoesNotExists = err
+ return ErrorMessage.collectionNotFound(collectionDoesNotExists.collectionName, collectionDoesNotExists.message)
+
+ case domainErrors.FieldAlreadyExists:
+ const fieldAlreadyExists: domainErrors.FieldAlreadyExists = err
+ return ErrorMessage.itemAlreadyExists(fieldAlreadyExists.fieldName, fieldAlreadyExists.collectionName, fieldAlreadyExists.message)
+
+ case domainErrors.FieldDoesNotExist:
+ const fieldDoesNotExist: domainErrors.FieldDoesNotExist = err
+ return ErrorMessage.invalidProperty(fieldDoesNotExist.collectionName, fieldDoesNotExist.propertyName, fieldDoesNotExist.message)
+
+ case domainErrors.UnsupportedSchemaOperation:
+ const unsupportedSchemaOperation: domainErrors.UnsupportedSchemaOperation = err
+ return ErrorMessage.operationIsNotSupportedByCollection(unsupportedSchemaOperation.collectionName, unsupportedSchemaOperation.operation, unsupportedSchemaOperation.message)
+
+ default:
+ return ErrorMessage.unknownError(err.message, err.status)
+ }
+ }
diff --git a/libs/velo-external-db-core/src/web/error-middleware.spec.ts b/libs/velo-external-db-core/src/web/error-middleware.spec.ts
index 402dd64b7..d4f8c770f 100644
--- a/libs/velo-external-db-core/src/web/error-middleware.spec.ts
+++ b/libs/velo-external-db-core/src/web/error-middleware.spec.ts
@@ -2,6 +2,8 @@ import * as Chance from 'chance'
import { errors } from '@wix-velo/velo-external-db-commons'
import { errorMiddleware } from './error-middleware'
import { Uninitialized } from '@wix-velo/test-commons'
+import { domainToSpiErrorTranslator } from './domain-to-spi-error-translator'
+
const chance = Chance()
describe('Error Middleware', () => {
@@ -24,7 +26,7 @@ describe('Error Middleware', () => {
errorMiddleware(err, null, ctx.res)
expect(ctx.res.status).toHaveBeenCalledWith(500)
- expect(ctx.res.send).toHaveBeenCalledWith( { message: err.message } )
+ expect(ctx.res.send).toHaveBeenCalledWith( { description: err.message, code: 'WDE0054' } )
})
test('converts exceptions to http error response', () => {
@@ -32,9 +34,9 @@ describe('Error Middleware', () => {
.forEach(Exception => {
const err = new Exception(chance.word())
errorMiddleware(err, null, ctx.res)
-
- expect(ctx.res.status).toHaveBeenCalledWith(err.status)
- expect(ctx.res.send).toHaveBeenCalledWith( { message: err.message } )
+ const spiError = domainToSpiErrorTranslator(err)
+ // expect(ctx.res.status).toHaveBeenCalledWith(err.status)
+ expect(ctx.res.send).toHaveBeenCalledWith( spiError.message )
ctx.res.status.mockClear()
ctx.res.send.mockClear()
diff --git a/libs/velo-external-db-core/src/web/error-middleware.ts b/libs/velo-external-db-core/src/web/error-middleware.ts
index 8be013790..b9104e450 100644
--- a/libs/velo-external-db-core/src/web/error-middleware.ts
+++ b/libs/velo-external-db-core/src/web/error-middleware.ts
@@ -1,9 +1,11 @@
import { NextFunction, Response } from 'express'
+import { domainToSpiErrorTranslator } from './domain-to-spi-error-translator'
export const errorMiddleware = (err: any, _req: any, res: Response, _next?: NextFunction) => {
if (process.env['NODE_ENV'] !== 'test') {
console.error(err)
}
- res.status(err.status || 500)
- .send({ message: err.message })
+
+ const errorMsg = domainToSpiErrorTranslator(err)
+ res.status(errorMsg.httpCode).send(errorMsg.message)
}
diff --git a/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts b/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts
new file mode 100644
index 000000000..497a02c4d
--- /dev/null
+++ b/libs/velo-external-db-core/src/web/jwt-auth-middleware.spec.ts
@@ -0,0 +1,132 @@
+import { sleep, Uninitialized } from '@wix-velo/test-commons'
+import * as driver from '../../test/drivers/auth_middleware_test_support'
+import { errors } from '@wix-velo/velo-external-db-commons'
+const { UnauthorizedError } = errors
+import * as Chance from 'chance'
+import { JwtAuthenticator, TOKEN_ISSUER } from './jwt-auth-middleware'
+import {
+ signedToken,
+ WixDataFacadeMock
+} from '../../test/drivers/auth_middleware_test_support'
+import { authConfig } from '@wix-velo/test-commons'
+import { PublicKeyMap } from './wix_data_facade'
+
+const chance = Chance()
+
+describe('JWT Auth Middleware', () => {
+
+ test('should authorize when JWT valid', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectAuthorized()
+ })
+
+ test('should authorize when JWT valid, only with second public key', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId)
+ env.auth = new JwtAuthenticator(ctx.externalDatabaseId, ctx.allowedMetasites, ctx.otherWixDataMock).authorizeJwt()
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+ expectAuthorized()
+ })
+
+ test('should throw when JWT siteId is not allowed', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: chance.word(), aud: ctx.externalDatabaseId }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT has no siteId claim', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, aud: ctx.externalDatabaseId }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT issuer is not Wix-Data', async() => {
+ const token = signedToken({ iss: chance.word(), siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT has no issuer', async() => {
+ const token = signedToken({ siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT audience is not externalDatabaseId of adapter', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: chance.word() }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT has no audience', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite }, ctx.keyId)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT kid is not found in Wix-Data keys', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, chance.word())
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT kid is absent', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId })
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ test('should throw when JWT is expired', async() => {
+ const token = signedToken({ iss: TOKEN_ISSUER, siteId: ctx.metasite, aud: ctx.externalDatabaseId }, ctx.keyId, '10ms')
+ await sleep(1000)
+ await env.auth(driver.requestBodyWith(Uninitialized, Uninitialized, `Bearer ${token}`), null, ctx.next)
+
+ expectUnauthorized()
+ })
+
+ const ctx = {
+ externalDatabaseId: Uninitialized,
+ metasite: Uninitialized,
+ allowedMetasites: Uninitialized,
+ next: Uninitialized,
+ keyId: Uninitialized,
+ otherWixDataMock: Uninitialized
+ }
+
+ const env = {
+ auth: Uninitialized,
+ }
+
+ const expectUnauthorized = () => {
+ expect(ctx.next).toHaveBeenCalledWith(new UnauthorizedError('You are not authorized'))
+ }
+
+ const expectAuthorized = () => {
+ expect(ctx.next).not.toHaveBeenCalledWith(new UnauthorizedError('You are not authorized'))
+ expect(ctx.next).toHaveBeenCalledWith()
+ }
+
+ beforeEach(() => {
+ ctx.externalDatabaseId = chance.word()
+ ctx.metasite = chance.word()
+ ctx.allowedMetasites = ctx.metasite
+ ctx.keyId = chance.word()
+ const otherKeyId = chance.word()
+ ctx.next = jest.fn().mockName('next')
+ const publicKeys: PublicKeyMap = {}
+ publicKeys[ctx.keyId] = authConfig.authPublicKey
+ const otherPublicKeys: PublicKeyMap = {}
+ otherPublicKeys[otherKeyId] = authConfig.otherAuthPublicKey
+ ctx.otherWixDataMock = new WixDataFacadeMock(otherPublicKeys, publicKeys)
+ env.auth = new JwtAuthenticator(ctx.externalDatabaseId, ctx.allowedMetasites, new WixDataFacadeMock(publicKeys)).authorizeJwt()
+ })
+})
diff --git a/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts b/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts
new file mode 100644
index 000000000..082d3f4fc
--- /dev/null
+++ b/libs/velo-external-db-core/src/web/jwt-auth-middleware.ts
@@ -0,0 +1,79 @@
+import { errors } from '@wix-velo/velo-external-db-commons'
+const { UnauthorizedError } = errors
+import { JwtHeader, JwtPayload, SigningKeyCallback, verify } from 'jsonwebtoken'
+import * as express from 'express'
+import { IWixDataFacade, PublicKeyMap } from './wix_data_facade'
+
+
+export const TOKEN_ISSUER = 'wix-data.wix.com'
+
+export class JwtAuthenticator {
+ publicKeys: PublicKeyMap | undefined
+ externalDatabaseId: string
+ allowedMetasites: string[]
+ wixDataFacade: IWixDataFacade
+
+ constructor(externalDatabaseId: string, allowedMetasites: string, wixDataFacade: IWixDataFacade) {
+ this.externalDatabaseId = externalDatabaseId
+ this.allowedMetasites = allowedMetasites ? allowedMetasites.split(',') : []
+ this.wixDataFacade = wixDataFacade
+ }
+
+ authorizeJwt() {
+ return async(req: express.Request, res: express.Response, next: express.NextFunction) => {
+ try {
+ const token = this.extractToken(req.header('authorization'))
+ this.publicKeys = this.publicKeys ?? await this.wixDataFacade.getPublicKeys(this.externalDatabaseId)
+ await this.verify(token)
+ } catch (err: any) {
+ console.error('Authorization failed: ' + err.message)
+ next(new UnauthorizedError('You are not authorized'))
+ }
+ next()
+ }
+ }
+
+ getKey(header: JwtHeader, callback: SigningKeyCallback) {
+ if (header.kid === undefined) {
+ callback(new UnauthorizedError('No kid set on JWT header'))
+ return
+ }
+ const publicKey = this.publicKeys![header.kid!]
+ if (publicKey === undefined) {
+ callback(new UnauthorizedError(`No public key fetched for kid ${header.kid}. Available keys: ${JSON.stringify(this.publicKeys)}`))
+ } else {
+ callback(null, publicKey)
+ }
+ }
+
+ verifyJwt(token: string) {
+ return new Promise<JwtPayload | string>((resolve, reject) =>
+ verify(token, this.getKey.bind(this), { audience: this.externalDatabaseId, issuer: TOKEN_ISSUER }, (err, decoded) =>
+ (err) ? reject(err) : resolve(decoded!)
+ ))
+ }
+
+
+ async verifyWithRetry(token: string): Promise<JwtPayload | string> {
+ try {
+ return await this.verifyJwt(token)
+ } catch (err) {
+ this.publicKeys = await this.wixDataFacade.getPublicKeys(this.externalDatabaseId)
+ return await this.verifyJwt(token)
+ }
+ }
+
+ async verify(token: string) {
+ const { siteId } = await this.verifyWithRetry(token) as JwtPayload
+ if (siteId === undefined || !this.allowedMetasites.includes(siteId)) {
+ throw new UnauthorizedError(`Unauthorized: ${siteId ? `site not allowed ${siteId}` : 'no siteId'}`)
+ }
+ }
+
+ private extractToken(header: string | undefined) {
+ if (header===undefined) {
+ throw new UnauthorizedError('No Authorization header')
+ }
+ return header.replace(/^(Bearer )/, '')
+ }
+}
diff --git a/libs/velo-external-db-core/src/web/wix_data_facade.ts b/libs/velo-external-db-core/src/web/wix_data_facade.ts
new file mode 100644
index 000000000..2ef8fa942
--- /dev/null
+++ b/libs/velo-external-db-core/src/web/wix_data_facade.ts
@@ -0,0 +1,40 @@
+import { errors } from '@wix-velo/velo-external-db-commons'
+const { UnauthorizedError } = errors
+import axios from 'axios'
+
+type PublicKeyResponse = {
+ publicKeys: {
+ id: string,
+ publicKeyPem: string
+ }[];
+};
+
+export type PublicKeyMap = { [key: string]: string }
+
+export interface IWixDataFacade {
+ getPublicKeys(externalDatabaseId: string): Promise<PublicKeyMap>
+}
+
+export class WixDataFacade implements IWixDataFacade {
+ baseUrl: string
+
+ constructor(baseUrl: string) {
+ this.baseUrl = baseUrl
+ }
+
+ async getPublicKeys(externalDatabaseId: string): Promise<PublicKeyMap> {
+ const url = `${this.baseUrl}/v1/external-databases/${externalDatabaseId}/public-keys`
+ const { data, status } = await axios.get<PublicKeyResponse>(url, {
+ headers: {
+ Accept: 'application/json',
+ },
+ })
+ if (status !== 200) {
+ throw new UnauthorizedError(`failed to get public keys: status ${status}`)
+ }
+ return data.publicKeys.reduce((m: PublicKeyMap, { id, publicKeyPem }) => {
+ m[id] = publicKeyPem
+ return m
+ }, {})
+ }
+}
diff --git a/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts b/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts
index 9d06a6642..c84f3a225 100644
--- a/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts
+++ b/libs/velo-external-db-core/test/drivers/auth_middleware_test_support.ts
@@ -1,9 +1,38 @@
+import { IWixDataFacade, PublicKeyMap } from '../../src/web/wix_data_facade'
+import * as jwt from 'jsonwebtoken'
+import { authConfig } from '@wix-velo/test-commons'
+import { SignOptions } from 'jsonwebtoken'
-export const requestBodyWith = (secretKey: string, role?: string | undefined, path?: string | undefined) => ({
+
+export const requestBodyWith = (role?: string | undefined, path?: string | undefined, authHeader?: string | undefined) => ({
path: path || '/',
body: {
requestContext: {
role: role || 'OWNER',
settings: {
- secretKey: secretKey
- } } } } )
+ } } },
+ header(_name: string) { return authHeader }
+} )
+
+export const signedToken = (payload: Record<string, unknown>, keyid?: string, expiration= '10000ms') => {
+ const options = keyid ? { algorithm: 'ES256', expiresIn: expiration, keyid: keyid } : { algorithm: 'ES256', expiresIn: expiration }
+ return jwt.sign(payload, authConfig.authPrivateKey, options as SignOptions)
+}
+
+export class WixDataFacadeMock implements IWixDataFacade {
+ publicKeys: PublicKeyMap[]
+ index: number
+
+ constructor(...publicKeys: PublicKeyMap[]) {
+ this.publicKeys = publicKeys
+ this.index = 0
+ }
+
+ getPublicKeys(_externalDatabaseId: string): Promise<PublicKeyMap> {
+ const publicKeyToReturn = this.publicKeys[this.index]
+ if (this.index < this.publicKeys.length-1) {
+ this.index++
+ }
+ return Promise.resolve(publicKeyToReturn)
+ }
+}
diff --git a/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts b/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts
index 3efb2e43d..8d019e4d8 100644
--- a/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts
+++ b/libs/velo-external-db-core/test/drivers/data_provider_test_support.ts
@@ -18,8 +18,8 @@ export const givenCountResult = (total: any, forCollectionName: any, filter: any
when(dataProvider.count).calledWith(forCollectionName, filter)
.mockResolvedValue(total)
-export const givenAggregateResult = (total: any, forCollectionName: any, filter: any, andAggregation: any) =>
- when(dataProvider.aggregate).calledWith(forCollectionName, filter, andAggregation)
+export const givenAggregateResult = (total: any, forCollectionName: any, filter: any, andAggregation: any, sort: any, skip: any, limit: any) =>
+ when(dataProvider.aggregate).calledWith(forCollectionName, filter, andAggregation, sort, skip, limit)
.mockResolvedValue(total)
export const expectInsertFor = (items: string | any[], forCollectionName: any) =>
diff --git a/libs/velo-external-db-core/test/drivers/data_service_test_support.ts b/libs/velo-external-db-core/test/drivers/data_service_test_support.ts
index 23983974a..2871991ac 100644
--- a/libs/velo-external-db-core/test/drivers/data_service_test_support.ts
+++ b/libs/velo-external-db-core/test/drivers/data_service_test_support.ts
@@ -17,8 +17,8 @@ export const dataService = {
const systemFields = SystemFields.map(({ name, type, subtype }) => ({ field: name, type, subtype }) )
-export const givenListResult = (entities: any, totalCount: any, forCollectionName: any, filter: any, sort: any, skip: any, limit: any, projection: any) =>
- when(dataService.find).calledWith(forCollectionName, filter, sort, skip, limit, projection)
+export const givenListResult = (entities: any, totalCount: any, forCollectionName: any, filter: any, sort: any, skip: any, limit: any, projection: any, omitTotalCount?: boolean) =>
+ when(dataService.find).calledWith(forCollectionName, filter, sort, skip, limit, projection, omitTotalCount)
.mockResolvedValue( { items: entities, totalCount } )
export const givenCountResult = (totalCount: any, forCollectionName: any, filter: any) =>
@@ -57,8 +57,8 @@ export const truncateResultTo = (forCollectionName: any) =>
when(dataService.truncate).calledWith(forCollectionName)
.mockResolvedValue(1)
-export const givenAggregateResult = (items: any, forCollectionName: any, filter: any, aggregation: any) =>
- when(dataService.aggregate).calledWith(forCollectionName, filter, aggregation)
+export const givenAggregateResult = (items: any, forCollectionName: any, filter: any, aggregation: any, sort: any, skip: any, limit: any) =>
+ when(dataService.aggregate).calledWith(forCollectionName, filter, aggregation, sort, skip, limit)
.mockResolvedValue({ items, totalCount: 0 })
export const reset = () => {
diff --git a/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts b/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts
index 67680fb5a..f3cc2a8fe 100644
--- a/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts
+++ b/libs/velo-external-db-core/test/drivers/filter_transformer_test_support.ts
@@ -11,6 +11,10 @@ export const stubEmptyFilterFor = (filter: any) => {
.mockReturnValue(EmptyFilter)
}
+export const stubEmptyFilterForUndefined = () => {
+ stubEmptyFilterFor(undefined)
+}
+
export const givenFilterByIdWith = (id: any, filter: any) => {
when(filterTransformer.transform).calledWith(filter)
.mockReturnValue({
diff --git a/libs/velo-external-db-core/test/drivers/schema_matchers.ts b/libs/velo-external-db-core/test/drivers/schema_matchers.ts
index cff91f1d8..508319743 100644
--- a/libs/velo-external-db-core/test/drivers/schema_matchers.ts
+++ b/libs/velo-external-db-core/test/drivers/schema_matchers.ts
@@ -1,4 +1,15 @@
+import {
+ Table,
+ CollectionCapabilities,
+ ResponseField
+} from '@wix-velo/velo-external-db-types'
import { asWixSchema, allowedOperationsFor, appendQueryOperatorsTo, asWixSchemaHeaders, ReadOnlyOperations } from '@wix-velo/velo-external-db-commons'
+import {
+ fieldTypeToWixDataEnum,
+ queryOperatorsToWixDataQueryOperators,
+ dataOperationsToWixDataQueryOperators,
+ collectionOperationsToWixDataCollectionOperations,
+} from '../../src/utils/schema_utils'
const appendAllowedOperationsToDbs = (dbs: any[], allowedSchemaOperations: any) => {
return dbs.map( (db: { fields: any }) => ({
@@ -25,3 +36,36 @@ export const schemaHeadersListFor = (collections: any) => toHaveSchemas(collecti
export const schemasWithReadOnlyCapabilitiesFor = (collections: any) => toHaveSchemas(collections, collectionToHaveReadOnlyCapability)
+export const fieldCapabilitiesObjectFor = (fieldCapabilities: { sortable: boolean, columnQueryOperators: string[] }) => expect.objectContaining({
+ sortable: fieldCapabilities.sortable,
+ queryOperators: expect.arrayContaining(fieldCapabilities.columnQueryOperators.map(c => queryOperatorsToWixDataQueryOperators(c)))
+})
+
+export const fieldInWixFormatFor = (field: ResponseField) => expect.objectContaining({
+ key: field.field,
+ type: fieldTypeToWixDataEnum(field.type),
+ capabilities: field.capabilities? fieldCapabilitiesObjectFor(field.capabilities) : undefined
+})
+
+export const fieldsToBeInWixFormat = (fields: ResponseField[]) => expect.arrayContaining(fields.map(f => fieldInWixFormatFor(f)))
+
+export const collectionCapabilitiesObjectFor = (collectionsCapabilities: CollectionCapabilities) => expect.objectContaining({
+ dataOperations: expect.arrayContaining(collectionsCapabilities.dataOperations.map(d => dataOperationsToWixDataQueryOperators(d))),
+ fieldTypes: expect.arrayContaining(collectionsCapabilities.fieldTypes.map(f => fieldTypeToWixDataEnum(f))),
+ collectionOperations: expect.arrayContaining(collectionsCapabilities.collectionOperations.map(c => collectionOperationsToWixDataCollectionOperations(c))),
+})
+
+export const collectionsInWixFormatFor = (collection: Table) => {
+ return expect.objectContaining({
+ id: collection.id,
+ fields: fieldsToBeInWixFormat(collection.fields),
+ capabilities: collection.capabilities? collectionCapabilitiesObjectFor(collection.capabilities): undefined
+ })
+}
+
+export const collectionsListFor = (collections: Table[]) => {
+ return expect.objectContaining({
+ collection: collections.map(collectionsInWixFormatFor)
+ })
+}
+
diff --git a/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts b/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts
index 3966326c2..2811da4d0 100644
--- a/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts
+++ b/libs/velo-external-db-core/test/drivers/schema_provider_test_support.ts
@@ -1,5 +1,7 @@
import { when } from 'jest-when'
-import { AllSchemaOperations } from '@wix-velo/velo-external-db-commons'
+import { AllSchemaOperations, AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { Table } from '@wix-velo/velo-external-db-types'
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
export const schemaProvider = {
list: jest.fn(),
@@ -8,7 +10,9 @@ export const schemaProvider = {
create: jest.fn(),
addColumn: jest.fn(),
removeColumn: jest.fn(),
- supportedOperations: jest.fn()
+ supportedOperations: jest.fn(),
+ columnCapabilitiesFor: jest.fn(),
+ changeColumnType: jest.fn(),
}
export const givenListResult = (dbs: any) =>
@@ -23,13 +27,17 @@ export const givenAdapterSupportedOperationsWith = (operations: any) =>
export const givenAllSchemaOperations = () =>
when(schemaProvider.supportedOperations).mockReturnValue(AllSchemaOperations)
-export const givenFindResults = (dbs: any[]) =>
- dbs.forEach((db: { id: any; fields: any }) => when(schemaProvider.describeCollection).calledWith(db.id).mockResolvedValue(db.fields) )
+export const givenFindResults = (tables: Table[]) =>
+ tables.forEach((table) => when(schemaProvider.describeCollection).calledWith(table.id).mockResolvedValue({ id: table.id, fields: table.fields, capabilities: table.capabilities }))
export const expectCreateOf = (collectionName: any) =>
when(schemaProvider.create).calledWith(collectionName)
.mockResolvedValue(undefined)
+export const expectCreateWithFieldsOf = (collectionName: any, column: any) =>
+ when(schemaProvider.create).calledWith(collectionName, column)
+ .mockResolvedValue(undefined)
+
export const expectCreateColumnOf = (column: any, collectionName: any) =>
when(schemaProvider.addColumn).calledWith(collectionName, column)
.mockResolvedValue(undefined)
@@ -38,6 +46,24 @@ export const expectRemoveColumnOf = (columnName: any, collectionName: any) =>
when(schemaProvider.removeColumn).calledWith(collectionName, columnName)
.mockResolvedValue(undefined)
+export const givenColumnCapabilities = () => {
+ when(schemaProvider.columnCapabilitiesFor).calledWith('text')
+ .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('number')
+ .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('boolean')
+ .mockReturnValue({ sortable: true, columnQueryOperators: [eq] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('url')
+ .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('datetime')
+ .mockReturnValue({ sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('image')
+ .mockReturnValue({ sortable: false, columnQueryOperators: [] })
+ when(schemaProvider.columnCapabilitiesFor).calledWith('object')
+ .mockReturnValue({ sortable: false, columnQueryOperators: [eq, ne] })
+}
+
+
export const reset = () => {
schemaProvider.list.mockClear()
schemaProvider.listHeaders.mockClear()
@@ -46,4 +72,6 @@ export const reset = () => {
schemaProvider.addColumn.mockClear()
schemaProvider.removeColumn.mockClear()
schemaProvider.supportedOperations.mockClear()
+ schemaProvider.columnCapabilitiesFor.mockClear()
+ schemaProvider.changeColumnType.mockClear()
}
diff --git a/libs/velo-external-db-core/test/gen.ts b/libs/velo-external-db-core/test/gen.ts
index cfb11c806..452c861da 100644
--- a/libs/velo-external-db-core/test/gen.ts
+++ b/libs/velo-external-db-core/test/gen.ts
@@ -1,6 +1,18 @@
import * as Chance from 'chance'
import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
import { gen as genCommon } from '@wix-velo/test-commons'
+import {
+ CollectionCapabilities,
+ CollectionOperation,
+ InputField,
+ FieldType,
+ ResponseField,
+ DataOperation,
+ Table,
+ Encryption,
+ } from '@wix-velo/velo-external-db-types'
+
+ const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
const chance = Chance()
@@ -8,13 +20,17 @@ export const invalidOperatorForType = (validOperators: string | string[]) => ran
Object.values(AdapterOperators).filter(x => !validOperators.includes(x))
)
-export const randomObjectFromArray = (array: any[]) => array[chance.integer({ min: 0, max: array.length - 1 })]
-
-export const randomColumn = () => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } )
+export const randomObjectFromArray = <T>(array: any[]): T => array[chance.integer({ min: 0, max: array.length - 1 })]
+export const randomColumn = (): InputField => ( { name: chance.word(), type: 'text', subtype: 'string', precision: '256', isPrimary: false } )
+// TODO: random the wix-type filed from the enum
export const randomWixType = () => randomObjectFromArray(['number', 'text', 'boolean', 'url', 'datetime', 'object'])
+export const randomFieldType = () => randomObjectFromArray<FieldType>(Object.values(FieldType))
+
+export const randomCollectionOperation = () => randomObjectFromArray<CollectionOperation>(Object.values(CollectionOperation))
+
export const randomOperator = () => (chance.pickone(['$ne', '$lt', '$lte', '$gt', '$gte', '$hasSome', '$eq', '$contains', '$startsWith', '$endsWith']))
export const randomFilter = () => {
@@ -26,7 +42,7 @@ export const randomFilter = () => {
}
}
-export const randomArrayOf = (gen: any) => {
+export const randomArrayOf= <T>(gen: any): T[] => {
const arr = []
const num = chance.natural({ min: 2, max: 20 })
for (let i = 0; i < num; i++) {
@@ -35,21 +51,41 @@ export const randomArrayOf = (gen: any) => {
return arr
}
-export const randomCollectionName = () => chance.word({ length: 5 })
+export const randomAdapterOperators = () => (chance.pickone([eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include]))
+
+export const randomDataOperations = () => (chance.pickone(Object.values(DataOperation)))
+
+export const randomColumnCapabilities = () => ({
+ sortable: chance.bool(),
+ columnQueryOperators: [ randomAdapterOperators() ]
+})
+
+export const randomCollectionCapabilities = (): CollectionCapabilities => ({
+ dataOperations: [ randomDataOperations() ],
+ fieldTypes: [ randomFieldType() ],
+ collectionOperations: [ randomCollectionOperation() ],
+ indexing: [],
+ encryption: Encryption.notSupported,
+ referenceCapabilities: {
+ supportedNamespaces: []
+ }
+})
+
+export const randomCollectionName = ():string => chance.word({ length: 5 })
-export const randomCollections = () => randomArrayOf( randomCollectionName )
+export const randomCollections = () => randomArrayOf<string>( randomCollectionName )
-export const randomWixDataType = () => chance.pickone(['number', 'text', 'boolean', 'url', 'datetime', 'image', 'object' ])
+export const randomWixDataType = () => chance.pickone(['number', 'text', 'boolean', 'datetime', 'object' ])
-export const randomDbField = () => ( { field: chance.word(), type: randomWixDataType(), subtype: chance.word(), isPrimary: chance.bool() } )
+export const randomDbField = (): ResponseField => ( { field: chance.word(), type: randomWixDataType(), subtype: chance.word(), isPrimary: chance.bool(), capabilities: randomColumnCapabilities() } )
-export const randomDbFields = () => randomArrayOf( randomDbField )
+export const randomDbFields = () => randomArrayOf<ResponseField>( randomDbField )
-export const randomDb = () => ( { id: randomCollectionName(), fields: randomDbFields() })
+export const randomDb = (): Table => ( { id: randomCollectionName(), fields: randomDbFields(), capabilities: randomCollectionCapabilities() })
-export const randomDbs = () => randomArrayOf( randomDb )
+export const randomDbs = (): Table[] => randomArrayOf( randomDb )
-export const randomDbsWithIdColumn = () => randomDbs().map(i => ({ ...i, fields: [ ...i.fields, { field: '_id', type: 'text' }] }))
+export const randomDbsWithIdColumn = (): Table[] => randomDbs().map(i => ({ ...i, fields: [ ...i.fields, { field: '_id', type: 'text', capabilities: randomColumnCapabilities() }] }))
export const truthyValue = () => chance.pickone(['true', '1', 1, true])
export const falsyValue = () => chance.pickone(['false', '0', 0, false])
diff --git a/libs/velo-external-db-types/src/collection_types.ts b/libs/velo-external-db-types/src/collection_types.ts
new file mode 100644
index 000000000..cf56870f6
--- /dev/null
+++ b/libs/velo-external-db-types/src/collection_types.ts
@@ -0,0 +1,124 @@
+export enum DataOperation {
+ query = 'query',
+ count = 'count',
+ queryReferenced = 'queryReferenced',
+ aggregate = 'aggregate',
+ insert = 'insert',
+ update = 'update',
+ remove = 'remove',
+ truncate = 'truncate',
+ insertReferences = 'insertReferences',
+ removeReferences = 'removeReferences',
+}
+
+export enum FieldType {
+ text = 'text',
+ number = 'number',
+ boolean = 'boolean',
+ datetime = 'datetime',
+ object = 'object',
+ longText = 'longText',
+ singleReference = 'singleReference',
+ multiReference = 'multiReference',
+}
+
+export enum CollectionOperation {
+ update = 'update',
+ remove = 'remove',
+}
+
+export enum Encryption {
+ notSupported = 'notSupported',
+ wixDataNative = 'wixDataNative',
+ dataSourceNative = 'dataSourceNative',
+}
+
+export type CollectionCapabilities = {
+ dataOperations: DataOperation[],
+ fieldTypes: FieldType[],
+ referenceCapabilities: ReferenceCapabilities,
+ collectionOperations: CollectionOperation[],
+ indexing: IndexingCapabilityEnum[],
+ encryption: Encryption,
+}
+
+export type ColumnCapabilities = {
+ sortable: boolean,
+ columnQueryOperators: string[],
+}
+
+export type FieldAttributes = {
+ type: string,
+ subtype?: string,
+ precision?: number | string,
+ isPrimary?: boolean,
+}
+
+export interface ReferenceCapabilities {
+ supportedNamespaces: string[],
+}
+
+export interface IndexingCapabilityEnum {
+}
+
+export enum SchemaOperations {
+ List = 'list',
+ ListHeaders = 'listHeaders',
+ Create = 'createCollection',
+ Drop = 'dropCollection',
+ AddColumn = 'addColumn',
+ RemoveColumn = 'removeColumn',
+ ChangeColumnType = 'changeColumnType',
+ Describe = 'describeCollection',
+ FindWithSort = 'findWithSort',
+ Aggregate = 'aggregate',
+ BulkDelete = 'bulkDelete',
+ Truncate = 'truncate',
+ UpdateImmediately = 'updateImmediately',
+ DeleteImmediately = 'deleteImmediately',
+ StartWithCaseSensitive = 'startWithCaseSensitive',
+ StartWithCaseInsensitive = 'startWithCaseInsensitive',
+ Projection = 'projection',
+ FindObject = 'findObject',
+ Matches = 'matches',
+ NotOperator = 'not',
+ IncludeOperator = 'include',
+ FilterByEveryField = 'filterByEveryField',
+ QueryNestedFields = 'queryNestedFields',
+ NonAtomicBulkInsert = 'NonAtomicBulkInsert',
+ AtomicBulkInsert = 'AtomicBulkInsert'
+}
+
+export type InputField = FieldAttributes & { name: string }
+
+export type ResponseField = FieldAttributes & {
+ field: string
+ capabilities?: {
+ sortable: boolean
+ columnQueryOperators: string[]
+ }
+}
+
+export type Table = {
+ id: string,
+ fields: ResponseField[]
+ capabilities?: CollectionCapabilities
+}
+export interface ISchemaProvider {
+ list(): Promise<Table[]>
+ listHeaders(): Promise<string[]>
+ supportedOperations(): SchemaOperations[]
+ create(collectionName: string, columns?: InputField[]): Promise<void>
+ addColumn(collectionName: string, column: InputField): Promise<void>
+ removeColumn(collectionName: string, columnName: string): Promise<void>
+ changeColumnType?(collectionName: string, column: InputField): Promise<void>
+ describeCollection(collectionName: string): Promise<ResponseField[]> | Promise<Table>
+ drop(collectionName: string): Promise<void>
+ translateDbTypes?(column: InputField | ResponseField | string): ResponseField | string
+ columnCapabilitiesFor?(columnType: string): ColumnCapabilities
+ capabilities?(): CollectionCapabilities
+}
+
+
+
+
diff --git a/libs/velo-external-db-types/src/index.ts b/libs/velo-external-db-types/src/index.ts
index 2fccbcdcf..2dce2a932 100644
--- a/libs/velo-external-db-types/src/index.ts
+++ b/libs/velo-external-db-types/src/index.ts
@@ -1,3 +1,11 @@
+import {
+ ResponseField,
+ SchemaOperations,
+ ISchemaProvider
+} from './collection_types'
+
+export * from './collection_types'
+
export enum AdapterOperator { //in velo-external-db-core
eq = 'eq',
gt = 'gt',
@@ -16,30 +24,6 @@ export enum AdapterOperator { //in velo-external-db-core
matches = 'matches'
}
-export enum SchemaOperations {
- List = 'list',
- ListHeaders = 'listHeaders',
- Create = 'createCollection',
- Drop = 'dropCollection',
- AddColumn = 'addColumn',
- RemoveColumn = 'removeColumn',
- Describe = 'describeCollection',
- FindWithSort = 'findWithSort',
- Aggregate = 'aggregate',
- BulkDelete = 'bulkDelete',
- Truncate = 'truncate',
- UpdateImmediately = 'updateImmediately',
- DeleteImmediately = 'deleteImmediately',
- StartWithCaseSensitive = 'startWithCaseSensitive',
- StartWithCaseInsensitive = 'startWithCaseInsensitive',
- Projection = 'projection',
- FindObject = 'findObject',
- Matches = 'matches',
- NotOperator = 'not',
- IncludeOperator = 'include',
- FilterByEveryField = 'filterByEveryField',
-}
-
export type FieldWithQueryOperators = ResponseField & { queryOperators: string[] }
export interface AsWixSchemaHeaders {
@@ -116,45 +100,15 @@ export type AdapterAggregation = {
export interface IDataProvider {
find(collectionName: string, filter: AdapterFilter, sort: any, skip: number, limit: number, projection: string[]): Promise<Item[]>;
count(collectionName: string, filter: AdapterFilter): Promise<number>;
- insert(collectionName: string, items: Item[], fields?: ResponseField[]): Promise<number>;
+ insert(collectionName: string, items: Item[], fields?: ResponseField[], upsert?: boolean): Promise<number>;
update(collectionName: string, items: Item[], fields?: any): Promise<number>;
delete(collectionName: string, itemIds: string[]): Promise<number>;
truncate(collectionName: string): Promise<void>;
- aggregate?(collectionName: string, filter: AdapterFilter, aggregation: AdapterAggregation): Promise<Item[]>;
+ // sort, skip, limit are not really optional, after we'll implement in all the data providers we can remove the ?
+ aggregate?(collectionName: string, filter: AdapterFilter, aggregation: AdapterAggregation, sort?: Sort[], skip?: number, limit?: number ): Promise<Item[]>;
}
-export type TableHeader = {
- id: string
-}
-
-export type Table = TableHeader & { fields: ResponseField[] }
-
-export type FieldAttributes = {
- type: string,
- subtype?: string,
- precision?: number | string,
- isPrimary?: boolean,
-}
-
-export type InputField = FieldAttributes & { name: string }
-
-export type ResponseField = FieldAttributes & { field: string }
-
-export interface ISchemaProvider {
- list(): Promise<Table[]>
- listHeaders(): Promise<string[]>
- supportedOperations(): SchemaOperations[]
- create(collectionName: string, columns?: InputField[]): Promise<void>
- addColumn(collectionName: string, column: InputField): Promise<void>
- removeColumn(collectionName: string, columnName: string): Promise<void>
- describeCollection(collectionName: string): Promise<ResponseField[]>
- drop(collectionName: string): Promise<void>
- translateDbTypes?(column: InputField | ResponseField | string): ResponseField | string
-}
-
-export interface IBaseHttpError extends Error {
- status: number;
-}
+export interface IBaseHttpError extends Error {}
type ValidConnectionResult = { valid: true }
type InvalidConnectionResult = { valid: false, error: IBaseHttpError }
@@ -226,15 +180,6 @@ export enum WixDataFunction {
$sum = '$sum',
}
-export type WixDataAggregation = {
- processingStep: {
- _id: string | { [key: string]: any }
- [key: string]: any
- // [fieldAlias: string]: {[key in WixDataFunction]: string | number },
- }
- postFilteringStep: WixDataFilter
-}
-
export type WixDataRole = 'OWNER' | 'BACKEND_CODE' | 'MEMBER' | 'VISITOR'
export type VeloRole = 'Admin' | 'Member' | 'Visitor'
diff --git a/package.json b/package.json
index e1be6ab4b..279cbd4bd 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,7 @@
"build:full-image": "nx run velo-external-db:build-image ",
"lint": "eslint --cache ./",
"lint:fix": "eslint --cache --fix ./",
- "test": "npm run test:core; npm run test:postgres; npm run test:spanner; npm run test:mysql; npm run test:mssql; npm run test:firestore; npm run test:mongo; npm run test:airtable; npm run test:dynamodb; npm run test:bigquery",
+ "test": "npm run test:core; npm run test:mysql; npm run test:postgres; npm run test:mssql; npm run test:spanner; npm run test:mongo; npm run test:dynamodb; npm run test:firestore; npm run test:google-sheets",
"test:core": "nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-config,@wix-velo/velo-external-db-core,@wix-velo/external-db-security",
"test:postgres": "TEST_ENGINE=postgres nx run-many --skip-nx-cache --target=test --projects=@wix-velo/external-db-postgres,velo-external-db",
"test:postgres13": "npm run test:postgres",
@@ -49,6 +49,7 @@
"ejs": "^3.1.8",
"express": "^4.17.2",
"google-spreadsheet": "^3.3.0",
+ "jsonwebtoken": "^8.5.1",
"moment": "^2.29.3",
"mongodb": "^4.6.0",
"mssql": "^8.1.0",
@@ -82,6 +83,7 @@
"@types/google-spreadsheet": "^3.3.0",
"@types/jest": "^27.4.1",
"@types/jest-when": "^3.5.0",
+ "@types/jsonwebtoken": "^8.5.9",
"@types/mssql": "^8.0.2",
"@types/mysql": "^2.15.21",
"@types/node": "^16.11.7",
diff --git a/workspace.json b/workspace.json
index 7a9d98698..fc54eac4c 100644
--- a/workspace.json
+++ b/workspace.json
@@ -6,15 +6,13 @@
"@wix-velo/external-db-mysql": "libs/external-db-mysql",
"@wix-velo/external-db-mssql": "libs/external-db-mssql",
"@wix-velo/external-db-spanner": "libs/external-db-spanner",
- "@wix-velo/external-db-mongo": "libs/external-db-mongo",
"@wix-velo/external-db-firestore": "libs/external-db-firestore",
- "@wix-velo/external-db-airtable": "libs/external-db-airtable",
- "@wix-velo/external-db-bigquery": "libs/external-db-bigquery",
- "@wix-velo/external-db-dynamodb": "libs/external-db-dynamodb",
"@wix-velo/external-db-google-sheets": "libs/external-db-google-sheets",
+ "@wix-velo/external-db-dynamodb": "libs/external-db-dynamodb",
"@wix-velo/external-db-security": "libs/external-db-security",
"@wix-velo/test-commons": "libs/test-commons",
"@wix-velo/velo-external-db-commons": "libs/velo-external-db-commons",
+ "@wix-velo/external-db-mongo": "libs/external-db-mongo",
"@wix-velo/velo-external-db-core": "libs/velo-external-db-core",
"@wix-velo/velo-external-db-types": "libs/velo-external-db-types",
"@wix-velo/external-db-testkit": "libs/external-db-testkit",