Mongo to V3 (#397)

* feat: new capabilities file

* feat: new capabilities property in the schema list

* fix: fixed the types in mongo_schema_provider

* feat: upsert insert implementation

* feat: aggregate implementation

* feat: enable query nested fields

* feat: new supported ops and test

* test: enable mongo tests on github cli

* fix: updated mssql supported ops

* refactor: some changes according to the review

* test: new supported operations

* test: spanner supported ops update

Author: MXPOL

.github/workflows/main.yml

@@ -41,7 +41,8 @@ jobs:
                    "postgres", "postgres13", "postgres12", "postgres11", "postgres10", "postgres9",
                    "mysql", "mysql5",
                    "mssql", "mssql17",
-                   spanner
+                   spanner,
+                   "mongo", "mongo4",
                    ]
 
     env:

apps/velo-external-db/src/storage/factory.ts

@@ -23,10 +23,10 @@ export const engineConnectorFor = async(_type: string, config: any): Promise<Dat
             const { mySqlFactory } = require('@wix-velo/external-db-mysql')
             return await mySqlFactory(config)
         }
-        // case 'mongo': {
-        //     const { mongoFactory } = require('@wix-velo/external-db-mongo')
-        //     return await mongoFactory(config)
-        // }
+        case 'mongo': {
+            const { mongoFactory } = require('@wix-velo/external-db-mongo')
+            return await mongoFactory(config)
+        }
         // case 'google-sheet': {
         //     const { googleSheetFactory } = require('@wix-velo/external-db-google-sheets')
         //     return await googleSheetFactory(config)

apps/velo-external-db/test/e2e/app_data.e2e.spec.ts

@@ -1,8 +1,8 @@
 import axios from 'axios'
 import each from 'jest-each'
-import Chance = require('chance')
+import * as Chance from 'chance'
 import { Uninitialized, gen as genCommon, testIfSupportedOperationsIncludes, streamToArray } from '@wix-velo/test-commons'
-import { SchemaOperations } from '@wix-velo/velo-external-db-types'
+import { InputField, SchemaOperations, Item } from '@wix-velo/velo-external-db-types'
 import { dataSpi } from '@wix-velo/velo-external-db-core'
 import { authAdmin, authOwner, authVisitor } from '@wix-velo/external-db-testkit'
 import * as gen from '../gen'
@@ -11,7 +11,7 @@ import * as matchers from '../drivers/schema_api_rest_matchers'
 import * as data from '../drivers/data_api_rest_test_support'
 import * as authorization from '../drivers/authorization_test_support'
 import { initApp, teardownApp, dbTeardown, setupDb, currentDbImplementationName, supportedOperations } from '../resources/e2e_resources'
-const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField, QueryNestedFields } = SchemaOperations
+const { UpdateImmediately, DeleteImmediately, Truncate, Aggregate, FindWithSort, Projection, FilterByEveryField, QueryNestedFields, NonAtomicBulkInsert, AtomicBulkInsert } = SchemaOperations
 
 const chance = Chance()
 
@@ -80,7 +80,7 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`,  ()
         )
     })
 
-    test('insert api should fail if item already exists', async() => {
+    testIfSupportedOperationsIncludes(supportedOperations, [AtomicBulkInsert])('insert api should fail if item already exists', async() => {
         await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
         await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin)
 
@@ -98,6 +98,23 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`,  ()
         )
     })
 
+    testIfSupportedOperationsIncludes(supportedOperations, [NonAtomicBulkInsert])('insert api should throw 409 error if item already exists and continue inserting the rest', async() => {
+        await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
+        await data.givenItems([ ctx.items[1] ], ctx.collectionName, authAdmin)
+
+        const response = axiosInstance.post('/data/insert', data.insertRequest(ctx.collectionName, ctx.items, false),  { responseType: 'stream', ...authAdmin })
+
+        const expectedItems = ctx.items.map(i => dataSpi.QueryResponsePart.item(i))
+                
+        await expect(response).rejects.toThrow('409')
+        await expect(data.queryCollectionAsArray(ctx.collectionName, [], undefined, authOwner)).resolves.toEqual(expect.toIncludeAllMembers(
+            [ 
+                ...expectedItems, 
+                data.pagingMetadata(expectedItems.length, expectedItems.length)
+            ])
+        )
+    })
+
     test('insert api should succeed if item already exists and overwriteExisting is on', async() => {
         await schema.givenCollection(ctx.collectionName, [ctx.column], authOwner)
         await data.givenItems([ ctx.item ], ctx.collectionName, authAdmin)
@@ -347,8 +364,24 @@ describe(`Velo External DB Data REST API: ${currentDbImplementationName()}`,  ()
         })
     })
 
+    interface Ctx {
+        collectionName: string
+        column: InputField
+        numberColumns: InputField[]
+        objectColumn: InputField
+        item:  Item
+        items:  Item[]
+        modifiedItem: Item
+        modifiedItems: Item[]
+        anotherItem: Item
+        numberItem: Item
+        anotherNumberItem: Item
+        objectItem: Item
+        nestedFieldName: string
+        pastVeloDate: { $date: string; }
+    }
 
-    const ctx = {
+    const ctx: Ctx  = {
         collectionName: Uninitialized,
         column: Uninitialized,
         numberColumns: Uninitialized,

apps/velo-external-db/test/env/env.db.setup.js

@@ -8,7 +8,7 @@ const { testResources: mysql } = require ('@wix-velo/external-db-mysql')
 const { testResources: spanner } = require ('@wix-velo/external-db-spanner')
 // const { testResources: firestore } = require ('@wix-velo/external-db-firestore')
 const { testResources: mssql } = require ('@wix-velo/external-db-mssql')
-// const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
+const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
 // const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets')
 // const { testResources: airtable } = require('@wix-velo/external-db-airtable')
 // const { testResources: dynamoDb } = require('@wix-velo/external-db-dynamodb')
@@ -39,9 +39,9 @@ const initEnv = async(testEngine) => {
             await mssql.initEnv()
             break
 
-        // case 'mongo':
-        //     await mongo.initEnv()
-        //     break
+        case 'mongo':
+            await mongo.initEnv()
+            break
         // case 'google-sheet':
         //     await googleSheet.initEnv()
         //     break
@@ -86,9 +86,9 @@ const cleanup = async(testEngine) => {
         //     await googleSheet.cleanup()
         //     break
 
-        // case 'mongo':
-        //     await mongo.cleanup()
-        //     break
+        case 'mongo':
+            await mongo.cleanup()
+            break
 
         // case 'dynamodb':
         //     await dynamoDb.cleanup()

apps/velo-external-db/test/env/env.db.teardown.js

@@ -3,7 +3,7 @@ const { testResources: mysql } = require ('@wix-velo/external-db-mysql')
 const { testResources: spanner } = require ('@wix-velo/external-db-spanner')
 // const { testResources: firestore } = require ('@wix-velo/external-db-firestore')
 const { testResources: mssql } = require ('@wix-velo/external-db-mssql')
-// const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
+const { testResources: mongo } = require ('@wix-velo/external-db-mongo')
 // const { testResources: googleSheet } = require('@wix-velo/external-db-google-sheets')
 // const { testResources: airtable } = require('@wix-velo/external-db-airtable')
 // const { testResources: dynamo } = require('@wix-velo/external-db-dynamodb')
@@ -45,9 +45,9 @@ const shutdownEnv = async(testEngine) => {
         //     await dynamo.shutdownEnv()
         //     break
 
-        // case 'mongo': 
-        //     await mongo.shutdownEnv()
-        //     break
+        case 'mongo': 
+            await mongo.shutdownEnv()
+            break
 
         // case 'bigquery':
         //     await bigquery.shutdownEnv()

apps/velo-external-db/test/resources/e2e_resources.ts

@@ -33,7 +33,7 @@ const testSuits = {
     spanner: new E2EResources(spanner, createAppWithWixDataBaseUrl),
     firestore: new E2EResources(firestore, createApp),
     mssql: new E2EResources(mssql, createAppWithWixDataBaseUrl),
-    mongo: new E2EResources(mongo, createApp),
+    mongo: new E2EResources(mongo, createAppWithWixDataBaseUrl),
     'google-sheet': new E2EResources(googleSheet, createApp),
     airtable: new E2EResources(airtable, createApp),
     dynamodb: new E2EResources(dynamo, createApp),

apps/velo-external-db/test/resources/provider_resources.ts

@@ -74,7 +74,7 @@ const testSuits = {
     spanner: suiteDef('Spanner', spannerTestEnvInit, spanner.testResources),
     firestore: suiteDef('Firestore', firestoreTestEnvInit, firestore.testResources.supportedOperations),
     mssql: suiteDef('Sql Server', mssqlTestEnvInit, mssql.testResources),
-    mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources.supportedOperations),
+    mongo: suiteDef('Mongo', mongoTestEnvInit, mongo.testResources),
     airtable: suiteDef('Airtable', airTableTestEnvInit, airtable.testResources.supportedOperations),
     dynamodb: suiteDef('DynamoDb', dynamoTestEnvInit, dynamo.testResources.supportedOperations),
     bigquery: suiteDef('BigQuery', bigqueryTestEnvInit, bigquery.testResources.supportedOperations),

libs/external-db-mongo/src/exception_translator.ts

@@ -1,15 +1,17 @@
 import { errors } from '@wix-velo/velo-external-db-commons'
 const { ItemAlreadyExists } = errors
 
-const notThrowingTranslateErrorCodes = (err: any) => {
+const extractItemIdFromError = (err: any) => err.message.split('"')[1]
+
+const notThrowingTranslateErrorCodes = (err: any, collectionName: string) => {
     switch (err.code) {
-        case 11000:
-            return new ItemAlreadyExists(`Item already exists: ${err.message}`)
+        case 11000:            
+            return new ItemAlreadyExists(`Item already exists: ${err.message}`, collectionName, extractItemIdFromError(err))
         default: 
             return new Error (`default ${err.message}`) 
     }
 }
 
-export const translateErrorCodes = (err: any) => {
-    throw notThrowingTranslateErrorCodes(err)
+export const translateErrorCodes = (err: any, collectionName: string) => {    
+    throw notThrowingTranslateErrorCodes(err, collectionName)
 }

libs/external-db-mongo/src/mongo_capabilities.ts

@@ -0,0 +1,19 @@
+import { AdapterOperators } from '@wix-velo/velo-external-db-commons'
+import { CollectionOperation, DataOperation, FieldType } from '@wix-velo/velo-external-db-types'
+
+const { query, count, queryReferenced, aggregate, } = DataOperation
+const { eq, ne, string_contains, string_begins, string_ends, gt, gte, lt, lte, include } = AdapterOperators
+
+export const ReadWriteOperations = Object.values(DataOperation)
+export const ReadOnlyOperations = [query, count, queryReferenced, aggregate]
+export const FieldTypes = Object.values(FieldType)
+export const CollectionOperations = Object.values(CollectionOperation)
+export const ColumnsCapabilities = {
+    text: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+    url: { sortable: true, columnQueryOperators: [eq, ne, string_contains, string_begins, string_ends, include, gt, gte, lt, lte] },
+    number: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte, include] },
+    boolean: { sortable: true, columnQueryOperators: [eq] },
+    image: { sortable: false, columnQueryOperators: [] },
+    object: { sortable: false, columnQueryOperators: [] },
+    datetime: { sortable: true, columnQueryOperators: [eq, ne, gt, gte, lt, lte] },
+}

libs/external-db-mongo/src/mongo_data_provider.ts

@@ -1,6 +1,6 @@
 import { translateErrorCodes } from './exception_translator'
-import { unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils'
-import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item } from '@wix-velo/velo-external-db-types'
+import { insertExpressionFor, isEmptyObject, unpackIdFieldForItem, updateExpressionFor, validateTable } from './mongo_utils'
+import { IDataProvider, AdapterFilter as Filter, AdapterAggregation as Aggregation, Item, Sort,  } from '@wix-velo/velo-external-db-types'
 import FilterParser from './sql_filter_transformer'
 import { MongoClient } from 'mongodb'
 
@@ -34,22 +34,22 @@ export default class DataProvider implements IDataProvider {
                                 .count(filterExpr)
     }
 
-    async insert(collectionName: string, items: Item[] ): Promise<number> {
+    async insert(collectionName: string, items: Item[], _fields: any[], upsert = false): Promise<number> {
         validateTable(collectionName)
-        const result = await this.client.db()
-                                        .collection(collectionName)
-                                        //@ts-ignore - Type 'string' is not assignable to type 'ObjectId', objectId Can be a 24 character hex string, 12 byte binary Buffer, or a number. and we cant assume that on the _id input
-                                        .insertMany(items)
-                                        .catch(translateErrorCodes)
-        return result.insertedCount
+        const { insertedCount, upsertedCount } = await this.client.db()
+                                                                  .collection(collectionName) 
+                                                                  .bulkWrite(insertExpressionFor(items, upsert), { ordered: false })
+                                                                  .catch(e => translateErrorCodes(e, collectionName))
+        
+        return insertedCount + upsertedCount
     }
 
     async update(collectionName: string, items: Item[]): Promise<number> {
         validateTable(collectionName)
         const result = await this.client.db()
                                         .collection(collectionName)
                                         .bulkWrite( updateExpressionFor(items) )
-        return result.nModified
+                                        return result.nModified
     }
 
     async delete(collectionName: string, itemIds: string[]): Promise<number> {
@@ -67,17 +67,26 @@ export default class DataProvider implements IDataProvider {
                          .deleteMany({})
     }
 
-    async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation): Promise<Item[]> {
+    async aggregate(collectionName: string, filter: Filter, aggregation: Aggregation, sort: Sort[], skip: number, limit: number): Promise<Item[]> {
         validateTable(collectionName)
+        const additionalAggregationStages = []
         const { fieldsStatement, havingFilter } = this.filterParser.parseAggregation(aggregation)
         const { filterExpr } = this.filterParser.transform(filter)
+        const sortExpr = this.filterParser.orderAggregationBy(sort)
+
+        !isEmptyObject(sortExpr.$sort)? additionalAggregationStages.push(sortExpr) : null
+        skip? additionalAggregationStages.push({ $skip: skip }) : null
+        limit? additionalAggregationStages.push({ $limit: limit }) : null
+        
         const result = await this.client.db()
-                                    .collection(collectionName)
-                                    .aggregate( [ { $match: filterExpr },
-                                         fieldsStatement,
-                                         havingFilter
-                                    ] )
-                                    .toArray()
+                                        .collection(collectionName)
+                                        .aggregate([ 
+                                            { $match: filterExpr },
+                                            fieldsStatement,
+                                            havingFilter,
+                                            ...additionalAggregationStages
+                                        ])
+                                        .toArray()
 
         return result.map( unpackIdFieldForItem )
     }