Improvement/cldsrv 724 sur utapi tests

package.json

@@ -19,6 +19,8 @@
   },
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.705.0",
+    "@aws-sdk/credential-providers": "^3.864.0",
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "arsenal": "git+https://github.com/scality/Arsenal#8.2.32",

tests/functional/aws-node-sdk/lib/utility/bucket-util.js

@@ -1,81 +1,95 @@
-const AWS = require('aws-sdk');
-AWS.config.logger = console;
-const { S3 } = require('aws-sdk');
+const {
+    S3Client,
+    HeadBucketCommand,
+    CreateBucketCommand,
+    DeleteBucketCommand,
+    ListObjectVersionsCommand,
+    DeleteObjectCommand,
+    ListBucketsCommand,
+} = require('@aws-sdk/client-s3');
 const projectFixture = require('../fixtures/project');
 const getConfig = require('../../test/support/config');
 
 class BucketUtility {
-    constructor(profile = 'default', config = {}) {
+    constructor(profile = 'default', config = {}, unauthenticated = false) {
         const s3Config = getConfig(profile, config);
-
-        this.s3 = new S3(s3Config);
-        this.s3.config.setPromisesDependency(Promise);
-        this.s3.config.update({
-            maxRetries: 0,
-        });
+        if (unauthenticated) {
+            this.s3 = new S3Client({
+                ...s3Config,
+                maxAttempts: 0,
+                credentials: { accessKeyId: '', secretAccessKey: '' },
+                forcePathStyle: true,
+                signer: { sign: async request => request },
+             });
+        }
+       else {
+            this.s3 = new S3Client({
+                ...s3Config,
+                maxAttempts: 0,
+             });
+       }
     }
 
     bucketExists(bucketName) {
-        return this.s3
-            .headBucket({ Bucket: bucketName }).promise()
+        return this.s3.send(new HeadBucketCommand({ Bucket: bucketName }))
             .then(() => true)
             .catch(err => {
-                if (err.code === 'NotFound') {
+                if (err.name === 'NotFound') {
                     return false;
                 }
                 throw err;
             });
     }
 
     createOne(bucketName) {
-        return this.s3
-            .createBucket({ Bucket: bucketName }).promise()
-            .then(() => bucketName);
+        return this.s3.send(new CreateBucketCommand({ Bucket: bucketName }))
+            .then(() => bucketName)
+            .catch(err => {
+                throw err;
+            });
     }
 
     createOneWithLock(bucketName) {
-        return this.s3.createBucket({
+        return this.s3.send(new CreateBucketCommand({
             Bucket: bucketName,
             ObjectLockEnabledForBucket: true,
-        }).promise()
-        .then(() => bucketName);
+        }))
+            .then(() => bucketName)
+            .catch(err => {
+                throw err;
+            });
     }
 
     createMany(bucketNames) {
         const promises = bucketNames.map(
             bucketName => this.createOne(bucketName)
         );
-
         return Promise.all(promises);
     }
-
     createRandom(nBuckets = 1) {
         if (nBuckets === 1) {
             const bucketName = projectFixture.generateBucketName();
-
             return this.createOne(bucketName);
         }
-
         const bucketNames = projectFixture
             .generateManyBucketNames(nBuckets)
-            .sort(() => 0.5 - Math.random()); // Simply shuffle array
-
+            .sort(() => 0.5 - Math.random());
         return this.createMany(bucketNames);
     }
 
     deleteOne(bucketName) {
-        return this.s3
-            .deleteBucket({ Bucket: bucketName }).promise();
+        return this.s3.send(new DeleteBucketCommand({ Bucket: bucketName }))
+            .catch(err => {
+                throw err;
+            });
     }
 
     deleteMany(bucketNames) {
         const promises = bucketNames.map(
             bucketName => this.deleteOne(bucketName)
         );
-
         return Promise.all(promises);
     }
-
     /**
      * Recursively delete all versions of all objects within the bucket
      * @param bucketName
@@ -87,46 +101,44 @@ class BucketUtility {
             Bucket: bucketName,
         };
 
-        return this.s3
-            .listObjectVersions(param).promise()
-            .then(data =>
-                Promise.all(
-                    data.Versions
-                        .filter(object => !object.Key.endsWith('/'))
-                        // remove all objects
+        return this.s3.send(new ListObjectVersionsCommand(param))
+            .then(data => Promise.all(
+                (data.Versions || [])
+                    .filter(object => !object.Key.endsWith('/'))
+                    .map(object =>
+                        this.s3.send(new DeleteObjectCommand({
+                            Bucket: bucketName,
+                            Key: object.Key,
+                            VersionId: object.VersionId,
+                            ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
+                        }))
+                            .then(() => object)
+                    )
+                    .concat((data.Versions || [])
+                        .filter(object => object.Key.endsWith('/'))
                         .map(object =>
-                            this.s3.deleteObject({
+                            this.s3.send(new DeleteObjectCommand({
                                 Bucket: bucketName,
                                 Key: object.Key,
                                 VersionId: object.VersionId,
                                 ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
-                            }).promise()
-                              .then(() => object)
+                            }))
+                            .then(() => object)
                         )
-                        .concat(data.Versions
-                            .filter(object => object.Key.endsWith('/'))
-                            // remove all directories
-                            .map(object =>
-                                this.s3.deleteObject({
-                                    Bucket: bucketName,
-                                    Key: object.Key,
-                                    VersionId: object.VersionId,
-                                    ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
-                                }).promise()
-                                .then(() => object)
-                            )
+                    )
+                    .concat((data.DeleteMarkers || [])
+                        .map(object =>
+                        this.s3.send(new DeleteObjectCommand({
+                            Bucket: bucketName,
+                            Key: object.Key,
+                            VersionId: object.VersionId,
+                            ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
+                            }))
+                            .then(() => object)
                         )
-                        .concat(data.DeleteMarkers
-                            .map(object =>
-                                 this.s3.deleteObject({
-                                     Bucket: bucketName,
-                                     Key: object.Key,
-                                     VersionId: object.VersionId,
-                                     ...(BypassGovernanceRetention && { BypassGovernanceRetention }),
-                                 }).promise()
-                                 .then(() => object)))
-                )
-            );
+                    )
+            )
+        );
     }
 
     emptyMany(bucketNames) {
@@ -136,7 +148,7 @@ class BucketUtility {
 
         return Promise.all(promises);
     }
-
+    
     emptyIfExists(bucketName) {
         return this.bucketExists(bucketName)
             .then(exists => {
@@ -146,19 +158,19 @@ class BucketUtility {
                 return undefined;
             });
     }
-
     emptyManyIfExists(bucketNames) {
         const promises = bucketNames.map(
             bucketName => this.emptyIfExists(bucketName)
         );
-
         return Promise.all(promises);
     }
 
     getOwner() {
-        return this.s3
-            .listBuckets().promise()
-            .then(data => data.Owner);
+        return this.s3.send(new ListBucketsCommand({}))
+            .then(data => data.Owner)
+            .catch(err => {
+                throw err;
+            });
     }
 }
 

tests/functional/aws-node-sdk/test/support/awsConfig.js

@@ -1,9 +1,10 @@
-const AWS = require('aws-sdk');
+const { fromIni } = require('@aws-sdk/credential-providers');
 const fs = require('fs');
 const path = require('path');
 const { config } = require('../../../../../lib/Config');
 const https = require('https');
 const http = require('http');
+
 function getAwsCredentials(profile, credFile) {
     const filename = path.join(process.env.HOME, credFile);
 
@@ -14,7 +15,7 @@ function getAwsCredentials(profile, credFile) {
         throw new Error(msg);
     }
 
-    return new AWS.SharedIniFileCredentials({ profile, filename });
+    return fromIni({ profile, filepath: filename });
 }
 
 function getRealAwsConfig(location) {
@@ -26,19 +27,18 @@ function getRealAwsConfig(location) {
     const params = {
         endpoint: gcpEndpoint ?
             `${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`,
-        signatureVersion: 'v4',
     };
     if (config.locationConstraints[location].type === 'gcp') {
         params.mainBucket = bucketName;
         params.mpuBucket = mpuBucketName;
     }
     if (useHTTPS) {
-        params.httpOptions = {
-            agent: new https.Agent({ keepAlive: true }),
+        params.requestHandler = {
+            httpsAgent: new https.Agent({ keepAlive: true }),
         };
     } else {
-        params.httpOptions = {
-            agent: new http.Agent({ keepAlive: true }),
+        params.requestHandler = {
+            httpAgent: new http.Agent({ keepAlive: true }),
         };
     }
     if (credentialsProfile) {
@@ -48,13 +48,12 @@ function getRealAwsConfig(location) {
         return params;
     }
     if (pathStyle) {
-        params.s3ForcePathStyle = true;
-    }
-    if (!useHTTPS) {
-        params.sslEnabled = false;
+        params.forcePathStyle = true;
     }
-    params.accessKeyId = locCredentials.accessKey;
-    params.secretAccessKey = locCredentials.secretKey;
+    params.credentials = {
+        accessKeyId: locCredentials.accessKey,
+        secretAccessKey: locCredentials.secretKey,
+    };
     return params;
 }
 

tests/functional/aws-node-sdk/test/support/config.js

@@ -1,5 +1,6 @@
 const https = require('https');
-const AWS = require('aws-sdk');
+const http = require('http');
+const { NodeHttpHandler } = require('@smithy/node-http-handler');
 
 const { getCredentials } = require('./credentials');
 const { getAwsCredentials } = require('./awsConfig');
@@ -19,19 +20,45 @@ if (ssl && ssl.ca) {
 
 const DEFAULT_GLOBAL_OPTIONS = {
     httpOptions,
-    apiVersions: { s3: '2006-03-01' },
-    signatureCache: false,
-    sslEnabled: ssl !== undefined,
 };
+
 const DEFAULT_MEM_OPTIONS = {
     endpoint: `${transport}://127.0.0.1:8000`,
-    s3ForcePathStyle: true,
+    port: 8000,
+    forcePathStyle: true,
+    region: 'us-east-1', 
+    maxAttempts: 3,
+    requestHandler: new NodeHttpHandler({
+        connectionTimeout: 5000,
+        socketTimeout: 5000,
+        httpAgent: new (ssl ? https : http).Agent({
+            maxSockets: 200,
+            keepAlive: true,
+            keepAliveMsecs: 1000,
+        }),
+    }),
+};
+
+const DEFAULT_AWS_OPTIONS = {
+    region: 'us-east-1',
+    maxAttempts: 3,
+    requestHandler: new NodeHttpHandler({
+        connectionTimeout: 5000,
+        socketTimeout: 5000,
+        httpAgent: new https.Agent({
+            maxSockets: 200,
+            keepAlive: true,
+            keepAliveMsecs: 1000,
+        }),
+    }),
 };
-const DEFAULT_AWS_OPTIONS = {};
 
 function _getMemCredentials(profile) {
     const { accessKeyId, secretAccessKey } = getCredentials(profile);
-    return new AWS.Credentials(accessKeyId, secretAccessKey);
+    return {
+        accessKeyId,
+        secretAccessKey,
+    };
 }
 
 function _getMemConfig(profile, config) {
@@ -58,11 +85,11 @@ function _getAwsConfig(profile, config) {
     return awsConfig;
 }
 
-function getConfig(profile = 'default', config = {}) {
-    const fn = process.env.AWS_ON_AIR && process.env.AWS_ON_AIR === 'true'
-        ? _getAwsConfig : _getMemConfig;
-
-    return fn.apply(this, [profile, config]);
+function getConfig(profile, config) {
+    if (process.env.AWS_ON_AIR) {
+        return _getAwsConfig(profile, config);
+    }
+    return _getMemConfig(profile, config);
 }
 
 module.exports = getConfig;