-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(storage): internal uploadData implementation (#13888)
* feat(storage): internal uploadData implementation * chore: update bundle size * fix: address feedbacks
- Loading branch information
1 parent
0bca04a
commit fb4237b
Showing
29 changed files
with
241 additions
and
108 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
67 changes: 67 additions & 0 deletions
67
packages/storage/__tests__/internals/apis/uploadData.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import { uploadData as advancedUploadData } from '../../../src/internals'; | ||
| import { uploadData as uploadDataInternal } from '../../../src/providers/s3/apis/internal/uploadData'; | ||
|
|
||
| jest.mock('../../../src/providers/s3/apis/internal/uploadData'); | ||
| const mockedUploadDataInternal = jest.mocked(uploadDataInternal); | ||
| const mockedUploadTask = 'UPLOAD_TASK'; | ||
|
|
||
| describe('uploadData (internal)', () => { | ||
| beforeEach(() => { | ||
| mockedUploadDataInternal.mockReturnValue(mockedUploadTask as any); | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| jest.clearAllMocks(); | ||
| }); | ||
|
|
||
| it('should pass advanced option locationCredentialsProvider to internal remove', async () => { | ||
| const useAccelerateEndpoint = true; | ||
| const bucket = { bucketName: 'bucket', region: 'us-east-1' }; | ||
| const locationCredentialsProvider = async () => ({ | ||
| credentials: { | ||
| accessKeyId: 'akid', | ||
| secretAccessKey: 'secret', | ||
| sessionToken: 'token', | ||
| expiration: new Date(), | ||
| }, | ||
| }); | ||
| const contentDisposition = { type: 'attachment', filename: 'foo' } as const; | ||
| const onProgress = jest.fn(); | ||
| const metadata = { foo: 'bar' }; | ||
|
|
||
| const result = advancedUploadData({ | ||
| path: 'input/path/to/mock/object', | ||
| data: 'data', | ||
| options: { | ||
| useAccelerateEndpoint, | ||
| bucket, | ||
| locationCredentialsProvider, | ||
| contentDisposition, | ||
| contentEncoding: 'gzip', | ||
| contentType: 'text/html', | ||
| onProgress, | ||
| metadata, | ||
| }, | ||
| }); | ||
|
|
||
| expect(mockedUploadDataInternal).toHaveBeenCalledTimes(1); | ||
| expect(mockedUploadDataInternal).toHaveBeenCalledWith({ | ||
| path: 'input/path/to/mock/object', | ||
| data: 'data', | ||
| options: { | ||
| useAccelerateEndpoint, | ||
| bucket, | ||
| locationCredentialsProvider, | ||
| contentDisposition, | ||
| contentEncoding: 'gzip', | ||
| contentType: 'text/html', | ||
| onProgress, | ||
| metadata, | ||
| }, | ||
| }); | ||
| expect(result).toEqual(mockedUploadTask); | ||
| }); | ||
| }); |
2 changes: 1 addition & 1 deletion
2
packages/storage/__tests__/providers/s3/apis/uploadData/byteLength.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import { UploadDataInput } from '../types/inputs'; | ||
| import { UploadDataOutput } from '../types/outputs'; | ||
| import { uploadData as uploadDataInternal } from '../../providers/s3/apis/internal/uploadData'; | ||
|
|
||
| /** | ||
| * @internal | ||
| */ | ||
| export const uploadData = (input: UploadDataInput) => { | ||
| const { data, path, options } = input; | ||
|
|
||
| return uploadDataInternal({ | ||
| path, | ||
| data, | ||
| options: { | ||
| useAccelerateEndpoint: options?.useAccelerateEndpoint, | ||
| bucket: options?.bucket, | ||
| onProgress: options?.onProgress, | ||
| contentDisposition: options?.contentDisposition, | ||
| contentEncoding: options?.contentEncoding, | ||
| contentType: options?.contentType, | ||
| metadata: options?.metadata, | ||
| preventOverwrite: options?.preventOverwrite, | ||
|
|
||
| // Advanced options | ||
| locationCredentialsProvider: options?.locationCredentialsProvider, | ||
| }, | ||
| // Type casting is necessary because `uploadDataInternal` supports both Gen1 and Gen2 signatures, but here | ||
| // given in input can only be Gen2 signature, the return can only ben Gen2 signature. | ||
| }) as UploadDataOutput; | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
53 changes: 53 additions & 0 deletions
53
packages/storage/src/providers/s3/apis/internal/uploadData/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import { UploadDataInput } from '../../../types'; | ||
| // TODO: Remove this interface when we move to public advanced APIs. | ||
| import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs'; | ||
| import { createUploadTask } from '../../../utils'; | ||
| import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; | ||
| import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; | ||
| import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../../utils/constants'; | ||
|
|
||
| import { byteLength } from './byteLength'; | ||
| import { putObjectJob } from './putObjectJob'; | ||
| import { getMultipartUploadHandlers } from './multipart'; | ||
|
|
||
| export const uploadData = ( | ||
| input: UploadDataInput | UploadDataWithPathInputWithAdvancedOptions, | ||
| ) => { | ||
| const { data } = input; | ||
|
|
||
| const dataByteLength = byteLength(data); | ||
| assertValidationError( | ||
| dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE, | ||
| StorageValidationErrorCode.ObjectIsTooLarge, | ||
| ); | ||
|
|
||
| if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) { | ||
| // Single part upload | ||
| const abortController = new AbortController(); | ||
|
|
||
| return createUploadTask({ | ||
| isMultipartUpload: false, | ||
| job: putObjectJob(input, abortController.signal, dataByteLength), | ||
| onCancel: (message?: string) => { | ||
| abortController.abort(message); | ||
| }, | ||
| }); | ||
| } else { | ||
| // Multipart upload | ||
| const { multipartUploadJob, onPause, onResume, onCancel } = | ||
| getMultipartUploadHandlers(input, dataByteLength); | ||
|
|
||
| return createUploadTask({ | ||
| isMultipartUpload: true, | ||
| job: multipartUploadJob, | ||
| onCancel: (message?: string) => { | ||
| onCancel(message); | ||
| }, | ||
| onPause, | ||
| onResume, | ||
| }); | ||
| } | ||
| }; |
5 changes: 4 additions & 1 deletion
5
...uploadData/multipart/calculatePartSize.ts → ...uploadData/multipart/calculatePartSize.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 3 additions & 3 deletions
6
...is/uploadData/multipart/getDataChunker.ts → ...al/uploadData/multipart/getDataChunker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.