Skip to content

Commit 9a51a17

Browse files
committed
update-aws-sdk-uploads-sdk-aws-region
1 parent 55701de commit 9a51a17

9 files changed

Lines changed: 130 additions & 52 deletions

File tree

CHANGELOG.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Changelog
2+
3+
All notable changes to this project will be documented in this file.
4+
5+
## [1.0.1]
6+
- Support for **private S3 buckets and private objects**. Private content is now migrated along with public content.
7+
- Internal handling of **S3 region** – no longer required as user input during migration setup.
8+
- Migration SDK updated to use the **latest FastPix Node.js Upload SDK** for improved performance and compatibility.
9+
10+
## [1.0.0]
11+
12+
### Features
13+
- **Automated video content migration** to FastPix.
14+
- **Bulk transfer capabilities** for large video datasets.
15+
- **Metadata mapping** during the migration process for consistency and alignment.

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ Key Features:
2323

2424
3. Latest releases
2525
- Current Version: 1.0.0
26-
- View our [changelog](link-to-changelog) for details on recent updates
27-
- Download the latest release from our [releases page](link-to-releases)
26+
- View our [changelog](https://github.com/FastPix/migration-tool/blob/main/CHANGELOG.md) for details on recent updates
27+
- Download the latest release from our [releases page](https://github.com/FastPix/migration-tool/releases/tag/v1.0.0)
2828

2929
4. Guides and Documentation
3030
- User Guide: Complete documentation on how to use the migration tool

package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010
},
1111
"dependencies": {
1212
"@aws-sdk/client-s3": "^3.693.0",
13+
"@aws-sdk/s3-request-presigner": "^3.840.0",
14+
"@fastpix/fastpix-node": "^1.0.2",
1315
"@mux/mux-node": "^8.8.0",
16+
"https": "^1.0.0",
1417
"immer": "^10.1.1",
1518
"next": "15.0.3",
1619
"react": "^18.2.0",

src/app/apicalls/amazonS3/route.ts

Lines changed: 76 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,103 @@
11
import { NextRequest, NextResponse } from 'next/server';
2-
import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3';
2+
import { S3Client, ListObjectsV2Command, GetObjectCommand, HeadBucketCommand } from '@aws-sdk/client-s3';
33

44
import { PlatformCredentials } from '../../components/Utils/types';
55
import processVideosForPlatform from '../../components/Utils/fastpix';
6+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
7+
import https from 'https';
8+
9+
export async function getBucketRegion(bucketUrl: string) {
10+
return new Promise((resolve, reject) => {
11+
const req = https.request(bucketUrl, { method: 'HEAD' }, (res) => {
12+
const region = res.headers['x-amz-bucket-region'];
13+
if (region) {
14+
resolve(region);
15+
} else {
16+
reject('Bucket region not found in headers');
17+
}
18+
});
19+
20+
req.on('error', reject);
21+
req.end();
22+
});
23+
}
24+
25+
const isPublicObject = (bucket: string, key: string): Promise<boolean> => {
26+
const url = `https://${bucket}.s3.amazonaws.com/${encodeURIComponent(key)}`;
27+
28+
return new Promise((resolve) => {
29+
https.request(url, { method: 'HEAD' }, (res) => {
30+
const status = res.statusCode || 0;
31+
if (status === 200) {
32+
resolve(true); // public
33+
} else if (status === 403 || status === 401) {
34+
resolve(false); // private
35+
} else {
36+
resolve(false); // treat unknown as private
37+
}
38+
}).on('error', (err) => {
39+
console.log("HEAD error:", err.message);
40+
resolve(false);
41+
}).end();
42+
});
43+
};
644

745
const fetchS3Media = async (sourcePlatform: PlatformCredentials) => {
846
const { publicKey, secretKey, additionalMetadata } = sourcePlatform.credentials;
947
const { bucket, region } = additionalMetadata;
48+
const bucketUrl = `https://${bucket}.s3.amazonaws.com`
49+
const s3Region = await getBucketRegion(bucketUrl);
1050

1151
const s3 = new S3Client({
1252
credentials: {
1353
accessKeyId: publicKey,
1454
secretAccessKey: secretKey,
1555
},
16-
region: region,
56+
region: s3Region,
1757
});
1858

1959
let videos: Array<any> = [];
2060
let continuationToken: string | undefined = undefined;
2161

2262
try {
2363
do {
24-
25-
// @ts-ignore
26-
const command = new ListObjectsV2Command({
64+
const command = new ListObjectsV2Command({
2765
Bucket: bucket.trim(),
2866
ContinuationToken: continuationToken,
2967
});
30-
31-
// @ts-ignore
32-
const response = await s3.send(command);
68+
69+
const response = await s3.send(command);
3370
const filteredVideos = response.Contents?.filter(file => file.Key?.endsWith('.mp4')) || [];
3471
videos = [...videos, ...filteredVideos];
35-
36-
// @ts-ignore
37-
continuationToken = response?.NextContinuationToken ?? "";
72+
continuationToken = response.NextContinuationToken ?? "";
3873
} while (continuationToken);
3974

75+
const signedUrls = await Promise.all(videos.map(async (video) => {
76+
const key = video.Key!;
77+
const isPublic = await isPublicObject(bucket, key);
78+
79+
let url: string;
80+
if (isPublic) {
81+
url = `https://${bucket}.s3.amazonaws.com/${encodeURIComponent(key)}`;
82+
} else {
83+
const getCommand = new GetObjectCommand({ Bucket: bucket, Key: key });
84+
url = await getSignedUrl(s3, getCommand, { expiresIn: 86400 });
85+
}
86+
87+
return {
88+
videoId: key,
89+
mp4_url: url,
90+
ETag: video.ETag ?? null
91+
};
92+
}));
93+
4094
return {
4195
success: true,
42-
videos: videos.map(video => ({
43-
videoId: video.Key ?? null,
44-
mp4_url: `https://${bucket}.s3.amazonaws.com/${video.Key}`,
45-
ETag: video.ETag ?? null
46-
})),
96+
videos: signedUrls
4797
};
48-
} catch (error) {
49-
98+
99+
} catch (error: any) {
100+
50101
// @ts-ignore
51102
return { success: false, status: 404, message: error?.message };
52103
}
@@ -65,28 +116,29 @@ export async function POST(request: NextRequest) {
65116
{ status: 404 }
66117
);
67118
}
68-
69-
const videos = amazonS3Response.videos ?? []; // Vidoes from amazon s3
119+
120+
const videos = amazonS3Response.videos ?? []; // Videos from Amazon S3
121+
70122
const result = await processVideosForPlatform(destinationPlatform, videos, "amazon-s3"); // process videos in fastpix
71123

72124
const createdMedia = result.createdMedia; // created vidoes in fastpix
73125
const failedMedia = result.failedMedia; // failed vidoes in fastpix
74126

75127
if (createdMedia.length > 0 || failedMedia.length > 0) { // Either video is created or failed to create we send the response
76-
128+
77129
return NextResponse.json(
78130
{ success: true, createdMedia, failedMedia },
79131
{ status: 200 }
80132
);
81133
} else { // If there are no vidoes in amazon s3 then we send 404
82134
const errorMsg = videos.length === 0 ? "No Vidoes found in AmazonS3 Video" : "Something went wrong";
83-
135+
84136
return NextResponse.json(
85-
{ message: errorMsg},
137+
{ message: errorMsg },
86138
{ status: videos.length === 0 ? 404 : 400 }
87139
);
88140
}
89-
141+
90142
} catch (error: any) {
91143

92144
return NextResponse.json(

src/app/apicalls/validatecredentials/route.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import Mux from "@mux/mux-node";
22
import { S3Client, HeadBucketCommand } from '@aws-sdk/client-s3';
3+
import { getBucketRegion } from "../amazonS3/route";
34

45
interface AdditionalMetaData {
56
environment?: string;
@@ -128,12 +129,16 @@ export async function POST(request: Request) {
128129
}
129130

130131
case 's3': {
132+
133+
const bucketUrl = `https://${data.additionalMetadata.bucket}.s3.amazonaws.com`;
134+
const region = await getBucketRegion(bucketUrl);
135+
131136
const client = new S3Client({
132137
credentials: {
133138
accessKeyId: data.publicKey,
134139
secretAccessKey: data.secretKey!,
135140
},
136-
region: data.additionalMetadata.region,
141+
region: region,
137142
});
138143

139144
const input = {

src/app/components/ErrorUI/index.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ interface ErrorInterface {
88
}
99

1010
const ErrorUI = ({title, description, code, icon}:ErrorInterface) => {
11-
11+
1212
return (
1313
<div className="w-full h-min-screen flex items-center justify-center">
1414
<div className="flex flex-col items-center justify-center gap-y-1">

src/app/components/Migration/index.tsx

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,9 @@ const MigrationStatus: React.FC = () => {
7474
{originPlatformVideos?.map((video, index) =>
7575
video?.data?.id ? (
7676
<div key={index} className="grid grid-cols-[1fr_2fr_1fr] border">
77-
<div className="px-4 py-2 text-sm sm:text-base">{index + 1}</div>
78-
<div className="px-4 py-2 text-sm sm:text-base">{video?.data?.id}</div>
79-
<div className="px-4 py-2 text-sm sm:text-base">
77+
<div className="px-4 py-2 text-[14px]">{index + 1}</div>
78+
<div className="px-4 py-2 text-[14px]">{video?.data?.id}</div>
79+
<div className="px-4 py-2 text-[14px]">
8080
<div className="bg-baby-blue w-[120px] p-[5px] flex gap-x-2 items-center rounded-lg">
8181
<span><CreatedIcon /></span>
8282
<span className="text-cobalt-blue text-[12px]">CREATED</span>

src/app/components/SourceCredentials/index.tsx

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ const PLATFORM_CREDENTIALS = [
1515
values: [
1616
{ label: 'Access Key ID', name: 'publicKey', type: 'text' },
1717
{ label: 'Secret Access Key', name: 'secretKey', type: 'text' },
18-
{ label: 'Region', name: 'region', type: 'select', values: ['us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'sa-east-1'] },
18+
// { label: 'Region', name: 'region', type: 'select',
19+
// values: ['us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1', 'eu-central-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'sa-east-1', 'us-east-2', 'af-south-1', 'ap-east-1', 'ap-south-2', 'ap-southeast-3', 'ap-southeast-5',
20+
// 'ap-southeast-4', 'ap-south-1', 'ap-northeast-3', 'ap-northeast-2', 'ap-east-2', 'ap-southeast-7', 'ap-northeast-1', 'ca-central-1', 'ca-west-1', 'eu-west-2', 'eu-south-1', 'eu-west-3', 'eu-south-2', 'eu-north-1', 'eu-central-2', 'il-central-1',
21+
// 'mx-central-1', 'me-south-1', 'me-central-1', 'sa-east-1'
22+
// ] },
1923
{ label: 'Bucket name', name: 'bucket', type: 'text' },
2024
],
2125
},

src/app/components/Utils/fastpix.ts

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import Client from "@fastpix/fastpix-node";
12
import { PlatformCredentials } from './types';
23

34
interface Videos {
@@ -15,7 +16,10 @@ const createMediaInFastPix = async (destinationPlatform: PlatformCredentials, mp
1516
const config = destinationPlatform?.config ? destinationPlatform.config : null;
1617
const maxResolutionTier = config?.maxResolutionTier ? config.maxResolutionTier : "1080p";
1718
const playbackPolicy = config?.playbackPolicy?.[0];
18-
19+
20+
const cleanedUrl = mp4_support.trim();
21+
const finalUrl = cleanedUrl.replace(/\s+/g, ''); // Removes all whitespace
22+
1923
const requestBody = {
2024
"metadata": {
2125
"originPlaformVideoId": videoId,
@@ -37,40 +41,35 @@ const createMediaInFastPix = async (destinationPlatform: PlatformCredentials, mp
3741
}),
3842
},
3943
"accessPolicy": playbackPolicy,
40-
"subtitles":{
41-
"name":"english",
42-
"languageCode":"en"
43-
},
44+
// "subtitles":{
45+
// "languageName":"english",
46+
// "languageCode":"en"
47+
// },
4448
"maxResolution": maxResolutionTier,
4549
"inputs": [
4650
{
4751
type: 'video',
48-
url: `${mp4_support}`,
52+
url: `${finalUrl}`,
4953
},
5054
],
5155
"mp4Support": "capped_4k",
5256
};
5357

54-
try {
55-
const response = await fetch(url, {
56-
method: 'POST',
57-
headers: {
58-
'Accept': 'application/json',
59-
'Content-Type': 'application/json',
60-
'Authorization': 'Basic ' + Buffer.from(`${credentials?.publicKey ? credentials.publicKey : null}:${credentials?.secretKey ? credentials.secretKey : null}`).toString('base64')
61-
},
62-
body: JSON.stringify(requestBody),
63-
});
58+
const fastpix = new Client({
59+
accessTokenId: credentials.publicKey ?? null,
60+
secretKey: credentials.secretKey ?? null,
61+
});
6462

65-
const fastPixCreateMediaRes = await response.json();
63+
try {
64+
const response = await fastpix.uploadMediaFromUrl(requestBody);
6665

67-
if (response.ok) {
66+
if (response.success) {
6867

69-
return { success: true, response: fastPixCreateMediaRes };
68+
return { success: true, response: response };
7069

7170
} else {
7271

73-
return { success: false, statusCode: response?.status, message: fastPixCreateMediaRes?.error?.message, fields: fastPixCreateMediaRes?.error?.fields, payload: requestBody };
72+
return { success: false, statusCode: response.success ? 200 : response?.error?.code, message: response?.error?.message, fields: response?.error?.fields, payload: requestBody };
7473
}
7574
} catch (error) {
7675

0 commit comments

Comments
 (0)