add concurrency limit (#4)

Co-authored-by: dave <aleksander.nau@cadolabs.io>

Author: qem19

Dockerfile

@@ -1,5 +1,5 @@
 FROM node:alpine
 
-RUN yarn global add @Cado-Labs/ucdn
+RUN yarn global add @cadolabs/ucdn
 
 ENTRYPOINT ["/usr/local/bin/ucdn"]

README.md

@@ -5,13 +5,13 @@ Tool for uploading assets to AWS S3.
 ## Installation
 
 ```sh
-$ yarn global add @Cado-Labs/ucdn
+$ yarn global add @cadolabs/ucdn
 ```
 
 or
 
 ```sh
-$ npm i -g @Cado-Labs/ucdn
+$ npm i -g @cadolabs/ucdn
 ```
 
 ## Usage
@@ -29,15 +29,16 @@ All following options except `config` and `config-key` can be configured using t
 Available AWS regions can be viewed at https://docs.aws.amazon.com/sns/latest/dg/sns-supported-regions-countries.html.
 
 ```
---config, -c                            path to config file           [default: "./ucdn.yml"]
---config-key, -C                        root config key               [default: null]
---region, -r                            AWS geographical area         [default: "eu-west-1"]
---dir, -d                               assets directory              [default: "dist/"]
---bucket, -b                            AWS bucket for upload         [required]
---exclude, -e                           excluded extenstions          [default: ["html","gz"]]
---accessKeyId, --access-key-id          AWS access key ID             [required]
---secretAccessKey, --secret-access-key  AWS secret access key         [required]
---targetDir, --target-dir               AWS bucket target directory   [default: ""]
+--config, -c                            path to config file                    [default: "./ucdn.yml"]
+--config-key, -C                        root config key                        [default: null]
+--region, -r                            AWS geographical area                  [default: "eu-west-1"]
+--dir, -d                               assets directory                       [default: "dist/"]
+--bucket, -b                            AWS bucket for upload                  [required]
+--exclude, -e                           excluded extenstions                   [default: ["html","gz"]]
+--accessKeyId, --access-key-id          AWS access key ID                      [required]
+--secretAccessKey, --secret-access-key  AWS secret access key                  [required]
+--targetDir, --target-dir               AWS bucket target directory            [default: ""]
+--concurrencyLimit, --processes         Limit of concurrent upload processes   [default: 100]
 ```
 
 ## Contributing

index.js

@@ -1,5 +1,5 @@
-const upload = require("./upload")
+import upload from "./upload.js"
 
-module.exports = {
+export default {
   upload,
 }

package.json

@@ -1,19 +1,21 @@
 {
-  "name": "@Cado-Labs/ucdn",
-  "version": "1.0.0",
+  "name": "@cadolabs/ucdn",
+  "version": "1.1.0",
   "bin": "ucdn",
   "main": "index.js",
   "repository": "git@github.com:Cado-Labs/ucdn.git",
   "license": "MIT",
   "author": "Aleksei Bespalov <nulldefiner@gmail.com>",
   "homepage": "https://github.com/Cado-Labs/ucdn",
+  "type": "module",
   "scripts": {
     "lint": "eslint . ucdn"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.421.0",
     "js-yaml": "^4.1.0",
     "mime-types": "^2.1.35",
+    "p-limit": "^6.2.0",
     "yargs": "^17.7.2"
   },
   "devDependencies": {

ucdn

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
-const yargs = require("yargs")
+import yargs from "yargs"
+import { hideBin } from 'yargs/helpers'
 
-const pkg = require("./package")
-const utils = require("./utils")
-const upload = require("./upload")
+import pkg from "./package.json" with { type: 'json' }
+import utils from "./utils.js"
+import upload from "./upload.js"
 
 const yargsOptions = {
   config: {
@@ -59,9 +60,15 @@ const yargsOptions = {
     type: "string",
     default: "",
   },
+  concurrencyLimit: {
+    alias: "processes",
+    describe: "Limit of concurrent upload processes",
+    type: "integer",
+    default: 100,
+  },
 }
 
-yargs
+yargs(hideBin(process.argv))
   .env("UCDN")
   .command("upload", "upload assets to AWS S3", {}, upload)
   .options(yargsOptions)
@@ -71,6 +78,7 @@ yargs
   .strict()
   .middleware(argv => {
     const config = utils.loadYamlFile(argv.config)
+
     const configObject = argv.C ? (config[argv.C] || {}) : config
 
     Object

upload.js

@@ -1,7 +1,8 @@
-const fs = require("fs")
-const path = require("path")
-const mime = require("mime-types")
-const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3")
+import fs from "fs"
+import path from "path"
+import mime from "mime-types"
+import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"
+import pLimit from "p-limit"
 
 const configDefaults = {
   region: null,
@@ -11,6 +12,7 @@ const configDefaults = {
   accessKeyId: null,
   secretAccessKey: null,
   targetDir: null,
+  concurrencyLimit: 100,
 }
 
 const getConfig = passedConfig => {
@@ -33,14 +35,16 @@ const getKey = (file, root, targetDir) => {
   return path.join(targetDir, filepath)
 }
 
-const uploadFile = (s3, config, file, key) => {
-  return fs.promises.readFile(file).then(data => {
+const uploadFileFunc = (s3, config, file, key) => {
+  return () => {
+    const data = fs.readFileSync(file)
     const { bucket } = config
     const basename = path.basename(file)
     const contentType = mime.contentType(basename)
     const params = { Bucket: bucket, Body: data, Key: key, ContentType: contentType }
+
     return s3.send(new PutObjectCommand(params))
-  })
+  }
 }
 
 const upload = argv => {
@@ -61,9 +65,11 @@ const upload = argv => {
     return !exclude.includes(ext)
   })
 
+  const limit = pLimit(config.concurrencyLimit);
+
   const promises = files.map(file => {
     const key = getKey(file, directory, targetDir)
-    return uploadFile(s3, config, file, key)
+    return limit(uploadFileFunc(s3, config, file, key))
       .then(() => console.log("Uploaded", file))
   })
 
@@ -78,4 +84,4 @@ const upload = argv => {
     })
 }
 
-module.exports = upload
+export default upload

utils.js

@@ -1,13 +1,13 @@
-const path = require("path")
-const fs = require("fs")
-const yaml = require("js-yaml")
+import path from "path"
+import fs from "fs"
+import yaml from "js-yaml"
 
 const loadYamlFile = filePath => {
   const configPath = path.resolve(filePath)
   const content = fs.readFileSync(configPath)
   return yaml.load(content)
 }
 
-module.exports = {
+export default {
   loadYamlFile,
 }

yarn.lock

@@ -2571,6 +2571,13 @@ p-limit@^3.0.2:
   dependencies:
     yocto-queue "^0.1.0"
 
+p-limit@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-6.2.0.tgz#c254d22ba6aeef441a3564c5e6c2f2da59268a0f"
+  integrity sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==
+  dependencies:
+    yocto-queue "^1.1.1"
+
 p-locate@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834"
@@ -3033,3 +3040,8 @@ yocto-queue@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
+
+yocto-queue@^1.1.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.1.tgz#36d7c4739f775b3cbc28e6136e21aa057adec418"
+  integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==