@uppy/companion: switch from aws-sdk v2 to @aws-sdk/* (v3) (#4285)

Co-authored-by: Antoine du Hamel <antoine@transloadit.com>
Co-authored-by: Mikael Finstad <finstaden@gmail.com>

packages/@uppy/companion/package.json

@@ -28,8 +28,11 @@
   ],
   "bin": "./bin/companion",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.338.0",
+    "@aws-sdk/lib-storage": "^3.338.0",
+    "@aws-sdk/s3-presigned-post": "^3.338.0",
+    "@aws-sdk/s3-request-presigner": "^3.338.0",
     "atob": "2.1.2",
-    "aws-sdk": "^2.1038.0",
     "body-parser": "1.20.0",
     "chalk": "4.1.2",
     "common-tags": "1.8.2",

packages/@uppy/companion/src/server/Uploader.js

@@ -10,6 +10,8 @@ const { promisify } = require('node:util')
 const FormData = require('form-data')
 const throttle = require('lodash/throttle')
 
+const { Upload } = require('@aws-sdk/lib-storage')
+
 // TODO move to `require('streams/promises').pipeline` when dropping support for Node.js 14.x.
 const pipeline = promisify(pipelineCb)
 
@@ -642,7 +644,11 @@ class Uploader {
     }
 
     const filename = this.uploadFileName
-    const { client, options } = this.options.s3
+    /**
+     * @type {{client: import('@aws-sdk/client-s3').S3Client, options: Record<string, any>}}
+     */
+    const s3Options = this.options.s3
+    const { client, options } = s3Options
 
     const params = {
       Bucket: options.bucket,
@@ -654,35 +660,31 @@ class Uploader {
 
     if (options.acl != null) params.ACL = options.acl
 
-    const upload = client.upload(params, {
+    const upload = new Upload({
+      client,
+      params,
       // using chunkSize as partSize too, see https://github.com/transloadit/uppy/pull/3511
       partSize: this.options.chunkSize,
+      leavePartsOnError: true, // https://github.com/aws/aws-sdk-js-v3/issues/2311
     })
 
     upload.on('httpUploadProgress', ({ loaded, total }) => {
       this.onProgress(loaded, total)
     })
 
-    return new Promise((resolve, reject) => {
-      upload.send((error, data) => {
-        if (error) {
-          reject(error)
-          return
-        }
-
-        resolve({
-          url: data && data.Location ? data.Location : null,
-          extraData: {
-            response: {
-              responseText: JSON.stringify(data),
-              headers: {
-                'content-type': 'application/json',
-              },
-            },
+    const data = await upload.done()
+    return {
+      // @ts-expect-error For some reason `|| null` is not enough for TS
+      url: data?.Location || null,
+      extraData: {
+        response: {
+          responseText: JSON.stringify(data),
+          headers: {
+            'content-type': 'application/json',
           },
-        })
-      })
-    })
+        },
+      },
+    }
   }
 }
 

packages/@uppy/companion/src/server/controllers/s3.js

@@ -1,4 +1,14 @@
 const express = require('express')
+const {
+  CreateMultipartUploadCommand,
+  ListPartsCommand,
+  UploadPartCommand,
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3')
+
+const { createPresignedPost } = require('@aws-sdk/s3-presigned-post')
+const { getSignedUrl } = require('@aws-sdk/s3-request-presigner')
 
 function rfc2047Encode (data) {
   // eslint-disable-next-line no-param-reassign
@@ -32,7 +42,9 @@ module.exports = function s3 (config) {
    *  - fields - Form fields to send along.
    */
   function getUploadParameters (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
 
     if (!client || typeof config.bucket !== 'string') {
@@ -48,7 +60,6 @@ module.exports = function s3 (config) {
     }
 
     const fields = {
-      key,
       success_action_status: '201',
       'content-type': req.query.type,
     }
@@ -59,23 +70,20 @@ module.exports = function s3 (config) {
       fields[`x-amz-meta-${metadataKey}`] = metadata[metadataKey]
     })
 
-    client.createPresignedPost({
+    createPresignedPost(client, {
       Bucket: config.bucket,
       Expires: config.expires,
       Fields: fields,
       Conditions: config.conditions,
-    }, (err, data) => {
-      if (err) {
-        next(err)
-        return
-      }
+      Key: key,
+    }).then(data => {
       res.json({
         method: 'post',
         url: data.url,
         fields: data.fields,
         expires: config.expires,
       })
-    })
+    }, next)
   }
 
   /**
@@ -93,7 +101,9 @@ module.exports = function s3 (config) {
    *  - uploadId - The ID of this multipart upload, to be used in later requests.
    */
   function createMultipartUpload (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const key = config.getKey(req, req.body.filename, req.body.metadata || {})
     const { type, metadata } = req.body
@@ -115,16 +125,12 @@ module.exports = function s3 (config) {
 
     if (config.acl != null) params.ACL = config.acl
 
-    client.createMultipartUpload(params, (err, data) => {
-      if (err) {
-        next(err)
-        return
-      }
+    client.send(new CreateMultipartUploadCommand(params)).then((data) => {
       res.json({
         key: data.Key,
         uploadId: data.UploadId,
       })
-    })
+    }, next)
   }
 
   /**
@@ -141,7 +147,9 @@ module.exports = function s3 (config) {
    *     - Size - size of this part.
    */
   function getUploadedParts (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const { uploadId } = req.params
     const { key } = req.query
@@ -154,17 +162,12 @@ module.exports = function s3 (config) {
     let parts = []
 
     function listPartsPage (startAt) {
-      client.listParts({
+      client.send(new ListPartsCommand({
         Bucket: config.bucket,
         Key: key,
         UploadId: uploadId,
         PartNumberMarker: startAt,
-      }, (err, data) => {
-        if (err) {
-          next(err)
-          return
-        }
-
+      })).then(data => {
         parts = parts.concat(data.Parts)
 
         if (data.IsTruncated) {
@@ -173,7 +176,7 @@ module.exports = function s3 (config) {
         } else {
           res.json(parts)
         }
-      })
+      }, next)
     }
     listPartsPage(0)
   }
@@ -190,7 +193,9 @@ module.exports = function s3 (config) {
    *  - url - The URL to upload to, including signed query parameters.
    */
   function signPartUpload (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const { uploadId, partNumber } = req.params
     const { key } = req.query
@@ -204,20 +209,15 @@ module.exports = function s3 (config) {
       return
     }
 
-    client.getSignedUrl('uploadPart', {
+    getSignedUrl(client, new UploadPartCommand({
       Bucket: config.bucket,
       Key: key,
       UploadId: uploadId,
       PartNumber: partNumber,
       Body: '',
-      Expires: config.expires,
-    }, (err, url) => {
-      if (err) {
-        next(err)
-        return
-      }
+    }), { expiresIn: config.expires }).then(url => {
       res.json({ url, expires: config.expires })
-    })
+    }, next)
   }
 
   /**
@@ -234,7 +234,9 @@ module.exports = function s3 (config) {
    *                    in an object mapped to part numbers.
    */
   function batchSignPartsUpload (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const { uploadId } = req.params
     const { key, partNumbers } = req.query
@@ -257,14 +259,13 @@ module.exports = function s3 (config) {
 
     Promise.all(
       partNumbersArray.map((partNumber) => {
-        return client.getSignedUrlPromise('uploadPart', {
+        return getSignedUrl(client, new UploadPartCommand({
           Bucket: config.bucket,
           Key: key,
           UploadId: uploadId,
-          PartNumber: partNumber,
+          PartNumber: Number(partNumber),
           Body: '',
-          Expires: config.expires,
-        })
+        }), { expiresIn: config.expires })
       }),
     ).then((urls) => {
       const presignedUrls = Object.create(null)
@@ -272,9 +273,7 @@ module.exports = function s3 (config) {
         presignedUrls[partNumbersArray[index]] = urls[index]
       }
       res.json({ presignedUrls })
-    }).catch((err) => {
-      next(err)
-    })
+    }).catch(next)
   }
 
   /**
@@ -288,7 +287,9 @@ module.exports = function s3 (config) {
    *   Empty.
    */
   function abortMultipartUpload (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const { uploadId } = req.params
     const { key } = req.query
@@ -298,17 +299,11 @@ module.exports = function s3 (config) {
       return
     }
 
-    client.abortMultipartUpload({
+    client.send(new AbortMultipartUploadCommand({
       Bucket: config.bucket,
       Key: key,
       UploadId: uploadId,
-    }, (err) => {
-      if (err) {
-        next(err)
-        return
-      }
-      res.json({})
-    })
+    })).then(() => res.json({}), next)
   }
 
   /**
@@ -324,7 +319,9 @@ module.exports = function s3 (config) {
    *  - location - The full URL to the object in the S3 bucket.
    */
   function completeMultipartUpload (req, res, next) {
-    // @ts-ignore The `companion` property is added by middleware before reaching here.
+    /**
+     * @type {import('@aws-sdk/client-s3').S3Client}
+     */
     const client = req.companion.s3Client
     const { uploadId } = req.params
     const { key } = req.query
@@ -342,22 +339,18 @@ module.exports = function s3 (config) {
       return
     }
 
-    client.completeMultipartUpload({
+    client.send(new CompleteMultipartUploadCommand({
       Bucket: config.bucket,
       Key: key,
       UploadId: uploadId,
       MultipartUpload: {
         Parts: parts,
       },
-    }, (err, data) => {
-      if (err) {
-        next(err)
-        return
-      }
+    })).then(data => {
       res.json({
         location: data.Location,
       })
-    })
+    }, next)
   }
 
   return express.Router()

packages/@uppy/companion/src/server/s3-client.js

@@ -1,5 +1,4 @@
-const S3 = require('aws-sdk/clients/s3')
-const AWS = require('aws-sdk')
+const { S3Client } = require('@aws-sdk/client-s3')
 
 /**
  * instantiates the aws-sdk s3 client that will be used for s3 uploads.
@@ -21,16 +20,19 @@ module.exports = (companionOptions) => {
     }
 
     let s3ClientOptions = {
-      signatureVersion: 'v4',
       endpoint: s3.endpoint,
       region: s3.region,
     }
+    if (typeof s3.endpoint === 'string') {
+      // TODO: deprecate those replacements in favor of what AWS SDK supports out of the box.
+      s3ClientOptions.endpoint = s3.endpoint.replace(/{service}/, 's3').replace(/{region}/, s3.region)
+    }
 
     if (s3.useAccelerateEndpoint && s3.bucket != null) {
       s3ClientOptions = {
         ...s3ClientOptions,
         useAccelerateEndpoint: true,
-        s3BucketEndpoint: true,
+        bucketEndpoint: true,
         // This is a workaround for lacking support for useAccelerateEndpoint in createPresignedPost
         // See https://github.com/transloadit/uppy/issues/4135#issuecomment-1276450023
         endpoint: `https://${s3.bucket}.s3-accelerate.amazonaws.com/`,
@@ -46,13 +48,13 @@ module.exports = (companionOptions) => {
     // If the user doesn't specify key and secret, the default credentials (process-env)
     // will be used by S3 in calls below.
     if (s3.key && s3.secret && !s3ClientOptions.credentials) {
-      s3ClientOptions.credentials = new AWS.Credentials(
-        s3.key,
-        s3.secret,
-        s3.sessionToken,
-      )
+      s3ClientOptions.credentials = {
+        accessKeyId: s3.key,
+        secretAccessKey: s3.secret,
+        sessionToken: s3.sessionToken,
+      }
     }
-    s3Client = new S3(s3ClientOptions)
+    s3Client = new S3Client(s3ClientOptions)
   }
 
   return s3Client