example: add multipart support to `aws-nodejs` (#4257)

examples/aws-nodejs/index.js

@@ -1,15 +1,33 @@
+'use strict'
+
+const path = require('node:path')
+const crypto = require('node:crypto')
 require('dotenv').config({ path: path.join(__dirname, '..', '..', '.env') })
 
 const express = require('express')
 
 const app = express()
-const path = require('node:path')
 
-const port = process.env.PORT
+const port = process.env.PORT ?? 8080
 const bodyParser = require('body-parser')
 
+let s3Client
 const aws = require('aws-sdk')
 
+const expires = 800 // Define how long until a S3 signature expires.
+
+function getS3Client () {
+  s3Client ??= new aws.S3({
+    signatureVersion: 'v4',
+    region: process.env.COMPANION_AWS_REGION,
+    credentials : new aws.Credentials(
+      process.env.COMPANION_AWS_KEY,
+      process.env.COMPANION_AWS_SECRET,
+    ),
+  })
+  return s3Client
+}
+
 app.use(bodyParser.json())
 
 app.get('/', (req, res) => {
@@ -23,21 +41,20 @@ app.get('/drag', (req, res) => {
 })
 
 app.post('/sign-s3', (req, res) => {
-  const s3 = new aws.S3()
-  const fileName = req.body.filename
+  const s3 = getS3Client()
+  const Key = `${crypto.randomUUID()}-${req.body.filename}`
   const { contentType } = req.body
   const s3Params = {
-    Bucket: process.env.S3_BUCKET,
-    Key: fileName,
-    Expires: 60,
+    Bucket: process.env.COMPANION_AWS_BUCKET,
+    Key,
+    Expires: expires,
     ContentType: contentType,
-    ACL: 'public-read',
   }
 
-  s3.getSignedUrl('putObject', s3Params, (err, data) => {
+  s3.getSignedUrl('putObject', s3Params, (err, data, next) => {
     if (err) {
-      console.log(err)
-      return res.end()
+      next(err)
+      return
     }
     const returnData = {
       url: data,
@@ -48,6 +65,166 @@ app.post('/sign-s3', (req, res) => {
   })
 })
 
+//  === <S3 Multipart> ===
+// You can remove those endpoints if you only want to support the non-multipart uploads.
+
+app.post('/s3/multipart', (req, res, next) => {
+  const client = getS3Client()
+  const { type, metadata, filename } = req.body
+  if (typeof filename !== 'string') {
+    return res.status(400).json({ error: 's3: content filename must be a string' })
+  }
+  if (typeof type !== 'string') {
+    return res.status(400).json({ error: 's3: content type must be a string' })
+  }
+  const Key = `${crypto.randomUUID()}-${filename}`
+
+  const params = {
+    Bucket: process.env.COMPANION_AWS_BUCKET,
+    Key,
+    ContentType: type,
+    Metadata: metadata,
+  }
+
+  return client.createMultipartUpload(params, (err, data) => {
+    if (err) {
+      next(err)
+      return
+    }
+    res.json({
+      key: data.Key,
+      uploadId: data.UploadId,
+    })
+  })
+})
+
+function validatePartNumber (partNumber) {
+  // eslint-disable-next-line no-param-reassign
+  partNumber = Number(partNumber)
+  return Number.isInteger(partNumber) && partNumber >= 1 && partNumber <= 10_000
+}
+app.get('/s3/multipart/:uploadId/:partNumber', (req, res, next) => {
+  const client = getS3Client()
+  const { uploadId, partNumber } = req.params
+  const { key } = req.query
+
+  if (!validatePartNumber(partNumber)) {
+    return res.status(400).json({ error: 's3: the part number must be an integer between 1 and 10000.' })
+  }
+  if (typeof key !== 'string') {
+    return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' })
+  }
+
+  return client.getSignedUrl('uploadPart', {
+    Bucket: process.env.COMPANION_AWS_BUCKET,
+    Key: key,
+    UploadId: uploadId,
+    PartNumber: partNumber,
+    Body: '',
+    Expires: expires,
+  }, (err, url) => {
+    if (err) {
+      next(err)
+      return
+    }
+    res.json({ url, expires })
+  })
+})
+
+app.get('/s3/multipart/:uploadId', (req, res, next) => {
+  const client = getS3Client()
+  const { uploadId } = req.params
+  const { key } = req.query
+
+  if (typeof key !== 'string') {
+    return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' })
+  }
+
+  const parts = []
+  listPartsPage(0)
+
+  function listPartsPage (startAt) {
+    client.listParts({
+      Bucket: process.env.COMPANION_AWS_BUCKET,
+      Key: key,
+      UploadId: uploadId,
+      PartNumberMarker: startAt,
+    }, (err, data) => {
+      if (err) {
+        next(err)
+        return
+      }
+
+      parts.push(...data.Parts)
+
+      if (data.IsTruncated) {
+        // Get the next page.
+        listPartsPage(data.NextPartNumberMarker)
+      } else {
+        res.json(parts)
+      }
+    })
+  }
+})
+
+function isValidPart (part) {
+  return part && typeof part === 'object' && typeof part.PartNumber === 'number' && typeof part.ETag === 'string'
+}
+app.post('/s3/multipart/:uploadId/complete', (req, res, next) => {
+  const client = getS3Client()
+  const { uploadId } = req.params
+  const { key } = req.query
+  const { parts } = req.body
+
+  if (typeof key !== 'string') {
+    return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' })
+  }
+  if (!Array.isArray(parts) || !parts.every(isValidPart)) {
+    return res.status(400).json({ error: 's3: `parts` must be an array of {ETag, PartNumber} objects.' })
+  }
+
+  return client.completeMultipartUpload({
+    Bucket: process.env.COMPANION_AWS_BUCKET,
+    Key: key,
+    UploadId: uploadId,
+    MultipartUpload: {
+      Parts: parts,
+    },
+  }, (err, data) => {
+    if (err) {
+      next(err)
+      return
+    }
+    res.json({
+      location: data.Location,
+    })
+  })
+})
+
+app.delete('/s3/multipart/:uploadId', (req, res, next) => {
+  const client = getS3Client()
+  const { uploadId } = req.params
+  const { key } = req.query
+
+  if (typeof key !== 'string') {
+    return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' })
+  }
+
+  return client.abortMultipartUpload({
+    Bucket: process.env.COMPANION_AWS_BUCKET,
+    Key: key,
+    UploadId: uploadId,
+  }, (err) => {
+    if (err) {
+      next(err)
+      return
+    }
+    res.json({})
+  })
+})
+
+// === </S3 MULTIPART> ===
+
 app.listen(port, () => {
   console.log(`Example app listening on port ${port}`)
 })

examples/aws-nodejs/package.json

@@ -4,6 +4,7 @@
   "description": "Uppy for AWS S3 with a custom Node.js backend for signing URLs",
   "main": "index.js",
   "scripts": {
+    "dev": "node --watch index.js",
     "start": "node index.js"
   },
   "private": true,

examples/aws-nodejs/public/index.html

@@ -2,36 +2,44 @@
 <html>
   <head>
     <meta charset="utf-8">
-    <title>Uppy</title>
+    <title>Uppy – AWS upload example</title>
     <link href="https://releases.transloadit.com/uppy/v3.3.1/uppy.min.css" rel="stylesheet">
   </head>
   <body>
-    <div id="drag-drop-area"></div>
+    <h1>AWS upload example</h1>
+    <h2>AWS S3 (non multipart)</h2>
+    <div id="aws-non-multipart"></div>
+    <h2>AWS S3 multipart</h2>
+    <div id="aws-multipart"></div>
     <script type="module">
-      import { Uppy, Dashboard, AwsS3 } from "https://releases.transloadit.com/uppy/v3.3.1/uppy.min.mjs"
-      var uppy = new Uppy()
-        .use(Dashboard, {
+      import { Uppy, Dashboard, AwsS3Multipart, AwsS3 } from "https://releases.transloadit.com/uppy/v3.3.1/uppy.min.mjs"
+      {
+        const uppy = new Uppy()
+          .use(Dashboard, {
             inline: true,
-            target: '#drag-drop-area',
+            target: '#aws-non-multipart',
           })
-        .use(AwsS3, {
-          getUploadParameters (file) {
-            // Send a request to our PHP signing endpoint.
-            return fetch('/sign-s3', {
-              method: 'post',
-              // Send and receive JSON.
-              headers: {
-                accept: 'application/json',
-                'content-type': 'application/json',
-              },
-              body: JSON.stringify({
-                filename: file.name,
-                contentType: file.type,
-              }),
-            }).then((response) => {
+          .use(AwsS3, {
+            async getUploadParameters (file) {
+              // Send a request to our Express.js signing endpoint.
+              const response = await fetch('/sign-s3', {
+                method: 'POST',
+                // Send and receive JSON.
+                headers: {
+                  accept: 'application/json',
+                  'content-type': 'application/json',
+                },
+                body: JSON.stringify({
+                  filename: file.name,
+                  contentType: file.type,
+                }),
+              })
+
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+
               // Parse the JSON response.
-              return response.json()
-            }).then((data) => {
+              const data = await response.json()
+
               // Return an object in the correct shape.
               return {
                 method: data.method,
@@ -42,17 +50,146 @@
                   'Content-Type': file.type,
                 },
               }
-            })
-          },
-      });
+            },
+          });
+        
+        uppy.on('complete', (result) => {
+          console.log('Upload complete! We’ve uploaded these files:', result.successful)
+        })
+        
+        uppy.on('upload-success', (file, data) => {
+          console.log('Upload success! We’ve uploaded this file:', file.meta['name'])
+        })
+      }
+      {
+        const uppy = new Uppy()
+          .use(Dashboard, {
+            inline: true,
+            target: '#aws-multipart',
+          })
+          .use(AwsS3Multipart, {
+            async createMultipartUpload(file, signal) {
+              if (signal?.aborted) {
+                const err = new DOMException('The operation was aborted', 'AbortError')
+                Object.defineProperty(err, 'cause', { __proto__: null, configurable: true, writable: true, value: signal.reason })
+                throw err
+              }
+
+              const metadata = {}
+
+              Object.keys(file.meta || {}).forEach(key => {
+                if (file.meta[key] != null) {
+                  metadata[key] = file.meta[key].toString()
+                }
+              })
 
-      uppy.on('complete', (result) => {
-        console.log('Upload complete! We’ve uploaded these files:', result.successful)
-      })
+              const response = await fetch('/s3/multipart', {
+                method: 'POST',
+                // Send and receive JSON.
+                headers: {
+                  accept: 'application/json',
+                  'content-type': 'application/json',
+                },
+                body: JSON.stringify({
+                  filename: file.name,
+                  type: file.type,
+                  metadata,
+                }),
+                signal,
+              })
+
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+
+              // Parse the JSON response.
+              const data = await response.json()
 
-      uppy.on('upload-success', (file, data) => {
-        console.log('Upload success! We’ve uploaded this file:', file.meta['name'])
-      })
+              return data
+            },
+
+            async abortMultipartUpload (file, { key, uploadId }, signal) {
+              const filename = encodeURIComponent(key)
+              const uploadIdEnc = encodeURIComponent(uploadId)
+              const response = await fetch(`/s3/multipart/${uploadIdEnc}?key=${filename}`, {
+                method: 'DELETE',
+                signal,
+              })
+              
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+            },
+
+            async signPart (file, { uploadId, key, partNumber, signal }) {
+              if (signal?.aborted) {
+                const err = new DOMException('The operation was aborted', 'AbortError')
+                Object.defineProperty(err, 'cause', { __proto__: null, configurable: true, writable: true, value: signal.reason })
+                throw err
+              }
+
+              if (uploadId == null || key == null || partNumber == null) {
+                throw new Error('Cannot sign without a key, an uploadId, and a partNumber')
+              }
+
+              const filename = encodeURIComponent(key)
+              const response = await fetch(`/s3/multipart/${uploadId}/${partNumber}?key=${filename}`, { signal })
+
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+
+              const data = await response.json()
+
+              return data
+            },
+
+            async listParts (file, { key, uploadId }, signal) {
+              if (signal?.aborted) {
+                const err = new DOMException('The operation was aborted', 'AbortError')
+                Object.defineProperty(err, 'cause', { __proto__: null, configurable: true, writable: true, value: signal.reason })
+                throw err
+              }
+
+              const filename = encodeURIComponent(key)
+              const response = await fetch(`/s3/multipart/${uploadId}?key=${filename}`, { signal })
+
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+
+              const data = await response.json()
+
+              return data
+            },
+
+            async completeMultipartUpload (file, { key, uploadId, parts }, signal) {
+              if (signal?.aborted) {
+                const err = new DOMException('The operation was aborted', 'AbortError')
+                Object.defineProperty(err, 'cause', { __proto__: null, configurable: true, writable: true, value: signal.reason })
+                throw err
+              }
+
+              const filename = encodeURIComponent(key)
+              const uploadIdEnc = encodeURIComponent(uploadId)
+              const response = await fetch(`s3/multipart/${uploadIdEnc}/complete?key=${filename}`, {
+                method: 'POST',
+                headers: {
+                  accept: 'application/json',
+                  'content-type': 'application/json',
+                },
+                body: JSON.stringify({ parts }),
+                signal,
+              })
+
+              if (!response.ok) throw new Error('Unsuccessful request', { cause: response })
+
+              const data = await response.json()
+
+              return data
+            }
+          })
+        
+        uppy.on('complete', (result) => {
+          console.log('Upload complete! We’ve uploaded these files:', result.successful)
+        })
+        
+        uppy.on('upload-success', (file, data) => {
+          console.log('Upload success! We’ve uploaded this file:', file.meta['name'])
+        })
+      }
     </script>
   </body>
 </html>