aws-s3-multipart: make chunk size configurable, fixes #1543 (#2253)

packages/@uppy/aws-s3-multipart/src/MultipartUploader.js

@@ -2,6 +2,9 @@ const MB = 1024 * 1024
 
 const defaultOptions = {
   limit: 1,
+  getChunkSize (file) {
+    return Math.ceil(file.size / 10000)
+  },
   onStart () {},
   onProgress () {},
   onPartComplete () {},
@@ -22,6 +25,11 @@ class MultipartUploader {
       ...defaultOptions,
       ...options
     }
+    // Use default `getChunkSize` if it was null or something
+    if (!this.options.getChunkSize) {
+      this.options.getChunkSize = defaultOptions.getChunkSize
+    }
+
     this.file = file
 
     this.key = this.options.key || null
@@ -48,7 +56,10 @@ class MultipartUploader {
 
   _initChunks () {
     const chunks = []
-    const chunkSize = Math.max(Math.ceil(this.file.size / 10000), 5 * MB)
+    const desiredChunkSize = this.options.getChunkSize(this.file)
+    // at least 5MB per request, at most 10k requests
+    const minChunkSize = Math.max(5 * MB, Math.ceil(this.file.size / 10000))
+    const chunkSize = Math.max(desiredChunkSize, minChunkSize)
 
     for (let i = 0; i < this.file.size; i += chunkSize) {
       const end = Math.min(this.file.size, i + chunkSize)

packages/@uppy/aws-s3-multipart/src/index.js

@@ -200,6 +200,7 @@ module.exports = class AwsS3Multipart extends Plugin {
         prepareUploadPart: this.opts.prepareUploadPart.bind(this, file),
         completeMultipartUpload: this.opts.completeMultipartUpload.bind(this, file),
         abortMultipartUpload: this.opts.abortMultipartUpload.bind(this, file),
+        getChunkSize: this.opts.getChunkSize ? this.opts.getChunkSize.bind(this) : null,
 
         onStart,
         onProgress,

packages/@uppy/aws-s3-multipart/types/index.d.ts

@@ -11,6 +11,7 @@ declare module AwsS3Multipart {
 
   interface AwsS3MultipartOptions extends Uppy.PluginOptions {
     companionUrl?: string
+    getChunkSize?: (file: Uppy.UppyFile) => number
     createMultipartUpload?: (
       file: Uppy.UppyFile
     ) => MaybePromise<{ uploadId: string; key: string }>

packages/@uppy/aws-s3-multipart/types/index.test-d.ts

@@ -1,4 +1,4 @@
-import { expectType } from 'tsd'
+import { expectError, expectType } from 'tsd'
 import Uppy = require('@uppy/core')
 import AwsS3Multipart = require('../')
 
@@ -37,3 +37,11 @@ import AwsS3Multipart = require('../')
     }
   })
 }
+
+{
+  const uppy = Uppy<Uppy.StrictTypes>()
+  expectError(uppy.use(AwsS3Multipart, { getChunkSize: 100 }))
+  expectError(uppy.use(AwsS3Multipart, { getChunkSize: () => 'not a number' }))
+  uppy.use(AwsS3Multipart, { getChunkSize: () => 100 })
+  uppy.use(AwsS3Multipart, { getChunkSize: (file) => file.size })
+}

packages/@uppy/aws-s3/types/index.d.ts

@@ -15,7 +15,7 @@ declare module AwsS3 {
     getUploadParameters?: (
       file: Uppy.UppyFile
     ) => MaybePromise<AwsS3UploadParameters>
-    metaFields?: string[],
+    metaFields?: string[]
     timeout?: number
     limit?: number
   }

website/src/docs/aws-s3-multipart.md

@@ -54,6 +54,14 @@ Custom headers that should be sent along to [Companion](/docs/companion) on ever
 
 This will be used by the default implementations of the upload-related functions below. If you provide your own implementations, these headers are not sent automatically.
 
+### getChunkSize(file)
+
+A function that returns the minimum chunk size to use when uploading the given file.
+
+The S3 Multipart plugin uploads files in chunks. Each chunk requires a signing request ([`prepareUploadPart()`](#prepareUploadPart-file-partData)). To reduce the amount of requests for large files, you can choose a larger chunk size, at the cost of having to re-upload more data if one chunk fails to upload.
+
+S3 requires a minimum chunk size of 5MB, and supports at most 10,000 chunks per multipart upload. If `getChunkSize()` returns a size that's too small, Uppy will increase it to S3's minimum requirements.
+
 ### createMultipartUpload(file)
 
 A function that calls the S3 Multipart API to create a new upload. `file` is the file object from Uppy's state. The most relevant keys are `file.name` and `file.type`.