Merge pull request #726 from transloadit/feature/s3-multipart

S3 Multipart upload

CHANGELOG.md

@@ -32,7 +32,7 @@ PRs are welcome! Please do open an issue to discuss first if it's a big feature,
 - [ ] maybe restrict system file picking dialog too https://github.com/transloadit/uppy/issues/253
 - [ ] uppy-server: what happens if access token expires amid an upload/download process.
 - [ ] good way to change plugin options at runtime—maybe `this.state.options`?
-- [ ] s3: multipart/"resumable" uploads for large files (@goto-bus-stop)
+- [x] s3: multipart/"resumable" uploads for large files (@goto-bus-stop)
 - [ ] DnD Bar: drag and drop + statusbar or progressbar ? (@arturi)
 - [ ] possibility to work on already uploaded / in progress files #112, #113
 - [ ] possibility to edit/delete more than one file at once #118, #97
@@ -129,6 +129,7 @@ To Be Released: 2018-05-31.
 - [ ] docs: improve on React docs https://uppy.io/docs/react/, add small example for each component maybe? Dashboard, DragDrop, ProgressBar? No need to make separate pages for all of them, just headings on the same page. Right now docs are confusing, because they focus on DashboardModal. Also problems with syntax highlight on https://uppy.io/docs/react/dashboard-modal/ (@goto-bus-stop)
 - [ ] core: customizing metadata fields, boolean metadata; see #809, #454 and related (@arturi)
 - [ ] providers: Add user/account names to Uppy provider views (@ifedapoolarewaju)
+- [x] s3: implement multipart uploads (#726, @goto-bus-stop)
 
 ## 0.24.4
 

src/plugins/AwsS3/Multipart.js

@@ -0,0 +1,424 @@
+const Plugin = require('../../core/Plugin')
+const RequestClient = require('../../server/RequestClient')
+const UppySocket = require('../../core/UppySocket')
+const {
+  emitSocketProgress,
+  getSocketHost,
+  limitPromises
+} = require('../../core/Utils')
+const Uploader = require('./MultipartUploader')
+
+/**
+ * Create a wrapper around an event emitter with a `remove` method to remove
+ * all events that were added using the wrapped emitter.
+ */
+function createEventTracker (emitter) {
+  const events = []
+  return {
+    on (event, fn) {
+      events.push([ event, fn ])
+      return emitter.on(event, fn)
+    },
+    remove () {
+      events.forEach(([ event, fn ]) => {
+        emitter.off(event, fn)
+      })
+    }
+  }
+}
+
+function assertServerError (res) {
+  if (res && res.error) {
+    const error = new Error(res.message)
+    Object.assign(error, res.error)
+    throw error
+  }
+  return res
+}
+
+module.exports = class AwsS3Multipart extends Plugin {
+  constructor (uppy, opts) {
+    super(uppy, opts)
+    this.type = 'uploader'
+    this.id = 'AwsS3Multipart'
+    this.title = 'AWS S3 Multipart'
+    this.server = new RequestClient(uppy, opts)
+
+    const defaultOptions = {
+      timeout: 30 * 1000,
+      limit: 0,
+      createMultipartUpload: this.createMultipartUpload.bind(this),
+      listParts: this.listParts.bind(this),
+      prepareUploadPart: this.prepareUploadPart.bind(this),
+      abortMultipartUpload: this.abortMultipartUpload.bind(this),
+      completeMultipartUpload: this.completeMultipartUpload.bind(this)
+    }
+
+    this.opts = Object.assign({}, defaultOptions, opts)
+
+    this.upload = this.upload.bind(this)
+
+    if (typeof this.opts.limit === 'number' && this.opts.limit !== 0) {
+      this.limitRequests = limitPromises(this.opts.limit)
+    } else {
+      this.limitRequests = (fn) => fn
+    }
+
+    this.uploaders = Object.create(null)
+    this.uploaderEvents = Object.create(null)
+    this.uploaderSockets = Object.create(null)
+  }
+
+  /**
+   * Clean up all references for a file's upload: the MultipartUploader instance,
+   * any events related to the file, and the uppy-server WebSocket connection.
+   */
+  resetUploaderReferences (fileID) {
+    if (this.uploaders[fileID]) {
+      this.uploaders[fileID].abort()
+      this.uploaders[fileID] = null
+    }
+    if (this.uploaderEvents[fileID]) {
+      this.uploaderEvents[fileID].remove()
+      this.uploaderEvents[fileID] = null
+    }
+    if (this.uploaderSockets[fileID]) {
+      this.uploaderSockets[fileID].close()
+      this.uploaderSockets[fileID] = null
+    }
+  }
+
+  assertHost () {
+    if (!this.opts.host) {
+      throw new Error('Expected a `host` option containing an uppy-server address.')
+    }
+  }
+
+  createMultipartUpload (file) {
+    this.assertHost()
+
+    return this.server.post('s3/multipart', {
+      filename: file.name,
+      type: file.type
+    }).then(assertServerError)
+  }
+
+  listParts (file, { key, uploadId }) {
+    this.assertHost()
+
+    const filename = encodeURIComponent(key)
+    return this.server.get(`s3/multipart/${uploadId}?key=${filename}`)
+      .then(assertServerError)
+  }
+
+  prepareUploadPart (file, { key, uploadId, number }) {
+    this.assertHost()
+
+    const filename = encodeURIComponent(key)
+    return this.server.get(`s3/multipart/${uploadId}/${number}?key=${filename}`)
+      .then(assertServerError)
+  }
+
+  completeMultipartUpload (file, { key, uploadId, parts }) {
+    this.assertHost()
+
+    const filename = encodeURIComponent(key)
+    const uploadIdEnc = encodeURIComponent(uploadId)
+    return this.server.post(`s3/multipart/${uploadIdEnc}/complete?key=${filename}`, { parts })
+      .then(assertServerError)
+  }
+
+  abortMultipartUpload (file, { key, uploadId }) {
+    this.assertHost()
+
+    const filename = encodeURIComponent(key)
+    const uploadIdEnc = encodeURIComponent(uploadId)
+    return this.server.delete(`s3/multipart/${uploadIdEnc}?key=${filename}`)
+      .then(assertServerError)
+  }
+
+  uploadFile (file) {
+    return new Promise((resolve, reject) => {
+      const upload = new Uploader(file.data, Object.assign({
+        // .bind to pass the file object to each handler.
+        createMultipartUpload: this.limitRequests(this.opts.createMultipartUpload.bind(this, file)),
+        listParts: this.limitRequests(this.opts.listParts.bind(this, file)),
+        prepareUploadPart: this.opts.prepareUploadPart.bind(this, file),
+        completeMultipartUpload: this.limitRequests(this.opts.completeMultipartUpload.bind(this, file)),
+        abortMultipartUpload: this.limitRequests(this.opts.abortMultipartUpload.bind(this, file)),
+
+        limit: this.opts.limit || 5,
+        onStart: (data) => {
+          const cFile = this.uppy.getFile(file.id)
+          this.uppy.setFileState(file.id, {
+            s3Multipart: Object.assign({}, cFile.s3Multipart, {
+              key: data.key,
+              uploadId: data.uploadId,
+              parts: []
+            })
+          })
+        },
+        onProgress: (bytesUploaded, bytesTotal) => {
+          this.uppy.emit('upload-progress', file, {
+            uploader: this,
+            bytesUploaded: bytesUploaded,
+            bytesTotal: bytesTotal
+          })
+        },
+        onError: (err) => {
+          this.uppy.log(err)
+          this.uppy.emit('upload-error', file, err)
+          err.message = `Failed because: ${err.message}`
+
+          this.resetUploaderReferences(file.id)
+          reject(err)
+        },
+        onSuccess: (result) => {
+          this.uppy.emit('upload-success', file, upload, result.location)
+
+          if (result.location) {
+            this.uppy.log('Download ' + upload.file.name + ' from ' + result.location)
+          }
+
+          this.resetUploaderReferences(file.id)
+          resolve(upload)
+        },
+        onPartComplete: (part) => {
+          // Store completed parts in state.
+          const cFile = this.uppy.getFile(file.id)
+          this.uppy.setFileState(file.id, {
+            s3Multipart: Object.assign({}, cFile.s3Multipart, {
+              parts: [
+                ...cFile.s3Multipart.parts,
+                part
+              ]
+            })
+          })
+
+          this.uppy.emit('s3-multipart:part-uploaded', cFile, part)
+        }
+      }, file.s3Multipart))
+
+      this.uploaders[file.id] = upload
+      this.uploaderEvents[file.id] = createEventTracker(this.uppy)
+
+      this.onFileRemove(file.id, (removed) => {
+        this.resetUploaderReferences(file.id)
+        resolve(`upload ${removed.id} was removed`)
+      })
+
+      this.onFilePause(file.id, (isPaused) => {
+        if (isPaused) {
+          upload.pause()
+        } else {
+          upload.start()
+        }
+      })
+
+      this.onPauseAll(file.id, () => {
+        upload.pause()
+      })
+
+      this.onCancelAll(file.id, () => {
+        upload.abort({ really: true })
+      })
+
+      this.onResumeAll(file.id, () => {
+        upload.start()
+      })
+
+      if (!file.isPaused) {
+        upload.start()
+      }
+
+      if (!file.isRestored) {
+        this.uppy.emit('upload-started', file, upload)
+      }
+    })
+  }
+
+  uploadRemote (file) {
+    this.resetUploaderReferences(file.id)
+
+    return new Promise((resolve, reject) => {
+      if (file.serverToken) {
+        return this.connectToServerSocket(file)
+          .then(() => resolve())
+          .catch(reject)
+      }
+
+      this.uppy.emit('upload-started', file)
+
+      fetch(file.remote.url, {
+        method: 'post',
+        credentials: 'include',
+        headers: {
+          'Accept': 'application/json',
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(Object.assign({}, file.remote.body, {
+          protocol: 's3-multipart',
+          size: file.data.size,
+          metadata: file.meta
+        }))
+      })
+      .then((res) => {
+        if (res.status < 200 || res.status > 300) {
+          return reject(res.statusText)
+        }
+
+        return res.json().then((data) => {
+          this.uppy.setFileState(file.id, { serverToken: data.token })
+          return this.uppy.getFile(file.id)
+        })
+      })
+      .then((file) => {
+        return this.connectToServerSocket(file)
+      })
+      .then(() => {
+        resolve()
+      })
+      .catch((err) => {
+        reject(new Error(err))
+      })
+    })
+  }
+
+  connectToServerSocket (file) {
+    return new Promise((resolve, reject) => {
+      const token = file.serverToken
+      const host = getSocketHost(file.remote.host)
+      const socket = new UppySocket({ target: `${host}/api/${token}` })
+      this.uploaderSockets[socket] = socket
+      this.uploaderEvents[file.id] = createEventTracker(this.uppy)
+
+      this.onFileRemove(file.id, (removed) => {
+        socket.send('pause', {})
+        resolve(`upload ${file.id} was removed`)
+      })
+
+      this.onFilePause(file.id, (isPaused) => {
+        socket.send(isPaused ? 'pause' : 'resume', {})
+      })
+
+      this.onPauseAll(file.id, () => socket.send('pause', {}))
+
+      this.onCancelAll(file.id, () => socket.send('pause', {}))
+
+      this.onResumeAll(file.id, () => {
+        if (file.error) {
+          socket.send('pause', {})
+        }
+        socket.send('resume', {})
+      })
+
+      this.onRetry(file.id, () => {
+        socket.send('pause', {})
+        socket.send('resume', {})
+      })
+
+      this.onRetryAll(file.id, () => {
+        socket.send('pause', {})
+        socket.send('resume', {})
+      })
+
+      if (file.isPaused) {
+        socket.send('pause', {})
+      }
+
+      socket.on('progress', (progressData) => emitSocketProgress(this, progressData, file))
+
+      socket.on('error', (errData) => {
+        this.uppy.emit('upload-error', file, new Error(errData.error))
+        reject(new Error(errData.error))
+      })
+
+      socket.on('success', (data) => {
+        this.uppy.emit('upload-success', file, data, data.url)
+        resolve()
+      })
+    })
+  }
+
+  upload (fileIDs) {
+    if (fileIDs.length === 0) return Promise.resolve()
+
+    const promises = fileIDs.map((id) => {
+      const file = this.uppy.getFile(id)
+      if (file.isRemote) {
+        return this.uploadRemote(file)
+      }
+      return this.uploadFile(file)
+    })
+
+    return Promise.all(promises)
+  }
+
+  addResumableUploadsCapabilityFlag () {
+    this.uppy.setState({
+      capabilities: Object.assign({}, this.uppy.getState().capabilities, {
+        resumableUploads: true
+      })
+    })
+  }
+
+  onFileRemove (fileID, cb) {
+    this.uploaderEvents[fileID].on('file-removed', (file) => {
+      if (fileID === file.id) cb(file.id)
+    })
+  }
+
+  onFilePause (fileID, cb) {
+    this.uploaderEvents[fileID].on('upload-pause', (targetFileID, isPaused) => {
+      if (fileID === targetFileID) {
+        // const isPaused = this.uppy.pauseResume(fileID)
+        cb(isPaused)
+      }
+    })
+  }
+
+  onRetry (fileID, cb) {
+    this.uploaderEvents[fileID].on('upload-retry', (targetFileID) => {
+      if (fileID === targetFileID) {
+        cb()
+      }
+    })
+  }
+
+  onRetryAll (fileID, cb) {
+    this.uploaderEvents[fileID].on('retry-all', (filesToRetry) => {
+      if (!this.uppy.getFile(fileID)) return
+      cb()
+    })
+  }
+
+  onPauseAll (fileID, cb) {
+    this.uploaderEvents[fileID].on('pause-all', () => {
+      if (!this.uppy.getFile(fileID)) return
+      cb()
+    })
+  }
+
+  onCancelAll (fileID, cb) {
+    this.uploaderEvents[fileID].on('cancel-all', () => {
+      if (!this.uppy.getFile(fileID)) return
+      cb()
+    })
+  }
+
+  onResumeAll (fileID, cb) {
+    this.uploaderEvents[fileID].on('resume-all', () => {
+      if (!this.uppy.getFile(fileID)) return
+      cb()
+    })
+  }
+
+  install () {
+    this.addResumableUploadsCapabilityFlag()
+    this.uppy.addUploader(this.upload)
+  }
+
+  uninstall () {
+    this.uppy.removeUploader(this.upload)
+  }
+}

src/plugins/AwsS3/MultipartUploader.js

@@ -0,0 +1,284 @@
+const MB = 1024 * 1024
+
+const defaultOptions = {
+  limit: 1,
+  onStart () {},
+  onProgress () {},
+  onPartComplete () {},
+  onSuccess () {},
+  onError (err) {
+    throw err
+  }
+}
+
+function remove (arr, el) {
+  const i = arr.indexOf(el)
+  if (i !== -1) arr.splice(i, 1)
+}
+
+class MultipartUploader {
+  constructor (file, options) {
+    this.options = Object.assign({}, defaultOptions, options)
+    this.file = file
+
+    this.key = this.options.key || null
+    this.uploadId = this.options.uploadId || null
+    this.parts = this.options.parts || []
+
+    this.isPaused = false
+    this.chunks = null
+    this.chunkState = null
+    this.uploading = []
+
+    this._initChunks()
+  }
+
+  _initChunks () {
+    const chunks = []
+    const chunkSize = Math.max(Math.ceil(this.file.size / 10000), 5 * MB)
+
+    for (let i = 0; i < this.file.size; i += chunkSize) {
+      const end = Math.min(this.file.size, i + chunkSize)
+      chunks.push(this.file.slice(i, end))
+    }
+
+    this.chunks = chunks
+    this.chunkState = chunks.map(() => ({
+      uploaded: 0,
+      busy: false,
+      done: false
+    }))
+  }
+
+  _createUpload () {
+    return Promise.resolve().then(() =>
+      this.options.createMultipartUpload()
+    ).then((result) => {
+      const valid = typeof result === 'object' && result &&
+        typeof result.uploadId === 'string' &&
+        typeof result.key === 'string'
+      if (!valid) {
+        throw new TypeError(`AwsS3/Multipart: Got incorrect result from 'createMultipartUpload()', expected an object '{ uploadId, key }'.`)
+      }
+      return result
+    }).then((result) => {
+      this.key = result.key
+      this.uploadId = result.uploadId
+
+      this.options.onStart(result)
+    }).then(() => {
+      this._uploadParts()
+    }).catch((err) => {
+      this._onError(err)
+    })
+  }
+
+  _resumeUpload () {
+    return Promise.resolve().then(() =>
+      this.options.listParts({
+        uploadId: this.uploadId,
+        key: this.key
+      })
+    ).then((parts) => {
+      parts.forEach((part) => {
+        const i = part.PartNumber - 1
+        this.chunkState[i] = {
+          uploaded: part.Size,
+          etag: part.ETag,
+          done: true
+        }
+
+        // Only add if we did not yet know about this part.
+        if (!this.parts.some((p) => p.PartNumber === part.PartNumber)) {
+          this.parts.push({
+            PartNumber: part.PartNumber,
+            ETag: part.ETag
+          })
+        }
+      })
+      this._uploadParts()
+    }).catch((err) => {
+      this._onError(err)
+    })
+  }
+
+  _uploadParts () {
+    if (this.isPaused) return
+
+    const need = this.options.limit - this.uploading.length
+    if (need === 0) return
+
+    // All parts are uploaded.
+    if (this.chunkState.every((state) => state.done)) {
+      this._completeUpload()
+      return
+    }
+
+    const candidates = []
+    for (let i = 0; i < this.chunkState.length; i++) {
+      const state = this.chunkState[i]
+      if (state.done || state.busy) continue
+
+      candidates.push(i)
+      if (candidates.length >= need) {
+        break
+      }
+    }
+
+    candidates.forEach((index) => {
+      this._uploadPart(index)
+    })
+  }
+
+  _uploadPart (index) {
+    const body = this.chunks[index]
+    this.chunkState[index].busy = true
+
+    return Promise.resolve().then(() =>
+      this.options.prepareUploadPart({
+        key: this.key,
+        uploadId: this.uploadId,
+        body,
+        number: index + 1
+      })
+    ).then((result) => {
+      const valid = typeof result === 'object' && result &&
+        typeof result.url === 'string'
+      if (!valid) {
+        throw new TypeError(`AwsS3/Multipart: Got incorrect result from 'prepareUploadPart()', expected an object '{ url }'.`)
+      }
+      return result
+    }).then(({ url }) => {
+      this._uploadPartBytes(index, url)
+    })
+  }
+
+  _onPartProgress (index, sent, total) {
+    this.chunkState[index].uploaded = sent
+
+    const totalUploaded = this.chunkState.reduce((n, c) => n + c.uploaded, 0)
+    this.options.onProgress(totalUploaded, this.file.size)
+  }
+
+  _onPartComplete (index, etag) {
+    this.chunkState[index].etag = etag
+    this.chunkState[index].done = true
+
+    const part = {
+      PartNumber: index + 1,
+      ETag: etag
+    }
+    this.parts.push(part)
+
+    this.options.onPartComplete(part)
+
+    this._uploadParts()
+  }
+
+  _uploadPartBytes (index, url) {
+    const body = this.chunks[index]
+    const xhr = new XMLHttpRequest()
+    xhr.open('PUT', url, true)
+    xhr.responseType = 'text'
+
+    this.uploading.push(xhr)
+
+    xhr.upload.addEventListener('progress', (ev) => {
+      if (!ev.lengthComputable) return
+
+      this._onPartProgress(index, ev.loaded, ev.total)
+    })
+
+    xhr.addEventListener('abort', (ev) => {
+      remove(this.uploading, ev.target)
+      this.chunkState[index].busy = false
+    })
+
+    xhr.addEventListener('load', (ev) => {
+      remove(this.uploading, ev.target)
+      this.chunkState[index].busy = false
+
+      if (ev.target.status < 200 || ev.target.status >= 300) {
+        this._onError(new Error('Non 2xx'))
+        return
+      }
+
+      this._onPartProgress(index, body.size, body.size)
+
+      // NOTE This must be allowed by CORS.
+      const etag = ev.target.getResponseHeader('ETag')
+      if (etag === null) {
+        this._onError(new Error('AwsS3/Multipart: Could not read the ETag header. This likely means CORS is not configured correctly on the S3 Bucket. Seee https://uppy.io/docs/aws-s3-multipart#S3-Bucket-Configuration for instructions.'))
+        return
+      }
+
+      this._onPartComplete(index, etag)
+    })
+
+    xhr.addEventListener('error', (ev) => {
+      remove(this.uploading, ev.target)
+      this.chunkState[index].busy = false
+
+      const error = new Error('Unknown error')
+      error.source = ev.target
+      this._onError(error)
+    })
+
+    xhr.send(body)
+  }
+
+  _completeUpload () {
+    // Parts may not have completed uploading in sorted order, if limit > 1.
+    this.parts.sort((a, b) => a.PartNumber - b.PartNumber)
+
+    return Promise.resolve().then(() =>
+      this.options.completeMultipartUpload({
+        key: this.key,
+        uploadId: this.uploadId,
+        parts: this.parts
+      })
+    ).then((result) => {
+      this.options.onSuccess(result)
+    }, (err) => {
+      this._onError(err)
+    })
+  }
+
+  _abortUpload () {
+    this.options.abortMultipartUpload({
+      key: this.key,
+      uploadId: this.uploadId
+    })
+  }
+
+  _onError (err) {
+    this.options.onError(err)
+  }
+
+  start () {
+    this.isPaused = false
+    if (this.uploadId) {
+      this._resumeUpload()
+    } else {
+      this._createUpload()
+    }
+  }
+
+  pause () {
+    const inProgress = this.uploading.slice()
+    inProgress.forEach((xhr) => {
+      xhr.abort()
+    })
+    this.isPaused = true
+  }
+
+  abort (opts = {}) {
+    const really = opts.really || false
+
+    if (!really) return this.pause()
+
+    this._abortUpload()
+  }
+}
+
+module.exports = MultipartUploader

src/server/RequestClient.js

@@ -58,4 +58,19 @@ module.exports = class RequestClient {
       // @todo validate response status before calling json
       .then((res) => res.json())
   }
+
+  delete (path, data) {
+    return fetch(`${this.hostname}/${path}`, {
+      method: 'delete',
+      credentials: 'include',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json'
+      },
+      body: data ? JSON.stringify(data) : null
+    })
+      .then(this.onReceiveResponse)
+      // @todo validate response status before calling json
+      .then((res) => res.json())
+  }
 }

website/src/docs/aws-s3-multipart.md

@@ -0,0 +1,119 @@
+---
+type: docs
+order: 33
+title: "AwsS3Multipart"
+permalink: docs/aws-s3-multipart/
+---
+
+The `AwsS3Multipart` plugin can be used to upload files directly to an S3 bucket using S3's Multipart upload strategy. With this strategy, files are chopped up in parts of 5MB+ each, so they can be uploaded concurrently. It's also very reliable: if a single part fails to upload, only that 5MB has to be retried.
+
+```js
+const AwsS3Multipart = require('uppy/lib/plugins/AwsS3/Multipart')
+uppy.use(AwsS3Multipart, {
+  limit: 4,
+  host: 'https://uppy-server.myapp.net/'
+})
+```
+
+## Options
+
+### limit: 0
+
+The maximum amount of chunks to upload simultaneously. `0` means unlimited.
+
+### host: null
+
+The Uppy Server URL to use to proxy calls to the S3 Multipart API.
+
+### createMultipartUpload(file)
+
+A function that calls the S3 Multipart API to create a new upload. `file` is the file object from Uppy's state. The most relevant keys are `file.name` and `file.type`.
+
+Return a Promise for an object with keys:
+
+ - `uploadId` - The UploadID returned by S3.
+ - `key` - The object key for the file. This needs to be returned to allow it to be different from the `file.name`.
+
+The default implementation calls out to Uppy Server's S3 signing endpoints.
+
+### listParts({ uploadId, key })
+
+A function that calls the S3 Multipart API to list the parts of a file that have already been uploaded. Receives an object with keys:
+
+ - `uploadId` - The UploadID of this Multipart upload.
+ - `key` - The object key of this Multipart upload.
+
+Return a Promise for an array of S3 Part objects, as returned by the S3 Multipart API. Each object has keys:
+
+ - `PartNumber` - The index in the file of the uploaded part.
+ - `Size` - The size of the part in bytes.
+ - `ETag` - The ETag of the part, used to identify it when completing the multipart upload and combining all parts into a single file.
+
+The default implementation calls out to Uppy Server's S3 signing endpoints.
+
+### prepareUploadPart(partData)
+
+A function that generates a signed URL to upload a single part. The `partData` argument is an object with keys:
+
+ - `uploadId` - The UploadID of this Multipart upload.
+ - `key` - The object key in the S3 bucket.
+ - `body` - A [`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) of this part's contents.
+ - `number` - The index of this part in the file (`PartNumber` in S3 terminology).
+
+Return a Promise for an object with keys:
+
+ - `url` - The presigned URL to upload a part. This can be generated using the S3 SDK like so:
+
+   ```js
+   sdkInstance.getSignedUrl('uploadPart', {
+     Bucket: 'target',
+     Key: partData.key,
+     UploadId: partData.uploadId,
+     PartNumber: partData.number,
+     Body: '', // Empty, because it's uploaded later
+     Expires: Date.now() + 5 * 60 * 1000
+   }, (err, url) => { /* there's the url! */ })
+   ```
+
+### abortMultipartUpload({ uploadId, key })
+
+A function that calls the S3 Multipart API to abort a Multipart upload, and delete all parts that have been uploaded so far. Receives an object with keys:
+
+ - `uploadId` - The UploadID of this Multipart upload.
+ - `key` - The object key of this Multipart upload.
+
+This is typically called when the user cancels an upload. Cancellation cannot fail in Uppy, so the result of this function is ignored.
+
+The default implementation calls out to Uppy Server's S3 signing endpoints.
+
+### completeMultipartUpload({ uploadId, key, parts })
+
+A function that calls the S3 Multipart API to complete a Multipart upload, combining all parts into a single object in the S3 bucket. Receives an object with keys:
+
+ - `uploadId` - The UploadID of this Multipart upload.
+ - `key` - The object key of this Multipart upload.
+ - `parts` - S3-style list of parts, an array of objects with `ETag` and `PartNumber` properties. This can be passed straight to S3's Multipart API.
+
+Return a Promise for an object with properties:
+
+ - `location` - **(Optional)** A publically accessible URL to the object in the S3 bucket.
+
+The default implementation calls out to Uppy Server's S3 signing endpoints.
+
+## S3 Bucket Configuration
+
+S3 buckets do not allow public uploads by default.  In order to allow Uppy to upload to a bucket directly, its CORS permissions need to be configured.
+
+This process is described in the [AwsS3 documentation](/docs/aws-s3/#S3-Bucket-configuration).
+
+On top of the configuration mentioned there, the `ETag` header must also be whitelisted:
+
+```xml
+<CORSRule>
+  <AllowedMethod>PUT</AllowedMethod>
+  <!-- ... all your existingCORS config goes here ... -->
+
+  <!-- The magic: -->
+  <ExposeHeader>ETag</ExposeHeader>
+</CORSRule>
+```

website/src/docs/aws-s3.md

@@ -9,13 +9,20 @@ The `AwsS3` plugin can be used to upload files directly to an S3 bucket.
 Uploads can be signed using [uppy-server][uppy-server docs] or a custom signing function.
 
 ```js
+const AwsS3 = require('uppy/lib/plugins/AwsS3')
+const ms = require('ms')
+
 uppy.use(AwsS3, {
-  // Options
+  limit: 2,
+  timeout: ms('1 minute'),
+  host: 'https://uppy-server.myapp.com/'
 })
 ```
 
 There are broadly two ways to upload to S3 in a browser. A server can generate a presigned URL for a [PUT upload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html), or a server can generate form data for a [POST upload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html). uppy-server uses a POST upload. See [POST uPloads](#post-uploads) for some caveats if you would like to use POST uploads without uppy-server. See [Generating a presigned upload URL server-side](#example-presigned-url) for an example of a PUT upload.
 
+There is also a separate plugin for S3 Multipart uploads. Multipart in this sense is Amazon's proprietary chunked, resumable upload mechanism for large files. See the [AwsS3Multipart](/docs/aws-s3-multipart) documentation.
+
 ## Options
 
 ### `host`

website/src/docs/transloadit.md

@@ -1,6 +1,6 @@
 ---
 type: docs
-order: 33
+order: 34
 title: "Transloadit"
 permalink: docs/transloadit/
 ---