Merge pull request #280 from goto-bus-stop/feature/tl-import-urls

transloadit: support uploading to a different service, then importing into assembly

src/plugins/Transloadit/Client.js

@@ -30,7 +30,7 @@ module.exports = class Client {
     Object.keys(fields).forEach((key) => {
       data.append(key, fields[key])
     })
-    data.append('tus_num_expected_upload_files', expectedFiles)
+    data.append('num_expected_upload_files', expectedFiles)
 
     return fetch(`${this.apiUrl}/assemblies`, {
       method: 'post',
@@ -47,6 +47,26 @@ module.exports = class Client {
     })
   }
 
+  reserveFile (assembly, file) {
+    const size = encodeURIComponent(file.size)
+    return fetch(`${assembly.assembly_ssl_url}/reserve_file?size=${size}`, { method: 'post' })
+      .then((response) => response.json())
+  }
+
+  addFile (assembly, file) {
+    if (!file.uploadURL) {
+      return Promise.reject(new Error('File does not have an `uploadURL`.'))
+    }
+    const size = encodeURIComponent(file.size)
+    const url = encodeURIComponent(file.uploadURL)
+    const filename = encodeURIComponent(file.name)
+    const fieldname = 'file'
+
+    const qs = `size=${size}&filename=${filename}&fieldname=${fieldname}&s3Url=${url}`
+    return fetch(`${assembly.assembly_ssl_url}/add_file?${qs}`, { method: 'post' })
+      .then((response) => response.json())
+  }
+
   /**
    * Get the current status for an assembly.
    *

src/plugins/Transloadit/index.js

@@ -24,6 +24,7 @@ module.exports = class Transloadit extends Plugin {
       waitForEncoding: false,
       waitForMetadata: false,
       alwaysRunAssembly: false, // TODO name
+      importFromUploadURLs: false,
       signature: null,
       params: null,
       fields: {},
@@ -44,6 +45,7 @@ module.exports = class Transloadit extends Plugin {
 
     this.prepareUpload = this.prepareUpload.bind(this)
     this.afterUpload = this.afterUpload.bind(this)
+    this.onFileUploadURLAvailable = this.onFileUploadURLAvailable.bind(this)
 
     if (this.opts.params) {
       this.validateParams(this.opts.params)
@@ -111,6 +113,8 @@ module.exports = class Transloadit extends Plugin {
   }
 
   createAssembly (fileIDs, uploadID, options) {
+    const pluginOptions = this.opts
+
     this.core.log('Transloadit: create assembly')
 
     return this.client.createAssembly({
@@ -136,25 +140,28 @@ module.exports = class Transloadit extends Plugin {
         // Attach meta parameters for the Tus plugin. See:
         // https://github.com/tus/tusd/wiki/Uploading-to-Transloadit-using-tus#uploading-using-tus
         // TODO Should this `meta` be moved to a `tus.meta` property instead?
-        // If the MetaData plugin can add eg. resize parameters, it doesn't
-        // make much sense to set those as upload-metadata for tus.
-        const meta = Object.assign({}, file.meta, {
+        const tlMeta = {
           assembly_url: assembly.assembly_url,
           filename: file.name,
           fieldname: 'file'
-        })
+        }
+        const meta = Object.assign({}, file.meta, tlMeta)
         // Add assembly-specific Tus endpoint.
         const tus = Object.assign({}, file.tus, {
-          endpoint: assembly.tus_url
+          endpoint: assembly.tus_url,
+          // Only send assembly metadata to the tus endpoint.
+          metaFields: Object.keys(tlMeta)
         })
         const transloadit = {
           assembly: assembly.assembly_id
         }
-        return Object.assign(
-          {},
-          file,
-          { meta, tus, transloadit }
-        )
+
+        const newFile = Object.assign({}, file, { transloadit })
+        // Only configure the Tus plugin if we are uploading straight to Transloadit (the default).
+        if (!pluginOptions.importFromUploadURLs) {
+          Object.assign(newFile, { meta, tus })
+        }
+        return newFile
       }
 
       const files = Object.assign({}, this.core.state.files)
@@ -167,10 +174,12 @@ module.exports = class Transloadit extends Plugin {
       this.core.emit('transloadit:assembly-created', assembly, fileIDs)
 
       return this.connectSocket(assembly)
-    }).then(() => {
+        .then(() => assembly)
+    }).then((assembly) => {
       this.core.log('Transloadit: Created assembly')
+      return assembly
     }).catch((err) => {
-      this.core.info(this.opts.locale.strings.creatingAssemblyFailed, 'error', 0)
+      this.core.info(pluginOptions.locale.strings.creatingAssemblyFailed, 'error', 0)
 
       // Reject the promise.
       throw err
@@ -181,6 +190,35 @@ module.exports = class Transloadit extends Plugin {
     return this.opts.waitForEncoding || this.opts.waitForMetadata
   }
 
+  /**
+   * Used when `importFromUploadURLs` is enabled: reserves all files in
+   * the assembly.
+   */
+  reserveFiles (assembly, fileIDs) {
+    return Promise.all(fileIDs.map((fileID) => {
+      const file = this.core.getFile(fileID)
+      return this.client.reserveFile(assembly, file)
+    }))
+  }
+
+  /**
+   * Used when `importFromUploadURLs` is enabled: adds files to the assembly
+   * once they have been fully uploaded.
+   */
+  onFileUploadURLAvailable (fileID) {
+    const file = this.core.getFile(fileID)
+    if (!file || !file.transloadit || !file.transloadit.assembly) {
+      return
+    }
+
+    const assembly = this.state.assemblies[file.transloadit.assembly]
+
+    this.client.addFile(assembly, file).catch((err) => {
+      this.core.log(err)
+      this.core.emit('transloadit:import-error', assembly, file.id, err)
+    })
+  }
+
   findFile (uploadedFile) {
     const files = this.core.state.files
     for (const id in files) {
@@ -272,7 +310,11 @@ module.exports = class Transloadit extends Plugin {
     })
 
     const createAssembly = ({ fileIDs, options }) => {
-      return this.createAssembly(fileIDs, uploadID, options).then(() => {
+      return this.createAssembly(fileIDs, uploadID, options).then((assembly) => {
+        if (this.opts.importFromUploadURLs) {
+          return this.reserveFiles(assembly, fileIDs)
+        }
+      }).then(() => {
         fileIDs.forEach((fileID) => {
           this.core.emit('core:preprocess-complete', fileID)
         })
@@ -382,13 +424,28 @@ module.exports = class Transloadit extends Plugin {
         reject(error)
       }
 
+      const onImportError = (assembly, fileID, error) => {
+        if (assemblyIDs.indexOf(assembly.assembly_id) === -1) {
+          return
+        }
+
+        // Not sure if we should be doing something when it's just one file failing.
+        // ATM, the only options are 1) ignoring or 2) failing the entire upload.
+        // I think failing the upload is better than silently ignoring.
+        // In the future we should maybe have a way to resolve uploads with some failures,
+        // like returning an object with `{ successful, failed }` uploads.
+        onAssemblyError(assembly, error)
+      }
+
       const removeListeners = () => {
         this.core.off('transloadit:complete', onAssemblyFinished)
         this.core.off('transloadit:assembly-error', onAssemblyError)
+        this.core.off('transloadit:import-error', onImportError)
       }
 
       this.core.on('transloadit:complete', onAssemblyFinished)
       this.core.on('transloadit:assembly-error', onAssemblyError)
+      this.core.on('transloadit:import-error', onImportError)
     }).then(() => {
       // Clean up uploadID → assemblyIDs, they're no longer going to be used anywhere.
       const uploadsAssemblies = Object.assign({}, this.state.uploadsAssemblies)
@@ -401,6 +458,10 @@ module.exports = class Transloadit extends Plugin {
     this.core.addPreProcessor(this.prepareUpload)
     this.core.addPostProcessor(this.afterUpload)
 
+    if (this.opts.importFromUploadURLs) {
+      this.core.on('core:upload-success', this.onFileUploadURLAvailable)
+    }
+
     this.updateState({
       // Contains assembly status objects, indexed by their ID.
       assemblies: {},
@@ -416,6 +477,10 @@ module.exports = class Transloadit extends Plugin {
   uninstall () {
     this.core.removePreProcessor(this.prepareUpload)
     this.core.removePostProcessor(this.afterUpload)
+
+    if (this.opts.importFromUploadURLs) {
+      this.core.off('core:upload-success', this.onFileUploadURLAvailable)
+    }
   }
 
   getAssembly (id) {

website/src/docs/transloadit.md

@@ -20,6 +20,8 @@ uppy.use(Transloadit, {
 })
 ```
 
+NB: It is not required to use the `Tus10` plugin if [importFromUploadURLs](#importFromUploadURLs) is enabled.
+
 ## Options
 
 ### `waitForEncoding`
@@ -31,6 +33,35 @@ Whether to wait for all assemblies to complete before completing the upload.
 Whether to wait for metadata to be extracted from uploaded files before completing the upload.
 If `waitForEncoding` is enabled, this has no effect.
 
+### `importFromUploadURLs`
+
+Instead of uploading to Transloadit's servers directly, allow another plugin to upload files, and then import those files into the Transloadit assembly.
+Default `false`.
+
+When enabling this option, Transloadit will *not* configure the Tus plugin to upload to Transloadit.
+Instead, a separate upload plugin must be used.
+Once the upload completes, the Transloadit plugin adds the uploaded file to the assembly.
+
+For example, to upload files to an S3 bucket and then transcode them:
+
+```js
+uppy.use(AwsS3, {
+  getUploadParameters (file) {
+    return { /* upload parameters */ }
+  }
+})
+uppy.use(Transloadit, {
+  importFromUploadURLs: true,
+  params: {
+    auth: { key: /* secret */ },
+    template_id: /* secret */
+  }
+})
+```
+
+In order for this to work, the upload plugin must assign a publically accessible `uploadURL` property to the uploaded file object.
+The Tus and S3 plugins both do this—for the XHRUpload plugin, you may have to specify a custom `getUploadResponse` function.
+
 ### `params`
 
 The assembly parameters to use for the upload.