Merge pull request #1 from transloadit/master

Merge

CHANGELOG.md

@@ -47,6 +47,7 @@ Ideas that will be planned and find their way into a release at one point
 - [ ] possibility to edit/delete more than one file at once #118, #97
 - [ ] optimize problematic filenames #72
 - [ ] an uploader plugin to receive files in a callback instead of uploading them
+- [ ] core: calling `upload` immediately after `addFile` does not upload all files (#249 @goto-bus-stop)
 
 ## 1.0 Goals
 
@@ -55,7 +56,7 @@ What we need to do to release Uppy 1.0
 - [x] feature: restrictions: by size, number of files, file type
 - [x] feature: beta file recovering after closed tab / browser crash
 - [ ] feature: improved UI for Provider, Google Drive and Instagram, grid/list views
-- [ ] feature: finish the direct-to-s3 upload plugin and test it with the flow to then upload to :transloadit: afterwards. This is because this might influence the inner flow of the plugin architecture quite a bit
+- [x] feature: finish the direct-to-s3 upload plugin and test it with the flow to then upload to :transloadit: afterwards. This is because this might influence the inner flow of the plugin architecture quite a bit
 - [ ] feature: Uppy should work well with React/Redux and React Native
 - [ ] feature: preset for Transloadit that mimics jQuery SDK
 - [ ] QA: test how everything works together: user experience from `npm install` to production build with Webpack, using in React/Redux environment (npm pack)
@@ -95,9 +96,8 @@ Theme: React and Retry
 
 - [ ] core: add error in file progress state? error UI, question mark button, `core:error` (@arturi)
 - [ ] core: retry or show error when upload can’t start / fails (offline, wrong endpoint) — now it just sits there (@arturi @goto-bus-stop)
-- [ ] core: calling `upload` immediately after `addFile` does not upload all files (#249 @goto-bus-stop)
 - [ ] core: React / Redux PRs (@arturi @goto-bus-stop)
-- [ ] transloadit: upload to S3, then import into :tl: assembly using `/add_file?s3url=${url}` (@goto-bus-stop)
+- [x] transloadit: upload to S3, then import into :tl: assembly using `/add_file?s3url=${url}` (@goto-bus-stop)
 - [ ] goldenretriver: add “ghost” files (@arturi @goto-bus-stop)
 - [ ] dashboard: cancel button for any kind of uploads? currently resume/pause only for tus, and cancel for XHR (@arturi @goto-bus-stop)
 - [x] informer: support “explanations”, a (?) button that shows more info on hover / click

src/core/Core.js

@@ -72,6 +72,7 @@ class Uppy {
     this.initSocket = this.initSocket.bind(this)
     this.log = this.log.bind(this)
     this.addFile = this.addFile.bind(this)
+    this.removeFile = this.removeFile.bind(this)
     this.calculateProgress = this.calculateProgress.bind(this)
     this.resetProgress = this.resetProgress.bind(this)
 
@@ -115,10 +116,8 @@ class Uppy {
    *
    */
   updateAll (state) {
-    Object.keys(this.plugins).forEach((pluginType) => {
-      this.plugins[pluginType].forEach((plugin) => {
-        plugin.update(state)
-      })
+    this.iteratePlugins(plugin => {
+      plugin.update(state)
     })
   }
 
@@ -444,7 +443,11 @@ class Uppy {
 
     this.on('core:upload-error', (fileID, error) => {
       const fileName = this.state.files[fileID].name
-      this.info(`Failed to upload: ${fileName}`, 'error', 5000)
+      let message = `Failed to upload ${fileName}`
+      if (typeof error === 'object' && error.message) {
+        message = `${message}: ${error.message}`
+      }
+      this.info(message, 'error', 5000)
     })
 
     this.on('core:upload', () => {
@@ -609,21 +612,21 @@ class Uppy {
 
     // Instantiate
     const plugin = new Plugin(this, opts)
-    const pluginName = plugin.id
+    const pluginId = plugin.id
     this.plugins[plugin.type] = this.plugins[plugin.type] || []
 
-    if (!pluginName) {
-      throw new Error('Your plugin must have a name')
+    if (!pluginId) {
+      throw new Error('Your plugin must have an id')
     }
 
     if (!plugin.type) {
       throw new Error('Your plugin must have a type')
     }
 
-    let existsPluginAlready = this.getPlugin(pluginName)
+    let existsPluginAlready = this.getPlugin(pluginId)
     if (existsPluginAlready) {
-      let msg = `Already found a plugin named '${existsPluginAlready.name}'.
-        Tried to use: '${pluginName}'.
+      let msg = `Already found a plugin named '${existsPluginAlready.id}'.
+        Tried to use: '${pluginId}'.
         Uppy is currently limited to running one of every plugin.
         Share your use case with us over at
         https://github.com/transloadit/uppy/issues/

src/plugins/AwsS3/index.js

@@ -66,6 +66,14 @@ module.exports = class AwsS3 extends Plugin {
             key: getValue('Key'),
             etag: getValue('ETag')
           }
+        },
+        getResponseError (xhr) {
+          // If no response, we don't have a specific error message, use the default.
+          if (!xhr.responseXML) {
+            return
+          }
+          const error = xhr.responseXML.querySelector('Error > Message')
+          return new Error(error.textContent)
         }
       })
     })

src/plugins/Dashboard/FileCard.js

@@ -6,9 +6,13 @@ module.exports = function fileCard (props) {
   const file = props.fileCardFor ? props.files[props.fileCardFor] : false
   const meta = {}
 
-  function tempStoreMeta (ev) {
+  const tempStoreMetaOrSubmit = (ev) => {
+    if (ev.keyCode === 13) {
+      props.done(meta, file.id)
+    }
+
     const value = ev.target.value
-    const name = ev.target.attributes.name.value
+    const name = ev.target.dataset.name
     meta[name] = value
   }
 
@@ -18,11 +22,11 @@ module.exports = function fileCard (props) {
       return html`<fieldset class="UppyDashboardFileCard-fieldset">
         <label class="UppyDashboardFileCard-label">${field.name}</label>
         <input class="UppyDashboardFileCard-input"
-               name="${field.id}"
                type="text"
+               data-name="${field.id}"
                value="${file.meta[field.id]}"
                placeholder="${field.placeholder || ''}"
-               onkeyup=${tempStoreMeta} /></fieldset>`
+               onkeyup=${tempStoreMetaOrSubmit} /></fieldset>`
     })
   }
 
@@ -46,8 +50,8 @@ module.exports = function fileCard (props) {
           <div class="UppyDashboardFileCard-info">
             <fieldset class="UppyDashboardFileCard-fieldset">
               <label class="UppyDashboardFileCard-label">Name</label>
-              <input class="UppyDashboardFileCard-input" name="name" type="text" value="${file.meta.name}"
-                     onkeyup=${tempStoreMeta} />
+              <input class="UppyDashboardFileCard-input" data-name="name" type="text" value="${file.meta.name}"
+                     onkeyup=${tempStoreMetaOrSubmit} />
             </fieldset>
             ${renderMetaFields(file)}
           </div>
@@ -60,5 +64,5 @@ module.exports = function fileCard (props) {
               title="Finish editing file"
               onclick=${() => props.done(meta, file.id)}>${checkIcon()}</button>
     </div>
-    </div>`
+  </div>`
 }

src/plugins/Dashboard/index.js

@@ -249,7 +249,7 @@ module.exports = class DashboardUI extends Plugin {
   }
 
   handleFileCard (fileId) {
-    const modal = this.core.getState().modal
+    const modal = this.core.state.modal
 
     this.core.setState({
       modal: Object.assign({}, modal, {
@@ -318,44 +318,36 @@ module.exports = class DashboardUI extends Plugin {
       return target.type === 'progressindicator'
     })
 
-    // const addFile = (file) => {
-    //   this.core.emitter.emit('core:file-add', file)
-    // }
-
-    const removeFile = (fileID) => {
-      this.core.emitter.emit('core:file-remove', fileID)
-    }
-
     const startUpload = (ev) => {
       this.core.upload().catch((err) => {
         // Log error.
-        console.error(err.stack || err.message || err)
+        this.core.log(err.stack || err.message || err)
       })
     }
 
     const pauseUpload = (fileID) => {
-      this.core.emitter.emit('core:upload-pause', fileID)
+      this.core.emit.emit('core:upload-pause', fileID)
     }
 
     const cancelUpload = (fileID) => {
-      this.core.emitter.emit('core:upload-cancel', fileID)
-      this.core.emitter.emit('core:file-remove', fileID)
+      this.core.emit('core:upload-cancel', fileID)
+      this.core.emit('core:file-remove', fileID)
     }
 
     const showFileCard = (fileID) => {
-      this.core.emitter.emit('dashboard:file-card', fileID)
+      this.core.emit('dashboard:file-card', fileID)
     }
 
     const fileCardDone = (meta, fileID) => {
-      this.core.emitter.emit('core:update-meta', meta, fileID)
-      this.core.emitter.emit('dashboard:file-card')
+      this.core.emit('core:update-meta', meta, fileID)
+      this.core.emit('dashboard:file-card')
     }
 
     const info = (text, type, duration) => {
       this.core.info(text, type, duration)
     }
 
-    const resumableUploads = this.core.getState().capabilities.resumableUploads || false
+    const resumableUploads = this.core.state.capabilities.resumableUploads || false
 
     return Dashboard({
       state: state,
@@ -382,7 +374,7 @@ module.exports = class DashboardUI extends Plugin {
       pauseAll: this.pauseAll,
       resumeAll: this.resumeAll,
       addFile: this.core.addFile,
-      removeFile: removeFile,
+      removeFile: this.core.removeFile,
       info: info,
       note: this.opts.note,
       metaFields: state.metaFields,

src/plugins/Plugin.js

@@ -17,7 +17,6 @@ module.exports = class Plugin {
   constructor (core, opts) {
     this.core = core
     this.opts = opts || {}
-    this.type = 'none'
 
     // clear everything inside the target selector
     this.opts.replaceTargetContent === this.opts.replaceTargetContent || true

src/plugins/Transloadit/Client.js

@@ -30,7 +30,7 @@ module.exports = class Client {
     Object.keys(fields).forEach((key) => {
       data.append(key, fields[key])
     })
-    data.append('tus_num_expected_upload_files', expectedFiles)
+    data.append('num_expected_upload_files', expectedFiles)
 
     return fetch(`${this.apiUrl}/assemblies`, {
       method: 'post',
@@ -47,6 +47,26 @@ module.exports = class Client {
     })
   }
 
+  reserveFile (assembly, file) {
+    const size = encodeURIComponent(file.size)
+    return fetch(`${assembly.assembly_ssl_url}/reserve_file?size=${size}`, { method: 'post' })
+      .then((response) => response.json())
+  }
+
+  addFile (assembly, file) {
+    if (!file.uploadURL) {
+      return Promise.reject(new Error('File does not have an `uploadURL`.'))
+    }
+    const size = encodeURIComponent(file.size)
+    const url = encodeURIComponent(file.uploadURL)
+    const filename = encodeURIComponent(file.name)
+    const fieldname = 'file'
+
+    const qs = `size=${size}&filename=${filename}&fieldname=${fieldname}&s3Url=${url}`
+    return fetch(`${assembly.assembly_ssl_url}/add_file?${qs}`, { method: 'post' })
+      .then((response) => response.json())
+  }
+
   /**
    * Get the current status for an assembly.
    *

src/plugins/Transloadit/index.js

@@ -24,6 +24,7 @@ module.exports = class Transloadit extends Plugin {
       waitForEncoding: false,
       waitForMetadata: false,
       alwaysRunAssembly: false, // TODO name
+      importFromUploadURLs: false,
       signature: null,
       params: null,
       fields: {},
@@ -44,6 +45,7 @@ module.exports = class Transloadit extends Plugin {
 
     this.prepareUpload = this.prepareUpload.bind(this)
     this.afterUpload = this.afterUpload.bind(this)
+    this.onFileUploadURLAvailable = this.onFileUploadURLAvailable.bind(this)
 
     if (this.opts.params) {
       this.validateParams(this.opts.params)
@@ -111,6 +113,8 @@ module.exports = class Transloadit extends Plugin {
   }
 
   createAssembly (fileIDs, uploadID, options) {
+    const pluginOptions = this.opts
+
     this.core.log('Transloadit: create assembly')
 
     return this.client.createAssembly({
@@ -136,25 +140,28 @@ module.exports = class Transloadit extends Plugin {
         // Attach meta parameters for the Tus plugin. See:
         // https://github.com/tus/tusd/wiki/Uploading-to-Transloadit-using-tus#uploading-using-tus
         // TODO Should this `meta` be moved to a `tus.meta` property instead?
-        // If the MetaData plugin can add eg. resize parameters, it doesn't
-        // make much sense to set those as upload-metadata for tus.
-        const meta = Object.assign({}, file.meta, {
+        const tlMeta = {
           assembly_url: assembly.assembly_url,
           filename: file.name,
           fieldname: 'file'
-        })
+        }
+        const meta = Object.assign({}, file.meta, tlMeta)
         // Add assembly-specific Tus endpoint.
         const tus = Object.assign({}, file.tus, {
-          endpoint: assembly.tus_url
+          endpoint: assembly.tus_url,
+          // Only send assembly metadata to the tus endpoint.
+          metaFields: Object.keys(tlMeta)
         })
         const transloadit = {
           assembly: assembly.assembly_id
         }
-        return Object.assign(
-          {},
-          file,
-          { meta, tus, transloadit }
-        )
+
+        const newFile = Object.assign({}, file, { transloadit })
+        // Only configure the Tus plugin if we are uploading straight to Transloadit (the default).
+        if (!pluginOptions.importFromUploadURLs) {
+          Object.assign(newFile, { meta, tus })
+        }
+        return newFile
       }
 
       const files = Object.assign({}, this.core.state.files)
@@ -167,10 +174,12 @@ module.exports = class Transloadit extends Plugin {
       this.core.emit('transloadit:assembly-created', assembly, fileIDs)
 
       return this.connectSocket(assembly)
-    }).then(() => {
+        .then(() => assembly)
+    }).then((assembly) => {
       this.core.log('Transloadit: Created assembly')
+      return assembly
     }).catch((err) => {
-      this.core.info(this.opts.locale.strings.creatingAssemblyFailed, 'error', 0)
+      this.core.info(pluginOptions.locale.strings.creatingAssemblyFailed, 'error', 0)
 
       // Reject the promise.
       throw err
@@ -181,6 +190,35 @@ module.exports = class Transloadit extends Plugin {
     return this.opts.waitForEncoding || this.opts.waitForMetadata
   }
 
+  /**
+   * Used when `importFromUploadURLs` is enabled: reserves all files in
+   * the assembly.
+   */
+  reserveFiles (assembly, fileIDs) {
+    return Promise.all(fileIDs.map((fileID) => {
+      const file = this.core.getFile(fileID)
+      return this.client.reserveFile(assembly, file)
+    }))
+  }
+
+  /**
+   * Used when `importFromUploadURLs` is enabled: adds files to the assembly
+   * once they have been fully uploaded.
+   */
+  onFileUploadURLAvailable (fileID) {
+    const file = this.core.getFile(fileID)
+    if (!file || !file.transloadit || !file.transloadit.assembly) {
+      return
+    }
+
+    const assembly = this.state.assemblies[file.transloadit.assembly]
+
+    this.client.addFile(assembly, file).catch((err) => {
+      this.core.log(err)
+      this.core.emit('transloadit:import-error', assembly, file.id, err)
+    })
+  }
+
   findFile (uploadedFile) {
     const files = this.core.state.files
     for (const id in files) {
@@ -272,7 +310,11 @@ module.exports = class Transloadit extends Plugin {
     })
 
     const createAssembly = ({ fileIDs, options }) => {
-      return this.createAssembly(fileIDs, uploadID, options).then(() => {
+      return this.createAssembly(fileIDs, uploadID, options).then((assembly) => {
+        if (this.opts.importFromUploadURLs) {
+          return this.reserveFiles(assembly, fileIDs)
+        }
+      }).then(() => {
         fileIDs.forEach((fileID) => {
           this.core.emit('core:preprocess-complete', fileID)
         })
@@ -382,13 +424,28 @@ module.exports = class Transloadit extends Plugin {
         reject(error)
       }
 
+      const onImportError = (assembly, fileID, error) => {
+        if (assemblyIDs.indexOf(assembly.assembly_id) === -1) {
+          return
+        }
+
+        // Not sure if we should be doing something when it's just one file failing.
+        // ATM, the only options are 1) ignoring or 2) failing the entire upload.
+        // I think failing the upload is better than silently ignoring.
+        // In the future we should maybe have a way to resolve uploads with some failures,
+        // like returning an object with `{ successful, failed }` uploads.
+        onAssemblyError(assembly, error)
+      }
+
       const removeListeners = () => {
         this.core.off('transloadit:complete', onAssemblyFinished)
         this.core.off('transloadit:assembly-error', onAssemblyError)
+        this.core.off('transloadit:import-error', onImportError)
       }
 
       this.core.on('transloadit:complete', onAssemblyFinished)
       this.core.on('transloadit:assembly-error', onAssemblyError)
+      this.core.on('transloadit:import-error', onImportError)
     }).then(() => {
       // Clean up uploadID → assemblyIDs, they're no longer going to be used anywhere.
       const uploadsAssemblies = Object.assign({}, this.state.uploadsAssemblies)
@@ -401,6 +458,10 @@ module.exports = class Transloadit extends Plugin {
     this.core.addPreProcessor(this.prepareUpload)
     this.core.addPostProcessor(this.afterUpload)
 
+    if (this.opts.importFromUploadURLs) {
+      this.core.on('core:upload-success', this.onFileUploadURLAvailable)
+    }
+
     this.updateState({
       // Contains assembly status objects, indexed by their ID.
       assemblies: {},
@@ -416,6 +477,10 @@ module.exports = class Transloadit extends Plugin {
   uninstall () {
     this.core.removePreProcessor(this.prepareUpload)
     this.core.removePostProcessor(this.afterUpload)
+
+    if (this.opts.importFromUploadURLs) {
+      this.core.off('core:upload-success', this.onFileUploadURLAvailable)
+    }
   }
 
   getAssembly (id) {

src/plugins/XHRUpload.js

@@ -20,6 +20,9 @@ module.exports = class XHRUpload extends Plugin {
       headers: {},
       getResponseData (xhr) {
         return JSON.parse(xhr.response)
+      },
+      getResponseError (xhr) {
+        return new Error('Upload error')
       }
     }
 
@@ -88,8 +91,10 @@ module.exports = class XHRUpload extends Plugin {
 
           return resolve(file)
         } else {
-          this.core.emit('core:upload-error', file.id, xhr)
-          return reject('Upload error')
+          const error = opts.getResponseError(xhr) || new Error('Upload error')
+          error.request = xhr
+          this.core.emit('core:upload-error', file.id, error)
+          return reject(error)
         }
 
         // var upload = {}
@@ -103,7 +108,7 @@ module.exports = class XHRUpload extends Plugin {
 
       xhr.addEventListener('error', (ev) => {
         this.core.emit('core:upload-error', file.id)
-        return reject('Upload error')
+        return reject(new Error('Upload error'))
       })
 
       xhr.open(opts.method.toUpperCase(), opts.endpoint, true)

website/src/docs/transloadit.md

@@ -20,6 +20,8 @@ uppy.use(Transloadit, {
 })
 ```
 
+NB: It is not required to use the `Tus10` plugin if [importFromUploadURLs](#importFromUploadURLs) is enabled.
+
 ## Options
 
 ### `waitForEncoding`
@@ -31,6 +33,35 @@ Whether to wait for all assemblies to complete before completing the upload.
 Whether to wait for metadata to be extracted from uploaded files before completing the upload.
 If `waitForEncoding` is enabled, this has no effect.
 
+### `importFromUploadURLs`
+
+Instead of uploading to Transloadit's servers directly, allow another plugin to upload files, and then import those files into the Transloadit assembly.
+Default `false`.
+
+When enabling this option, Transloadit will *not* configure the Tus plugin to upload to Transloadit.
+Instead, a separate upload plugin must be used.
+Once the upload completes, the Transloadit plugin adds the uploaded file to the assembly.
+
+For example, to upload files to an S3 bucket and then transcode them:
+
+```js
+uppy.use(AwsS3, {
+  getUploadParameters (file) {
+    return { /* upload parameters */ }
+  }
+})
+uppy.use(Transloadit, {
+  importFromUploadURLs: true,
+  params: {
+    auth: { key: /* secret */ },
+    template_id: /* secret */
+  }
+})
+```
+
+In order for this to work, the upload plugin must assign a publically accessible `uploadURL` property to the uploaded file object.
+The Tus and S3 plugins both do this—for the XHRUpload plugin, you may have to specify a custom `getUploadResponse` function.
+
 ### `params`
 
 The assembly parameters to use for the upload.