Merge pull request #1497 from transloadit/non-parallel

Non parallel uploads

CHANGELOG.md

@@ -91,6 +91,8 @@ PRs are welcome! Please do open an issue to discuss first if it's a big feature,
 
 ## 1.1
 
+- [ ] ! core: _calculateTotalProgress results in incorrectly high (1038%) progress with files that don’t have size (like from Instagram) (@goto-bus-stop)
+- [ ] companion: restore deferredLength — parallel upload/download, 423 and 500 issues (@ife)
 - [ ] dashboard: optional alert `onbeforeunload` while upload is in progress, safeguarding from accidentaly navigating away from a page with an ongoing upload
 - [x] dashboard: Bring back "Drop Here" screen for dragged URLs without introducing flickering (tricky! see PR #1400)
 - [ ] a11y: Fix remaining issues (https://github.com/transloadit/uppy/issues/created_by/nqst)
@@ -99,7 +101,7 @@ PRs are welcome! Please do open an issue to discuss first if it's a big feature,
 - [ ] dashboard: add option to use `body` or `window` or CSS selector as drop zone / paste zone as well (@arturi)
 - [ ] chore: fix up all code using the prettier branch. work is done, just needs an execute and review/okay by the team
 - [ ] !!! dashboard: Remove the Authorization required tooltip on the authentication screen https://github.com/transloadit/uppy/issues/1425
-- [ ] @uppy/companion: investigate 423 and 500 issues with React Native + Url plugin when pause/resuming an upload
+- [x] @uppy/companion: investigate 423 and 500 issues with React Native + Url plugin when pause/resuming an upload
 - [ ] docs: add docs on locales — how to use from NPM and CDN
 - [ ] @uppy/transloadit: finish Transloadit-Client header on https://github.com/transloadit/uppy/tree/feature/transloadit-client
 - [ ] companion: reports an error at first sign in. we did a hotfix in https://github.com/transloadit/uppy/pull/1478#issuecomment-485937942 but need a proper fix for that (@ife). Also: what about changing the location of that tooltip? So legit errors also don't block buttons?

packages/@uppy/companion/src/server/Uploader.js

@@ -1,5 +1,4 @@
 const fs = require('fs')
-const stream = require('stream')
 const path = require('path')
 const tus = require('tus-js-client')
 const uuid = require('uuid')
@@ -51,15 +50,33 @@ class Uploader {
     this.options.path = `${this.options.pathPrefix}/${Uploader.FILE_NAME_PREFIX}-${this.token}`
     this.streamsEnded = false
     this.duplexStream = null
-    if (this.options.protocol === PROTOCOLS.tus) {
-      this.duplexStream = new stream.PassThrough()
-        .on('error', (err) => logger.error(`${this.shortToken} ${err}`, 'uploader.duplex.error'))
-    }
+    // @TODO disabling parallel uploads and downloads for now
+    // if (this.options.protocol === PROTOCOLS.tus) {
+    //   this.duplexStream = new stream.PassThrough()
+    //     .on('error', (err) => logger.error(`${this.shortToken} ${err}`, 'uploader.duplex.error'))
+    // }
     this.writeStream = fs.createWriteStream(this.options.path, { mode: 0o666 }) // no executable files
       .on('error', (err) => logger.error(`${this.shortToken} ${err}`, 'uploader.write.error'))
     /** @type {number} */
     this.emittedProgress = 0
     this.storage = options.storage
+    this._paused = false
+
+    if (this.options.protocol === PROTOCOLS.tus) {
+      emitter().on(`pause:${this.token}`, () => {
+        this._paused = true
+        if (this.tus) {
+          this.tus.abort()
+        }
+      })
+
+      emitter().on(`resume:${this.token}`, () => {
+        this._paused = false
+        if (this.tus) {
+          this.tus.start()
+        }
+      })
+    }
   }
 
   /**
@@ -150,25 +167,30 @@ class Uploader {
         if (this.options.endpoint && protocol === PROTOCOLS.multipart) {
           this.uploadMultipart()
         }
+
+        if (protocol === PROTOCOLS.tus && !this.tus) {
+          return this.uploadTus()
+        }
       })
 
       return this.endStreams()
     }
 
-    this.writeToStreams(chunk, () => {
+    this.writeStream.write(chunk, () => {
       logger.debug(`${this.shortToken} ${this.bytesWritten} bytes`, 'uploader.download.progress')
-      if (protocol === PROTOCOLS.multipart) {
+      if (protocol === PROTOCOLS.multipart || protocol === PROTOCOLS.tus) {
         return this.emitIllusiveProgress()
       }
 
       if (protocol === PROTOCOLS.s3Multipart && !this.s3Upload) {
         return this.uploadS3()
       }
-      if (!this.options.endpoint) return
+      // @TODO disabling parallel uploads and downloads for now
+      // if (!this.options.endpoint) return
 
-      if (protocol === PROTOCOLS.tus && !this.tus) {
-        return this.uploadTus()
-      }
+      // if (protocol === PROTOCOLS.tus && !this.tus) {
+      //   return this.uploadTus()
+      // }
     })
   }
 
@@ -223,7 +245,14 @@ class Uploader {
    * @param {number=} bytesUploaded the bytes actually Uploaded so far
    */
   emitIllusiveProgress (bytesUploaded) {
-    const bytesTotal = this.streamsEnded ? this.bytesWritten : this.options.size
+    if (this._paused) {
+      return
+    }
+
+    let bytesTotal = this.streamsEnded ? this.bytesWritten : this.options.size
+    if (!this.streamsEnded) {
+      bytesTotal = Math.max(bytesTotal, this.bytesWritten)
+    }
     bytesUploaded = bytesUploaded || 0
     // for a 10MB file, 10MB of download will account for 5MB upload progress
     // and 10MB of actual upload will account for the other 5MB upload progress.
@@ -303,23 +332,19 @@ class Uploader {
     const fname = path.basename(this.options.path)
     const ftype = this.options.metadata.type
     const metadata = Object.assign({ filename: fname, filetype: ftype }, this.options.metadata || {})
-    const file = this.duplexStream
+    const file = fs.createReadStream(this.options.path)
     const uploader = this
-    const oneGB = 1024 * 1024 * 1024  // 1 GB
-    // chunk size can't be infinity with deferred length.
-    // cap value to 1GB to avoid buffer allocation error (RangeError)
-    const chunkSize = Math.min(this.options.size || oneGB, oneGB)
 
     // @ts-ignore
     this.tus = new tus.Upload(file, {
       endpoint: this.options.endpoint,
       uploadUrl: this.options.uploadUrl,
       // @ts-ignore
-      uploadLengthDeferred: true,
+      uploadLengthDeferred: false,
       resume: true,
-      uploadSize: null,
+      retryDelays: [0, 1000, 3000, 5000],
+      uploadSize: this.bytesWritten,
       metadata,
-      chunkSize,
       /**
        *
        * @param {Error} error
@@ -334,7 +359,7 @@ class Uploader {
        * @param {number} bytesTotal
        */
       onProgress (bytesUploaded, bytesTotal) {
-        uploader.emitProgress(bytesUploaded, bytesTotal)
+        uploader.emitIllusiveProgress(bytesUploaded)
       },
       onSuccess () {
         uploader.emitSuccess(uploader.tus.url)
@@ -342,15 +367,9 @@ class Uploader {
       }
     })
 
-    this.tus.start()
-
-    emitter().on(`pause:${this.token}`, () => {
-      this.tus.abort()
-    })
-
-    emitter().on(`resume:${this.token}`, () => {
+    if (!this._paused) {
       this.tus.start()
-    })
+    }
   }
 
   uploadMultipart () {

packages/@uppy/core/src/index.js

@@ -699,10 +699,16 @@ class Uppy {
       uploadedSize += averageSize * (file.progress.percentage || 0)
     })
 
-    const totalProgress = totalSize === 0
+    let totalProgress = totalSize === 0
       ? 0
       : Math.round(uploadedSize / totalSize * 100)
 
+    // hot fix, because:
+    // uploadedSize ended up larger than totalSize, resulting in 1325% total
+    if (totalProgress > 100) {
+      totalProgress = 100
+    }
+
     this.setState({ totalProgress })
     this.emit('progress', totalProgress)
   }