Prechádzať zdrojové kódy

@uppy/core: improve performance of validating & uploading files (#4402)

* show how many files are added when loading

remake of https://github.com/transloadit/uppy/pull/4388

* add french (cherry pick)

* implement concurrent file listing

* refactor / fix lint

* refactor/reduce duplication

* pull out totals validation

don't do it for every file added, as it's very slow
instead do the check at the end when all files are added.
this allows us to easily work with 10k+ files
fixes #4389

* Update packages/@uppy/core/src/Uppy.js

Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com>

* make restricter.validate validate everything

instead make more specific methods for sub-validation
also rename validateTotals to validateAggregateRestrictions

* improve errors and user feedback

- handle errors centrally so that we can limit the amount of toasts (informers) sent to the users (prevent flooding hundreds/thousands of them)
- introduce FileRestrictionError which is a restriction error for a specific file
- introduce isUserFacing field for RestrictionError

* fix performance issue reintroduced

* improvements

- show "%{count} additional restrictions were not fulfilled" for any restriction errors more than 4
- refactor/rename methods
- improve ghost logic/comments

* improve performance when uploading

- introduce new event "upload-start"  that can contain multiple files
- make a new patchFilesState method to allow updating more files
- unify "upload-start" logic in all plugins (send it before files start uploading)
- defer slicing buffer until we need the data
- refactor to reuse code

* fix e2e build issue

* try to upgrade cypress

maybe it fixes the error

* Revert "fix e2e build issue"

This reverts commit ff3e580c0f34ddd635adb666e88a1482c7ce7293.

* upgrade parcel

* move mutation logic to end

* remove FileRestrictionError

merge it with RestrictionError

* fix silly bug

looks like the e2e tests are doing its job 👏

---------

Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
Mikael Finstad 2 rokov pred
rodič
commit
90f7fb9197

+ 2 - 2
e2e/package.json

@@ -45,11 +45,11 @@
     "@uppy/zoom": "workspace:^"
   },
   "devDependencies": {
-    "cypress": "^10.0.0",
+    "cypress": "^12.9.0",
     "cypress-terminal-report": "^4.1.2",
     "deep-freeze": "^0.0.1",
     "execa": "^6.1.0",
-    "parcel": "^2.0.1",
+    "parcel": "2.0.0-nightly.1278",
     "prompts": "^2.4.2",
     "react": "^18.1.0",
     "react-dom": "^18.1.0",

+ 1 - 1
package.json

@@ -45,7 +45,7 @@
     "@babel/preset-env": "^7.14.7",
     "@babel/register": "^7.10.5",
     "@babel/types": "^7.17.0",
-    "@parcel/transformer-vue": "2.7.0",
+    "@parcel/transformer-vue": "2.8.4-nightly.2903+5b901a317",
     "@types/jasmine": "file:./private/@types/jasmine",
     "@types/jasminewd2": "file:./private/@types/jasmine",
     "@typescript-eslint/eslint-plugin": "^5.0.0",

+ 19 - 9
packages/@uppy/aws-s3-multipart/src/MultipartUploader.js

@@ -70,19 +70,29 @@ class MultipartUploader {
 
     // Upload zero-sized files in one zero-sized chunk
     if (this.#data.size === 0) {
-      this.#chunks = [this.#data]
-      this.#data.onProgress = this.#onPartProgress(0)
-      this.#data.onComplete = this.#onPartComplete(0)
+      this.#chunks = [{
+        getData: () => this.#data,
+        onProgress: this.#onPartProgress(0),
+        onComplete: this.#onPartComplete(0),
+      }]
     } else {
       const arraySize = Math.ceil(fileSize / chunkSize)
       this.#chunks = Array(arraySize)
-      let j = 0
-      for (let i = 0; i < fileSize; i += chunkSize) {
+
+      for (let i = 0, j = 0; i < fileSize; i += chunkSize, j++) {
         const end = Math.min(fileSize, i + chunkSize)
-        const chunk = this.#data.slice(i, end)
-        chunk.onProgress = this.#onPartProgress(j)
-        chunk.onComplete = this.#onPartComplete(j)
-        this.#chunks[j++] = chunk
+
+        // Defer data fetching/slicing until we actually need the data, because it's slow if we have a lot of files
+        const getData = () => {
+          const i2 = i
+          return this.#data.slice(i2, end)
+        }
+
+        this.#chunks[j] = {
+          getData,
+          onProgress: this.#onPartProgress(j),
+          onComplete: this.#onPartComplete(j),
+        }
       }
     }
 

+ 34 - 39
packages/@uppy/aws-s3-multipart/src/index.js

@@ -4,7 +4,7 @@ import EventTracker from '@uppy/utils/lib/EventTracker'
 import emitSocketProgress from '@uppy/utils/lib/emitSocketProgress'
 import getSocketHost from '@uppy/utils/lib/getSocketHost'
 import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'
-
+import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'
 import { createAbortError } from '@uppy/utils/lib/AbortController'
 import packageJson from '../package.json'
 import MultipartUploader from './MultipartUploader.js'
@@ -200,17 +200,23 @@ class HTTPCommunicationQueue {
     return this.#sendCompletionRequest(file, { key, uploadId, parts, signal }).abortOn(signal)
   }
 
-  async uploadChunk (file, partNumber, body, signal) {
+  async uploadChunk (file, partNumber, chunk, signal) {
     throwIfAborted(signal)
     const { uploadId, key } = await this.getUploadId(file, signal)
     throwIfAborted(signal)
     for (;;) {
-      const signature = await this.#fetchSignature(file, { uploadId, key, partNumber, body, signal }).abortOn(signal)
+      const chunkData = chunk.getData()
+      const { onProgress, onComplete } = chunk
+
+      const signature = await this.#fetchSignature(file, {
+        uploadId, key, partNumber, body: chunkData, signal,
+      }).abortOn(signal)
+
       throwIfAborted(signal)
       try {
         return {
           PartNumber: partNumber,
-          ...await this.#uploadPartBytes(signature, body, signal).abortOn(signal),
+          ...await this.#uploadPartBytes({ signature, body: chunkData, onProgress, onComplete, signal }).abortOn(signal),
         }
       } catch (err) {
         if (!await this.#shouldRetry(err)) throw err
@@ -258,8 +264,6 @@ export default class AwsS3Multipart extends BasePlugin {
       }
     }
 
-    this.upload = this.upload.bind(this)
-
     /**
      * Simultaneous upload limiting is shared across all uploads with this plugin.
      *
@@ -369,7 +373,7 @@ export default class AwsS3Multipart extends BasePlugin {
       .then(assertServerError)
   }
 
-  static async uploadPartBytes ({ url, expires, headers }, body, signal) {
+  static async uploadPartBytes ({ signature: { url, expires, headers }, body, onProgress, onComplete, signal }) {
     throwIfAborted(signal)
 
     if (url == null) {
@@ -397,7 +401,7 @@ export default class AwsS3Multipart extends BasePlugin {
       }
       signal.addEventListener('abort', onabort)
 
-      xhr.upload.addEventListener('progress', body.onProgress)
+      xhr.upload.addEventListener('progress', onProgress)
 
       xhr.addEventListener('abort', () => {
         cleanup()
@@ -427,7 +431,7 @@ export default class AwsS3Multipart extends BasePlugin {
           return
         }
 
-        body.onProgress?.(body.size)
+        onProgress?.(body.size)
 
         // NOTE This must be allowed by CORS.
         const etag = ev.target.getResponseHeader('ETag')
@@ -437,7 +441,7 @@ export default class AwsS3Multipart extends BasePlugin {
           return
         }
 
-        body.onComplete?.(etag)
+        onComplete?.(etag)
         resolve({
           ETag: etag,
         })
@@ -466,8 +470,10 @@ export default class AwsS3Multipart extends BasePlugin {
     })
   }
 
-  uploadFile (file) {
+  #uploadFile (file) {
     return new Promise((resolve, reject) => {
+      const getFile = () => this.uppy.getFile(file.id) || file
+
       const onProgress = (bytesUploaded, bytesTotal) => {
         this.uppy.emit('upload-progress', file, {
           uploader: this,
@@ -485,7 +491,6 @@ export default class AwsS3Multipart extends BasePlugin {
       }
 
       const onSuccess = (result) => {
-        const uploadObject = upload // eslint-disable-line no-use-before-define
         const uploadResp = {
           body: {
             ...result,
@@ -495,23 +500,17 @@ export default class AwsS3Multipart extends BasePlugin {
 
         this.resetUploaderReferences(file.id)
 
-        const cFile = this.uppy.getFile(file.id)
-        this.uppy.emit('upload-success', cFile || file, uploadResp)
+        this.uppy.emit('upload-success', getFile(), uploadResp)
 
         if (result.location) {
           this.uppy.log(`Download ${file.name} from ${result.location}`)
         }
 
-        resolve(uploadObject)
+        resolve()
       }
 
       const onPartComplete = (part) => {
-        const cFile = this.uppy.getFile(file.id)
-        if (!cFile) {
-          return
-        }
-
-        this.uppy.emit('s3-multipart:part-uploaded', cFile, part)
+        this.uppy.emit('s3-multipart:part-uploaded', getFile(), part)
       }
 
       const upload = new MultipartUploader(file.data, {
@@ -564,11 +563,7 @@ export default class AwsS3Multipart extends BasePlugin {
         upload.start()
       })
 
-      // Don't double-emit upload-started for Golden Retriever-restored files that were already started
-      if (!file.progress.uploadStarted || !file.isRestored) {
-        upload.start()
-        this.uppy.emit('upload-started', file)
-      }
+      upload.start()
     })
   }
 
@@ -597,14 +592,9 @@ export default class AwsS3Multipart extends BasePlugin {
 
   // NOTE! Keep this duplicated code in sync with other plugins
   // TODO we should probably abstract this into a common function
-  async uploadRemote (file) {
+  async #uploadRemote (file) {
     this.resetUploaderReferences(file.id)
 
-    // Don't double-emit upload-started for Golden Retriever-restored files that were already started
-    if (!file.progress.uploadStarted || !file.isRestored) {
-      this.uppy.emit('upload-started', file)
-    }
-
     try {
       if (file.serverToken) {
         return await this.connectToServerSocket(file)
@@ -733,15 +723,20 @@ export default class AwsS3Multipart extends BasePlugin {
     })
   }
 
-  async upload (fileIDs) {
+  #upload = async (fileIDs) => {
     if (fileIDs.length === 0) return undefined
 
-    const promises = fileIDs.map((id) => {
-      const file = this.uppy.getFile(id)
+    const files = this.uppy.getFilesByIds(fileIDs)
+
+    const filesFiltered = filterNonFailedFiles(files)
+    const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered)
+    this.uppy.emit('upload-start', filesToEmit)
+
+    const promises = filesFiltered.map((file) => {
       if (file.isRemote) {
-        return this.uploadRemote(file)
+        return this.#uploadRemote(file)
       }
-      return this.uploadFile(file)
+      return this.#uploadFile(file)
     })
 
     return Promise.all(promises)
@@ -810,7 +805,7 @@ export default class AwsS3Multipart extends BasePlugin {
       },
     })
     this.uppy.addPreProcessor(this.#setCompanionHeaders)
-    this.uppy.addUploader(this.upload)
+    this.uppy.addUploader(this.#upload)
   }
 
   uninstall () {
@@ -822,6 +817,6 @@ export default class AwsS3Multipart extends BasePlugin {
       },
     })
     this.uppy.removePreProcessor(this.#setCompanionHeaders)
-    this.uppy.removeUploader(this.upload)
+    this.uppy.removeUploader(this.#upload)
   }
 }

+ 7 - 5
packages/@uppy/aws-s3/src/index.js

@@ -28,6 +28,7 @@
 import BasePlugin from '@uppy/core/lib/BasePlugin.js'
 import { RateLimitedQueue, internalRateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'
 import { RequestClient } from '@uppy/companion-client'
+import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'
 
 import packageJson from '../package.json'
 import MiniXHRUpload from './MiniXHRUpload.js'
@@ -163,7 +164,7 @@ export default class AwsS3 extends BasePlugin {
       .then(assertServerError)
   }
 
-  #handleUpload = (fileIDs) => {
+  #handleUpload = async (fileIDs) => {
     /**
      * keep track of `getUploadParameters()` responses
      * so we can cancel the calls individually using just a file ID
@@ -178,10 +179,11 @@ export default class AwsS3 extends BasePlugin {
     }
     this.uppy.on('file-removed', onremove)
 
-    fileIDs.forEach((id) => {
-      const file = this.uppy.getFile(id)
-      this.uppy.emit('upload-started', file)
-    })
+    const files = this.uppy.getFilesByIds(fileIDs)
+
+    const filesFiltered = filterNonFailedFiles(files)
+    const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered)
+    this.uppy.emit('upload-start', filesToEmit)
 
     const getUploadParameters = this.#requests.wrapPromiseFunction((file) => {
       return this.opts.getUploadParameters(file)

+ 42 - 19
packages/@uppy/core/src/Restricter.js

@@ -13,6 +13,12 @@ const defaultOptions = {
 }
 
 class RestrictionError extends Error {
+  constructor (message, { isUserFacing = true, file } = {}) {
+    super(message)
+    this.isUserFacing = isUserFacing
+    if (file != null) this.file = file // only some restriction errors are related to a particular file
+  }
+
   isRestriction = true
 }
 
@@ -30,16 +36,38 @@ class Restricter {
     }
   }
 
-  validate (file, files) {
-    const { maxFileSize, minFileSize, maxTotalFileSize, maxNumberOfFiles, allowedFileTypes } = this.getOpts().restrictions
+  // Because these operations are slow, we cannot run them for every file (if we are adding multiple files)
+  validateAggregateRestrictions (existingFiles, addingFiles) {
+    const { maxTotalFileSize, maxNumberOfFiles } = this.getOpts().restrictions
 
     if (maxNumberOfFiles) {
-      const nonGhostFiles = files.filter(f => !f.isGhost)
-      if (nonGhostFiles.length + 1 > maxNumberOfFiles) {
+      const nonGhostFiles = existingFiles.filter(f => !f.isGhost)
+      if (nonGhostFiles.length + addingFiles.length > maxNumberOfFiles) {
         throw new RestrictionError(`${this.i18n('youCanOnlyUploadX', { smart_count: maxNumberOfFiles })}`)
       }
     }
 
+    if (maxTotalFileSize) {
+      let totalFilesSize = existingFiles.reduce((total, f) => (total + f.size), 0)
+
+      for (const addingFile of addingFiles) {
+        if (addingFile.size != null) { // We can't check maxTotalFileSize if the size is unknown.
+          totalFilesSize += addingFile.size
+
+          if (totalFilesSize > maxTotalFileSize) {
+            throw new RestrictionError(this.i18n('exceedsSize', {
+              size: prettierBytes(maxTotalFileSize),
+              file: addingFile.name,
+            }))
+          }
+        }
+      }
+    }
+  }
+
+  validateSingleFile (file) {
+    const { maxFileSize, minFileSize, allowedFileTypes } = this.getOpts().restrictions
+
     if (allowedFileTypes) {
       const isCorrectFileType = allowedFileTypes.some((type) => {
         // check if this is a mime-type
@@ -57,19 +85,7 @@ class Restricter {
 
       if (!isCorrectFileType) {
         const allowedFileTypesString = allowedFileTypes.join(', ')
-        throw new RestrictionError(this.i18n('youCanOnlyUploadFileTypes', { types: allowedFileTypesString }))
-      }
-    }
-
-    // We can't check maxTotalFileSize if the size is unknown.
-    if (maxTotalFileSize && file.size != null) {
-      const totalFilesSize = files.reduce((total, f) => (total + f.size), file.size)
-
-      if (totalFilesSize > maxTotalFileSize) {
-        throw new RestrictionError(this.i18n('exceedsSize', {
-          size: prettierBytes(maxTotalFileSize),
-          file: file.name,
-        }))
+        throw new RestrictionError(this.i18n('youCanOnlyUploadFileTypes', { types: allowedFileTypesString }), { file })
       }
     }
 
@@ -78,17 +94,24 @@ class Restricter {
       throw new RestrictionError(this.i18n('exceedsSize', {
         size: prettierBytes(maxFileSize),
         file: file.name,
-      }))
+      }), { file })
     }
 
     // We can't check minFileSize if the size is unknown.
     if (minFileSize && file.size != null && file.size < minFileSize) {
       throw new RestrictionError(this.i18n('inferiorSize', {
         size: prettierBytes(minFileSize),
-      }))
+      }), { file })
     }
   }
 
+  validate (existingFiles, addingFiles) {
+    addingFiles.forEach((addingFile) => {
+      this.validateSingleFile(addingFile)
+    })
+    this.validateAggregateRestrictions(existingFiles, addingFiles)
+  }
+
   validateMinNumberOfFiles (files) {
     const { minNumberOfFiles } = this.getOpts().restrictions
     if (Object.keys(files).length < minNumberOfFiles) {

+ 219 - 161
packages/@uppy/core/src/Uppy.js

@@ -178,6 +178,23 @@ class Uppy {
     return this.store.getState()
   }
 
+  patchFilesState (filesWithNewState) {
+    const existingFilesState = this.getState().files
+
+    this.setState({
+      files: {
+        ...existingFilesState,
+        ...Object.fromEntries(Object.entries(filesWithNewState).map(([fileID, newFileState]) => ([
+          fileID,
+          {
+            ...existingFilesState[fileID],
+            ...newFileState,
+          },
+        ]))),
+      },
+    })
+  }
+
   /**
    * Shorthand to set state for a specific file.
    */
@@ -186,9 +203,7 @@ class Uppy {
       throw new Error(`Can’t set state for ${fileID} (the file could have been removed)`)
     }
 
-    this.setState({
-      files: { ...this.getState().files, [fileID]: { ...this.getState().files[fileID], ...state } },
-    })
+    this.patchFilesState({ [fileID]: state })
   }
 
   i18nInit () {
@@ -323,6 +338,10 @@ class Uppy {
     return Object.values(files)
   }
 
+  getFilesByIds (ids) {
+    return ids.map((id) => this.getFile(id))
+  }
+
   getObjectOfFilesPerState () {
     const { files: filesObject, totalProgress, error } = this.getState()
     const files = Object.values(filesObject)
@@ -362,29 +381,43 @@ class Uppy {
 
   /*
   * @constructs
-  * @param { Error } error
+  * @param { Error[] } errors
   * @param { undefined } file
   */
   /*
   * @constructs
   * @param { RestrictionError } error
-  * @param { UppyFile | undefined } file
   */
-  #informAndEmit (error, file) {
-    const { message, details = '' } = error
+  #informAndEmit (errors) {
+    for (const error of errors) {
+      const { file, isRestriction } = error
 
-    if (error.isRestriction) {
-      this.emit('restriction-failed', file, error)
-    } else {
-      this.emit('error', error)
+      if (isRestriction) {
+        this.emit('restriction-failed', file, error)
+      } else {
+        this.emit('error', error)
+      }
+      this.log(error, 'warning')
+    }
+
+    const userFacingErrors = errors.filter((error) => error.isUserFacing)
+
+    // don't flood the user: only show the first 4 toasts
+    const maxNumToShow = 4
+    const firstErrors = userFacingErrors.slice(0, maxNumToShow)
+    const additionalErrors = userFacingErrors.slice(maxNumToShow)
+    firstErrors.forEach(({ message, details = '' }) => {
+      this.info({ message, details }, 'error', this.opts.infoTimeout)
+    })
+
+    if (additionalErrors.length > 0) {
+      this.info({ message: this.i18n('additionalRestrictionsFailed', { count: additionalErrors.length }) })
     }
-    this.info({ message, details }, 'error', this.opts.infoTimeout)
-    this.log(error, 'warning')
   }
 
   validateRestrictions (file, files = this.getFiles()) {
     try {
-      this.#restricter.validate(file, files)
+      this.#restricter.validate(files, [file])
     } catch (err) {
       return err
     }
@@ -417,8 +450,8 @@ class Uppy {
     const { allowNewUpload } = this.getState()
 
     if (allowNewUpload === false) {
-      const error = new RestrictionError(this.i18n('noMoreFilesAllowed'))
-      this.#informAndEmit(error, file)
+      const error = new RestrictionError(this.i18n('noMoreFilesAllowed'), { file })
+      this.#informAndEmit([error])
       throw error
     }
   }
@@ -434,25 +467,17 @@ class Uppy {
 
   /**
    * Create a file state object based on user-provided `addFile()` options.
-   *
-   * Note this is extremely side-effectful and should only be done when a file state object
-   * will be added to state immediately afterward!
-   *
-   * The `files` value is passed in because it may be updated by the caller without updating the store.
    */
-  #checkAndCreateFileStateObject (files, fileDescriptor) {
+  #transformFile (fileDescriptorOrFile) {
     // Uppy expects files in { name, type, size, data } format.
     // If the actual File object is passed from input[type=file] or drag-drop,
     // we normalize it to match Uppy file object
-    if (fileDescriptor instanceof File) {
-      // eslint-disable-next-line no-param-reassign
-      fileDescriptor = {
-        name: fileDescriptor.name,
-        type: fileDescriptor.type,
-        size: fileDescriptor.size,
-        data: fileDescriptor,
-      }
-    }
+    const fileDescriptor = fileDescriptorOrFile instanceof File ? {
+      name: fileDescriptorOrFile.name,
+      type: fileDescriptorOrFile.type,
+      size: fileDescriptorOrFile.size,
+      data: fileDescriptorOrFile,
+    } : fileDescriptorOrFile
 
     const fileType = getFileType(fileDescriptor)
     const fileName = getFileName(fileType, fileDescriptor)
@@ -460,12 +485,6 @@ class Uppy {
     const isRemote = Boolean(fileDescriptor.isRemote)
     const id = getSafeFileId(fileDescriptor)
 
-    if (this.checkIfFileAlreadyExists(id)) {
-      const error = new RestrictionError(this.i18n('noDuplicates', { fileName }))
-      this.#informAndEmit(error, fileDescriptor)
-      throw error
-    }
-
     const meta = fileDescriptor.meta || {}
     meta.name = fileName
     meta.type = fileType
@@ -473,7 +492,7 @@ class Uppy {
     // `null` means the size is unknown.
     const size = Number.isFinite(fileDescriptor.data.size) ? fileDescriptor.data.size : null
 
-    let newFile = {
+    return {
       source: fileDescriptor.source || '',
       id,
       name: fileName,
@@ -496,27 +515,6 @@ class Uppy {
       remote: fileDescriptor.remote || '',
       preview: fileDescriptor.preview,
     }
-
-    const onBeforeFileAddedResult = this.opts.onBeforeFileAdded(newFile, files)
-
-    if (onBeforeFileAddedResult === false) {
-      // Don’t show UI info for this error, as it should be done by the developer
-      const error = new RestrictionError('Cannot add the file because onBeforeFileAdded returned false.')
-      this.emit('restriction-failed', fileDescriptor, error)
-      throw error
-    } else if (typeof onBeforeFileAddedResult === 'object' && onBeforeFileAddedResult !== null) {
-      newFile = onBeforeFileAddedResult
-    }
-
-    try {
-      const filesArray = Object.keys(files).map(i => files[i])
-      this.#restricter.validate(newFile, filesArray)
-    } catch (err) {
-      this.#informAndEmit(err, newFile)
-      throw err
-    }
-
-    return newFile
   }
 
   // Schedule an upload if `autoProceed` is enabled.
@@ -533,6 +531,76 @@ class Uppy {
     }
   }
 
+  #checkAndUpdateFileState (filesToAdd) {
+    const { files: existingFiles } = this.getState()
+
+    // create a copy of the files object only once
+    const nextFilesState = { ...existingFiles }
+    const validFilesToAdd = []
+    const errors = []
+
+    for (const fileToAdd of filesToAdd) {
+      try {
+        let newFile = this.#transformFile(fileToAdd)
+
+        // If a file has been recovered (Golden Retriever), but we were unable to recover its data (probably too large),
+        // users are asked to re-select these half-recovered files and then this method will be called again.
+        // In order to keep the progress, meta and everthing else, we keep the existing file,
+        // but we replace `data`, and we remove `isGhost`, because the file is no longer a ghost now
+        if (existingFiles[newFile.id]?.isGhost) {
+          const { isGhost, ...existingFileState } = existingFiles[newFile.id]
+          newFile = {
+            ...existingFileState,
+            data: fileToAdd.data,
+          }
+          this.log(`Replaced the blob in the restored ghost file: ${newFile.name}, ${newFile.id}`)
+        }
+
+        if (this.checkIfFileAlreadyExists(newFile.id)) {
+          throw new RestrictionError(this.i18n('noDuplicates', { fileName: newFile.name }), { file: fileToAdd })
+        }
+
+        const onBeforeFileAddedResult = this.opts.onBeforeFileAdded(newFile, nextFilesState)
+
+        if (onBeforeFileAddedResult === false) {
+          // Don’t show UI info for this error, as it should be done by the developer
+          throw new RestrictionError('Cannot add the file because onBeforeFileAdded returned false.', { isUserFacing: false, file: fileToAdd })
+        } else if (typeof onBeforeFileAddedResult === 'object' && onBeforeFileAddedResult !== null) {
+          newFile = onBeforeFileAddedResult
+        }
+
+        this.#restricter.validateSingleFile(newFile)
+
+        // need to add it to the new local state immediately, so we can use the state to validate the next files too
+        nextFilesState[newFile.id] = newFile
+        validFilesToAdd.push(newFile)
+      } catch (err) {
+        errors.push(err)
+      }
+    }
+
+    try {
+      // need to run this separately because it's much more slow, so if we run it inside the for-loop it will be very slow
+      // when many files are added
+      this.#restricter.validateAggregateRestrictions(Object.values(existingFiles), validFilesToAdd)
+    } catch (err) {
+      errors.push(err)
+
+      // If we have any aggregate error, don't allow adding this batch
+      return {
+        nextFilesState: existingFiles,
+        validFilesToAdd: [],
+        errors,
+      }
+    }
+
+    return {
+      nextFilesState,
+      validFilesToAdd,
+      errors,
+    }
+  }
+
   /**
    * Add a new file to `state.files`. This will run `onBeforeFileAdded`,
    * try to guess file type in a clever way, check file against restrictions,
@@ -544,34 +612,24 @@ class Uppy {
   addFile (file) {
     this.#assertNewUploadAllowed(file)
 
-    const { files } = this.getState()
-    let newFile = this.#checkAndCreateFileStateObject(files, file)
-
-    // Users are asked to re-select recovered files without data,
-    // and to keep the progress, meta and everthing else, we only replace said data
-    if (files[newFile.id] && files[newFile.id].isGhost) {
-      newFile = {
-        ...files[newFile.id],
-        data: file.data,
-        isGhost: false,
-      }
-      this.log(`Replaced the blob in the restored ghost file: ${newFile.name}, ${newFile.id}`)
-    }
+    const { nextFilesState, validFilesToAdd, errors } = this.#checkAndUpdateFileState([file])
 
-    this.setState({
-      files: {
-        ...files,
-        [newFile.id]: newFile,
-      },
-    })
+    const restrictionErrors = errors.filter((error) => error.isRestriction)
+    this.#informAndEmit(restrictionErrors)
+
+    if (errors.length > 0) throw errors[0]
 
-    this.emit('file-added', newFile)
-    this.emit('files-added', [newFile])
-    this.log(`Added file: ${newFile.name}, ${newFile.id}, mime type: ${newFile.type}`)
+    this.setState({ files: nextFilesState })
+
+    const [firstValidFileToAdd] = validFilesToAdd
+
+    this.emit('file-added', firstValidFileToAdd)
+    this.emit('files-added', validFilesToAdd)
+    this.log(`Added file: ${firstValidFileToAdd.name}, ${firstValidFileToAdd.id}, mime type: ${firstValidFileToAdd.type}`)
 
     this.#startIfAutoProceed()
 
-    return newFile.id
+    return firstValidFileToAdd.id
   }
 
   /**
@@ -584,71 +642,54 @@ class Uppy {
   addFiles (fileDescriptors) {
     this.#assertNewUploadAllowed()
 
-    // create a copy of the files object only once
-    const files = { ...this.getState().files }
-    const newFiles = []
-    const errors = []
-    for (let i = 0; i < fileDescriptors.length; i++) {
-      try {
-        let newFile = this.#checkAndCreateFileStateObject(files, fileDescriptors[i])
-        // Users are asked to re-select recovered files without data,
-        // and to keep the progress, meta and everthing else, we only replace said data
-        if (files[newFile.id] && files[newFile.id].isGhost) {
-          newFile = {
-            ...files[newFile.id],
-            data: fileDescriptors[i].data,
-            isGhost: false,
-          }
-          this.log(`Replaced blob in a ghost file: ${newFile.name}, ${newFile.id}`)
-        }
-        files[newFile.id] = newFile
-        newFiles.push(newFile)
-      } catch (err) {
-        if (!err.isRestriction) {
-          errors.push(err)
-        }
-      }
-    }
-
-    this.setState({ files })
-
-    newFiles.forEach((newFile) => {
-      this.emit('file-added', newFile)
-    })
-
-    this.emit('files-added', newFiles)
+    const { nextFilesState, validFilesToAdd, errors } = this.#checkAndUpdateFileState(fileDescriptors)
 
-    if (newFiles.length > 5) {
-      this.log(`Added batch of ${newFiles.length} files`)
-    } else {
-      Object.keys(newFiles).forEach(fileID => {
-        this.log(`Added file: ${newFiles[fileID].name}\n id: ${newFiles[fileID].id}\n type: ${newFiles[fileID].type}`)
-      })
-    }
+    const restrictionErrors = errors.filter((error) => error.isRestriction)
+    this.#informAndEmit(restrictionErrors)
 
-    if (newFiles.length > 0) {
-      this.#startIfAutoProceed()
-    }
+    const nonRestrictionErrors = errors.filter((error) => !error.isRestriction)
 
-    if (errors.length > 0) {
+    if (nonRestrictionErrors.length > 0) {
       let message = 'Multiple errors occurred while adding files:\n'
-      errors.forEach((subError) => {
+      nonRestrictionErrors.forEach((subError) => {
         message += `\n * ${subError.message}`
       })
 
       this.info({
-        message: this.i18n('addBulkFilesFailed', { smart_count: errors.length }),
+        message: this.i18n('addBulkFilesFailed', { smart_count: nonRestrictionErrors.length }),
         details: message,
       }, 'error', this.opts.infoTimeout)
 
       if (typeof AggregateError === 'function') {
-        throw new AggregateError(errors, message)
+        throw new AggregateError(nonRestrictionErrors, message)
       } else {
         const err = new Error(message)
-        err.errors = errors
+        err.errors = nonRestrictionErrors
         throw err
       }
     }
+
+    // OK, we haven't thrown an error, we can start updating state and emitting events now:
+
+    this.setState({ files: nextFilesState })
+
+    validFilesToAdd.forEach((file) => {
+      this.emit('file-added', file)
+    })
+
+    this.emit('files-added', validFilesToAdd)
+
+    if (validFilesToAdd.length > 5) {
+      this.log(`Added batch of ${validFilesToAdd.length} files`)
+    } else {
+      Object.values(validFilesToAdd).forEach((file) => {
+        this.log(`Added file: ${file.name}\n id: ${file.id}\n type: ${file.type}`)
+      })
+    }
+
+    if (validFilesToAdd.length > 0) {
+      this.#startIfAutoProceed()
+    }
   }
 
   removeFiles (fileIDs, reason) {
@@ -965,14 +1006,15 @@ class Uppy {
 
       if (typeof error === 'object' && error.message) {
         const newError = new Error(error.message)
+        newError.isUserFacing = true // todo maybe don't do this with all errors?
         newError.details = error.message
         if (error.details) {
           newError.details += ` ${error.details}`
         }
         newError.message = this.i18n('failedToUpload', { file: file?.name })
-        this.#informAndEmit(newError)
+        this.#informAndEmit([newError])
       } else {
-        this.#informAndEmit(error)
+        this.#informAndEmit([error])
       }
     })
 
@@ -993,20 +1035,35 @@ class Uppy {
       this.setState({ error: null })
     })
 
-    this.on('upload-started', (file) => {
-      if (file == null || !this.getFile(file.id)) {
-        this.log(`Not setting progress for a file that has been removed: ${file?.id}`)
-        return
-      }
-      this.setFileState(file.id, {
-        progress: {
-          uploadStarted: Date.now(),
-          uploadComplete: false,
-          percentage: 0,
-          bytesUploaded: 0,
-          bytesTotal: file.size,
+    const onUploadStarted = (files) => {
+      const filesFiltered = files.filter((file) => {
+        const exists = (file != null && this.getFile(file.id))
+        if (!exists) this.log(`Not setting progress for a file that has been removed: ${file?.id}`)
+        return exists
+      })
+
+      const filesState = Object.fromEntries(filesFiltered.map((file) => ([
+        file.id,
+        {
+          progress: {
+            uploadStarted: Date.now(),
+            uploadComplete: false,
+            percentage: 0,
+            bytesUploaded: 0,
+            bytesTotal: file.size,
+          },
         },
+      ])))
+
+      this.patchFilesState(filesState)
+    }
+
+    this.on('upload-start', (files) => {
+      files.forEach((file) => {
+        // todo backward compat, remove this event in a next major
+        this.emit('upload-started', file)
       })
+      onUploadStarted(files)
     })
 
     this.on('upload-progress', this.calculateProgress)
@@ -1415,9 +1472,12 @@ class Uppy {
    * @private
    */
   async #runUpload (uploadID) {
-    let { currentUploads } = this.getState()
-    let currentUpload = currentUploads[uploadID]
-    const restoreStep = currentUpload.step || 0
+    const getCurrentUpload = () => {
+      const { currentUploads } = this.getState()
+      return currentUploads[uploadID]
+    }
+
+    let currentUpload = getCurrentUpload()
 
     const steps = [
       ...this.#preProcessors,
@@ -1425,31 +1485,30 @@ class Uppy {
       ...this.#postProcessors,
     ]
     try {
-      for (let step = restoreStep; step < steps.length; step++) {
+      for (let step = currentUpload.step || 0; step < steps.length; step++) {
         if (!currentUpload) {
           break
         }
         const fn = steps[step]
 
-        const updatedUpload = {
-          ...currentUpload,
-          step,
-        }
-
         this.setState({
           currentUploads: {
-            ...currentUploads,
-            [uploadID]: updatedUpload,
+            ...this.getState().currentUploads,
+            [uploadID]: {
+              ...currentUpload,
+              step,
+            },
           },
         })
 
+        const { fileIDs } = currentUpload
+
         // TODO give this the `updatedUpload` object as its only parameter maybe?
         // Otherwise when more metadata may be added to the upload this would keep getting more parameters
-        await fn(updatedUpload.fileIDs, uploadID)
+        await fn(fileIDs, uploadID)
 
         // Update currentUpload value in case it was modified asynchronously.
-        currentUploads = this.getState().currentUploads
-        currentUpload = currentUploads[uploadID]
+        currentUpload = getCurrentUpload()
       }
     } catch (err) {
       this.#removeUpload(uploadID)
@@ -1481,8 +1540,7 @@ class Uppy {
       await this.addResultData(uploadID, { successful, failed, uploadID })
 
       // Update currentUpload value in case it was modified asynchronously.
-      currentUploads = this.getState().currentUploads
-      currentUpload = currentUploads[uploadID]
+      currentUpload = getCurrentUpload()
     }
     // Emit completion events.
     // This is in a separate function so that the `currentUploads` variable
@@ -1531,7 +1589,7 @@ class Uppy {
     return Promise.resolve()
       .then(() => this.#restricter.validateMinNumberOfFiles(files))
       .catch((err) => {
-        this.#informAndEmit(err)
+        this.#informAndEmit([err])
         throw err
       })
       .then(() => {

+ 11 - 5
packages/@uppy/core/src/Uppy.test.js

@@ -684,8 +684,10 @@ describe('src/Core', () => {
   describe('adding a file', () => {
     it('should call onBeforeFileAdded if it was specified in the options when initialising the class', () => {
       const onBeforeFileAdded = jest.fn()
+
       const core = new Core({
-        onBeforeFileAdded,
+        // need to capture a snapshot of files, because files will change in the next tick, thus failing the expect below
+        onBeforeFileAdded: (file, files) => onBeforeFileAdded(file, { ...files }),
       })
 
       core.addFile({
@@ -1428,7 +1430,7 @@ describe('src/Core', () => {
       const promise = new Promise((resolve) => { proceedUpload = resolve })
       const finishPromise = new Promise((resolve) => { finishUpload = resolve })
       core.addUploader(async ([id]) => {
-        core.emit('upload-started', core.getFile(id))
+        core.emit('upload-start', [core.getFile(id)])
         await promise
         core.emit('upload-progress', core.getFile(id), {
           bytesTotal: 3456,
@@ -1448,7 +1450,11 @@ describe('src/Core', () => {
       core.calculateTotalProgress()
 
       const uploadPromise = core.upload()
-      await new Promise((resolve) => core.once('upload-started', resolve))
+      await Promise.all([
+        new Promise((resolve) => core.once('upload-start', resolve)),
+        // todo backward compat: remove in next major
+        new Promise((resolve) => core.once('upload-started', resolve)),
+      ])
 
       expect(core.getFiles()[0].size).toBeNull()
       expect(core.getFiles()[0].progress).toMatchObject({
@@ -1491,7 +1497,7 @@ describe('src/Core', () => {
       const core = new Core()
 
       core.once('file-added', (file) => {
-        core.emit('upload-started', file)
+        core.emit('upload-start', [file])
         core.emit('upload-progress', file, {
           bytesTotal: 3456,
           bytesUploaded: 1234,
@@ -1505,7 +1511,7 @@ describe('src/Core', () => {
       })
 
       core.once('file-added', (file) => {
-        core.emit('upload-started', file)
+        core.emit('upload-start', [file])
         core.emit('upload-progress', file, {
           bytesTotal: null,
           bytesUploaded: null,

+ 1 - 0
packages/@uppy/core/src/locale.js

@@ -57,5 +57,6 @@ export default {
       0: 'Added %{smart_count} file from %{folder}',
       1: 'Added %{smart_count} files from %{folder}',
     },
+    additionalRestrictionsFailed: '%{count} additional restrictions were not fulfilled',
   },
 }

+ 2 - 0
packages/@uppy/dashboard/src/Dashboard.jsx

@@ -655,6 +655,8 @@ export default class Dashboard extends UIPlugin {
       }
     }
 
+    this.uppy.log('[Dashboard] Processing dropped files')
+
     // Add all dropped files
     const files = await getDroppedFiles(event.dataTransfer, { logDropError })
     if (files.length > 0) {

+ 22 - 2
packages/@uppy/dashboard/src/components/Dashboard.jsx

@@ -144,8 +144,28 @@ export default function Dashboard (props) {
 
           {showFileList ? (
             <FileList
-              // eslint-disable-next-line react/jsx-props-no-spreading
-              {...props}
+              id={props.id}
+              error={props.error}
+              i18n={props.i18n}
+              uppy={props.uppy}
+              files={props.files}
+              acquirers={props.acquirers}
+              resumableUploads={props.resumableUploads}
+              hideRetryButton={props.hideRetryButton}
+              hidePauseResumeButton={props.hidePauseResumeButton}
+              hideCancelButton={props.hideCancelButton}
+              showLinkToFileUploadResult={props.showLinkToFileUploadResult}
+              showRemoveButtonAfterComplete={props.showRemoveButtonAfterComplete}
+              isWide={props.isWide}
+              metaFields={props.metaFields}
+              toggleFileCard={props.toggleFileCard}
+              handleRequestThumbnail={props.handleRequestThumbnail}
+              handleCancelThumbnail={props.handleCancelThumbnail}
+              recoveredState={props.recoveredState}
+              individualCancellation={props.individualCancellation}
+              openFileEditor={props.openFileEditor}
+              canEditFile={props.canEditFile}
+              toggleAddFilesPanel={props.toggleAddFilesPanel}
               isSingleFile={isSingleFile}
               itemsPerRow={itemsPerRow}
             />

+ 46 - 45
packages/@uppy/dashboard/src/components/FileList.jsx

@@ -1,4 +1,5 @@
 import { h } from 'preact'
+import { useMemo } from 'preact/hooks'
 import FileItem from './FileItem/index.jsx'
 import VirtualList from './VirtualList.jsx'
 
@@ -17,52 +18,28 @@ function chunks (list, size) {
   return chunked
 }
 
-export default (props) => {
+export default ({
+  id, error, i18n, uppy, files, acquirers, resumableUploads, hideRetryButton, hidePauseResumeButton, hideCancelButton,
+  showLinkToFileUploadResult, showRemoveButtonAfterComplete, isWide, metaFields, isSingleFile, toggleFileCard,
+  handleRequestThumbnail, handleCancelThumbnail, recoveredState, individualCancellation, itemsPerRow, openFileEditor,
+  canEditFile, toggleAddFilesPanel, containerWidth, containerHeight,
+}) => {
   // It's not great that this is hardcoded!
   // It's ESPECIALLY not great that this is checking against `itemsPerRow`!
-  const rowHeight = props.itemsPerRow === 1
+  const rowHeight = itemsPerRow === 1
     // Mobile
     ? 71
     // 190px height + 2 * 5px margin
     : 200
 
-  const fileProps = {
-    // FIXME This is confusing, it's actually the Dashboard's plugin ID
-    id: props.id,
-    error: props.error,
-    // TODO move this to context
-    i18n: props.i18n,
-    uppy: props.uppy,
-    // features
-    acquirers: props.acquirers,
-    resumableUploads: props.resumableUploads,
-    individualCancellation: props.individualCancellation,
-    // visual options
-    hideRetryButton: props.hideRetryButton,
-    hidePauseResumeButton: props.hidePauseResumeButton,
-    hideCancelButton: props.hideCancelButton,
-    showLinkToFileUploadResult: props.showLinkToFileUploadResult,
-    showRemoveButtonAfterComplete: props.showRemoveButtonAfterComplete,
-    isWide: props.isWide,
-    metaFields: props.metaFields,
-    recoveredState: props.recoveredState,
-    isSingleFile: props.isSingleFile,
-    containerWidth: props.containerWidth,
-    containerHeight: props.containerHeight,
-    // callbacks
-    toggleFileCard: props.toggleFileCard,
-    handleRequestThumbnail: props.handleRequestThumbnail,
-    handleCancelThumbnail: props.handleCancelThumbnail,
-  }
-
-  const sortByGhostComesFirst = (file1, file2) => {
-    return props.files[file2].isGhost - props.files[file1].isGhost
-  }
-
   // Sort files by file.isGhost, ghost files first, only if recoveredState is present
-  const files = Object.keys(props.files)
-  if (props.recoveredState) files.sort(sortByGhostComesFirst)
-  const rows = chunks(files, props.itemsPerRow)
+  const rows = useMemo(() => {
+    const sortByGhostComesFirst = (file1, file2) => files[file2].isGhost - files[file1].isGhost
+
+    const fileIds = Object.keys(files)
+    if (recoveredState) fileIds.sort(sortByGhostComesFirst)
+    return chunks(fileIds, itemsPerRow)
+  }, [files, itemsPerRow, recoveredState])
 
   const renderRow = (row) => (
     // The `role="presentation` attribute ensures that the list items are properly
@@ -72,19 +49,43 @@ export default (props) => {
       {row.map((fileID) => (
         <FileItem
           key={fileID}
-          uppy={props.uppy}
-          {...fileProps} // eslint-disable-line react/jsx-props-no-spreading
+          uppy={uppy}
+          // FIXME This is confusing, it's actually the Dashboard's plugin ID
+          id={id}
+          error={error}
+          // TODO move this to context
+          i18n={i18n}
+          // features
+          acquirers={acquirers}
+          resumableUploads={resumableUploads}
+          individualCancellation={individualCancellation}
+          // visual options
+          hideRetryButton={hideRetryButton}
+          hidePauseResumeButton={hidePauseResumeButton}
+          hideCancelButton={hideCancelButton}
+          showLinkToFileUploadResult={showLinkToFileUploadResult}
+          showRemoveButtonAfterComplete={showRemoveButtonAfterComplete}
+          isWide={isWide}
+          metaFields={metaFields}
+          recoveredState={recoveredState}
+          isSingleFile={isSingleFile}
+          containerWidth={containerWidth}
+          containerHeight={containerHeight}
+          // callbacks
+          toggleFileCard={toggleFileCard}
+          handleRequestThumbnail={handleRequestThumbnail}
+          handleCancelThumbnail={handleCancelThumbnail}
           role="listitem"
-          openFileEditor={props.openFileEditor}
-          canEditFile={props.canEditFile}
-          toggleAddFilesPanel={props.toggleAddFilesPanel}
-          file={props.files[fileID]}
+          openFileEditor={openFileEditor}
+          canEditFile={canEditFile}
+          toggleAddFilesPanel={toggleAddFilesPanel}
+          file={files[fileID]}
         />
       ))}
     </div>
   )
 
-  if (props.isSingleFile) {
+  if (isSingleFile) {
     return (
       <div class="uppy-Dashboard-files">
         {renderRow(rows[0])}

+ 1 - 0
packages/@uppy/locales/src/en_US.js

@@ -81,6 +81,7 @@ en_US.strings = {
     '0': 'Added %{smart_count} file from %{folder}',
     '1': 'Added %{smart_count} files from %{folder}',
   },
+  additionalRestrictionsFailed: '%{count} additional restrictions were not fulfilled',
   folderAlreadyAdded: 'The folder "%{folder}" was already added',
   generatingThumbnails: 'Generating thumbnails...',
   import: 'Import',

+ 3 - 0
packages/@uppy/provider-views/src/ProviderView/ProviderView.jsx

@@ -301,6 +301,9 @@ export default class ProviderView extends View {
               folder: name,
             })
           } else {
+            // TODO we don't really know at this point whether any files were actually added
+            // (only later after addFiles has been called) so we should probably rewrite this.
+            // Example: If all files fail to add due to restriction error, it will still say "Added 100 files from folder"
             message = this.plugin.uppy.i18n('folderAdded', {
               smart_count: numNewFiles, folder: name,
             })

+ 19 - 38
packages/@uppy/tus/src/index.js

@@ -3,12 +3,12 @@ import * as tus from 'tus-js-client'
 import { Provider, RequestClient, Socket } from '@uppy/companion-client'
 import emitSocketProgress from '@uppy/utils/lib/emitSocketProgress'
 import getSocketHost from '@uppy/utils/lib/getSocketHost'
-import settle from '@uppy/utils/lib/settle'
 import EventTracker from '@uppy/utils/lib/EventTracker'
 import NetworkError from '@uppy/utils/lib/NetworkError'
 import isNetworkError from '@uppy/utils/lib/isNetworkError'
 import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'
 import hasProperty from '@uppy/utils/lib/hasProperty'
+import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'
 import getFingerprint from './getFingerprint.js'
 
 import packageJson from '../package.json'
@@ -102,7 +102,6 @@ export default class Tus extends BasePlugin {
     this.uploaderSockets = Object.create(null)
 
     this.handleResetProgress = this.handleResetProgress.bind(this)
-    this.handleUpload = this.handleUpload.bind(this)
     this.#queueRequestSocketToken = this.requests.wrapPromiseFunction(this.#requestSocketToken, { priority: -1 })
   }
 
@@ -183,7 +182,7 @@ export default class Tus extends BasePlugin {
    * @param {UppyFile} file for use with upload
    * @returns {Promise<void>}
    */
-  upload (file) {
+  #upload (file) {
     this.resetUploaderReferences(file.id)
 
     // Create a new tus upload
@@ -192,8 +191,6 @@ export default class Tus extends BasePlugin {
       let qRequest
       let upload
 
-      this.uppy.emit('upload-started', file)
-
       const opts = {
         ...this.opts,
         ...(file.tus || {}),
@@ -462,14 +459,9 @@ export default class Tus extends BasePlugin {
    * @param {UppyFile} file for use with upload
    * @returns {Promise<void>}
    */
-  async uploadRemote (file) {
+  async #uploadRemote (file) {
     this.resetUploaderReferences(file.id)
 
-    // Don't double-emit upload-started for Golden Retriever-restored files that were already started
-    if (!file.progress.uploadStarted || !file.isRestored) {
-      this.uppy.emit('upload-started', file)
-    }
-
     try {
       if (file.serverToken) {
         return await this.connectToServerSocket(file)
@@ -730,39 +722,29 @@ export default class Tus extends BasePlugin {
   /**
    * @param {(UppyFile | FailedUppyFile)[]} files
    */
-  uploadFiles (files) {
-    const promises = files.map((file, i) => {
+  async #uploadFiles (files) {
+    const filesFiltered = filterNonFailedFiles(files)
+    const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered)
+    this.uppy.emit('upload-start', filesToEmit)
+
+    await Promise.allSettled(filesFiltered.map((file, i) => {
       const current = i + 1
       const total = files.length
 
-      if ('error' in file && file.error) {
-        return Promise.reject(new Error(file.error))
-      } if (file.isRemote) {
-        // We emit upload-started here, so that it's also emitted for files
-        // that have to wait due to the `limit` option.
-        // Don't double-emit upload-started for Golden Retriever-restored files that were already started
-        if (!file.progress.uploadStarted || !file.isRestored) {
-          this.uppy.emit('upload-started', file)
-        }
-        return this.uploadRemote(file, current, total)
-      }
-      // Don't double-emit upload-started for Golden Retriever-restored files that were already started
-      if (!file.progress.uploadStarted || !file.isRestored) {
-        this.uppy.emit('upload-started', file)
+      if (file.isRemote) {
+        return this.#uploadRemote(file, current, total)
       }
-      return this.upload(file, current, total)
-    })
-
-    return settle(promises)
+      return this.#upload(file, current, total)
+    }))
   }
 
   /**
    * @param {string[]} fileIDs
    */
-  handleUpload (fileIDs) {
+  #handleUpload = async (fileIDs) => {
     if (fileIDs.length === 0) {
       this.uppy.log('[Tus] No files to upload')
-      return Promise.resolve()
+      return
     }
 
     if (this.opts.limit === 0) {
@@ -773,17 +755,16 @@ export default class Tus extends BasePlugin {
     }
 
     this.uppy.log('[Tus] Uploading...')
-    const filesToUpload = fileIDs.map((fileID) => this.uppy.getFile(fileID))
+    const filesToUpload = this.uppy.getFilesByIds(fileIDs)
 
-    return this.uploadFiles(filesToUpload)
-      .then(() => null)
+    await this.#uploadFiles(filesToUpload)
   }
 
   install () {
     this.uppy.setState({
       capabilities: { ...this.uppy.getState().capabilities, resumableUploads: true },
     })
-    this.uppy.addUploader(this.handleUpload)
+    this.uppy.addUploader(this.#handleUpload)
 
     this.uppy.on('reset-progress', this.handleResetProgress)
   }
@@ -792,6 +773,6 @@ export default class Tus extends BasePlugin {
     this.uppy.setState({
       capabilities: { ...this.uppy.getState().capabilities, resumableUploads: false },
     })
-    this.uppy.removeUploader(this.handleUpload)
+    this.uppy.removeUploader(this.#handleUpload)
   }
 }

+ 1 - 0
packages/@uppy/utils/package.json

@@ -61,6 +61,7 @@
     "./lib/mimeTypes": "./lib/mimeTypes.js",
     "./lib/getDroppedFiles": "./lib/getDroppedFiles/index.js",
     "./lib/FOCUSABLE_ELEMENTS.js": "./lib/FOCUSABLE_ELEMENTS.js",
+    "./lib/fileFilters": "./lib/fileFilters.js",
     "./src/microtip.scss": "./src/microtip.scss"
   },
   "dependencies": {

+ 10 - 0
packages/@uppy/utils/src/fileFilters.js

@@ -0,0 +1,10 @@
+export function filterNonFailedFiles (files) {
+  const hasError = (file) => 'error' in file && file.error
+
+  return files.filter((file) => !hasError(file))
+}
+
+// Don't double-emit upload-started for Golden Retriever-restored files that were already started
+export function filterFilesToEmitUploadStarted (files) {
+  return files.filter((file) => !file.progress.uploadStarted || !file.isRestored)
+}

+ 1 - 0
packages/@uppy/utils/src/settle.js

@@ -1,3 +1,4 @@
+// TODO remove (no longer in use)
 export default function settle (promises) {
   const resolutions = []
   const rejections = []

+ 23 - 34
packages/@uppy/xhr-upload/src/index.js

@@ -3,12 +3,12 @@ import { nanoid } from 'nanoid/non-secure'
 import { Provider, RequestClient, Socket } from '@uppy/companion-client'
 import emitSocketProgress from '@uppy/utils/lib/emitSocketProgress'
 import getSocketHost from '@uppy/utils/lib/getSocketHost'
-import settle from '@uppy/utils/lib/settle'
 import EventTracker from '@uppy/utils/lib/EventTracker'
 import ProgressTimeout from '@uppy/utils/lib/ProgressTimeout'
 import { RateLimitedQueue, internalRateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'
 import NetworkError from '@uppy/utils/lib/NetworkError'
 import isNetworkError from '@uppy/utils/lib/isNetworkError'
+import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'
 
 import packageJson from '../package.json'
 import locale from './locale.js'
@@ -113,8 +113,6 @@ export default class XHRUpload extends BasePlugin {
     this.opts = { ...defaultOptions, ...opts }
     this.i18nInit()
 
-    this.handleUpload = this.handleUpload.bind(this)
-
     // Simultaneous upload limiting is shared across all uploads with this plugin.
     if (internalRateLimitedQueue in this.opts) {
       this.requests = this.opts[internalRateLimitedQueue]
@@ -214,13 +212,11 @@ export default class XHRUpload extends BasePlugin {
     return formPost
   }
 
-  upload (file, current, total) {
+  async #upload (file, current, total) {
     const opts = this.getOptions(file)
 
     this.uppy.log(`uploading ${current} of ${total}`)
     return new Promise((resolve, reject) => {
-      this.uppy.emit('upload-started', file)
-
       const data = opts.formData
         ? this.createFormDataUpload(file, opts)
         : file.data
@@ -317,8 +313,6 @@ export default class XHRUpload extends BasePlugin {
       }
 
       queuedRequest = this.requests.run(() => {
-        this.uppy.emit('upload-started', file)
-
         // When using an authentication system like JWT, the bearer token goes as a header. This
         // header needs to be fresh each time the token is refreshed so computing and setting the
         // headers just before the upload starts enables this kind of authentication to work properly.
@@ -375,10 +369,9 @@ export default class XHRUpload extends BasePlugin {
 
   // NOTE! Keep this duplicated code in sync with other plugins
   // TODO we should probably abstract this into a common function
-  async uploadRemote (file) {
+  async #uploadRemote (file) {
     // TODO: we could rewrite this to use server-sent events instead of creating WebSockets.
     try {
-      this.uppy.emit('upload-started', file)
       if (file.serverToken) {
         return await this.connectToServerSocket(file)
       }
@@ -496,7 +489,7 @@ export default class XHRUpload extends BasePlugin {
     })
   }
 
-  uploadBundle (files) {
+  #uploadBundle (files) {
     return new Promise((resolve, reject) => {
       const { endpoint } = this.opts
       const { method } = this.opts
@@ -587,27 +580,19 @@ export default class XHRUpload extends BasePlugin {
       })
 
       xhr.send(formData)
-
-      files.forEach((file) => {
-        this.uppy.emit('upload-started', file)
-      })
     })
   }
 
-  uploadFiles (files) {
-    const promises = files.map((file, i) => {
+  async #uploadFiles (files) {
+    await Promise.allSettled(files.map((file, i) => {
       const current = parseInt(i, 10) + 1
       const total = files.length
 
-      if (file.error) {
-        return Promise.reject(new Error(file.error))
-      } if (file.isRemote) {
-        return this.uploadRemote(file, current, total)
+      if (file.isRemote) {
+        return this.#uploadRemote(file, current, total)
       }
-      return this.upload(file, current, total)
-    })
-
-    return settle(promises)
+      return this.#upload(file, current, total)
+    }))
   }
 
   onFileRemove (fileID, cb) {
@@ -638,10 +623,10 @@ export default class XHRUpload extends BasePlugin {
     })
   }
 
-  handleUpload (fileIDs) {
+  #handleUpload = async (fileIDs) => {
     if (fileIDs.length === 0) {
       this.uppy.log('[XHRUpload] No files to upload!')
-      return Promise.resolve()
+      return
     }
 
     // No limit configured by the user, and no RateLimitedQueue passed in by a "parent" plugin
@@ -654,11 +639,15 @@ export default class XHRUpload extends BasePlugin {
     }
 
     this.uppy.log('[XHRUpload] Uploading...')
-    const files = fileIDs.map((fileID) => this.uppy.getFile(fileID))
+    const files = this.uppy.getFilesByIds(fileIDs)
+
+    const filesFiltered = filterNonFailedFiles(files)
+    const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered)
+    this.uppy.emit('upload-start', filesToEmit)
 
     if (this.opts.bundle) {
       // if bundle: true, we don’t support remote uploads
-      const isSomeFileRemote = files.some(file => file.isRemote)
+      const isSomeFileRemote = filesFiltered.some(file => file.isRemote)
       if (isSomeFileRemote) {
         throw new Error('Can’t upload remote files when the `bundle: true` option is set')
       }
@@ -667,10 +656,10 @@ export default class XHRUpload extends BasePlugin {
         throw new TypeError('`headers` may not be a function when the `bundle: true` option is set')
       }
 
-      return this.uploadBundle(files)
+      await this.#uploadBundle(filesFiltered)
+    } else {
+      await this.#uploadFiles(filesFiltered)
     }
-
-    return this.uploadFiles(files).then(() => null)
   }
 
   install () {
@@ -684,7 +673,7 @@ export default class XHRUpload extends BasePlugin {
       })
     }
 
-    this.uppy.addUploader(this.handleUpload)
+    this.uppy.addUploader(this.#handleUpload)
   }
 
   uninstall () {
@@ -698,6 +687,6 @@ export default class XHRUpload extends BasePlugin {
       })
     }
 
-    this.uppy.removeUploader(this.handleUpload)
+    this.uppy.removeUploader(this.#handleUpload)
   }
 }

+ 5 - 1
private/dev/Dashboard.js

@@ -40,6 +40,7 @@ console.log(import.meta.env)
 // DEV CONFIG: enable or disable Golden Retriever
 
 const RESTORE = false
+const COMPRESS = false
 
 async function assemblyOptions () {
   return generateSignatureIfSecret(TRANSLOADIT_SECRET, {
@@ -108,7 +109,10 @@ export default () => {
     .use(DropTarget, {
       target: document.body,
     })
-    .use(Compressor)
+
+  if (COMPRESS) {
+    uppyDashboard.use(Compressor)
+  }
 
   switch (UPLOADER) {
     case 'tus':

Rozdielové dáta súboru neboli zobrazené, pretože súbor je príliš veľký
+ 1033 - 463
yarn.lock


Niektoré súbory nie sú zobrazené, pretože je v týchto rozdielových dátach zmenené mnoho súborov