Load Google Drive / OneDrive lists 5-10x faster & always load all files (#4513)

Co-authored-by: Mikael Finstad <finstaden@gmail.com>
Co-authored-by: Artur Paikin <artur@arturpaikin.com>

packages/@uppy/box/src/Box.jsx

@@ -44,6 +44,7 @@ export default class Box extends UIPlugin {
   install () {
     this.view = new ProviderViews(this, {
       provider: this.provider,
+      loadAllFiles: true,
     })
 
     const { target } = this.opts

packages/@uppy/companion-client/src/Provider.js

@@ -131,8 +131,8 @@ export default class Provider extends RequestClient {
     }
   }
 
-  list (directory) {
-    return this.get(`${this.id}/list/${directory || ''}`)
+  list (directory, options) {
+    return this.get(`${this.id}/list/${directory || ''}`, options)
   }
 
   async logout () {

packages/@uppy/companion/src/server/provider/box/index.js

@@ -21,7 +21,8 @@ async function getUserInfo ({ token }) {
 
 async function list ({ directory, query, token }) {
   const rootFolderID = '0'
-  return getClient({ token }).get(`folders/${directory || rootFolderID}/items`, { searchParams: { fields: BOX_FILES_FIELDS, offset: query.cursor }, responseType: 'json' }).json()
+  // https://developer.box.com/reference/resources/items/
+  return getClient({ token }).get(`folders/${directory || rootFolderID}/items`, { searchParams: { fields: BOX_FILES_FIELDS, offset: query.cursor, limit: 1000 }, responseType: 'json' }).json()
 }
 
 /**

packages/@uppy/companion/src/server/provider/drive/adapter.js

@@ -1,18 +1,7 @@
 const querystring = require('node:querystring')
 
-// @todo use the "about" endpoint to get the username instead
-// see: https://developers.google.com/drive/api/v2/reference/about/get
 const getUsername = (data) => {
-  for (const item of data.files) {
-    if (item.ownedByMe && item.permissions) {
-      for (const permission of item.permissions) {
-        if (permission.role === 'owner') {
-          return permission.emailAddress
-        }
-      }
-    }
-  }
-  return undefined
+  return data.user.emailAddress
 }
 
 exports.isGsuiteFile = (mimeType) => {
@@ -151,7 +140,7 @@ const getVideoDurationMillis = (item) => item.videoMediaMetadata && item.videoMe
 // Hopefully this name will not be used by Google
 exports.VIRTUAL_SHARED_DIR = 'shared-with-me'
 
-exports.adaptData = (listFilesResp, sharedDrivesResp, directory, query, showSharedWithMe) => {
+exports.adaptData = (listFilesResp, sharedDrivesResp, directory, query, showSharedWithMe, about) => {
   const adaptItem = (item) => ({
     isFolder: isFolder(item),
     icon: getItemIcon(item),
@@ -194,7 +183,7 @@ exports.adaptData = (listFilesResp, sharedDrivesResp, directory, query, showShar
   ]
 
   return {
-    username: getUsername(listFilesResp),
+    username: getUsername(about),
     items: adaptedItems,
     nextPagePath: getNextPagePath(listFilesResp, query, directory),
   }

packages/@uppy/companion/src/server/provider/drive/index.js

@@ -6,7 +6,7 @@ const { VIRTUAL_SHARED_DIR, adaptData, isShortcut, isGsuiteFile, getGsuiteExport
 const { withProviderErrorHandling } = require('../providerErrors')
 const { prepareStream } = require('../../helpers/utils')
 
-const DRIVE_FILE_FIELDS = 'kind,id,imageMediaMetadata,name,mimeType,ownedByMe,permissions(role,emailAddress),size,modifiedTime,iconLink,thumbnailLink,teamDriveId,videoMediaMetadata,shortcutDetails(targetId,targetMimeType)'
+const DRIVE_FILE_FIELDS = 'kind,id,imageMediaMetadata,name,mimeType,ownedByMe,size,modifiedTime,iconLink,thumbnailLink,teamDriveId,videoMediaMetadata,shortcutDetails(targetId,targetMimeType)'
 const DRIVE_FILES_FIELDS = `kind,nextPageToken,incompleteSearch,files(${DRIVE_FILE_FIELDS})`
 // using wildcard to get all 'drive' fields because specifying fields seems no to work for the /drives endpoint
 const SHARED_DRIVE_FIELDS = '*'
@@ -86,18 +86,9 @@ class Drive extends Provider {
           fields: DRIVE_FILES_FIELDS,
           pageToken: query.cursor,
           q,
-          // pageSize: The maximum number of files to return per page.
-          // Partial or empty result pages are possible even before the end of the files list has been reached.
-          // Acceptable values are 1 to 1000, inclusive. (Default: 100)
-          //
-          // @TODO:
-          // SAD WARNING: doesn’t work if you have multiple `fields`, defaults to 100 anyway.
-          // Works if we remove `permissions(role,emailAddress)`, which we use to set the email address
-          // of logged in user in the Provider View header on the frontend.
-          // See https://stackoverflow.com/questions/42592125/list-request-page-size-being-ignored
-          //
-          // pageSize: 1000,
-          // pageSize: 10, // can be used for testing pagination if you don't have many files
+          // We can only do a page size of 1000 because we do not request permissions in DRIVE_FILES_FIELDS.
+          // Otherwise we are limited to 100. Instead we get the user info from `this.user()`
+          pageSize: 1000,
           orderBy: 'folder,name',
           includeItemsFromAllDrives: true,
           supportsAllDrives: true,
@@ -106,8 +97,13 @@ class Drive extends Provider {
         return client.get('files', { searchParams, responseType: 'json' }).json()
       }
 
-      const [sharedDrives, filesResponse] = await Promise.all([fetchSharedDrives(), fetchFiles()])
-      // console.log({ directory, sharedDrives, filesResponse })
+      async function fetchAbout () {
+        const searchParams = { fields: 'user' }
+
+        return client.get('about', { searchParams, responseType: 'json' }).json()
+      }
+
+      const [sharedDrives, filesResponse, about] = await Promise.all([fetchSharedDrives(), fetchFiles(), fetchAbout()])
 
       return adaptData(
         filesResponse,
@@ -115,6 +111,7 @@ class Drive extends Provider {
         directory,
         query,
         isRoot && !query.cursor, // we can only show it on the first page request, or else we will have duplicates of it
+        about,
       )
     })
   }

packages/@uppy/companion/src/server/provider/onedrive/index.js

@@ -40,7 +40,8 @@ class OneDrive extends Provider {
   async list ({ directory, query, token }) {
     return this.#withErrorHandling('provider.onedrive.list.error', async () => {
       const path = directory ? `items/${directory}` : 'root'
-      const qs = { $expand: 'thumbnails' }
+      // https://learn.microsoft.com/en-us/graph/query-parameters?tabs=http#top-parameter
+      const qs = { $expand: 'thumbnails', $top: 999 }
       if (query.cursor) {
         qs.$skiptoken = query.cursor
       }

packages/@uppy/companion/test/__tests__/providers.js

@@ -137,7 +137,7 @@ describe('list provider files', () => {
     nock('https://api.box.com').get('/2.0/users/me').reply(200, {
       login: defaults.USERNAME,
     })
-    nock('https://api.box.com').get('/2.0/folders/0/items?fields=id%2Cmodified_at%2Cname%2Cpermissions%2Csize%2Ctype').reply(200, {
+    nock('https://api.box.com').get('/2.0/folders/0/items?fields=id%2Cmodified_at%2Cname%2Cpermissions%2Csize%2Ctype&limit=1000').reply(200, {
       entries: [
         {
           type: 'file',
@@ -157,7 +157,7 @@ describe('list provider files', () => {
       kind: 'drive#driveList', drives: [],
     })
 
-    nock('https://www.googleapis.com').get('/drive/v3/files?fields=kind%2CnextPageToken%2CincompleteSearch%2Cfiles%28kind%2Cid%2CimageMediaMetadata%2Cname%2CmimeType%2CownedByMe%2Cpermissions%28role%2CemailAddress%29%2Csize%2CmodifiedTime%2CiconLink%2CthumbnailLink%2CteamDriveId%2CvideoMediaMetadata%2CshortcutDetails%28targetId%2CtargetMimeType%29%29&q=%28%27root%27+in+parents%29+and+trashed%3Dfalse&orderBy=folder%2Cname&includeItemsFromAllDrives=true&supportsAllDrives=true').reply(200, {
+    nock('https://www.googleapis.com').get('/drive/v3/files?fields=kind%2CnextPageToken%2CincompleteSearch%2Cfiles%28kind%2Cid%2CimageMediaMetadata%2Cname%2CmimeType%2CownedByMe%2Csize%2CmodifiedTime%2CiconLink%2CthumbnailLink%2CteamDriveId%2CvideoMediaMetadata%2CshortcutDetails%28targetId%2CtargetMimeType%29%29&q=%28%27root%27+in+parents%29+and+trashed%3Dfalse&pageSize=1000&orderBy=folder%2Cname&includeItemsFromAllDrives=true&supportsAllDrives=true').reply(200, {
       kind: 'drive#fileList',
       nextPageToken: defaults.NEXT_PAGE_TOKEN,
       files: [
@@ -176,6 +176,8 @@ describe('list provider files', () => {
       ],
     })
 
+    nock('https://www.googleapis.com').get((uri) => uri.includes('about')).reply(200, { user: { emailAddress: 'john.doe@transloadit.com' } })
+
     await runTest('drive')
   })
 
@@ -231,7 +233,7 @@ describe('list provider files', () => {
       userPrincipalName: defaults.USERNAME,
       mail: defaults.USERNAME,
     })
-    nock('https://graph.microsoft.com').get('/v1.0/me/drive/root/children?%24expand=thumbnails').reply(200, {
+    nock('https://graph.microsoft.com').get('/v1.0/me/drive/root/children?%24expand=thumbnails&%24top=999').reply(200, {
       value: [
         {
           createdDateTime: '2020-01-31T15:40:26.197Z',

packages/@uppy/companion/test/fixtures/drive.js

@@ -4,7 +4,7 @@ const defaults = require('./constants')
 module.exports.expects = {}
 
 module.exports.nockGoogleDownloadFile = ({ times = 1 } = {}) => {
-  nock('https://www.googleapis.com').get(`/drive/v3/files/${defaults.ITEM_ID}?fields=kind%2Cid%2CimageMediaMetadata%2Cname%2CmimeType%2CownedByMe%2Cpermissions%28role%2CemailAddress%29%2Csize%2CmodifiedTime%2CiconLink%2CthumbnailLink%2CteamDriveId%2CvideoMediaMetadata%2CshortcutDetails%28targetId%2CtargetMimeType%29&supportsAllDrives=true`).times(times).reply(200, {
+  nock('https://www.googleapis.com').get(`/drive/v3/files/${defaults.ITEM_ID}?fields=kind%2Cid%2CimageMediaMetadata%2Cname%2CmimeType%2CownedByMe%2Csize%2CmodifiedTime%2CiconLink%2CthumbnailLink%2CteamDriveId%2CvideoMediaMetadata%2CshortcutDetails%28targetId%2CtargetMimeType%29&supportsAllDrives=true`).times(times).reply(200, {
     kind: 'drive#file',
     id: defaults.ITEM_ID,
     name: 'MY DUMMY FILE NAME.mp4',
@@ -17,4 +17,5 @@ module.exports.nockGoogleDownloadFile = ({ times = 1 } = {}) => {
     size: '758051',
   })
   nock('https://www.googleapis.com').get(`/drive/v3/files/${defaults.ITEM_ID}?alt=media&supportsAllDrives=true`).reply(200, {})
+  nock('https://www.googleapis.com').get((uri) => uri.includes('about')).reply(200, { user: { emailAddress: 'john.doe@transloadit.com' } })
 }

packages/@uppy/core/src/locale.js

@@ -42,6 +42,7 @@ export default {
     filter: 'Filter',
     resetFilter: 'Reset filter',
     loading: 'Loading...',
+    loadedXFiles: 'Loaded %{numFiles} files',
     authenticateWithTitle:
       'Please authenticate with %{pluginName} to select files',
     authenticateWith: 'Connect to %{pluginName}',

packages/@uppy/dashboard/src/Dashboard.jsx

@@ -172,6 +172,8 @@ export default class Dashboard extends UIPlugin {
     }
 
     this.setPluginState(update)
+
+    this.uppy.emit('dashboard:close-panel', state.activePickerPanel.id)
   }
 
   showPanel = (id) => {

packages/@uppy/dropbox/src/Dropbox.jsx

@@ -41,6 +41,7 @@ export default class Dropbox extends UIPlugin {
   install () {
     this.view = new ProviderViews(this, {
       provider: this.provider,
+      loadAllFiles: true,
     })
 
     const { target } = this.opts

packages/@uppy/google-drive/src/GoogleDrive.jsx

@@ -55,6 +55,7 @@ export default class GoogleDrive extends UIPlugin {
   install () {
     this.view = new DriveProviderViews(this, {
       provider: this.provider,
+      loadAllFiles: true,
     })
 
     const { target } = this.opts

packages/@uppy/onedrive/src/OneDrive.jsx

@@ -46,6 +46,7 @@ export default class OneDrive extends UIPlugin {
   install () {
     this.view = new ProviderViews(this, {
       provider: this.provider,
+      loadAllFiles: true,
     })
 
     const { target } = this.opts

packages/@uppy/provider-views/src/Item/components/ItemIcon.jsx

@@ -38,7 +38,7 @@ export default (props) => {
       return <VideoIcon />
     default: {
       const { alt } = props
-      return <img src={itemIconString} alt={alt} />
+      return <img src={itemIconString} alt={alt} loading="lazy" />
     }
   }
 }

packages/@uppy/provider-views/src/ProviderView/ProviderView.jsx

@@ -50,6 +50,7 @@ export default class ProviderView extends View {
       showTitles: true,
       showFilter: true,
       showBreadcrumbs: true,
+      loadAllFiles: false,
     }
 
     // merge default options with the ones set by user
@@ -105,30 +106,59 @@ export default class ProviderView extends View {
    * @returns {Promise}   Folders/files in folder
    */
   async getFolder (id, name) {
+    const controller = new AbortController()
+    const cancelRequest = () => {
+      controller.abort()
+      this.clearSelection()
+    }
+    const getNewBreadcrumbsDirectories = () => {
+      const state = this.plugin.getPluginState()
+      const index = state.directories.findIndex((dir) => id === dir.id)
+
+      if (index !== -1) {
+        return state.directories.slice(0, index + 1)
+      }
+      return state.directories.concat([{ id, title: name }])
+    }
+
+    this.plugin.uppy.on('dashboard:close-panel', cancelRequest)
+    this.plugin.uppy.on('cancel-all', cancelRequest)
     this.setLoading(true)
+
     try {
-      const res = await this.provider.list(id)
       const folders = []
       const files = []
-      let updatedDirectories
+      this.nextPagePath = id
 
-      const state = this.plugin.getPluginState()
-      const index = state.directories.findIndex((dir) => id === dir.id)
+      do {
+        const res = await this.provider.list(this.nextPagePath, { signal: controller.signal })
 
-      if (index !== -1) {
-        updatedDirectories = state.directories.slice(0, index + 1)
-      } else {
-        updatedDirectories = state.directories.concat([{ id, title: name }])
-      }
+        for (const f of res.items) {
+          if (f.isFolder) folders.push(f)
+          else files.push(f)
+        }
 
-      this.username = res.username || this.username
-      this.#updateFilesAndFolders(res, files, folders)
-      this.plugin.setPluginState({ directories: updatedDirectories, filterInput: '' })
+        this.nextPagePath = res.nextPagePath
+        if (res.username) this.username = res.username
+        this.setLoading(this.plugin.uppy.i18n('loadedXFiles', { numFiles: files.length + folders.length }))
+      } while (
+        this.nextPagePath && this.opts.loadAllFiles
+      )
+
+      const directories = getNewBreadcrumbsDirectories(this.nextPagePath)
+
+      this.plugin.setPluginState({ files, folders, directories, filterInput: '' })
       this.lastCheckbox = undefined
     } catch (err) {
+      if (err.cause?.name === 'AbortError') {
+        // Expected, user clicked “cancel”
+        return
+      }
       this.handleError(err)
     } finally {
       this.setLoading(false)
+      this.plugin.uppy.off('dashboard:close-panel', cancelRequest)
+      this.plugin.uppy.off('cancel-all', cancelRequest)
     }
   }