webcam: Try to respect restrictions (#2090)

* webcam: request a video mime type listed in `allowedFileTypes`

* webcam: stop recording when we are about to reach the max file size

* webcam: request an image mime type listed in `allowedFileTypes`

* docs: add Webcam#preferredImageMimeType

* locale: add `recordingStoppedMaxSize` string

* changelog: check webcam restrictions

Co-authored-by: Artur Paikin <artur@arturpaikin.com>

CHANGELOG.md

@@ -98,11 +98,11 @@ PRs are welcome! Please do open an issue to discuss first if it's a big feature,
 ## 1.11
 
 - [ ] plugins: Transformations, cropping, filters for images, study https://github.com/MattKetmo/darkroomjs/, https://github.com/fengyuanchen/cropperjs #151 #53 (@arturi)
-- [ ] google-drive: Google Drive - Google Docs https://github.com/transloadit/uppy/issues/1554#issuecomment-554904049 (@ife)
+- [x] google-drive: Google Drive - Google Docs https://github.com/transloadit/uppy/issues/1554#issuecomment-554904049 (@ife)
 - [ ] core: add maxTotalFileSize restriction #514 (@arturi)
-- [ ] dashboard: Dark Mode & Redesign by Alex & Artur (@arturi)
+- [x] dashboard: Dark Mode & Redesign by Alex & Artur (@arturi)
+- [x] webcam: Pick format based on `restrictions.allowedFileTypes`, eg. use PNG for snapshot instead of JPG if `allowedFileTypes: ['.png']` is set, you can probably ask for the correct filetype. In addition, we should stop recording video once the max allowed file size is exceeded. should be possible given how the MediaRecorder API works (@goto-bus-stop)
 - [ ] companion: what happens if access token expires during/between an download & upload (@ife)
-- [ ] webcam: Pick format based on `restrictions.allowedFileTypes`, eg. use PNG for snapshot instead of JPG if `allowedFileTypes: ['.png']` is set, you can probably ask for the correct filetype. In addition, we should stop recording video once the max allowed file size is exceeded. should be possible given how the MediaRecorder API works (@goto-bus-stop)
 - [ ] plugins: review & merge screenshot+screencast support similar to Webcam #148 (@arturi)
 - [ ] core: report information about the device --^ (@arturi)
 - [ ] providers: Provider Browser don't handle uppy restrictions, can we hide things that don't match the restrictions in Google Drive and Instagram? #1827 (@arturi)

packages/@uppy/locales/src/en_US.js

@@ -78,6 +78,7 @@ en_US.strings = {
     '1': 'Processing %{smart_count} files'
   },
   recordingLength: 'Recording length %{recording_length}',
+  recordingStoppedMaxSize: 'Recording stopped because the file size is about to exceed the limit',
   removeFile: 'Remove file',
   resetFilter: 'Reset filter',
   resume: 'Resume',

packages/@uppy/locales/src/nl_NL.js

@@ -72,6 +72,7 @@ nl_NL.strings = {
     '1': 'Bezig met %{smart_count} bestanden te verwerken'
   },
   recordingLength: 'Opnameduur %{recording_length}',
+  recordingStoppedMaxSize: 'Opname gestopt omdat de bestandsgrootte de limiet bijna overschrijdt',
   removeFile: 'Bestand verwijderen',
   resetFilter: 'Filter resetten',
   resume: 'Hervatten',

packages/@uppy/utils/src/getFileTypeExtension.js

@@ -3,13 +3,21 @@
 // We could use a library but they tend to contain dozens of KBs of mappings,
 // most of which will go unused, so not sure if that's worth it.
 const mimeToExtensions = {
-  'video/ogg': 'ogv',
+  'audio/mp3': 'mp3',
   'audio/ogg': 'ogg',
-  'video/webm': 'webm',
   'audio/webm': 'webm',
-  'video/x-matroska': 'mkv',
+  'image/gif': 'gif',
+  'image/heic': 'heic',
+  'image/heif': 'heif',
+  'image/jpeg': 'jpg',
+  'image/png': 'png',
+  'image/svg+xml': 'svg',
   'video/mp4': 'mp4',
-  'audio/mp3': 'mp3'
+  'video/ogg': 'ogv',
+  'video/quicktime': 'mov',
+  'video/webm': 'webm',
+  'video/x-matroska': 'mkv',
+  'video/x-msvideo': 'avi'
 }
 
 module.exports = function getFileTypeExtension (mimeType) {

packages/@uppy/webcam/src/index.js

@@ -2,14 +2,50 @@ const { h } = require('preact')
 const { Plugin } = require('@uppy/core')
 const Translator = require('@uppy/utils/lib/Translator')
 const getFileTypeExtension = require('@uppy/utils/lib/getFileTypeExtension')
+const mimeTypes = require('@uppy/utils/lib/mimeTypes')
 const canvasToBlob = require('@uppy/utils/lib/canvasToBlob')
 const supportsMediaRecorder = require('./supportsMediaRecorder')
 const CameraIcon = require('./CameraIcon')
 const CameraScreen = require('./CameraScreen')
 const PermissionsScreen = require('./PermissionsScreen')
 
-// Setup getUserMedia, with polyfill for older browsers
-// Adapted from: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
+/**
+ * Normalize a MIME type or file extension into a MIME type.
+ *
+ * @param {string} fileType - MIME type or a file extension prefixed with `.`.
+ * @returns {string|undefined} The MIME type or `undefined` if the fileType is an extension and is not known.
+ */
+function toMimeType (fileType) {
+  if (fileType[0] === '.') {
+    return mimeTypes[fileType.slice(1)]
+  }
+  return fileType
+}
+
+/**
+ * Is this MIME type a video?
+ *
+ * @param {string} mimeType - MIME type.
+ * @returns {boolean}
+ */
+function isVideoMimeType (mimeType) {
+  return /^video\/[^*]+$/.test(mimeType)
+}
+
+/**
+ * Is this MIME type an image?
+ *
+ * @param {string} mimeType - MIME type.
+ * @returns {boolean}
+ */
+function isImageMimeType (mimeType) {
+  return /^image\/[^*]+$/.test(mimeType)
+}
+
+/**
+ * Setup getUserMedia, with polyfill for older browsers
+ * Adapted from: https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
+ */
 function getMediaDevices () {
   // eslint-disable-next-line compat/compat
   if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
@@ -62,6 +98,7 @@ module.exports = class Webcam extends Plugin {
         stopRecording: 'Stop video recording',
         allowAccessTitle: 'Please allow access to your camera',
         allowAccessDescription: 'In order to take pictures or record video with your camera, please allow camera access for this site.',
+        recordingStoppedMaxSize: 'Recording stopped because the file size is about to exceed the limit',
         recordingLength: 'Recording length %{recording_length}'
       }
     }
@@ -78,6 +115,7 @@ module.exports = class Webcam extends Plugin {
       ],
       mirror: true,
       facingMode: 'user',
+      preferredImageMimeType: null,
       preferredVideoMimeType: null,
       showRecordingLength: false
     }
@@ -92,18 +130,18 @@ module.exports = class Webcam extends Plugin {
     this.render = this.render.bind(this)
 
     // Camera controls
-    this.start = this.start.bind(this)
-    this.stop = this.stop.bind(this)
-    this.takeSnapshot = this.takeSnapshot.bind(this)
-    this.startRecording = this.startRecording.bind(this)
-    this.stopRecording = this.stopRecording.bind(this)
-    this.oneTwoThreeSmile = this.oneTwoThreeSmile.bind(this)
-    this.focus = this.focus.bind(this)
+    this._start = this._start.bind(this)
+    this._stop = this._stop.bind(this)
+    this._takeSnapshot = this._takeSnapshot.bind(this)
+    this._startRecording = this._startRecording.bind(this)
+    this._stopRecording = this._stopRecording.bind(this)
+    this._oneTwoThreeSmile = this._oneTwoThreeSmile.bind(this)
+    this._focus = this._focus.bind(this)
 
     this.webcamActive = false
 
     if (this.opts.countdown) {
-      this.opts.onBeforeSnapshot = this.oneTwoThreeSmile
+      this.opts.onBeforeSnapshot = this._oneTwoThreeSmile
     }
   }
 
@@ -136,7 +174,7 @@ module.exports = class Webcam extends Plugin {
     }
   }
 
-  start () {
+  _start () {
     if (!this.isSupported()) {
       return Promise.reject(new Error('Webcam access not supported'))
     }
@@ -161,26 +199,63 @@ module.exports = class Webcam extends Plugin {
       })
   }
 
-  startRecording () {
+  /**
+   * @returns {object}
+   */
+  _getMediaRecorderOptions () {
     const options = {}
 
-    // Safari don't have support for isTypeSupported api.
+    // Try to use the `opts.preferredVideoMimeType` or one of the `allowedFileTypes` for the recording.
+    // If the browser doesn't support it, we'll fall back to the browser default instead.
+    // Safari doesn't have the `isTypeSupported` API.
     if (MediaRecorder.isTypeSupported) {
-      const preferredVideoMimeType = this.opts.preferredVideoMimeType
+      const { restrictions } = this.uppy.opts
+      let preferredVideoMimeTypes = []
+      if (this.opts.preferredVideoMimeType) {
+        preferredVideoMimeTypes = [this.opts.preferredVideoMimeType]
+      } else if (restrictions.allowedFileTypes) {
+        preferredVideoMimeTypes = restrictions.allowedFileTypes.map(toMimeType).filter(isVideoMimeType)
+      }
 
-      // Attempt to use the passed preferredVideoMimeType (if any) during recording.
-      // If the browser doesn't support it, we'll fall back to the browser default instead
-      if (preferredVideoMimeType && MediaRecorder.isTypeSupported(preferredVideoMimeType) && getFileTypeExtension(preferredVideoMimeType)) {
-        options.mimeType = preferredVideoMimeType
+      const acceptableMimeTypes = preferredVideoMimeTypes.filter((candidateType) =>
+        MediaRecorder.isTypeSupported(candidateType) &&
+          getFileTypeExtension(candidateType))
+      if (acceptableMimeTypes.length > 0) {
+        options.mimeType = acceptableMimeTypes[0]
       }
     }
 
-    this.recorder = new MediaRecorder(this.stream, options)
+    return options
+  }
+
+  _startRecording () {
+    this.recorder = new MediaRecorder(this.stream, this._getMediaRecorderOptions())
     this.recordingChunks = []
+    let stoppingBecauseOfMaxSize = false
     this.recorder.addEventListener('dataavailable', (event) => {
       this.recordingChunks.push(event.data)
+
+      const { restrictions } = this.uppy.opts
+      if (this.recordingChunks.length > 1 &&
+          restrictions.maxFileSize != null &&
+          !stoppingBecauseOfMaxSize) {
+        const totalSize = this.recordingChunks.reduce((acc, chunk) => acc + chunk.size, 0)
+        // Exclude the initial chunk from the average size calculation because it is likely to be a very small outlier
+        const averageChunkSize = (totalSize - this.recordingChunks[0].size) / (this.recordingChunks.length - 1)
+        const expectedEndChunkSize = averageChunkSize * 3
+        const maxSize = Math.max(0, restrictions.maxFileSize - expectedEndChunkSize)
+
+        if (totalSize > maxSize) {
+          stoppingBecauseOfMaxSize = true
+          this.uppy.info(this.i18n('recordingStoppedMaxSize'), 'warning', 4000)
+          this._stopRecording()
+        }
+      }
     })
-    this.recorder.start()
+
+    // use a "time slice" of 500ms: ondataavailable will be called each 500ms
+    // smaller time slices mean we can more accurately check the max file size restriction
+    this.recorder.start(500)
 
     if (this.opts.showRecordingLength) {
       // Start the recordingLengthTimer if we are showing the recording length.
@@ -195,7 +270,7 @@ module.exports = class Webcam extends Plugin {
     })
   }
 
-  stopRecording () {
+  _stopRecording () {
     const stopped = new Promise((resolve, reject) => {
       this.recorder.addEventListener('stop', () => {
         resolve()
@@ -226,12 +301,6 @@ module.exports = class Webcam extends Plugin {
     }).then(() => {
       this.recordingChunks = null
       this.recorder = null
-
-      // Close the Dashboard panel if plugin is installed
-      // into Dashboard (could be other parent UI plugin)
-      // if (this.parent && this.parent.hideAllPanels) {
-      //   this.parent.hideAllPanels()
-      // }
     }, (error) => {
       this.recordingChunks = null
       this.recorder = null
@@ -239,7 +308,7 @@ module.exports = class Webcam extends Plugin {
     })
   }
 
-  stop () {
+  _stop () {
     this.stream.getAudioTracks().forEach((track) => {
       track.stop()
     })
@@ -250,11 +319,11 @@ module.exports = class Webcam extends Plugin {
     this.stream = null
   }
 
-  getVideoElement () {
+  _getVideoElement () {
     return this.el.querySelector('.uppy-Webcam-video')
   }
 
-  oneTwoThreeSmile () {
+  _oneTwoThreeSmile () {
     return new Promise((resolve, reject) => {
       let count = this.opts.countdown
 
@@ -277,7 +346,7 @@ module.exports = class Webcam extends Plugin {
     })
   }
 
-  takeSnapshot () {
+  _takeSnapshot () {
     if (this.captureInProgress) return
     this.captureInProgress = true
 
@@ -286,18 +355,13 @@ module.exports = class Webcam extends Plugin {
       this.uppy.info(message, 'error', 5000)
       return Promise.reject(new Error(`onBeforeSnapshot: ${message}`))
     }).then(() => {
-      return this.getImage()
+      return this._getImage()
     }).then((tagFile) => {
       this.captureInProgress = false
-      // Close the Dashboard panel if plugin is installed
-      // into Dashboard (could be other parent UI plugin)
-      // if (this.parent && this.parent.hideAllPanels) {
-      //   this.parent.hideAllPanels()
-      // }
       try {
         this.uppy.addFile(tagFile)
       } catch (err) {
-        // Logging the error, exept restrictions, which is handled in Core
+        // Logging the error, except restrictions, which is handled in Core
         if (!err.isRestriction) {
           this.uppy.log(err)
         }
@@ -308,32 +372,32 @@ module.exports = class Webcam extends Plugin {
     })
   }
 
-  getImage () {
-    const video = this.getVideoElement()
+  _getImage () {
+    const video = this._getVideoElement()
     if (!video) {
       return Promise.reject(new Error('No video element found, likely due to the Webcam tab being closed.'))
     }
 
-    const name = `cam-${Date.now()}.jpg`
-    const mimeType = 'image/jpeg'
-
     const width = video.videoWidth
     const height = video.videoHeight
 
-    // const scaleH = this.opts.mirror ? -1 : 1 // Set horizontal scale to -1 if flip horizontal
-    // const scaleV = 1
-    // const posX = this.opts.mirror ? width * -1 : 0 // Set x position to -100% if flip horizontal
-    // const posY = 0
-
     const canvas = document.createElement('canvas')
     canvas.width = width
     canvas.height = height
     const ctx = canvas.getContext('2d')
     ctx.drawImage(video, 0, 0)
-    // ctx.save() // Save the current state
-    // ctx.scale(scaleH, scaleV) // Set scale to flip the image
-    // ctx.drawImage(video, posX, posY, width, height) // draw the image
-    // ctx.restore() // Restore the last saved state
+
+    const { restrictions } = this.uppy.opts
+    let preferredImageMimeTypes = []
+    if (this.opts.preferredImageMimeType) {
+      preferredImageMimeTypes = [this.opts.preferredImageMimeType]
+    } else if (restrictions.allowedFileTypes) {
+      preferredImageMimeTypes = restrictions.allowedFileTypes.map(toMimeType).filter(isImageMimeType)
+    }
+
+    const mimeType = preferredImageMimeTypes[0] || 'image/jpeg'
+    const ext = getFileTypeExtension(mimeType) || 'jpg'
+    const name = `cam-${Date.now()}.${ext}`
 
     return canvasToBlob(canvas, mimeType).then((blob) => {
       return {
@@ -365,7 +429,7 @@ module.exports = class Webcam extends Plugin {
     return Promise.resolve(file)
   }
 
-  focus () {
+  _focus () {
     if (!this.opts.countdown) return
     setTimeout(() => {
       this.uppy.info(this.i18n('smile'), 'success', 1500)
@@ -374,7 +438,7 @@ module.exports = class Webcam extends Plugin {
 
   render (state) {
     if (!this.webcamActive) {
-      this.start()
+      this._start()
     }
 
     const webcamState = this.getPluginState()
@@ -388,11 +452,11 @@ module.exports = class Webcam extends Plugin {
     return (
       <CameraScreen
         {...webcamState}
-        onSnapshot={this.takeSnapshot}
-        onStartRecording={this.startRecording}
-        onStopRecording={this.stopRecording}
-        onFocus={this.focus}
-        onStop={this.stop}
+        onSnapshot={this._takeSnapshot}
+        onStartRecording={this._startRecording}
+        onStopRecording={this._stopRecording}
+        onFocus={this._focus}
+        onStop={this._stop}
         i18n={this.i18n}
         modes={this.opts.modes}
         showRecordingLength={this.opts.showRecordingLength}
@@ -418,7 +482,7 @@ module.exports = class Webcam extends Plugin {
 
   uninstall () {
     if (this.stream) {
-      this.stop()
+      this._stop()
     }
 
     this.unmount()

packages/@uppy/webcam/src/index.test.js

@@ -0,0 +1,92 @@
+const Uppy = require('@uppy/core')
+const Webcam = require('./index')
+
+describe('Webcam', () => {
+  describe('_getMediaRecorderOptions', () => {
+    it('should not have a mimeType set if no preferences given', () => {
+      global.MediaRecorder = {
+        isTypeSupported: () => true
+      }
+
+      const uppy = Uppy().use(Webcam)
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).not.toBeDefined()
+    })
+
+    it('should use preferredVideoMimeType', () => {
+      global.MediaRecorder = {
+        isTypeSupported: (ty) => ty === 'video/webm'
+      }
+
+      const uppy = Uppy().use(Webcam, { preferredVideoMimeType: 'video/webm' })
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).toEqual('video/webm')
+    })
+
+    it('should not use preferredVideoMimeType if it is not supported', () => {
+      global.MediaRecorder = {
+        isTypeSupported: (ty) => ty === 'video/webm'
+      }
+
+      const uppy = Uppy().use(Webcam, { preferredVideoMimeType: 'video/mp4' })
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).not.toBeDefined()
+    })
+
+    it('should pick type based on `allowedFileTypes`', () => {
+      global.MediaRecorder = {
+        isTypeSupported: () => true
+      }
+
+      const uppy = Uppy({
+        restrictions: { allowedFileTypes: ['video/mp4', 'video/webm'] }
+      }).use(Webcam)
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).toEqual('video/mp4')
+    })
+
+    it('should use first supported type from allowedFileTypes', () => {
+      global.MediaRecorder = {
+        isTypeSupported: (ty) => ty === 'video/webm'
+      }
+
+      const uppy = Uppy({
+        restrictions: { allowedFileTypes: ['video/mp4', 'video/webm'] }
+      }).use(Webcam)
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).toEqual('video/webm')
+    })
+
+    it('should prefer preferredVideoMimeType over allowedFileTypes', () => {
+      global.MediaRecorder = {
+        isTypeSupported: () => true
+      }
+
+      const uppy = Uppy({
+        restrictions: { allowedFileTypes: ['video/mp4', 'video/webm'] }
+      })
+        .use(Webcam, { preferredVideoMimeType: 'video/webm' })
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).toEqual('video/webm')
+    })
+
+    it('should not use allowedFileTypes if they are unsupported', () => {
+      global.MediaRecorder = {
+        isTypeSupported: () => false
+      }
+
+      const uppy = Uppy({
+        restrictions: { allowedFileTypes: ['video/mp4', 'video/webm'] }
+      }).use(Webcam)
+      expect(
+        uppy.getPlugin('Webcam')._getMediaRecorderOptions().mimeType
+      ).toEqual(undefined)
+    })
+  })
+})

website/src/docs/webcam.md

@@ -10,7 +10,7 @@ tagline: "upload selfies or audio / video recordings"
 
 The `@uppy/webcam` plugin lets you take photos and record videos with a built-in camera on desktop and mobile devices.
 
-> To use the Webcam plugin in Chrome, [your site should be served over https](https://developers.google.com/web/updates/2015/10/chrome-47-webrtc#public_service_announcements). This restriction does not apply on `localhost`, so you don't have to jump through many hoops during development.
+> To use the Webcam plugin in Chrome, [your site must be served over https](https://developers.google.com/web/updates/2015/10/chrome-47-webrtc#public_service_announcements). This restriction does not apply on `localhost`, so you don't have to jump through many hoops during development.
 
 ```js
 const Webcam = require('@uppy/webcam')
@@ -66,6 +66,8 @@ uppy.use(Webcam, {
   mirror: true,
   facingMode: 'user',
   showRecordingLength: false,
+  preferredVideoMimeType: null,
+  preferredImageMimeType: null,
   locale: {}
 })
 ```
@@ -122,6 +124,14 @@ Configures whether or not to show the length of the recording while the recordin
 
 Set the preferred mime type for video recordings, for example `'video/webm'`. If the browser supports the given mime type, the video will be recorded in this format. If the browser does not support it, it will use the browser default.
 
+If no preferred video mime type is given, the Webcam plugin will prefer types listed in the [`allowedFileTypes` restriction](/docs/uppy/#restrictions), if any.
+
+### `preferredImageMimeType: null`
+
+Set the preferred mime type for images, for example `'image/png'`. If the browser supports rendering the given mime type, the image will be stored in this format. Else `image/jpeg` is used by default.
+
+If no preferred image mime type is given, the Webcam plugin will prefer types listed in the [`allowedFileTypes` restriction](/docs/uppy/#restrictions), if any.
+
 ### `locale: {}`
 
 Localize text that is shown to the user.