Companion refactor (#3542)

* modernise code

* fix lint

* pull out config related functions and middleware

* fix lint in uploader

packages/@uppy/companion/src/companion.js

@@ -1,19 +1,15 @@
-const fs = require('fs')
 const express = require('express')
-const ms = require('ms')
 // @ts-ignore
 const Grant = require('grant').express()
 const merge = require('lodash.merge')
 const cookieParser = require('cookie-parser')
 const interceptor = require('express-interceptor')
-const { isURL } = require('validator')
 const uuid = require('uuid')
 
 const grantConfig = require('./config/grant')()
 const providerManager = require('./server/provider')
 const controllers = require('./server/controllers')
 const s3 = require('./server/controllers/s3')
-const getS3Client = require('./server/s3-client')
 const url = require('./server/controllers/url')
 const emitter = require('./server/emitter')
 const redis = require('./server/redis')
@@ -21,31 +17,37 @@ const { getURLBuilder } = require('./server/helpers/utils')
 const jobs = require('./server/jobs')
 const logger = require('./server/logger')
 const middlewares = require('./server/middlewares')
+const { getMaskableSecrets, defaultOptions, validateConfig } = require('./config/companion')
 const { ProviderApiError, ProviderAuthError } = require('./server/provider/error')
 const { getCredentialsOverrideMiddleware } = require('./server/provider/credentials')
 // @ts-ignore
 const { version } = require('../package.json')
 
-const defaultOptions = {
-  server: {
-    protocol: 'http',
-    path: '',
-  },
-  providerOptions: {
-    s3: {
-      acl: 'public-read',
-      endpoint: 'https://{service}.{region}.amazonaws.com',
-      conditions: [],
-      useAccelerateEndpoint: false,
-      getKey: (req, filename) => filename,
-      expires: ms('5 minutes') / 1000,
+// intercepts grantJS' default response error when something goes
+// wrong during oauth process.
+const interceptGrantErrorResponse = interceptor((req, res) => {
+  return {
+    isInterceptable: () => {
+      // match grant.js' callback url
+      return /^\/connect\/\w+\/callback/.test(req.path)
     },
-  },
-  allowLocalUrls: false,
-  logClientVersion: true,
-  periodicPingUrls: [],
-  streamingUpload: false,
-}
+    intercept: (body, send) => {
+      const unwantedBody = 'error=Grant%3A%20missing%20session%20or%20misconfigured%20provider'
+      if (body === unwantedBody) {
+        logger.error(`grant.js responded with error: ${body}`, 'grant.oauth.error', req.id)
+        res.set('Content-Type', 'text/plain')
+        const reqHint = req.id ? `Request ID: ${req.id}` : ''
+        send([
+          'Companion was unable to complete the OAuth process :(',
+          'Error: User session is missing or the Provider was misconfigured',
+          reqHint,
+        ].join('\n'))
+      } else {
+        send(body)
+      }
+    },
+  }
+})
 
 // make the errors available publicly for custom providers
 module.exports.errors = { ProviderApiError, ProviderAuthError }
@@ -54,13 +56,13 @@ module.exports.socket = require('./server/socket')
 /**
  * Entry point into initializing the Companion app.
  *
- * @param {object} options
+ * @param {object} optionsArg
  * @returns {import('express').Express}
  */
-module.exports.app = (options = {}) => {
-  validateConfig(options)
+module.exports.app = (optionsArg = {}) => {
+  validateConfig(optionsArg)
 
-  options = merge({}, defaultOptions, options)
+  const options = merge({}, defaultOptions, optionsArg)
   const providers = providerManager.getDefaultProviders()
   const searchProviders = providerManager.getSearchProviders()
   providerManager.addProviderOptions(options, grantConfig)
@@ -71,7 +73,7 @@ module.exports.app = (options = {}) => {
   }
 
   // mask provider secrets from log messages
-  maskLogger(options)
+  logger.setMaskables(getMaskableSecrets(options))
 
   // create singleton redis client
   if (options.redisUrl) {
@@ -114,7 +116,7 @@ module.exports.app = (options = {}) => {
   app.use(middlewares.cors(options))
 
   // add uppy options to the request object so it can be accessed by subsequent handlers.
-  app.use('*', getOptionsMiddleware(options))
+  app.use('*', middlewares.getCompanionMiddleware(options))
   app.use('/s3', s3(options.providerOptions.s3))
   app.use('/url', url())
 
@@ -152,141 +154,3 @@ module.exports.app = (options = {}) => {
 
   return app
 }
-
-// intercepts grantJS' default response error when something goes
-// wrong during oauth process.
-const interceptGrantErrorResponse = interceptor((req, res) => {
-  return {
-    isInterceptable: () => {
-      // match grant.js' callback url
-      return /^\/connect\/\w+\/callback/.test(req.path)
-    },
-    intercept: (body, send) => {
-      const unwantedBody = 'error=Grant%3A%20missing%20session%20or%20misconfigured%20provider'
-      if (body === unwantedBody) {
-        logger.error(`grant.js responded with error: ${body}`, 'grant.oauth.error', req.id)
-        res.set('Content-Type', 'text/plain')
-        const reqHint = req.id ? `Request ID: ${req.id}` : ''
-        send([
-          'Companion was unable to complete the OAuth process :(',
-          'Error: User session is missing or the Provider was misconfigured',
-          reqHint,
-        ].join('\n'))
-      } else {
-        send(body)
-      }
-    },
-  }
-})
-
-/**
- *
- * @param {object} options
- */
-const getOptionsMiddleware = (options) => {
-  /**
-   * @param {object} req
-   * @param {object} res
-   * @param {Function} next
-   */
-  const middleware = (req, res, next) => {
-    const versionFromQuery = req.query.uppyVersions ? decodeURIComponent(req.query.uppyVersions) : null
-    req.companion = {
-      options,
-      s3Client: getS3Client(options),
-      authToken: req.header('uppy-auth-token') || req.query.uppyAuthToken,
-      clientVersion: req.header('uppy-versions') || versionFromQuery || '1.0.0',
-      buildURL: getURLBuilder(options),
-    }
-
-    if (options.logClientVersion) {
-      logger.info(`uppy client version ${req.companion.clientVersion}`, 'companion.client.version')
-    }
-    next()
-  }
-
-  return middleware
-}
-
-/**
- * Informs the logger about all provider secrets that should be masked
- * if they are found in a log message
- *
- * @param {object} companionOptions
- */
-const maskLogger = (companionOptions) => {
-  const secrets = []
-  const { providerOptions, customProviders } = companionOptions
-  Object.keys(providerOptions).forEach((provider) => {
-    if (providerOptions[provider].secret) {
-      secrets.push(providerOptions[provider].secret)
-    }
-  })
-
-  if (customProviders) {
-    Object.keys(customProviders).forEach((provider) => {
-      if (customProviders[provider].config && customProviders[provider].config.secret) {
-        secrets.push(customProviders[provider].config.secret)
-      }
-    })
-  }
-
-  logger.setMaskables(secrets)
-}
-
-/**
- * validates that the mandatory companion options are set.
- * If it is invalid, it will console an error of unset options and exits the process.
- * If it is valid, nothing happens.
- *
- * @param {object} companionOptions
- */
-const validateConfig = (companionOptions) => {
-  const mandatoryOptions = ['secret', 'filePath', 'server.host']
-  /** @type {string[]} */
-  const unspecified = []
-
-  mandatoryOptions.forEach((i) => {
-    const value = i.split('.').reduce((prev, curr) => (prev ? prev[curr] : undefined), companionOptions)
-
-    if (!value) unspecified.push(`"${i}"`)
-  })
-
-  // vaidate that all required config is specified
-  if (unspecified.length) {
-    const messagePrefix = 'Please specify the following options to use companion:'
-    throw new Error(`${messagePrefix}\n${unspecified.join(',\n')}`)
-  }
-
-  // validate that specified filePath is writeable/readable.
-  try {
-    // @ts-ignore
-    fs.accessSync(`${companionOptions.filePath}`, fs.R_OK | fs.W_OK) // eslint-disable-line no-bitwise
-  } catch (err) {
-    throw new Error(
-      `No access to "${companionOptions.filePath}". Please ensure the directory exists and with read/write permissions.`,
-    )
-  }
-
-  const { providerOptions, periodicPingUrls } = companionOptions
-
-  if (providerOptions) {
-    const deprecatedOptions = { microsoft: 'onedrive', google: 'drive' }
-    Object.keys(deprecatedOptions).forEach((deprected) => {
-      if (providerOptions[deprected]) {
-        throw new Error(`The Provider option "${deprected}" is no longer supported. Please use the option "${deprecatedOptions[deprected]}" instead.`)
-      }
-    })
-  }
-
-  if (companionOptions.uploadUrls == null || companionOptions.uploadUrls.length === 0) {
-    logger.warn('Running without uploadUrls specified is a security risk if running in production', 'startup.uploadUrls')
-  }
-
-  if (periodicPingUrls != null && (
-    !Array.isArray(periodicPingUrls)
-    || periodicPingUrls.some((url2) => !isURL(url2, { protocols: ['http', 'https'], require_protocol: true, require_tld: false }))
-  )) {
-    throw new TypeError('Invalid periodicPingUrls')
-  }
-}

packages/@uppy/companion/src/config/companion.js

@@ -0,0 +1,111 @@
+const ms = require('ms')
+const fs = require('fs')
+const { isURL } = require('validator')
+const logger = require('../server/logger')
+
+const defaultOptions = {
+  server: {
+    protocol: 'http',
+    path: '',
+  },
+  providerOptions: {
+    s3: {
+      acl: 'public-read',
+      endpoint: 'https://{service}.{region}.amazonaws.com',
+      conditions: [],
+      useAccelerateEndpoint: false,
+      getKey: (req, filename) => filename,
+      expires: ms('5 minutes') / 1000,
+    },
+  },
+  allowLocalUrls: false,
+  logClientVersion: true,
+  periodicPingUrls: [],
+  streamingUpload: false,
+}
+
+/**
+ * @param {object} companionOptions
+ */
+function getMaskableSecrets (companionOptions) {
+  const secrets = []
+  const { providerOptions, customProviders } = companionOptions
+  Object.keys(providerOptions).forEach((provider) => {
+    if (providerOptions[provider].secret) {
+      secrets.push(providerOptions[provider].secret)
+    }
+  })
+
+  if (customProviders) {
+    Object.keys(customProviders).forEach((provider) => {
+      if (customProviders[provider].config && customProviders[provider].config.secret) {
+        secrets.push(customProviders[provider].config.secret)
+      }
+    })
+  }
+
+  return secrets
+}
+
+/**
+ * validates that the mandatory companion options are set.
+ * If it is invalid, it will console an error of unset options and exits the process.
+ * If it is valid, nothing happens.
+ *
+ * @param {object} companionOptions
+ */
+const validateConfig = (companionOptions) => {
+  const mandatoryOptions = ['secret', 'filePath', 'server.host']
+  /** @type {string[]} */
+  const unspecified = []
+
+  mandatoryOptions.forEach((i) => {
+    const value = i.split('.').reduce((prev, curr) => (prev ? prev[curr] : undefined), companionOptions)
+
+    if (!value) unspecified.push(`"${i}"`)
+  })
+
+  // vaidate that all required config is specified
+  if (unspecified.length) {
+    const messagePrefix = 'Please specify the following options to use companion:'
+    throw new Error(`${messagePrefix}\n${unspecified.join(',\n')}`)
+  }
+
+  // validate that specified filePath is writeable/readable.
+  try {
+    // @ts-ignore
+    fs.accessSync(`${companionOptions.filePath}`, fs.R_OK | fs.W_OK) // eslint-disable-line no-bitwise
+  } catch (err) {
+    throw new Error(
+      `No access to "${companionOptions.filePath}". Please ensure the directory exists and with read/write permissions.`,
+    )
+  }
+
+  const { providerOptions, periodicPingUrls } = companionOptions
+
+  if (providerOptions) {
+    const deprecatedOptions = { microsoft: 'onedrive', google: 'drive' }
+    Object.keys(deprecatedOptions).forEach((deprected) => {
+      if (providerOptions[deprected]) {
+        throw new Error(`The Provider option "${deprected}" is no longer supported. Please use the option "${deprecatedOptions[deprected]}" instead.`)
+      }
+    })
+  }
+
+  if (companionOptions.uploadUrls == null || companionOptions.uploadUrls.length === 0) {
+    logger.warn('Running without uploadUrls specified is a security risk if running in production', 'startup.uploadUrls')
+  }
+
+  if (periodicPingUrls != null && (
+    !Array.isArray(periodicPingUrls)
+    || periodicPingUrls.some((url2) => !isURL(url2, { protocols: ['http', 'https'], require_protocol: true, require_tld: false }))
+  )) {
+    throw new TypeError('Invalid periodicPingUrls')
+  }
+}
+
+module.exports = {
+  defaultOptions,
+  getMaskableSecrets,
+  validateConfig,
+}

packages/@uppy/companion/src/server/Uploader.js

@@ -55,6 +55,77 @@ class AbortError extends Error {}
 
 class ValidationError extends Error {}
 
+/**
+ * Validate the options passed down to the uplaoder
+ *
+ * @param {UploaderOptions} options
+ */
+function validateOptions (options) {
+  // validate HTTP Method
+  if (options.httpMethod) {
+    if (typeof options.httpMethod !== 'string') {
+      throw new ValidationError('unsupported HTTP METHOD specified')
+    }
+
+    const method = options.httpMethod.toLowerCase()
+    if (method !== 'put' && method !== 'post') {
+      throw new ValidationError('unsupported HTTP METHOD specified')
+    }
+  }
+
+  if (exceedsMaxFileSize(options.companionOptions.maxFileSize, options.size)) {
+    throw new ValidationError('maxFileSize exceeded')
+  }
+
+  // validate fieldname
+  if (options.fieldname && typeof options.fieldname !== 'string') {
+    throw new ValidationError('fieldname must be a string')
+  }
+
+  // validate metadata
+  if (options.metadata != null) {
+    if (!isObject(options.metadata)) throw new ValidationError('metadata must be an object')
+  }
+
+  // validate headers
+  if (options.headers && !isObject(options.headers)) {
+    throw new ValidationError('headers must be an object')
+  }
+
+  // validate protocol
+  // @todo this validation should not be conditional once the protocol field is mandatory
+  if (options.protocol && !Object.keys(PROTOCOLS).some((key) => PROTOCOLS[key] === options.protocol)) {
+    throw new ValidationError('unsupported protocol specified')
+  }
+
+  // s3 uploads don't require upload destination
+  // validation, because the destination is determined
+  // by the server's s3 config
+  if (options.protocol !== PROTOCOLS.s3Multipart) {
+    if (!options.endpoint && !options.uploadUrl) {
+      throw new ValidationError('no destination specified')
+    }
+
+    const validateUrl = (url) => {
+      const validatorOpts = { require_protocol: true, require_tld: false }
+      if (url && !validator.isURL(url, validatorOpts)) {
+        throw new ValidationError('invalid destination url')
+      }
+
+      const allowedUrls = options.companionOptions.uploadUrls
+      if (allowedUrls && url && !hasMatch(url, allowedUrls)) {
+        throw new ValidationError('upload destination does not match any allowed destinations')
+      }
+    }
+
+    [options.endpoint, options.uploadUrl].forEach(validateUrl)
+  }
+
+  if (options.chunkSize != null && typeof options.chunkSize !== 'number') {
+    throw new ValidationError('incorrect chunkSize')
+  }
+}
+
 class Uploader {
   /**
    * Uploads file to destination based on the supplied protocol (tus, s3-multipart, multipart)
@@ -80,7 +151,7 @@ class Uploader {
    * @param {UploaderOptions} options
    */
   constructor (options) {
-    this.validateOptions(options)
+    validateOptions(options)
 
     this.options = options
     this.token = uuid.v4()
@@ -278,77 +349,6 @@ class Uploader {
     }
   }
 
-  /**
-   * Validate the options passed down to the uplaoder
-   *
-   * @param {UploaderOptions} options
-   */
-  validateOptions (options) {
-    // validate HTTP Method
-    if (options.httpMethod) {
-      if (typeof options.httpMethod !== 'string') {
-        throw new ValidationError('unsupported HTTP METHOD specified')
-      }
-
-      const method = options.httpMethod.toLowerCase()
-      if (method !== 'put' && method !== 'post') {
-        throw new ValidationError('unsupported HTTP METHOD specified')
-      }
-    }
-
-    if (exceedsMaxFileSize(options.companionOptions.maxFileSize, options.size)) {
-      throw new ValidationError('maxFileSize exceeded')
-    }
-
-    // validate fieldname
-    if (options.fieldname && typeof options.fieldname !== 'string') {
-      throw new ValidationError('fieldname must be a string')
-    }
-
-    // validate metadata
-    if (options.metadata != null) {
-      if (!isObject(options.metadata)) throw new ValidationError('metadata must be an object')
-    }
-
-    // validate headers
-    if (options.headers && !isObject(options.headers)) {
-      throw new ValidationError('headers must be an object')
-    }
-
-    // validate protocol
-    // @todo this validation should not be conditional once the protocol field is mandatory
-    if (options.protocol && !Object.keys(PROTOCOLS).some((key) => PROTOCOLS[key] === options.protocol)) {
-      throw new ValidationError('unsupported protocol specified')
-    }
-
-    // s3 uploads don't require upload destination
-    // validation, because the destination is determined
-    // by the server's s3 config
-    if (options.protocol !== PROTOCOLS.s3Multipart) {
-      if (!options.endpoint && !options.uploadUrl) {
-        throw new ValidationError('no destination specified')
-      }
-
-      const validateUrl = (url) => {
-        const validatorOpts = { require_protocol: true, require_tld: false }
-        if (url && !validator.isURL(url, validatorOpts)) {
-          throw new ValidationError('invalid destination url')
-        }
-
-        const allowedUrls = options.companionOptions.uploadUrls
-        if (allowedUrls && url && !hasMatch(url, allowedUrls)) {
-          throw new ValidationError('upload destination does not match any allowed destinations')
-        }
-      }
-
-      [options.endpoint, options.uploadUrl].forEach(validateUrl)
-    }
-
-    if (options.chunkSize != null && typeof options.chunkSize !== 'number') {
-      throw new ValidationError('incorrect chunkSize')
-    }
-  }
-
   /**
    * returns a substring of the token. Used as traceId for logging
    * we avoid using the entire token because this is meant to be a short term
@@ -427,10 +427,10 @@ class Uploader {
    * @param {string} url
    * @param {object} extraData
    */
-  emitSuccess (url, extraData = {}) {
+  emitSuccess (url, extraData) {
     const emitData = {
       action: 'success',
-      payload: Object.assign(extraData, { complete: true, url }),
+      payload: { ...extraData, complete: true, url },
     }
     this.saveState(emitData)
     emitter().emit(this.token, emitData)
@@ -441,13 +441,13 @@ class Uploader {
    * @param {Error} err
    */
   emitError (err) {
-    const serializedErr = serializeError(err)
     // delete stack to avoid sending server info to client
-    delete serializedErr.stack
+    // todo remove also extraData from serializedErr in next major
+    const { stack, ...serializedErr } = serializeError(err)
     const dataToEmit = {
       action: 'error',
       // @ts-ignore
-      payload: Object.assign(err.extraData || {}, { error: serializedErr }),
+      payload: { ...err.extraData, error: serializedErr },
     }
     this.saveState(dataToEmit)
     emitter().emit(this.token, dataToEmit)

packages/@uppy/companion/src/server/middlewares.js

@@ -6,6 +6,8 @@ const promBundle = require('express-prom-bundle')
 const { version } = require('../../package.json')
 const tokenService = require('./helpers/jwt')
 const logger = require('./logger')
+const getS3Client = require('./s3-client')
+const { getURLBuilder } = require('./helpers/utils')
 
 exports.hasSessionAndProvider = (req, res, next) => {
   if (!req.session || !req.body) {
@@ -144,3 +146,32 @@ exports.metrics = ({ path = undefined } = {}) => {
   versionGauge.set(numberVersion)
   return metricsMiddleware
 }
+
+/**
+ *
+ * @param {object} options
+ */
+exports.getCompanionMiddleware = (options) => {
+  /**
+   * @param {object} req
+   * @param {object} res
+   * @param {Function} next
+   */
+  const middleware = (req, res, next) => {
+    const versionFromQuery = req.query.uppyVersions ? decodeURIComponent(req.query.uppyVersions) : null
+    req.companion = {
+      options,
+      s3Client: getS3Client(options),
+      authToken: req.header('uppy-auth-token') || req.query.uppyAuthToken,
+      clientVersion: req.header('uppy-versions') || versionFromQuery || '1.0.0',
+      buildURL: getURLBuilder(options),
+    }
+
+    if (options.logClientVersion) {
+      logger.info(`uppy client version ${req.companion.clientVersion}`, 'companion.client.version')
+    }
+    next()
+  }
+
+  return middleware
+}