Default to HEAD requests when the Companion looks to get meta information about a URL (#3417)

* Update the url companion to default to a HEAD request to get file meta information, falling back to a GET request on failure. Add Tus chunk size env var option.

* refactor

Keep original signature and rewrite code a bit

* add docs for COMPANION_CHUNK_SIZE

Co-authored-by: Mikael Finstad <finstaden@gmail.com>

packages/@uppy/companion/src/server/helpers/request.js

@@ -104,11 +104,11 @@ module.exports.getProtectedHttpAgent = (protocol, blockPrivateIPs) => {
  * @param {boolean} blockLocalIPs
  * @returns {Promise<{type: string, size: number}>}
  */
-exports.getURLMeta = (url, blockLocalIPs = false) => {
-  return new Promise((resolve, reject) => {
+exports.getURLMeta = async (url, blockLocalIPs = false) => {
+  const requestWithMethod = async (method) => new Promise((resolve, reject) => {
     const opts = {
       uri: url,
-      method: 'GET',
+      method,
       followRedirect: exports.getRedirectEvaluator(url, blockLocalIPs),
       agentClass: exports.getProtectedHttpAgent((new URL(url)).protocol, blockLocalIPs),
     }
@@ -117,20 +117,39 @@ exports.getURLMeta = (url, blockLocalIPs = false) => {
       if (err) reject(err)
     })
     req.on('response', (response) => {
-      if (response.statusCode >= 300) {
-        // @todo possibly set a status code in the error object to get a more helpful
-        // hint at what the cause of error is.
-        reject(new Error(`URL server responded with status: ${response.statusCode}`))
-      } else {
-        req.abort() // No need to get the rest of the response, as we only want header
-
-        // Can be undefined for unknown length URLs, e.g. transfer-encoding: chunked
-        const contentLength = parseInt(response.headers['content-length'], 10)
-        resolve({
-          type: response.headers['content-type'],
-          size: Number.isNaN(contentLength) ? null : contentLength,
-        })
-      }
+      // Can be undefined for unknown length URLs, e.g. transfer-encoding: chunked
+      const contentLength = parseInt(response.headers['content-length'], 10)
+
+      // No need to get the rest of the response, as we only want header (not really relevant for HEAD, but why not)
+      req.abort()
+
+      resolve({
+        type: response.headers['content-type'],
+        size: Number.isNaN(contentLength) ? null : contentLength,
+        statusCode: response.statusCode,
+      })
     })
   })
+
+  // We prefer to use a HEAD request, as it doesn't download the content. If the URL doesn't
+  // support HEAD, or doesn't follow the spec and provide the correct Content-Length, we
+  // fallback to GET.
+  let urlMeta = await requestWithMethod('HEAD')
+
+  // If HTTP error response, we retry with GET, which may work on non-compliant servers
+  // (e.g. HEAD doesn't work on signed S3 URLs)
+  // We look for status codes in the 400 and 500 ranges here, as 3xx errors are
+  // unlikely to have to do with our choice of method
+  if (urlMeta.statusCode >= 400 || urlMeta.size === 0 || urlMeta.size == null) {
+    urlMeta = await requestWithMethod('GET')
+  }
+
+  if (urlMeta.statusCode >= 300) {
+    // @todo possibly set a status code in the error object to get a more helpful
+    // hint at what the cause of error is.
+    throw new Error(`URL server responded with status: ${urlMeta.statusCode}`)
+  }
+
+  const { size, type } = urlMeta
+  return { size, type }
 }

packages/@uppy/companion/src/standalone/helper.js

@@ -104,6 +104,7 @@ const getConfigFromEnv = () => {
     multipleInstances: true,
     streamingUpload: process.env.COMPANION_STREAMING_UPLOAD === 'true',
     maxFileSize: process.env.COMPANION_MAX_FILE_SIZE ? parseInt(process.env.COMPANION_MAX_FILE_SIZE, 10) : undefined,
+    chunkSize: process.env.COMPANION_CHUNK_SIZE ? parseInt(process.env.COMPANION_CHUNK_SIZE, 10) : undefined,
   }
 }
 

website/src/docs/companion.md

@@ -248,6 +248,9 @@ export COMPANION_STREAMING_UPLOAD=true
 
 # corresponds to the maxFileSize option
 export COMPANION_MAX_FILE_SIZE="100000000"
+
+# corresponds to the chunkSize option
+export COMPANION_CHUNK_SIZE="50000000"
 ```
 
 See [env.example.sh](https://github.com/transloadit/uppy/blob/main/env.example.sh) for an example configuration script.