Просмотр исходного кода

@uppy/aws-s3-multipart: add support for signing on the client (#4519)

Antoine du Hamel 1 год назад
Родитель
Сommit
a83373b83d

+ 1 - 1
.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x, 16.x, 18.x]
+        node-version: [16.x, 18.x, 20.x]
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3

+ 25 - 0
.yarn/patches/jest-environment-jsdom-npm-29.5.0-fc600add1e.patch

@@ -0,0 +1,25 @@
+diff --git a/build/index.js b/build/index.js
+index ec8a133f3fbea56f4b395c96ca2b27bd4903c62e..dfd3852248f5008f82794ce61e3dca10ee8f8389 100644
+--- a/build/index.js
++++ b/build/index.js
+@@ -49,7 +49,7 @@ class JSDOMEnvironment {
+   global;
+   errorEventListener;
+   moduleMocker;
+-  customExportConditions = ['browser'];
++  customExportConditions = ['jest','browser'];
+   constructor(config, context) {
+     const {projectConfig} = config;
+     const virtualConsole = new (_jsdom().VirtualConsole)();
+@@ -93,6 +93,11 @@ class JSDOMEnvironment {
+     // TODO: remove this ASAP, but it currently causes tests to run really slow
+     global.Buffer = Buffer;
+ 
++    // JSDOM does not provide the following, thankfully Node.js ships with spec-compliant equivalent:
++    global.TextDecoder = TextDecoder;
++    global.TextEncoder = TextEncoder;
++    Object.defineProperty(global, 'crypto', {__proto__:null, value:require('node:crypto').webcrypto});
++
+     // Report uncaught errors.
+     this.errorEventListener = event => {
+       if (userErrorListenerCount === 0 && event.error != null) {

+ 12 - 0
.yarn/patches/uuid-npm-8.3.2-eca0baba53.patch

@@ -0,0 +1,12 @@
+diff --git a/package.json b/package.json
+index f0ab3711ee4f490cbf961ebe6283ce2a28b6824b..644235a3ef52c974e946403a3fcdd137d01fad0c 100644
+--- a/package.json
++++ b/package.json
+@@ -25,6 +25,7 @@
+         "require": "./dist/index.js",
+         "import": "./wrapper.mjs"
+       },
++      "jest": "./dist/index.js",
+       "default": "./dist/esm-browser/index.js"
+     },
+     "./package.json": "./package.json"

+ 4 - 2
package.json

@@ -172,9 +172,11 @@
     "@types/eslint@^7.2.13": "^8.2.0",
     "@types/react": "^17",
     "@types/webpack-dev-server": "^4",
-    "preact": "patch:preact@npm:10.10.0#.yarn/patches/preact-npm-10.10.0-dd04de05e8.patch",
+    "jest-environment-jsdom": "patch:jest-environment-jsdom@npm:29.5.0#.yarn/patches/jest-environment-jsdom-npm-29.5.0-fc600add1e.patch",
     "pre-commit": "patch:pre-commit@npm:1.2.2#.yarn/patches/pre-commit-npm-1.2.2-f30af83877.patch",
+    "preact": "patch:preact@npm:10.10.0#.yarn/patches/preact-npm-10.10.0-dd04de05e8.patch",
     "start-server-and-test": "patch:start-server-and-test@npm:1.14.0#.yarn/patches/start-server-and-test-npm-1.14.0-841aa34fdf.patch",
-    "stylelint-config-rational-order": "patch:stylelint-config-rational-order@npm%3A0.1.2#./.yarn/patches/stylelint-config-rational-order-npm-0.1.2-d8336e84ed.patch"
+    "stylelint-config-rational-order": "patch:stylelint-config-rational-order@npm%3A0.1.2#./.yarn/patches/stylelint-config-rational-order-npm-0.1.2-d8336e84ed.patch",
+    "uuid@^8.3.2": "patch:uuid@npm:8.3.2#.yarn/patches/uuid-npm-8.3.2-eca0baba53.patch"
   }
 }

+ 2 - 0
packages/@uppy/aws-s3-multipart/package.json

@@ -28,6 +28,8 @@
     "@uppy/utils": "workspace:^"
   },
   "devDependencies": {
+    "@aws-sdk/client-s3": "^3.362.0",
+    "@aws-sdk/s3-request-presigner": "^3.362.0",
     "@jest/globals": "^29.0.0",
     "nock": "^13.1.0",
     "whatwg-fetch": "3.6.2"

+ 145 - 0
packages/@uppy/aws-s3-multipart/src/createSignedURL.js

@@ -0,0 +1,145 @@
+/**
+ * Create a canonical request by concatenating the following strings, separated
+ * by newline characters. This helps ensure that the signature that you
+ * calculate and the signature that AWS calculates can match.
+ *
+ * @see https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html#create-canonical-request
+ *
+ * @param {object} param0
+ * @param {string} param0.method – The HTTP method.
+ * @param {string} param0.CanonicalUri – The URI-encoded version of the absolute
+ * path component URL (everything between the host and the question mark
+ * character (?) that starts the query string parameters). If the absolute path
+ * is empty, use a forward slash character (/).
+ * @param {string} param0.CanonicalQueryString – The URL-encoded query string
+ * parameters, separated by ampersands (&). Percent-encode reserved characters,
+ * including the space character. Encode names and values separately. If there
+ * are empty parameters, append the equals sign to the parameter name before
+ * encoding. After encoding, sort the parameters alphabetically by key name. If
+ * there is no query string, use an empty string ("").
+ * @param {Record<string, string>} param0.SignedHeaders – The request headers,
+ * that will be signed, and their values, separated by newline characters.
+ * For the values, trim any leading or trailing spaces, convert sequential
+ * spaces to a single space, and separate the values for a multi-value header
+ * using commas. You must include the host header (HTTP/1.1), and any x-amz-*
+ * headers in the signature. You can optionally include other standard headers
+ * in the signature, such as content-type.
+ * @param {string} param0.HashedPayload – A string created using the payload in
+ * the body of the HTTP request as input to a hash function. This string uses
+ * lowercase hexadecimal characters. If the payload is empty, use an empty
+ * string as the input to the hash function.
+ * @returns {string}
+ */
+function createCanonicalRequest ({
+  method = 'PUT',
+  CanonicalUri = '/',
+  CanonicalQueryString = '',
+  SignedHeaders,
+  HashedPayload,
+}) {
+  const headerKeys = Object.keys(SignedHeaders).map(k => k.toLowerCase()).sort()
+  return [
+    method,
+    CanonicalUri,
+    CanonicalQueryString,
+    ...headerKeys.map(k => `${k}:${SignedHeaders[k]}`),
+    '',
+    headerKeys.join(';'),
+    HashedPayload,
+  ].join('\n')
+}
+
+const { subtle } = globalThis.crypto
+const ec = new TextEncoder()
+const algorithm = { name: 'HMAC', hash: 'SHA-256' }
+
+async function digest (data) {
+  return subtle.digest(algorithm.hash, ec.encode(data))
+}
+
+async function generateHmacKey (secret) {
+  return subtle.importKey('raw', typeof secret === 'string' ? ec.encode(secret) : secret, algorithm, false, ['sign'])
+}
+
+function arrayBufferToHexString (arrayBuffer) {
+  const byteArray = new Uint8Array(arrayBuffer)
+  let hexString = ''
+  for (let i = 0; i < byteArray.length; i++) {
+    hexString += byteArray[i].toString(16).padStart(2, '0')
+  }
+  return hexString
+}
+
+async function hash (key, data) {
+  return subtle.sign(algorithm, await generateHmacKey(key), ec.encode(data))
+}
+
+/**
+ * @see https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
+ * @param {Record<string,string>} param0
+ * @returns {Promise<URL>} the signed URL
+ */
+export default async function createSignedURL ({
+  accountKey, accountSecret, sessionToken,
+  bucketName,
+  Key, Region,
+  expires,
+  uploadId, partNumber,
+}) {
+  const Service = 's3'
+  const host = `${bucketName}.${Service}.${Region}.amazonaws.com`
+  const CanonicalUri = `/${encodeURI(Key)}`
+  const payload = 'UNSIGNED-PAYLOAD'
+
+  const requestDateTime = new Date().toISOString().replace(/[-:]|\.\d+/g, '') // YYYYMMDDTHHMMSSZ
+  const date = requestDateTime.slice(0, 8) // YYYYMMDD
+  const scope = `${date}/${Region}/${Service}/aws4_request`
+
+  const url = new URL(`https://${host}${CanonicalUri}`)
+  // N.B.: URL search params needs to be added in the ASCII order
+  url.searchParams.set('X-Amz-Algorithm', 'AWS4-HMAC-SHA256')
+  url.searchParams.set('X-Amz-Content-Sha256', payload)
+  url.searchParams.set('X-Amz-Credential', `${accountKey}/${scope}`)
+  url.searchParams.set('X-Amz-Date', requestDateTime)
+  url.searchParams.set('X-Amz-Expires', expires)
+  // We are signing on the client, so we expect there's going to be a session token:
+  url.searchParams.set('X-Amz-Security-Token', sessionToken)
+  url.searchParams.set('X-Amz-SignedHeaders', 'host')
+  // Those two are present only for Multipart Uploads:
+  if (partNumber) url.searchParams.set('partNumber', partNumber)
+  if (uploadId) url.searchParams.set('uploadId', uploadId)
+  url.searchParams.set('x-id', partNumber && uploadId ? 'UploadPart' : 'PutObject')
+
+  // Step 1: Create a canonical request
+  const canonical = createCanonicalRequest({
+    CanonicalUri,
+    CanonicalQueryString: url.search.slice(1),
+    SignedHeaders: {
+      host,
+    },
+    HashedPayload: payload,
+  })
+
+  // Step 2: Create a hash of the canonical request
+  const hashedCanonical = arrayBufferToHexString(await digest(canonical))
+
+  // Step 3: Create a string to sign
+  const stringToSign = [
+    `AWS4-HMAC-SHA256`, // The algorithm used to create the hash of the canonical request.
+    requestDateTime, // The date and time used in the credential scope.
+    scope, // The credential scope. This restricts the resulting signature to the specified Region and service.
+    hashedCanonical, // The hash of the canonical request.
+  ].join('\n')
+
+  // Step 4: Calculate the signature
+  const kDate = await hash(`AWS4${accountSecret}`, date)
+  const kRegion = await hash(kDate, Region)
+  const kService = await hash(kRegion, Service)
+  const kSigning = await hash(kService, 'aws4_request')
+  const signature = arrayBufferToHexString(await hash(kSigning, stringToSign))
+
+  // Step 5: Add the signature to the request
+  url.searchParams.set('X-Amz-Signature', signature)
+
+  return url
+}

+ 77 - 0
packages/@uppy/aws-s3-multipart/src/createSignedURL.test.js

@@ -0,0 +1,77 @@
+import { describe, it, beforeEach, afterEach } from '@jest/globals'
+import assert from 'node:assert'
+import { S3Client, UploadPartCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import createSignedURL from './createSignedURL.js'
+
+const bucketName = 'some-bucket'
+const s3ClientOptions = {
+  region: 'us-bar-1',
+  credentials: {
+    accessKeyId: 'foo',
+    secretAccessKey: 'bar',
+    sessionToken: 'foobar',
+  },
+}
+const { Date: OriginalDate } = globalThis
+
+describe('createSignedURL', () => {
+  beforeEach(() => {
+    const now_ms = OriginalDate.now()
+    globalThis.Date = function Date () {
+      if (new.target) {
+        return Reflect.construct(OriginalDate, [now_ms])
+      }
+      return Reflect.apply(OriginalDate, this, [now_ms])
+    }
+    globalThis.Date.now = function now () {
+      return now_ms
+    }
+  })
+  afterEach(() => {
+    globalThis.Date = OriginalDate
+  })
+  it('should be able to sign non-multipart upload', async () => {
+    const client = new S3Client(s3ClientOptions)
+    assert.strictEqual(
+      (await createSignedURL({
+        accountKey: s3ClientOptions.credentials.accessKeyId,
+        accountSecret: s3ClientOptions.credentials.secretAccessKey,
+        sessionToken: s3ClientOptions.credentials.sessionToken,
+        bucketName,
+        Key: 'some/key',
+        Region: s3ClientOptions.region,
+        expires: 900,
+      })).searchParams.get('X-Amz-Signature'),
+      new URL(await getSignedUrl(client, new PutObjectCommand({
+        Bucket: bucketName,
+        Fields: {},
+        Key: 'some/key',
+      }, { expiresIn: 900 }))).searchParams.get('X-Amz-Signature'),
+    )
+  })
+  it('should be able to sign multipart upload', async () => {
+    const client = new S3Client(s3ClientOptions)
+    const partNumber = 99
+    const uploadId = 'dummyUploadId'
+    assert.strictEqual(
+      (await createSignedURL({
+        accountKey: s3ClientOptions.credentials.accessKeyId,
+        accountSecret: s3ClientOptions.credentials.secretAccessKey,
+        sessionToken: s3ClientOptions.credentials.sessionToken,
+        uploadId,
+        partNumber,
+        bucketName,
+        Key: 'some/key',
+        Region: s3ClientOptions.region,
+        expires: 900,
+      })).searchParams.get('X-Amz-Signature'),
+      new URL(await getSignedUrl(client, new UploadPartCommand({
+        Bucket: bucketName,
+        UploadId: uploadId,
+        PartNumber: partNumber,
+        Key: 'some/key',
+      }, { expiresIn: 900 }))).searchParams.get('X-Amz-Signature'),
+    )
+  })
+})

+ 91 - 3
packages/@uppy/aws-s3-multipart/src/index.js

@@ -6,8 +6,10 @@ import getSocketHost from '@uppy/utils/lib/getSocketHost'
 import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue'
 import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters'
 import { createAbortError } from '@uppy/utils/lib/AbortController'
-import packageJson from '../package.json'
+
 import MultipartUploader, { pausingUploadReason } from './MultipartUploader.js'
+import createSignedURL from './createSignedURL.js'
+import packageJson from '../package.json'
 
 function assertServerError (res) {
   if (res && res.error) {
@@ -18,6 +20,27 @@ function assertServerError (res) {
   return res
 }
 
+/**
+ * Computes the expiry time for a request signed with temporary credentials. If
+ * no expiration was provided, or an invalid value (e.g. in the past) is
+ * provided, undefined is returned. This function assumes the client clock is in
+ * sync with the remote server, which is a requirement for the signature to be
+ * validated for AWS anyway.
+ *
+ * @param {import('../types/index.js').AwsS3STSResponse['credentials']} credentials
+ * @returns {number | undefined}
+ */
+function getExpiry (credentials) {
+  const expirationDate = credentials.Expiration
+  if (expirationDate) {
+    const timeUntilExpiry = Math.floor((new Date(expirationDate) - Date.now()) / 1000)
+    if (timeUntilExpiry > 9) {
+      return timeUntilExpiry
+    }
+  }
+  return undefined
+}
+
 function getAllowedMetadata ({ meta, allowedMetaFields, querify = false }) {
   const metaFields = allowedMetaFields ?? Object.keys(meta)
 
@@ -365,9 +388,12 @@ export default class AwsS3Multipart extends UploaderPlugin {
       listParts: this.listParts.bind(this),
       abortMultipartUpload: this.abortMultipartUpload.bind(this),
       completeMultipartUpload: this.completeMultipartUpload.bind(this),
-      signPart: this.signPart.bind(this),
+      getTemporarySecurityCredentials: false,
+      signPart: opts?.getTemporarySecurityCredentials ? this.createSignedURL.bind(this) : this.signPart.bind(this),
       uploadPartBytes: AwsS3Multipart.uploadPartBytes,
-      getUploadParameters: this.getUploadParameters.bind(this),
+      getUploadParameters: opts?.getTemporarySecurityCredentials
+        ? this.createSignedURL.bind(this)
+        : this.getUploadParameters.bind(this),
       companionHeaders: {},
     }
 
@@ -463,6 +489,68 @@ export default class AwsS3Multipart extends UploaderPlugin {
       .then(assertServerError)
   }
 
+  /**
+   * @type {import("../types").AwsS3STSResponse | Promise<import("../types").AwsS3STSResponse>}
+   */
+  #cachedTemporaryCredentials
+
+  async #getTemporarySecurityCredentials (options) {
+    throwIfAborted(options?.signal)
+
+    if (this.#cachedTemporaryCredentials == null) {
+      // We do not await it just yet, so concurrent calls do not try to override it:
+      if (this.opts.getTemporarySecurityCredentials === true) {
+        this.assertHost('getTemporarySecurityCredentials')
+        this.#cachedTemporaryCredentials = this.#client.get('s3/sts', null, options).then(assertServerError)
+      } else {
+        this.#cachedTemporaryCredentials = this.opts.getTemporarySecurityCredentials(options)
+      }
+      this.#cachedTemporaryCredentials = await this.#cachedTemporaryCredentials
+      setTimeout(() => {
+        // At half the time left before expiration, we clear the cache. That's
+        // an arbitrary tradeoff to limit the number of requests made to the
+        // remote while limiting the risk of using an expired token in case the
+        // clocks are not exactly synced.
+        // The HTTP cache should be configured to ensure a client doesn't request
+        // more tokens than it needs, but this timeout provides a second layer of
+        // security in case the HTTP cache is disabled or misconfigured.
+        this.#cachedTemporaryCredentials = null
+      }, (getExpiry(this.#cachedTemporaryCredentials.credentials) || 0) * 500)
+    }
+
+    return this.#cachedTemporaryCredentials
+  }
+
+  async createSignedURL (file, options) {
+    const data = await this.#getTemporarySecurityCredentials(options)
+    const expires = getExpiry(data.credentials) || 604_800 // 604 800 is the max value accepted by AWS.
+
+    const { uploadId, key, partNumber, signal } = options
+
+    // Return an object in the correct shape.
+    return {
+      method: 'PUT',
+      expires,
+      fields: {},
+      url: `${await createSignedURL({
+        accountKey: data.credentials.AccessKeyId,
+        accountSecret: data.credentials.SecretAccessKey,
+        sessionToken: data.credentials.SessionToken,
+        expires,
+        bucketName: data.bucket,
+        Region: data.region,
+        Key: key ?? `${crypto.randomUUID()}-${file.name}`,
+        uploadId,
+        partNumber,
+        signal,
+      })}`,
+      // Provide content type header required by S3
+      headers: {
+        'Content-Type': file.type,
+      },
+    }
+  }
+
   signPart (file, { uploadId, key, partNumber, signal }) {
     this.assertHost('signPart')
     throwIfAborted(signal)

+ 11 - 0
packages/@uppy/aws-s3-multipart/types/index.d.ts

@@ -11,6 +11,16 @@ export interface AwsS3SignedPart {
   url: string
   headers?: Record<string, string>
 }
+export interface AwsS3STSResponse {
+  credentials: {
+    AccessKeyId: string
+    SecretAccessKey: string
+    SessionToken: string
+    Expiration?: string
+  }
+  bucket: string
+  region: string
+}
 
 export interface AwsS3MultipartOptions extends PluginOptions {
     companionHeaders?: { [type: string]: string }
@@ -25,6 +35,7 @@ export interface AwsS3MultipartOptions extends PluginOptions {
       file: UppyFile,
       opts: { uploadId: string; key: string; signal: AbortSignal }
     ) => MaybePromise<AwsS3Part[]>
+    getTemporarySecurityCredentials?: boolean | (() => MaybePromise<AwsS3STSResponse>)
     signPart?: (
       file: UppyFile,
       opts: { uploadId: string; key: string; partNumber: number; body: Blob, signal: AbortSignal }

+ 1 - 0
packages/@uppy/companion/package.json

@@ -29,6 +29,7 @@
   "bin": "./bin/companion",
   "dependencies": {
     "@aws-sdk/client-s3": "^3.338.0",
+    "@aws-sdk/client-sts": "^3.338.0",
     "@aws-sdk/lib-storage": "^3.338.0",
     "@aws-sdk/s3-presigned-post": "^3.338.0",
     "@aws-sdk/s3-request-presigner": "^3.338.0",

+ 76 - 0
packages/@uppy/companion/src/server/controllers/s3.js

@@ -6,6 +6,10 @@ const {
   AbortMultipartUploadCommand,
   CompleteMultipartUploadCommand,
 } = require('@aws-sdk/client-s3')
+const {
+  STSClient,
+  GetFederationTokenCommand,
+} = require('@aws-sdk/client-sts')
 
 const { createPresignedPost } = require('@aws-sdk/s3-presigned-post')
 const { getSignedUrl } = require('@aws-sdk/s3-request-presigner')
@@ -353,7 +357,79 @@ module.exports = function s3 (config) {
     }, next)
   }
 
+  const policy = {
+    Version: '2012-10-17', // latest at the time of writing
+    Statement: [
+      {
+        Effect: 'Allow',
+        Action: [
+          's3:PutObject',
+        ],
+        Resource: [
+          `arn:aws:s3:::${config.bucket}/*`,
+          `arn:aws:s3:::${config.bucket}`,
+        ],
+      },
+    ],
+  }
+
+  let stsClient
+  function getSTSClient () {
+    if (stsClient == null) {
+      stsClient = new STSClient({
+        region: config.region,
+        credentials : {
+          accessKeyId: config.key,
+          secretAccessKey: config.secret,
+        },
+      })
+    }
+    return stsClient
+  }
+
+  /**
+   * Create STS credentials with the permission for sending PutObject/UploadPart to the bucket.
+   *
+   * Clients should cache the response and re-use it until they can reasonably
+   * expect uploads to complete before the token expires. To this effect, the
+   * Cache-Control header is set to invalidate the cache 5 minutes before the
+   * token expires.
+   *
+   * Response JSON:
+   * - credentials: the credentials including the SessionToken.
+   * - bucket: the S3 bucket name.
+   * - region: the region where that bucket is stored.
+   */
+  function getTemporarySecurityCredentials (req, res, next) {
+    getSTSClient().send(new GetFederationTokenCommand({
+      // Name of the federated user. The name is used as an identifier for the
+      // temporary security credentials (such as Bob). For example, you can
+      // reference the federated user name in a resource-based policy, such as
+      // in an Amazon S3 bucket policy.
+      // Companion is configured by default as an unprotected public endpoint,
+      // if you implement your own custom endpoint with user authentication you
+      // should probably use different names for each of your users.
+      Name: 'companion',
+      // The duration, in seconds, of the role session. The value specified
+      // can range from 900 seconds (15 minutes) up to the maximum session
+      // duration set for the role.
+      DurationSeconds: config.expires,
+      Policy: JSON.stringify(policy),
+    })).then(response => {
+      // This is a public unprotected endpoint.
+      // If you implement your own custom endpoint with user authentication you
+      // should probably use `private` instead of `public`.
+      res.setHeader('Cache-Control', `public,max-age=${config.expires - 300}`) // 300s is 5min.
+      res.json({
+        credentials: response.Credentials,
+        bucket: config.bucket,
+        region: config.region,
+      })
+    }, next)
+  }
+
   return express.Router()
+    .get('/sts', getTemporarySecurityCredentials)
     .get('/params', getUploadParameters)
     .post('/multipart', express.json(), createMultipartUpload)
     .get('/multipart/:uploadId', getUploadedParts)

Разница между файлами не показана из-за своего большого размера
+ 785 - 477
yarn.lock


Некоторые файлы не были показаны из-за большого количества измененных файлов