core: adjust ID generation to keep non-latin characters (#1722)

* core: add test for ID generation with non-latin names

* core: adjust ID generation to keep non-latin characters

This shouldn't be bc-breaking!

Now, non-latin characters are encoded as their charcode in base 32, so
files that only differ by name in a non-latin language will generate
different IDs.

packages/@uppy/core/src/__snapshots__/index.test.js.snap

@@ -18,7 +18,7 @@ Object {
     Object {
       "data": Uint8Array [],
       "extension": "jpg",
-      "id": "uppy-foojpg-image/jpeg",
+      "id": "uppy-foo/jpg-1e-image/jpeg",
       "isRemote": false,
       "meta": Object {
         "name": "foo.jpg",
@@ -41,7 +41,7 @@ Object {
     Object {
       "data": Uint8Array [],
       "extension": "jpg",
-      "id": "uppy-barjpg-image/jpeg",
+      "id": "uppy-bar/jpg-1e-image/jpeg",
       "isRemote": false,
       "meta": Object {
         "name": "bar.jpg",

packages/@uppy/core/src/index.test.js

@@ -483,7 +483,7 @@ describe('src/Core', () => {
         expect(postprocessor1.mock.calls.length).toEqual(1)
         // const lastModifiedTime = new Date()
         // const fileId = 'foojpg' + lastModifiedTime.getTime()
-        const fileId = 'uppy-foojpg-image'
+        const fileId = 'uppy-foo/jpg-1e-image'
 
         expect(postprocessor1.mock.calls[0][0].length).toEqual(1)
         expect(postprocessor1.mock.calls[0][0][0].substring(0, 17)).toEqual(
@@ -688,6 +688,29 @@ describe('src/Core', () => {
         /Cannot add new files: already uploading\./
       )
     })
+
+    it('does not dedupe different files', async () => {
+      const core = new Core()
+      const data = new Blob([sampleImage], { type: 'image/jpeg' })
+      data.lastModified = 1562770350937
+
+      core.addFile({
+        source: 'jest',
+        name: 'foo.jpg',
+        type: 'image/jpeg',
+        data
+      })
+      core.addFile({
+        source: 'jest',
+        name: 'foo푸.jpg',
+        type: 'image/jpeg',
+        data
+      })
+
+      expect(core.getFiles()).toHaveLength(2)
+      expect(core.getFile('uppy-foo/jpg-1e-image/jpeg-17175-1562770350937')).toBeDefined()
+      expect(core.getFile('uppy-foo//jpg-1l3o-1e-image/jpeg-17175-1562770350937')).toBeDefined()
+    })
   })
 
   describe('uploading a file', () => {
@@ -736,10 +759,10 @@ describe('src/Core', () => {
       const core = new Core()
       core.store.state.currentUploads = {
         upload1: {
-          fileIDs: ['uppy-file1jpg-image/jpeg', 'uppy-file2jpg-image/jpeg', 'uppy-file3jpg-image/jpeg']
+          fileIDs: ['uppy-file1/jpg-1e-image/jpeg', 'uppy-file2/jpg-1e-image/jpeg', 'uppy-file3/jpg-1e-image/jpeg']
         },
         upload2: {
-          fileIDs: ['uppy-file4jpg-image/jpeg', 'uppy-file5jpg-image/jpeg', 'uppy-file6jpg-image/jpeg']
+          fileIDs: ['uppy-file4/jpg-1e-image/jpeg', 'uppy-file5/jpg-1e-image/jpeg', 'uppy-file6/jpg-1e-image/jpeg']
         }
       }
       core.addUploader((fileIDs) => Promise.resolve())

packages/@uppy/utils/src/generateFileID.js

@@ -10,9 +10,21 @@ module.exports = function generateFileID (file) {
   // filter is needed to not join empty values with `-`
   return [
     'uppy',
-    file.name ? file.name.toLowerCase().replace(/[^A-Z0-9]/ig, '') : '',
+    file.name ? encodeFilename(file.name.toLowerCase()) : '',
     file.type,
     file.data.size,
     file.data.lastModified
   ].filter(val => val).join('-')
 }
+
+function encodeFilename (name) {
+  let suffix = ''
+  return name.replace(/[^A-Z0-9]/ig, (character) => {
+    suffix += '-' + encodeCharacter(character)
+    return '/'
+  }) + suffix
+}
+
+function encodeCharacter (character) {
+  return character.charCodeAt(0).toString(32)
+}

packages/@uppy/utils/src/generateFileID.test.js

@@ -12,7 +12,18 @@ describe('generateFileID', () => {
     }
 
     expect(generateFileID(fileObj)).toEqual(
-      'uppy-foo0fijpg-image/jpeg-2271173-1498510508000'
+      'uppy-foo0fi////jpg-20-53-14-1e-image/jpeg-2271173-1498510508000'
+    )
+
+    expect(generateFileID({
+      name: 'джумла-джpумлатест.jpg',
+      type: 'image/jpeg',
+      data: {
+        lastModified: 1498510508000,
+        size: 2271173
+      }
+    })).toEqual(
+      'uppy-/////////p/////////jpg-11k-11m-123-11s-11r-11g-1d-11k-11m-123-11s-11r-11g-122-11l-121-122-1e-image/jpeg-2271173-1498510508000'
     )
   })
 })