Feat:csv & docx support (#1139)

Co-authored-by: jyong <jyong@dify.ai>

api/controllers/console/datasets/file.py

@@ -26,7 +26,7 @@ from models.model import UploadFile
 
 cache = TTLCache(maxsize=None, ttl=30)
 
-ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx']
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
 PREVIEW_WORDS_LIMIT = 3000
 
 

web/app/components/datasets/create/assets/docx.svg

@@ -0,0 +1,23 @@
+<svg width="24" height="26" viewBox="0 0 24 26" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g filter="url(#filter0_d_6785_286)">
+<path d="M3 5.8C3 4.11984 3 3.27976 3.32698 2.63803C3.6146 2.07354 4.07354 1.6146 4.63803 1.32698C5.27976 1 6.11984 1 7.8 1H14L21 8V18.2C21 19.8802 21 20.7202 20.673 21.362C20.3854 21.9265 19.9265 22.3854 19.362 22.673C18.7202 23 17.8802 23 16.2 23H7.8C6.11984 23 5.27976 23 4.63803 22.673C4.07354 22.3854 3.6146 21.9265 3.32698 21.362C3 20.7202 3 19.8802 3 18.2V5.8Z" fill="#2349A9"/>
+</g>
+<path opacity="0.5" d="M14 1L21 8H16C14.8954 8 14 7.10457 14 6V1Z" fill="white"/>
+<g opacity="0.96">
+<path d="M8.13338 16C8.13338 16.6188 8.18659 17.1215 7.69589 17.5801C7.4062 17.8508 6.99827 17.9669 6.56078 17.9669H5V14.0331H6.56078C6.99827 14.0331 7.4062 14.1492 7.69589 14.4199C8.18659 14.8785 8.13338 15.3812 8.13338 16ZM7.09877 16C7.09877 15.337 7.06921 15.2265 6.98644 15.116C6.89185 14.9834 6.74996 14.895 6.48983 14.895H6.03461V17.105H6.48983C6.74996 17.105 6.89185 17.0166 6.98644 16.884C7.06921 16.7735 7.09877 16.6685 7.09877 16Z" fill="white"/>
+<path d="M11.9192 16C11.9192 16.5912 11.937 17.1436 11.4936 17.558C11.1862 17.8453 10.8314 18 10.3171 18C9.80274 18 9.44802 17.8453 9.14059 17.558C8.69719 17.1436 8.71493 16.5912 8.71493 16C8.71493 15.4088 8.69719 14.8564 9.14059 14.442C9.44802 14.1547 9.80274 14 10.3171 14C10.8314 14 11.1862 14.1547 11.4936 14.442C11.937 14.8564 11.9192 15.4088 11.9192 16ZM10.8846 16C10.8846 15.2818 10.8255 15.1492 10.7309 15.0331C10.6541 14.9392 10.5063 14.8619 10.3171 14.8619C10.1279 14.8619 9.9801 14.9392 9.90325 15.0331C9.80865 15.1492 9.74953 15.2818 9.74953 16C9.74953 16.7182 9.80865 16.8453 9.90325 16.9613C9.9801 17.0552 10.1279 17.1381 10.3171 17.1381C10.5063 17.1381 10.6541 17.0552 10.7309 16.9613C10.8255 16.8453 10.8846 16.7182 10.8846 16Z" fill="white"/>
+<path d="M15.689 16.7182C15.5353 17.5856 14.8909 18 14.0928 18C13.6021 18 13.2296 17.8453 12.9222 17.558C12.4788 17.1436 12.4965 16.5912 12.4965 16C12.4965 15.4088 12.4788 14.8564 12.9222 14.442C13.2296 14.1547 13.6021 14 14.0928 14C14.8909 14 15.5353 14.4144 15.689 15.2818H14.6367C14.5717 15.0608 14.4416 14.8619 14.0987 14.8619C13.9095 14.8619 13.7676 14.9337 13.6908 15.0276C13.5962 15.1436 13.5312 15.2818 13.5312 16C13.5312 16.7182 13.5962 16.8564 13.6908 16.9724C13.7676 17.0663 13.9095 17.1381 14.0987 17.1381C14.4416 17.1381 14.5717 16.9392 14.6367 16.7182H15.689Z" fill="white"/>
+<path d="M19.5 17.9669H18.3176L17.6259 16.7569L16.9342 17.9669H15.7518L17.0642 15.9503L15.8345 14.0331H17.011L17.6259 15.1436L18.2407 14.0331H19.4172L18.1875 15.9503L19.5 17.9669Z" fill="white"/>
+</g>
+<defs>
+<filter id="filter0_d_6785_286" x="1" y="0" width="22" height="26" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset dy="1"/>
+<feGaussianBlur stdDeviation="1"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0.0627451 0 0 0 0 0.0941176 0 0 0 0 0.156863 0 0 0 0.05 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_6785_286"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_6785_286" result="shape"/>
+</filter>
+</defs>
+</svg>

web/app/components/datasets/create/embedding-process/index.module.css

@@ -89,7 +89,9 @@
 .fileIcon.csv {
   background-image: url(../assets/csv.svg);
 }
-
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/create/embedding-process/index.tsx

@@ -194,7 +194,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
       </div>
       <div className={s.progressContainer}>
         {indexingStatusBatchDetail.map(indexingStatusDetail => (
-          <div className={cn(
+          <div key={indexingStatusDetail.id} className={cn(
             s.sourceItem,
             indexingStatusDetail.indexing_status === 'error' && s.error,
             indexingStatusDetail.indexing_status === 'completed' && s.success,

web/app/components/datasets/create/file-uploader/index.module.css

@@ -98,7 +98,9 @@
 .fileIcon.csv {
   background-image: url(../assets/csv.svg);
 }
-
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/create/file-uploader/index.tsx

@@ -29,9 +29,9 @@ const ACCEPTS = [
   '.md',
   '.markdown',
   '.txt',
-  // '.xls',
   '.xlsx',
-  // '.csv',
+  '.csv',
+  '.docx',
 ]
 
 const FileUploader = ({
@@ -250,7 +250,6 @@ const FileUploader = ({
             className={cn(
               s.file,
               fileItem.progress < 100 && s.uploading,
-              // s.active,
             )}
           >
             {fileItem.progress < 100 && (
@@ -274,33 +273,6 @@ const FileUploader = ({
             </div>
           </div>
         ))}
-        {/* {currentFile && (
-          <div
-            // onClick={() => onPreview(currentFile)}
-            className={cn(
-              s.file,
-              uploading && s.uploading,
-              // s.active,
-            )}
-          >
-            {uploading && (
-              <div className={s.progressbar} style={{ width: `${percent}%` }}/>
-            )}
-            <div className={s.fileInfo}>
-              <div className={cn(s.fileIcon, s[getFileType(currentFile)])}/>
-              <div className={s.filename}>{currentFile.name}</div>
-              <div className={s.size}>{getFileSize(currentFile.size)}</div>
-            </div>
-            <div className={s.actionWrapper}>
-              {uploading && (
-                <div className={s.percent}>{`${percent}%`}</div>
-              )}
-              {!uploading && (
-                <div className={s.remove} onClick={() => removeFile(index)}/>
-              )}
-            </div>
-          </div>
-        )} */}
       </div>
     </div>
   )

web/app/components/datasets/create/step-one/index.tsx

@@ -65,7 +65,7 @@ const StepOne = ({
   const { dataset } = useDatasetDetailContext()
   const [showModal, setShowModal] = useState(false)
   const [currentFile, setCurrentFile] = useState<File | undefined>()
-  const [currentNotionPage, setCurrentNotionPage] = useState<Page | undefined>()
+  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
   const { t } = useTranslation()
 
   const modalShowHandle = () => setShowModal(true)
@@ -78,7 +78,7 @@ const StepOne = ({
     setCurrentFile(undefined)
   }
 
-  const updateCurrentPage = (page: Page) => {
+  const updateCurrentPage = (page: NotionPage) => {
     setCurrentNotionPage(page)
   }
 

web/app/components/datasets/create/step-two/index.module.css

@@ -290,6 +290,10 @@
   background-image: url(../assets/csv.svg);
 }
 
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
+
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/documents/detail/completed/SegmentCard.tsx

@@ -43,7 +43,7 @@ type ISegmentCardProps = {
   scene?: UsageScene
   className?: string
   archived?: boolean
-  embeddingAvailable: boolean
+  embeddingAvailable?: boolean
 }
 
 const SegmentCard: FC<ISegmentCardProps> = ({

web/app/components/datasets/documents/style.module.css

@@ -87,6 +87,9 @@
 .csvIcon {
   background-image: url(~@/assets/csv.svg);
 }
+.docxIcon {
+  background-image: url(~@/assets/docx.svg);
+}
 .statusItemDetail {
   @apply h-8 font-medium border border-gray-200 inline-flex items-center rounded-lg pl-3 pr-4 mr-2;
 }

web/assets/docx.svg

@@ -0,0 +1,23 @@
+<svg width="24" height="26" viewBox="0 0 24 26" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g filter="url(#filter0_d_6785_286)">
+<path d="M3 5.8C3 4.11984 3 3.27976 3.32698 2.63803C3.6146 2.07354 4.07354 1.6146 4.63803 1.32698C5.27976 1 6.11984 1 7.8 1H14L21 8V18.2C21 19.8802 21 20.7202 20.673 21.362C20.3854 21.9265 19.9265 22.3854 19.362 22.673C18.7202 23 17.8802 23 16.2 23H7.8C6.11984 23 5.27976 23 4.63803 22.673C4.07354 22.3854 3.6146 21.9265 3.32698 21.362C3 20.7202 3 19.8802 3 18.2V5.8Z" fill="#2349A9"/>
+</g>
+<path opacity="0.5" d="M14 1L21 8H16C14.8954 8 14 7.10457 14 6V1Z" fill="white"/>
+<g opacity="0.96">
+<path d="M8.13338 16C8.13338 16.6188 8.18659 17.1215 7.69589 17.5801C7.4062 17.8508 6.99827 17.9669 6.56078 17.9669H5V14.0331H6.56078C6.99827 14.0331 7.4062 14.1492 7.69589 14.4199C8.18659 14.8785 8.13338 15.3812 8.13338 16ZM7.09877 16C7.09877 15.337 7.06921 15.2265 6.98644 15.116C6.89185 14.9834 6.74996 14.895 6.48983 14.895H6.03461V17.105H6.48983C6.74996 17.105 6.89185 17.0166 6.98644 16.884C7.06921 16.7735 7.09877 16.6685 7.09877 16Z" fill="white"/>
+<path d="M11.9192 16C11.9192 16.5912 11.937 17.1436 11.4936 17.558C11.1862 17.8453 10.8314 18 10.3171 18C9.80274 18 9.44802 17.8453 9.14059 17.558C8.69719 17.1436 8.71493 16.5912 8.71493 16C8.71493 15.4088 8.69719 14.8564 9.14059 14.442C9.44802 14.1547 9.80274 14 10.3171 14C10.8314 14 11.1862 14.1547 11.4936 14.442C11.937 14.8564 11.9192 15.4088 11.9192 16ZM10.8846 16C10.8846 15.2818 10.8255 15.1492 10.7309 15.0331C10.6541 14.9392 10.5063 14.8619 10.3171 14.8619C10.1279 14.8619 9.9801 14.9392 9.90325 15.0331C9.80865 15.1492 9.74953 15.2818 9.74953 16C9.74953 16.7182 9.80865 16.8453 9.90325 16.9613C9.9801 17.0552 10.1279 17.1381 10.3171 17.1381C10.5063 17.1381 10.6541 17.0552 10.7309 16.9613C10.8255 16.8453 10.8846 16.7182 10.8846 16Z" fill="white"/>
+<path d="M15.689 16.7182C15.5353 17.5856 14.8909 18 14.0928 18C13.6021 18 13.2296 17.8453 12.9222 17.558C12.4788 17.1436 12.4965 16.5912 12.4965 16C12.4965 15.4088 12.4788 14.8564 12.9222 14.442C13.2296 14.1547 13.6021 14 14.0928 14C14.8909 14 15.5353 14.4144 15.689 15.2818H14.6367C14.5717 15.0608 14.4416 14.8619 14.0987 14.8619C13.9095 14.8619 13.7676 14.9337 13.6908 15.0276C13.5962 15.1436 13.5312 15.2818 13.5312 16C13.5312 16.7182 13.5962 16.8564 13.6908 16.9724C13.7676 17.0663 13.9095 17.1381 14.0987 17.1381C14.4416 17.1381 14.5717 16.9392 14.6367 16.7182H15.689Z" fill="white"/>
+<path d="M19.5 17.9669H18.3176L17.6259 16.7569L16.9342 17.9669H15.7518L17.0642 15.9503L15.8345 14.0331H17.011L17.6259 15.1436L18.2407 14.0331H19.4172L18.1875 15.9503L19.5 17.9669Z" fill="white"/>
+</g>
+<defs>
+<filter id="filter0_d_6785_286" x="1" y="0" width="22" height="26" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset dy="1"/>
+<feGaussianBlur stdDeviation="1"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0.0627451 0 0 0 0 0.0941176 0 0 0 0 0.156863 0 0 0 0.05 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow_6785_286"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect1_dropShadow_6785_286" result="shape"/>
+</filter>
+</defs>
+</svg>

web/i18n/lang/dataset-creation.en.ts

@@ -23,7 +23,7 @@ const translation = {
       title: 'Upload text file',
       button: 'Drag and drop file, or',
       browse: 'Browse',
-      tip: 'Supports txt, html, markdown, xlsx, and pdf. Max {{size}}MB each.',
+      tip: 'Supports txt, html, markdown, xlsx, csv, docx and pdf. Max {{size}}MB each.',
       validation: {
         typeError: 'File type not supported',
         size: 'File too large. Maximum is {{size}}MB',

web/i18n/lang/dataset-creation.zh.ts

@@ -23,7 +23,7 @@ const translation = {
       title: '上传文本文件',
       button: '拖拽文件至此，或者',
       browse: '选择文件',
-      tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX，每个文件不超过 {{size}}MB。',
+      tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX、CSV、DOCX，每个文件不超过 {{size}}MB。',
       validation: {
         typeError: '文件类型不支持',
         size: '文件太大了，不能超过 {{size}}MB',