|
@@ -1,36 +1,36 @@
|
|
|
+/* eslint-disable no-mixed-operators */
|
|
|
'use client'
|
|
|
-import React, { useState, useRef, useEffect, useLayoutEffect } from 'react'
|
|
|
+import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
|
|
|
import { useTranslation } from 'react-i18next'
|
|
|
import { useBoolean } from 'ahooks'
|
|
|
-import type { File, PreProcessingRule, Rules, FileIndexingEstimateResponse as IndexingEstimateResponse } from '@/models/datasets'
|
|
|
+import { XMarkIcon } from '@heroicons/react/20/solid'
|
|
|
+import cn from 'classnames'
|
|
|
+import Link from 'next/link'
|
|
|
+import PreviewItem from './preview-item'
|
|
|
+import s from './index.module.css'
|
|
|
+import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
|
|
|
import {
|
|
|
- fetchDefaultProcessRule,
|
|
|
- createFirstDocument,
|
|
|
createDocument,
|
|
|
+ createFirstDocument,
|
|
|
fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
|
|
|
+ fetchDefaultProcessRule,
|
|
|
} from '@/service/datasets'
|
|
|
-import type { CreateDocumentReq, createDocumentResponse, FullDocumentDetail } from '@/models/datasets'
|
|
|
import Button from '@/app/components/base/button'
|
|
|
-import PreviewItem from './preview-item'
|
|
|
import Loading from '@/app/components/base/loading'
|
|
|
-import { XMarkIcon } from '@heroicons/react/20/solid'
|
|
|
|
|
|
-import cn from 'classnames'
|
|
|
-import s from './index.module.css'
|
|
|
-import Link from 'next/link'
|
|
|
import Toast from '@/app/components/base/toast'
|
|
|
import { formatNumber } from '@/utils/format'
|
|
|
|
|
|
type StepTwoProps = {
|
|
|
- isSetting?: boolean,
|
|
|
+ isSetting?: boolean
|
|
|
documentDetail?: FullDocumentDetail
|
|
|
- hasSetAPIKEY: boolean,
|
|
|
- onSetting: () => void,
|
|
|
- datasetId?: string,
|
|
|
- indexingType?: string,
|
|
|
- file?: File,
|
|
|
- onStepChange?: (delta: number) => void,
|
|
|
- updateIndexingTypeCache?: (type: string) => void,
|
|
|
+ hasSetAPIKEY: boolean
|
|
|
+ onSetting: () => void
|
|
|
+ datasetId?: string
|
|
|
+ indexingType?: string
|
|
|
+ file?: File
|
|
|
+ onStepChange?: (delta: number) => void
|
|
|
+ updateIndexingTypeCache?: (type: string) => void
|
|
|
updateResultCache?: (res: createDocumentResponse) => void
|
|
|
onSave?: () => void
|
|
|
onCancel?: () => void
|
|
@@ -71,8 +71,10 @@ const StepTwo = ({
|
|
|
const [defaultConfig, setDefaultConfig] = useState<Rules>()
|
|
|
const hasSetIndexType = !!indexingType
|
|
|
const [indexType, setIndexType] = useState<IndexingType>(
|
|
|
- indexingType ||
|
|
|
- hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL
|
|
|
+ indexingType
|
|
|
+ || hasSetAPIKEY
|
|
|
+ ? IndexingType.QUALIFIED
|
|
|
+ : IndexingType.ECONOMICAL,
|
|
|
)
|
|
|
const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
|
|
|
const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
|
|
@@ -82,19 +84,19 @@ const StepTwo = ({
|
|
|
})()
|
|
|
|
|
|
const scrollHandle = (e: any) => {
|
|
|
- if (e.target.scrollTop > 0) {
|
|
|
+ if (e.target.scrollTop > 0)
|
|
|
setScrolled(true)
|
|
|
- } else {
|
|
|
+
|
|
|
+ else
|
|
|
setScrolled(false)
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
const previewScrollHandle = (e: any) => {
|
|
|
- if (e.target.scrollTop > 0) {
|
|
|
+ if (e.target.scrollTop > 0)
|
|
|
setPreviewScrolled(true)
|
|
|
- } else {
|
|
|
+
|
|
|
+ else
|
|
|
setPreviewScrolled(false)
|
|
|
- }
|
|
|
}
|
|
|
const getFileName = (name: string) => {
|
|
|
const arr = name.split('.')
|
|
@@ -102,18 +104,17 @@ const StepTwo = ({
|
|
|
}
|
|
|
|
|
|
const getRuleName = (key: string) => {
|
|
|
- if (key === 'remove_extra_spaces') {
|
|
|
+ if (key === 'remove_extra_spaces')
|
|
|
return t('datasetCreation.stepTwo.removeExtraSpaces')
|
|
|
- }
|
|
|
- if (key === 'remove_urls_emails') {
|
|
|
+
|
|
|
+ if (key === 'remove_urls_emails')
|
|
|
return t('datasetCreation.stepTwo.removeUrlEmails')
|
|
|
- }
|
|
|
- if (key === 'remove_stopwords') {
|
|
|
+
|
|
|
+ if (key === 'remove_stopwords')
|
|
|
return t('datasetCreation.stepTwo.removeStopwords')
|
|
|
- }
|
|
|
}
|
|
|
const ruleChangeHandle = (id: string) => {
|
|
|
- const newRules = rules.map(rule => {
|
|
|
+ const newRules = rules.map((rule) => {
|
|
|
if (rule.id === id) {
|
|
|
return {
|
|
|
id: rule.id,
|
|
@@ -132,13 +133,23 @@ const StepTwo = ({
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ const fetchFileIndexingEstimate = async () => {
|
|
|
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
|
|
|
+ const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams())
|
|
|
+ if (segmentationType === SegmentType.CUSTOM)
|
|
|
+ setCustomFileIndexingEstimate(res)
|
|
|
+
|
|
|
+ else
|
|
|
+ setAutomaticFileIndexingEstimate(res)
|
|
|
+ }
|
|
|
+
|
|
|
const confirmChangeCustomConfig = async () => {
|
|
|
setCustomFileIndexingEstimate(null)
|
|
|
setShowPreview()
|
|
|
await fetchFileIndexingEstimate()
|
|
|
}
|
|
|
|
|
|
- const getIndexing_technique = () => indexingType ? indexingType : indexType
|
|
|
+ const getIndexing_technique = () => indexingType || indexType
|
|
|
|
|
|
const getProcessRule = () => {
|
|
|
const processRule: any = {
|
|
@@ -168,16 +179,6 @@ const StepTwo = ({
|
|
|
return params
|
|
|
}
|
|
|
|
|
|
- const fetchFileIndexingEstimate = async () => {
|
|
|
- const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams())
|
|
|
- if (segmentationType === SegmentType.CUSTOM) {
|
|
|
- setCustomFileIndexingEstimate(res)
|
|
|
- }
|
|
|
- else {
|
|
|
- setAutomaticFileIndexingEstimate(res)
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
const getCreationParams = () => {
|
|
|
let params
|
|
|
if (isSetting) {
|
|
@@ -185,7 +186,8 @@ const StepTwo = ({
|
|
|
original_document_id: documentDetail?.id,
|
|
|
process_rule: getProcessRule(),
|
|
|
} as CreateDocumentReq
|
|
|
- } else {
|
|
|
+ }
|
|
|
+ else {
|
|
|
params = {
|
|
|
data_source: {
|
|
|
type: 'upload_file',
|
|
@@ -226,25 +228,25 @@ const StepTwo = ({
|
|
|
}
|
|
|
|
|
|
const getDefaultMode = () => {
|
|
|
- if (documentDetail) {
|
|
|
+ if (documentDetail)
|
|
|
setSegmentationType(documentDetail.dataset_process_rule.mode)
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
const createHandle = async () => {
|
|
|
try {
|
|
|
- let res;
|
|
|
+ let res
|
|
|
const params = getCreationParams()
|
|
|
if (!datasetId) {
|
|
|
res = await createFirstDocument({
|
|
|
- body: params
|
|
|
+ body: params,
|
|
|
})
|
|
|
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
|
|
|
updateResultCache && updateResultCache(res)
|
|
|
- } else {
|
|
|
+ }
|
|
|
+ else {
|
|
|
res = await createDocument({
|
|
|
datasetId,
|
|
|
- body: params
|
|
|
+ body: params,
|
|
|
})
|
|
|
updateIndexingTypeCache && updateIndexingTypeCache(indexType)
|
|
|
updateResultCache && updateResultCache({
|
|
@@ -257,7 +259,7 @@ const StepTwo = ({
|
|
|
catch (err) {
|
|
|
Toast.notify({
|
|
|
type: 'error',
|
|
|
- message: err + '',
|
|
|
+ message: `${err}`,
|
|
|
})
|
|
|
}
|
|
|
}
|
|
@@ -266,35 +268,36 @@ const StepTwo = ({
|
|
|
// fetch rules
|
|
|
if (!isSetting) {
|
|
|
getRules()
|
|
|
- } else {
|
|
|
+ }
|
|
|
+ else {
|
|
|
getRulesFromDetail()
|
|
|
getDefaultMode()
|
|
|
}
|
|
|
}, [])
|
|
|
|
|
|
useEffect(() => {
|
|
|
- scrollRef.current?.addEventListener('scroll', scrollHandle);
|
|
|
+ scrollRef.current?.addEventListener('scroll', scrollHandle)
|
|
|
return () => {
|
|
|
- scrollRef.current?.removeEventListener('scroll', scrollHandle);
|
|
|
+ scrollRef.current?.removeEventListener('scroll', scrollHandle)
|
|
|
}
|
|
|
}, [])
|
|
|
|
|
|
useLayoutEffect(() => {
|
|
|
if (showPreview) {
|
|
|
- previewScrollRef.current?.addEventListener('scroll', previewScrollHandle);
|
|
|
+ previewScrollRef.current?.addEventListener('scroll', previewScrollHandle)
|
|
|
return () => {
|
|
|
- previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle);
|
|
|
+ previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle)
|
|
|
}
|
|
|
}
|
|
|
}, [showPreview])
|
|
|
|
|
|
useEffect(() => {
|
|
|
// get indexing type by props
|
|
|
- if (indexingType) {
|
|
|
+ if (indexingType)
|
|
|
setIndexType(indexingType as IndexingType)
|
|
|
- } else {
|
|
|
+
|
|
|
+ else
|
|
|
setIndexType(hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
|
|
|
- }
|
|
|
}, [hasSetAPIKEY, indexingType, datasetId])
|
|
|
|
|
|
useEffect(() => {
|
|
@@ -302,7 +305,8 @@ const StepTwo = ({
|
|
|
setAutomaticFileIndexingEstimate(null)
|
|
|
setShowPreview()
|
|
|
fetchFileIndexingEstimate()
|
|
|
- } else {
|
|
|
+ }
|
|
|
+ else {
|
|
|
hidePreview()
|
|
|
setCustomFileIndexingEstimate(null)
|
|
|
}
|
|
@@ -320,7 +324,7 @@ const StepTwo = ({
|
|
|
className={cn(
|
|
|
s.radioItem,
|
|
|
s.segmentationItem,
|
|
|
- segmentationType === SegmentType.AUTO && s.active
|
|
|
+ segmentationType === SegmentType.AUTO && s.active,
|
|
|
)}
|
|
|
onClick={() => setSegmentationType(SegmentType.AUTO)}
|
|
|
>
|
|
@@ -355,7 +359,7 @@ const StepTwo = ({
|
|
|
type="text"
|
|
|
className={s.input}
|
|
|
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={segmentIdentifier}
|
|
|
- onChange={(e) => setSegmentIdentifier(e.target.value)}
|
|
|
+ onChange={e => setSegmentIdentifier(e.target.value)}
|
|
|
/>
|
|
|
</div>
|
|
|
</div>
|
|
@@ -366,7 +370,7 @@ const StepTwo = ({
|
|
|
type="number"
|
|
|
className={s.input}
|
|
|
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={max}
|
|
|
- onChange={(e) => setMax(Number(e.target.value))}
|
|
|
+ onChange={e => setMax(Number(e.target.value))}
|
|
|
/>
|
|
|
</div>
|
|
|
</div>
|
|
@@ -403,9 +407,8 @@ const StepTwo = ({
|
|
|
hasSetIndexType && '!w-full',
|
|
|
)}
|
|
|
onClick={() => {
|
|
|
- if (hasSetAPIKEY) {
|
|
|
+ if (hasSetAPIKEY)
|
|
|
setIndexType(IndexingType.QUALIFIED)
|
|
|
- }
|
|
|
}}
|
|
|
>
|
|
|
<span className={cn(s.typeIcon, s.qualified)} />
|
|
@@ -418,11 +421,13 @@ const StepTwo = ({
|
|
|
<div className={s.tip}>{t('datasetCreation.stepTwo.qualifiedTip')}</div>
|
|
|
<div className='pb-0.5 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateCost')}</div>
|
|
|
{
|
|
|
- !!fileIndexingEstimate ? (
|
|
|
- <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
|
|
|
- ) : (
|
|
|
- <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
|
|
|
- )
|
|
|
+ fileIndexingEstimate
|
|
|
+ ? (
|
|
|
+ <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
|
|
|
+ )
|
|
|
+ : (
|
|
|
+ <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
|
|
|
+ )
|
|
|
}
|
|
|
</div>
|
|
|
{!hasSetAPIKEY && (
|
|
@@ -434,7 +439,6 @@ const StepTwo = ({
|
|
|
</div>
|
|
|
)}
|
|
|
|
|
|
-
|
|
|
{(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
|
|
|
<div
|
|
|
className={cn(
|
|
@@ -476,51 +480,58 @@ const StepTwo = ({
|
|
|
<div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateSegment')}</div>
|
|
|
<div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
|
|
|
{
|
|
|
- !!fileIndexingEstimate ? (
|
|
|
- <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.total_segments)} </div>
|
|
|
- ) : (
|
|
|
- <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
|
|
|
- )
|
|
|
+ fileIndexingEstimate
|
|
|
+ ? (
|
|
|
+ <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.total_segments)} </div>
|
|
|
+ )
|
|
|
+ : (
|
|
|
+ <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
|
|
|
+ )
|
|
|
}
|
|
|
</div>
|
|
|
</div>
|
|
|
</div>
|
|
|
- {!isSetting ? (
|
|
|
- <div className='flex items-center mt-8 py-2'>
|
|
|
- <Button onClick={() => onStepChange && onStepChange(-1)}>{t('datasetCreation.stepTwo.lastStep')}</Button>
|
|
|
- <div className={s.divider} />
|
|
|
- <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
|
|
|
- </div>
|
|
|
- ) : (
|
|
|
- <div className='flex items-center mt-8 py-2'>
|
|
|
- <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
|
|
|
- <Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
|
|
|
- </div>
|
|
|
- )}
|
|
|
+ {!isSetting
|
|
|
+ ? (
|
|
|
+ <div className='flex items-center mt-8 py-2'>
|
|
|
+ <Button onClick={() => onStepChange && onStepChange(-1)}>{t('datasetCreation.stepTwo.lastStep')}</Button>
|
|
|
+ <div className={s.divider} />
|
|
|
+ <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
|
|
|
+ </div>
|
|
|
+ )
|
|
|
+ : (
|
|
|
+ <div className='flex items-center mt-8 py-2'>
|
|
|
+ <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
|
|
|
+ <Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
|
|
|
+ </div>
|
|
|
+ )}
|
|
|
</div>
|
|
|
</div>
|
|
|
</div>
|
|
|
- {(showPreview) ? (
|
|
|
- <div ref={previewScrollRef} className={cn(s.previewWrap, 'relativeh-full overflow-y-scroll border-l border-[#F2F4F7]')}>
|
|
|
- <div className={cn(s.previewHeader, previewScrolled && `${s.fixed} pb-3`, ' flex items-center justify-between px-8')}>
|
|
|
- <span>{t('datasetCreation.stepTwo.previewTitle')}</span>
|
|
|
- <div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
|
|
|
- <XMarkIcon className='h-4 w-4'></XMarkIcon>
|
|
|
+ {(showPreview)
|
|
|
+ ? (
|
|
|
+ <div ref={previewScrollRef} className={cn(s.previewWrap, 'relativeh-full overflow-y-scroll border-l border-[#F2F4F7]')}>
|
|
|
+ <div className={cn(s.previewHeader, previewScrolled && `${s.fixed} pb-3`, ' flex items-center justify-between px-8')}>
|
|
|
+ <span>{t('datasetCreation.stepTwo.previewTitle')}</span>
|
|
|
+ <div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
|
|
|
+ <XMarkIcon className='h-4 w-4'></XMarkIcon>
|
|
|
+ </div>
|
|
|
+ </div>
|
|
|
+ <div className='my-4 px-8 space-y-4'>
|
|
|
+ {fileIndexingEstimate?.preview
|
|
|
+ ? (
|
|
|
+ <>
|
|
|
+ {fileIndexingEstimate?.preview.map((item, index) => (
|
|
|
+ <PreviewItem key={item} content={item} index={index + 1} />
|
|
|
+ ))}
|
|
|
+ </>
|
|
|
+ )
|
|
|
+ : <div className='flex items-center justify-center h-[200px]'><Loading type='area'></Loading></div>
|
|
|
+ }
|
|
|
</div>
|
|
|
</div>
|
|
|
- <div className='my-4 px-8 space-y-4'>
|
|
|
- {fileIndexingEstimate?.preview ? (
|
|
|
- <>
|
|
|
- {fileIndexingEstimate?.preview.map((item, index) => (
|
|
|
- <PreviewItem key={item} content={item} index={index + 1} />
|
|
|
- ))}
|
|
|
- </>
|
|
|
- ) : <div className='flex items-center justify-center h-[200px]'><Loading type='area'></Loading></div>
|
|
|
- }
|
|
|
- </div>
|
|
|
- </div>
|
|
|
- ) :
|
|
|
- (<div className={cn(s.sideTip)}>
|
|
|
+ )
|
|
|
+ : (<div className={cn(s.sideTip)}>
|
|
|
<div className={s.tipCard}>
|
|
|
<span className={s.icon} />
|
|
|
<div className={s.title}>{t('datasetCreation.stepTwo.sideTipTitle')}</div>
|