index.tsx 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732
  1. /* eslint-disable no-mixed-operators */
  2. 'use client'
  3. import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
  4. import { useTranslation } from 'react-i18next'
  5. import { useBoolean } from 'ahooks'
  6. import { XMarkIcon } from '@heroicons/react/20/solid'
  7. import cn from 'classnames'
  8. import Link from 'next/link'
  9. import { groupBy } from 'lodash-es'
  10. import PreviewItem, { PreviewType } from './preview-item'
  11. import s from './index.module.css'
  12. import type { CreateDocumentReq, File, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
  13. import {
  14. createDocument,
  15. createFirstDocument,
  16. fetchFileIndexingEstimate as didFetchFileIndexingEstimate,
  17. fetchDefaultProcessRule,
  18. } from '@/service/datasets'
  19. import Button from '@/app/components/base/button'
  20. import Loading from '@/app/components/base/loading'
  21. import Toast from '@/app/components/base/toast'
  22. import { formatNumber } from '@/utils/format'
  23. import type { DataSourceNotionPage } from '@/models/common'
  24. import { DataSourceType } from '@/models/datasets'
  25. import NotionIcon from '@/app/components/base/notion-icon'
  26. import Switch from '@/app/components/base/switch'
  27. import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
  28. import { useDatasetDetailContext } from '@/context/dataset-detail'
  29. import { IS_CE_EDITION } from '@/config'
  30. type Page = DataSourceNotionPage & { workspace_id: string }
  31. type StepTwoProps = {
  32. isSetting?: boolean
  33. documentDetail?: FullDocumentDetail
  34. hasSetAPIKEY: boolean
  35. onSetting: () => void
  36. datasetId?: string
  37. indexingType?: string
  38. dataSourceType: DataSourceType
  39. files: File[]
  40. notionPages?: Page[]
  41. onStepChange?: (delta: number) => void
  42. updateIndexingTypeCache?: (type: string) => void
  43. updateResultCache?: (res: createDocumentResponse) => void
  44. onSave?: () => void
  45. onCancel?: () => void
  46. }
  47. enum SegmentType {
  48. AUTO = 'automatic',
  49. CUSTOM = 'custom',
  50. }
  51. enum IndexingType {
  52. QUALIFIED = 'high_quality',
  53. ECONOMICAL = 'economy',
  54. }
  55. enum DocForm {
  56. TEXT = 'text_model',
  57. QA = 'qa_model',
  58. }
  59. const StepTwo = ({
  60. isSetting,
  61. documentDetail,
  62. hasSetAPIKEY,
  63. onSetting,
  64. datasetId,
  65. indexingType,
  66. dataSourceType,
  67. files,
  68. notionPages = [],
  69. onStepChange,
  70. updateIndexingTypeCache,
  71. updateResultCache,
  72. onSave,
  73. onCancel,
  74. }: StepTwoProps) => {
  75. const { t } = useTranslation()
  76. const { mutateDatasetRes } = useDatasetDetailContext()
  77. const scrollRef = useRef<HTMLDivElement>(null)
  78. const [scrolled, setScrolled] = useState(false)
  79. const previewScrollRef = useRef<HTMLDivElement>(null)
  80. const [previewScrolled, setPreviewScrolled] = useState(false)
  81. const [segmentationType, setSegmentationType] = useState<SegmentType>(SegmentType.AUTO)
  82. const [segmentIdentifier, setSegmentIdentifier] = useState('\\n')
  83. const [max, setMax] = useState(1000)
  84. const [rules, setRules] = useState<PreProcessingRule[]>([])
  85. const [defaultConfig, setDefaultConfig] = useState<Rules>()
  86. const hasSetIndexType = !!indexingType
  87. const [indexType, setIndexType] = useState<IndexingType>(
  88. indexingType
  89. || hasSetAPIKEY
  90. ? IndexingType.QUALIFIED
  91. : IndexingType.ECONOMICAL,
  92. )
  93. const [docForm, setDocForm] = useState<DocForm | string>(
  94. datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
  95. )
  96. const [previewSwitched, setPreviewSwitched] = useState(false)
  97. const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
  98. const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
  99. const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
  100. const fileIndexingEstimate = (() => {
  101. return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate
  102. })()
  103. const scrollHandle = (e: any) => {
  104. if (e.target.scrollTop > 0)
  105. setScrolled(true)
  106. else
  107. setScrolled(false)
  108. }
  109. const previewScrollHandle = (e: any) => {
  110. if (e.target.scrollTop > 0)
  111. setPreviewScrolled(true)
  112. else
  113. setPreviewScrolled(false)
  114. }
  115. const getFileName = (name: string) => {
  116. const arr = name.split('.')
  117. return arr.slice(0, -1).join('.')
  118. }
  119. const getRuleName = (key: string) => {
  120. if (key === 'remove_extra_spaces')
  121. return t('datasetCreation.stepTwo.removeExtraSpaces')
  122. if (key === 'remove_urls_emails')
  123. return t('datasetCreation.stepTwo.removeUrlEmails')
  124. if (key === 'remove_stopwords')
  125. return t('datasetCreation.stepTwo.removeStopwords')
  126. }
  127. const ruleChangeHandle = (id: string) => {
  128. const newRules = rules.map((rule) => {
  129. if (rule.id === id) {
  130. return {
  131. id: rule.id,
  132. enabled: !rule.enabled,
  133. }
  134. }
  135. return rule
  136. })
  137. setRules(newRules)
  138. }
  139. const resetRules = () => {
  140. if (defaultConfig) {
  141. setSegmentIdentifier(defaultConfig.segmentation.separator === '\n' ? '\\n' : defaultConfig.segmentation.separator || '\\n')
  142. setMax(defaultConfig.segmentation.max_tokens)
  143. setRules(defaultConfig.pre_processing_rules)
  144. }
  145. }
  146. const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT) => {
  147. // eslint-disable-next-line @typescript-eslint/no-use-before-define
  148. const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm))
  149. if (segmentationType === SegmentType.CUSTOM)
  150. setCustomFileIndexingEstimate(res)
  151. else
  152. setAutomaticFileIndexingEstimate(res)
  153. }
  154. const confirmChangeCustomConfig = () => {
  155. setCustomFileIndexingEstimate(null)
  156. setShowPreview()
  157. fetchFileIndexingEstimate()
  158. setPreviewSwitched(false)
  159. }
  160. const getIndexing_technique = () => indexingType || indexType
  161. const getProcessRule = () => {
  162. const processRule: any = {
  163. rules: {}, // api will check this. It will be removed after api refactored.
  164. mode: segmentationType,
  165. }
  166. if (segmentationType === SegmentType.CUSTOM) {
  167. const ruleObj = {
  168. pre_processing_rules: rules,
  169. segmentation: {
  170. separator: segmentIdentifier === '\\n' ? '\n' : segmentIdentifier,
  171. max_tokens: max,
  172. },
  173. }
  174. processRule.rules = ruleObj
  175. }
  176. return processRule
  177. }
  178. const getNotionInfo = () => {
  179. const workspacesMap = groupBy(notionPages, 'workspace_id')
  180. const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
  181. return {
  182. workspaceId,
  183. pages: workspacesMap[workspaceId],
  184. }
  185. })
  186. return workspaces.map((workspace) => {
  187. return {
  188. workspace_id: workspace.workspaceId,
  189. pages: workspace.pages.map((page) => {
  190. const { page_id, page_name, page_icon, type } = page
  191. return {
  192. page_id,
  193. page_name,
  194. page_icon,
  195. type,
  196. }
  197. }),
  198. }
  199. }) as NotionInfo[]
  200. }
  201. const getFileIndexingEstimateParams = (docForm: DocForm) => {
  202. let params
  203. if (dataSourceType === DataSourceType.FILE) {
  204. params = {
  205. info_list: {
  206. data_source_type: dataSourceType,
  207. file_info_list: {
  208. file_ids: files.map(file => file.id),
  209. },
  210. },
  211. indexing_technique: getIndexing_technique(),
  212. process_rule: getProcessRule(),
  213. doc_form: docForm,
  214. }
  215. }
  216. if (dataSourceType === DataSourceType.NOTION) {
  217. params = {
  218. info_list: {
  219. data_source_type: dataSourceType,
  220. notion_info_list: getNotionInfo(),
  221. },
  222. indexing_technique: getIndexing_technique(),
  223. process_rule: getProcessRule(),
  224. doc_form: docForm,
  225. }
  226. }
  227. return params
  228. }
  229. const getCreationParams = () => {
  230. let params
  231. if (isSetting) {
  232. params = {
  233. original_document_id: documentDetail?.id,
  234. doc_form: docForm,
  235. process_rule: getProcessRule(),
  236. } as CreateDocumentReq
  237. }
  238. else {
  239. params = {
  240. data_source: {
  241. type: dataSourceType,
  242. info_list: {
  243. data_source_type: dataSourceType,
  244. },
  245. },
  246. indexing_technique: getIndexing_technique(),
  247. process_rule: getProcessRule(),
  248. doc_form: docForm,
  249. } as CreateDocumentReq
  250. if (dataSourceType === DataSourceType.FILE) {
  251. params.data_source.info_list.file_info_list = {
  252. file_ids: files.map(file => file.id),
  253. }
  254. }
  255. if (dataSourceType === DataSourceType.NOTION)
  256. params.data_source.info_list.notion_info_list = getNotionInfo()
  257. }
  258. return params
  259. }
  260. const getRules = async () => {
  261. try {
  262. const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' })
  263. const separator = res.rules.segmentation.separator
  264. setSegmentIdentifier(separator === '\n' ? '\\n' : separator || '\\n')
  265. setMax(res.rules.segmentation.max_tokens)
  266. setRules(res.rules.pre_processing_rules)
  267. setDefaultConfig(res.rules)
  268. }
  269. catch (err) {
  270. console.log(err)
  271. }
  272. }
  273. const getRulesFromDetail = () => {
  274. if (documentDetail) {
  275. const rules = documentDetail.dataset_process_rule.rules
  276. const separator = rules.segmentation.separator
  277. const max = rules.segmentation.max_tokens
  278. setSegmentIdentifier(separator === '\n' ? '\\n' : separator || '\\n')
  279. setMax(max)
  280. setRules(rules.pre_processing_rules)
  281. setDefaultConfig(rules)
  282. }
  283. }
  284. const getDefaultMode = () => {
  285. if (documentDetail)
  286. setSegmentationType(documentDetail.dataset_process_rule.mode)
  287. }
  288. const createHandle = async () => {
  289. try {
  290. let res
  291. const params = getCreationParams()
  292. if (!datasetId) {
  293. res = await createFirstDocument({
  294. body: params,
  295. })
  296. updateIndexingTypeCache && updateIndexingTypeCache(indexType)
  297. updateResultCache && updateResultCache(res)
  298. }
  299. else {
  300. res = await createDocument({
  301. datasetId,
  302. body: params,
  303. })
  304. updateIndexingTypeCache && updateIndexingTypeCache(indexType)
  305. updateResultCache && updateResultCache(res)
  306. }
  307. if (mutateDatasetRes)
  308. mutateDatasetRes()
  309. onStepChange && onStepChange(+1)
  310. isSetting && onSave && onSave()
  311. }
  312. catch (err) {
  313. Toast.notify({
  314. type: 'error',
  315. message: `${err}`,
  316. })
  317. }
  318. }
  319. const handleSwitch = (state: boolean) => {
  320. if (state)
  321. setDocForm(DocForm.QA)
  322. else
  323. setDocForm(DocForm.TEXT)
  324. }
  325. const changeToEconomicalType = () => {
  326. if (!hasSetIndexType) {
  327. setIndexType(IndexingType.ECONOMICAL)
  328. setDocForm(DocForm.TEXT)
  329. }
  330. }
  331. const previewSwitch = async () => {
  332. setPreviewSwitched(true)
  333. if (segmentationType === SegmentType.AUTO)
  334. setAutomaticFileIndexingEstimate(null)
  335. else
  336. setCustomFileIndexingEstimate(null)
  337. await fetchFileIndexingEstimate(DocForm.QA)
  338. }
  339. useEffect(() => {
  340. // fetch rules
  341. if (!isSetting) {
  342. getRules()
  343. }
  344. else {
  345. getRulesFromDetail()
  346. getDefaultMode()
  347. }
  348. }, [])
  349. useEffect(() => {
  350. scrollRef.current?.addEventListener('scroll', scrollHandle)
  351. return () => {
  352. scrollRef.current?.removeEventListener('scroll', scrollHandle)
  353. }
  354. }, [])
  355. useLayoutEffect(() => {
  356. if (showPreview) {
  357. previewScrollRef.current?.addEventListener('scroll', previewScrollHandle)
  358. return () => {
  359. previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle)
  360. }
  361. }
  362. }, [showPreview])
  363. useEffect(() => {
  364. if (indexingType === IndexingType.ECONOMICAL && docForm === DocForm.QA)
  365. setDocForm(DocForm.TEXT)
  366. }, [indexingType, docForm])
  367. useEffect(() => {
  368. // get indexing type by props
  369. if (indexingType)
  370. setIndexType(indexingType as IndexingType)
  371. else
  372. setIndexType(hasSetAPIKEY ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL)
  373. }, [hasSetAPIKEY, indexingType, datasetId])
  374. useEffect(() => {
  375. if (segmentationType === SegmentType.AUTO) {
  376. setAutomaticFileIndexingEstimate(null)
  377. setShowPreview()
  378. fetchFileIndexingEstimate()
  379. setPreviewSwitched(false)
  380. }
  381. else {
  382. hidePreview()
  383. setCustomFileIndexingEstimate(null)
  384. setPreviewSwitched(false)
  385. }
  386. }, [segmentationType, indexType])
  387. return (
  388. <div className='flex w-full h-full'>
  389. <div ref={scrollRef} className='relative h-full w-full overflow-y-scroll'>
  390. <div className={cn(s.pageHeader, scrolled && s.fixed)}>{t('datasetCreation.steps.two')}</div>
  391. <div className={cn(s.form)}>
  392. <div className={s.label}>{t('datasetCreation.stepTwo.segmentation')}</div>
  393. <div className='max-w-[640px]'>
  394. <div
  395. className={cn(
  396. s.radioItem,
  397. s.segmentationItem,
  398. segmentationType === SegmentType.AUTO && s.active,
  399. )}
  400. onClick={() => setSegmentationType(SegmentType.AUTO)}
  401. >
  402. <span className={cn(s.typeIcon, s.auto)} />
  403. <span className={cn(s.radio)} />
  404. <div className={s.typeHeader}>
  405. <div className={s.title}>{t('datasetCreation.stepTwo.auto')}</div>
  406. <div className={s.tip}>{t('datasetCreation.stepTwo.autoDescription')}</div>
  407. </div>
  408. </div>
  409. <div
  410. className={cn(
  411. s.radioItem,
  412. s.segmentationItem,
  413. segmentationType === SegmentType.CUSTOM && s.active,
  414. segmentationType === SegmentType.CUSTOM && s.custom,
  415. )}
  416. onClick={() => setSegmentationType(SegmentType.CUSTOM)}
  417. >
  418. <span className={cn(s.typeIcon, s.customize)} />
  419. <span className={cn(s.radio)} />
  420. <div className={s.typeHeader}>
  421. <div className={s.title}>{t('datasetCreation.stepTwo.custom')}</div>
  422. <div className={s.tip}>{t('datasetCreation.stepTwo.customDescription')}</div>
  423. </div>
  424. {segmentationType === SegmentType.CUSTOM && (
  425. <div className={s.typeFormBody}>
  426. <div className={s.formRow}>
  427. <div className='w-full'>
  428. <div className={s.label}>{t('datasetCreation.stepTwo.separator')}</div>
  429. <input
  430. type="text"
  431. className={s.input}
  432. placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={segmentIdentifier}
  433. onChange={e => setSegmentIdentifier(e.target.value)}
  434. />
  435. </div>
  436. </div>
  437. <div className={s.formRow}>
  438. <div className='w-full'>
  439. <div className={s.label}>{t('datasetCreation.stepTwo.maxLength')}</div>
  440. <input
  441. type="number"
  442. className={s.input}
  443. placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={max}
  444. onChange={e => setMax(Number(e.target.value))}
  445. />
  446. </div>
  447. </div>
  448. <div className={s.formRow}>
  449. <div className='w-full'>
  450. <div className={s.label}>{t('datasetCreation.stepTwo.rules')}</div>
  451. {rules.map(rule => (
  452. <div key={rule.id} className={s.ruleItem}>
  453. <input id={rule.id} type="checkbox" defaultChecked={rule.enabled} onChange={() => ruleChangeHandle(rule.id)} className="w-4 h-4 rounded border-gray-300 text-blue-700 focus:ring-blue-700" />
  454. <label htmlFor={rule.id} className="ml-2 text-sm font-normal cursor-pointer text-gray-800">{getRuleName(rule.id)}</label>
  455. </div>
  456. ))}
  457. </div>
  458. </div>
  459. <div className={s.formFooter}>
  460. <Button type="primary" className={cn(s.button, '!h-8 text-primary-600')} onClick={confirmChangeCustomConfig}>{t('datasetCreation.stepTwo.preview')}</Button>
  461. <Button className={cn(s.button, 'ml-2 !h-8')} onClick={resetRules}>{t('datasetCreation.stepTwo.reset')}</Button>
  462. </div>
  463. </div>
  464. )}
  465. </div>
  466. </div>
  467. <div className={s.label}>{t('datasetCreation.stepTwo.indexMode')}</div>
  468. <div className='max-w-[640px]'>
  469. <div className='flex items-center gap-3'>
  470. {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
  471. <div
  472. className={cn(
  473. s.radioItem,
  474. s.indexItem,
  475. !hasSetAPIKEY && s.disabled,
  476. !hasSetIndexType && indexType === IndexingType.QUALIFIED && s.active,
  477. hasSetIndexType && s.disabled,
  478. hasSetIndexType && '!w-full',
  479. )}
  480. onClick={() => {
  481. if (hasSetAPIKEY)
  482. setIndexType(IndexingType.QUALIFIED)
  483. }}
  484. >
  485. <span className={cn(s.typeIcon, s.qualified)} />
  486. {!hasSetIndexType && <span className={cn(s.radio)} />}
  487. <div className={s.typeHeader}>
  488. <div className={s.title}>
  489. {t('datasetCreation.stepTwo.qualified')}
  490. {!hasSetIndexType && <span className={s.recommendTag}>{t('datasetCreation.stepTwo.recommend')}</span>}
  491. </div>
  492. <div className={s.tip}>{t('datasetCreation.stepTwo.qualifiedTip')}</div>
  493. <div className='pb-0.5 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateCost')}</div>
  494. {
  495. fileIndexingEstimate
  496. ? (
  497. <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.tokens)} tokens(<span className='text-yellow-500'>${formatNumber(fileIndexingEstimate.total_price)}</span>)</div>
  498. )
  499. : (
  500. <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
  501. )
  502. }
  503. </div>
  504. {!hasSetAPIKEY && (
  505. <div className={s.warningTip}>
  506. <span>{t('datasetCreation.stepTwo.warning')}&nbsp;</span>
  507. <span className={s.click} onClick={onSetting}>{t('datasetCreation.stepTwo.click')}</span>
  508. </div>
  509. )}
  510. </div>
  511. )}
  512. {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && (
  513. <div
  514. className={cn(
  515. s.radioItem,
  516. s.indexItem,
  517. !hasSetIndexType && indexType === IndexingType.ECONOMICAL && s.active,
  518. hasSetIndexType && s.disabled,
  519. hasSetIndexType && '!w-full',
  520. )}
  521. onClick={changeToEconomicalType}
  522. >
  523. <span className={cn(s.typeIcon, s.economical)} />
  524. {!hasSetIndexType && <span className={cn(s.radio)} />}
  525. <div className={s.typeHeader}>
  526. <div className={s.title}>{t('datasetCreation.stepTwo.economical')}</div>
  527. <div className={s.tip}>{t('datasetCreation.stepTwo.economicalTip')}</div>
  528. <div className='pb-0.5 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateCost')}</div>
  529. <div className='text-xs font-medium text-gray-800'>0 tokens</div>
  530. </div>
  531. </div>
  532. )}
  533. </div>
  534. {hasSetIndexType && (
  535. <div className='mt-2 text-xs text-gray-500 font-medium'>
  536. {t('datasetCreation.stepTwo.indexSettedTip')}
  537. <Link className='text-[#155EEF]' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
  538. </div>
  539. )}
  540. {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
  541. <div className='flex justify-between items-center mt-3 px-5 py-4 rounded-xl bg-gray-50 border border-gray-100'>
  542. <div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
  543. <MessageChatSquare className='w-4 h-4' />
  544. </div>
  545. <div className='grow mx-3'>
  546. <div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
  547. <div className='text-[13px] leading-[18px] text-gray-500'>{t('datasetCreation.stepTwo.QATip')}</div>
  548. </div>
  549. <div className='shrink-0'>
  550. <Switch
  551. defaultValue={docForm === DocForm.QA}
  552. onChange={handleSwitch}
  553. size='md'
  554. />
  555. </div>
  556. </div>
  557. )}
  558. <div className={s.source}>
  559. <div className={s.sourceContent}>
  560. {dataSourceType === DataSourceType.FILE && (
  561. <>
  562. <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.fileSource')}</div>
  563. <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
  564. <span className={cn(s.fileIcon, files.length && s[files[0].extension])} />
  565. {getFileName(files[0].name || '')}
  566. {files.length > 1 && (
  567. <span className={s.sourceCount}>
  568. <span>{t('datasetCreation.stepTwo.other')}</span>
  569. <span>{files.length - 1}</span>
  570. <span>{t('datasetCreation.stepTwo.fileUnit')}</span>
  571. </span>
  572. )}
  573. </div>
  574. </>
  575. )}
  576. {dataSourceType === DataSourceType.NOTION && (
  577. <>
  578. <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.notionSource')}</div>
  579. <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
  580. <NotionIcon
  581. className='shrink-0 mr-1'
  582. type='page'
  583. src={notionPages[0]?.page_icon}
  584. />
  585. {notionPages[0]?.page_name}
  586. {notionPages.length > 1 && (
  587. <span className={s.sourceCount}>
  588. <span>{t('datasetCreation.stepTwo.other')}</span>
  589. <span>{notionPages.length - 1}</span>
  590. <span>{t('datasetCreation.stepTwo.notionUnit')}</span>
  591. </span>
  592. )}
  593. </div>
  594. </>
  595. )}
  596. </div>
  597. <div className={s.divider} />
  598. <div className={s.segmentCount}>
  599. <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.emstimateSegment')}</div>
  600. <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
  601. {
  602. fileIndexingEstimate
  603. ? (
  604. <div className='text-xs font-medium text-gray-800'>{formatNumber(fileIndexingEstimate.total_segments)} </div>
  605. )
  606. : (
  607. <div className={s.calculating}>{t('datasetCreation.stepTwo.calculating')}</div>
  608. )
  609. }
  610. </div>
  611. </div>
  612. </div>
  613. {!isSetting
  614. ? (
  615. <div className='flex items-center mt-8 py-2'>
  616. <Button onClick={() => onStepChange && onStepChange(-1)}>{t('datasetCreation.stepTwo.lastStep')}</Button>
  617. <div className={s.divider} />
  618. <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.nextStep')}</Button>
  619. </div>
  620. )
  621. : (
  622. <div className='flex items-center mt-8 py-2'>
  623. <Button type='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
  624. <Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
  625. </div>
  626. )}
  627. </div>
  628. </div>
  629. </div>
  630. {(showPreview)
  631. ? (
  632. <div ref={previewScrollRef} className={cn(s.previewWrap, 'relativeh-full overflow-y-scroll border-l border-[#F2F4F7]')}>
  633. <div className={cn(s.previewHeader, previewScrolled && `${s.fixed} pb-3`)}>
  634. <div className='flex items-center justify-between px-8'>
  635. <div className='grow flex items-center'>
  636. <div>{t('datasetCreation.stepTwo.previewTitle')}</div>
  637. {docForm === DocForm.QA && !previewSwitched && (
  638. <Button className='ml-2 !h-[26px] !py-[3px] !px-2 !text-xs !font-medium !text-primary-600' onClick={previewSwitch}>{t('datasetCreation.stepTwo.previewButton')}</Button>
  639. )}
  640. </div>
  641. <div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
  642. <XMarkIcon className='h-4 w-4'></XMarkIcon>
  643. </div>
  644. </div>
  645. {docForm === DocForm.QA && !previewSwitched && (
  646. <div className='px-8 pr-12 text-xs text-gray-500'>
  647. <span>{t('datasetCreation.stepTwo.previewSwitchTipStart')}</span>
  648. <span className='text-amber-600'>{t('datasetCreation.stepTwo.previewSwitchTipEnd')}</span>
  649. </div>
  650. )}
  651. </div>
  652. <div className='my-4 px-8 space-y-4'>
  653. {previewSwitched && docForm === DocForm.QA && fileIndexingEstimate?.qa_preview && (
  654. <>
  655. {fileIndexingEstimate?.qa_preview.map((item, index) => (
  656. <PreviewItem type={PreviewType.QA} key={item.question} qa={item} index={index + 1} />
  657. ))}
  658. </>
  659. )}
  660. {(docForm === DocForm.TEXT || !previewSwitched) && fileIndexingEstimate?.preview && (
  661. <>
  662. {fileIndexingEstimate?.preview.map((item, index) => (
  663. <PreviewItem type={PreviewType.TEXT} key={item} content={item} index={index + 1} />
  664. ))}
  665. </>
  666. )}
  667. {previewSwitched && docForm === DocForm.QA && !fileIndexingEstimate?.qa_preview && (
  668. <div className='flex items-center justify-center h-[200px]'>
  669. <Loading type='area' />
  670. </div>
  671. )}
  672. {!previewSwitched && !fileIndexingEstimate?.preview && (
  673. <div className='flex items-center justify-center h-[200px]'>
  674. <Loading type='area' />
  675. </div>
  676. )}
  677. </div>
  678. </div>
  679. )
  680. : (<div className={cn(s.sideTip)}>
  681. <div className={s.tipCard}>
  682. <span className={s.icon} />
  683. <div className={s.title}>{t('datasetCreation.stepTwo.sideTipTitle')}</div>
  684. <div className={s.content}>
  685. <p className='mb-3'>{t('datasetCreation.stepTwo.sideTipP1')}</p>
  686. <p className='mb-3'>{t('datasetCreation.stepTwo.sideTipP2')}</p>
  687. <p className='mb-3'>{t('datasetCreation.stepTwo.sideTipP3')}</p>
  688. <p>{t('datasetCreation.stepTwo.sideTipP4')}</p>
  689. </div>
  690. </div>
  691. </div>)}
  692. </div>
  693. )
  694. }
  695. export default StepTwo