dify
This commit is contained in:
@@ -0,0 +1,238 @@
|
||||
import React, { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import PreviewContainer from '../../../preview/container'
|
||||
import { PreviewHeader } from '../../../preview/header'
|
||||
import type { CrawlResultItem, CustomFile, DocumentItem, FileIndexingEstimateResponse } from '@/models/datasets'
|
||||
import { ChunkingMode } from '@/models/datasets'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import PreviewDocumentPicker from '../../../common/document-picker/preview-document-picker'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { ChunkContainer, QAPreview } from '../../../chunk'
|
||||
import { FormattedText } from '../../../formatted-text/formatted'
|
||||
import { PreviewSlice } from '../../../formatted-text/flavours/preview-slice'
|
||||
import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
|
||||
import { RiSearchEyeLine } from '@remixicon/react'
|
||||
import Badge from '@/app/components/base/badge'
|
||||
import Button from '@/app/components/base/button'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { getFileExtension } from '../data-source/online-drive/file-list/list/utils'
|
||||
|
||||
type ChunkPreviewProps = {
|
||||
dataSourceType: DatasourceType
|
||||
localFiles: CustomFile[]
|
||||
onlineDocuments: NotionPage[]
|
||||
websitePages: CrawlResultItem[]
|
||||
onlineDriveFiles: OnlineDriveFile[]
|
||||
isIdle: boolean
|
||||
isPending: boolean
|
||||
estimateData: FileIndexingEstimateResponse | undefined
|
||||
onPreview: () => void
|
||||
handlePreviewFileChange: (file: DocumentItem) => void
|
||||
handlePreviewOnlineDocumentChange: (page: NotionPage) => void
|
||||
handlePreviewWebsitePageChange: (page: CrawlResultItem) => void
|
||||
handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void
|
||||
}
|
||||
|
||||
const ChunkPreview = ({
|
||||
dataSourceType,
|
||||
localFiles,
|
||||
onlineDocuments,
|
||||
websitePages,
|
||||
onlineDriveFiles,
|
||||
isIdle,
|
||||
isPending,
|
||||
estimateData,
|
||||
onPreview,
|
||||
handlePreviewFileChange,
|
||||
handlePreviewOnlineDocumentChange,
|
||||
handlePreviewWebsitePageChange,
|
||||
handlePreviewOnlineDriveFileChange,
|
||||
}: ChunkPreviewProps) => {
|
||||
const { t } = useTranslation()
|
||||
const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form)
|
||||
|
||||
const [previewFile, setPreviewFile] = useState<DocumentItem>(localFiles[0] as DocumentItem)
|
||||
const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
|
||||
const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
|
||||
const [previewOnlineDriveFile, setPreviewOnlineDriveFile] = useState<OnlineDriveFile>(onlineDriveFiles[0])
|
||||
|
||||
return (
|
||||
<PreviewContainer
|
||||
header={<PreviewHeader
|
||||
title={t('datasetCreation.stepTwo.preview')}
|
||||
>
|
||||
<div className='flex items-center gap-1'>
|
||||
{dataSourceType === DatasourceType.localFile
|
||||
&& <PreviewDocumentPicker
|
||||
files={localFiles as Array<Required<CustomFile>>}
|
||||
onChange={(selected) => {
|
||||
setPreviewFile(selected)
|
||||
handlePreviewFileChange(selected)
|
||||
}}
|
||||
value={previewFile}
|
||||
/>
|
||||
}
|
||||
{dataSourceType === DatasourceType.onlineDocument
|
||||
&& <PreviewDocumentPicker
|
||||
files={
|
||||
onlineDocuments.map(page => ({
|
||||
id: page.page_id,
|
||||
name: page.page_name,
|
||||
extension: 'md',
|
||||
}))
|
||||
}
|
||||
onChange={(selected) => {
|
||||
const selectedPage = onlineDocuments.find(page => page.page_id === selected.id)
|
||||
setPreviewOnlineDocument(selectedPage!)
|
||||
handlePreviewOnlineDocumentChange(selectedPage!)
|
||||
}}
|
||||
value={{
|
||||
id: previewOnlineDocument?.page_id || '',
|
||||
name: previewOnlineDocument?.page_name || '',
|
||||
extension: 'md',
|
||||
}}
|
||||
/>
|
||||
}
|
||||
{dataSourceType === DatasourceType.websiteCrawl
|
||||
&& <PreviewDocumentPicker
|
||||
files={
|
||||
websitePages.map(page => ({
|
||||
id: page.source_url,
|
||||
name: page.title,
|
||||
extension: 'md',
|
||||
}))
|
||||
}
|
||||
onChange={(selected) => {
|
||||
const selectedPage = websitePages.find(page => page.source_url === selected.id)
|
||||
setPreviewWebsitePage(selectedPage!)
|
||||
handlePreviewWebsitePageChange(selectedPage!)
|
||||
}}
|
||||
value={
|
||||
{
|
||||
id: previewWebsitePage?.source_url || '',
|
||||
name: previewWebsitePage?.title || '',
|
||||
extension: 'md',
|
||||
}
|
||||
}
|
||||
/>
|
||||
}
|
||||
{dataSourceType === DatasourceType.onlineDrive
|
||||
&& <PreviewDocumentPicker
|
||||
files={
|
||||
onlineDriveFiles.map(file => ({
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
extension: getFileExtension(previewOnlineDriveFile?.name),
|
||||
}))
|
||||
}
|
||||
onChange={(selected) => {
|
||||
const selectedFile = onlineDriveFiles.find(file => file.id === selected.id)
|
||||
setPreviewOnlineDriveFile(selectedFile!)
|
||||
handlePreviewOnlineDriveFileChange(selectedFile!)
|
||||
}}
|
||||
value={
|
||||
{
|
||||
id: previewOnlineDriveFile?.id || '',
|
||||
name: previewOnlineDriveFile?.name || '',
|
||||
extension: getFileExtension(previewOnlineDriveFile?.name),
|
||||
}
|
||||
}
|
||||
/>
|
||||
}
|
||||
{
|
||||
currentDocForm !== ChunkingMode.qa
|
||||
&& <Badge text={t('datasetCreation.stepTwo.previewChunkCount', {
|
||||
count: estimateData?.total_segments || 0,
|
||||
}) as string}
|
||||
/>
|
||||
}
|
||||
</div>
|
||||
</PreviewHeader>}
|
||||
className='relative flex h-full w-full shrink-0'
|
||||
mainClassName='space-y-6'
|
||||
>
|
||||
{!isPending && currentDocForm === ChunkingMode.qa && estimateData?.qa_preview && (
|
||||
estimateData?.qa_preview.map((item, index) => (
|
||||
<ChunkContainer
|
||||
key={`${item.question}-${index}`}
|
||||
label={`Chunk-${index + 1}`}
|
||||
characterCount={item.question.length + item.answer.length}
|
||||
>
|
||||
<QAPreview qa={item} />
|
||||
</ChunkContainer>
|
||||
))
|
||||
)}
|
||||
{!isPending && currentDocForm === ChunkingMode.text && estimateData?.preview && (
|
||||
estimateData?.preview.map((item, index) => (
|
||||
<ChunkContainer
|
||||
key={`${item.content}-${index}`}
|
||||
label={`Chunk-${index + 1}`}
|
||||
characterCount={item.content.length}
|
||||
>
|
||||
{item.content}
|
||||
</ChunkContainer>
|
||||
))
|
||||
)}
|
||||
{!isPending && currentDocForm === ChunkingMode.parentChild && estimateData?.preview && (
|
||||
estimateData?.preview?.map((item, index) => {
|
||||
const indexForLabel = index + 1
|
||||
return (
|
||||
<ChunkContainer
|
||||
key={`${item.content}-${index}`}
|
||||
label={`Chunk-${indexForLabel}`}
|
||||
characterCount={item.content.length}
|
||||
>
|
||||
<FormattedText>
|
||||
{item.child_chunks.map((child, index) => {
|
||||
const indexForLabel = index + 1
|
||||
return (
|
||||
<PreviewSlice
|
||||
key={child}
|
||||
label={`C-${indexForLabel}`}
|
||||
text={child}
|
||||
tooltip={`Child-chunk-${indexForLabel} · ${child.length} Characters`}
|
||||
labelInnerClassName='text-[10px] font-semibold align-bottom leading-7'
|
||||
dividerClassName='leading-7'
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</FormattedText>
|
||||
</ChunkContainer>
|
||||
)
|
||||
})
|
||||
)}
|
||||
{isIdle && (
|
||||
<div className='flex h-full w-full items-center justify-center'>
|
||||
<div className='flex flex-col items-center justify-center gap-3 pb-4'>
|
||||
<RiSearchEyeLine className='size-10 text-text-empty-state-icon' />
|
||||
<p className='text-sm text-text-tertiary'>
|
||||
{t('datasetCreation.stepTwo.previewChunkTip')}
|
||||
</p>
|
||||
<Button onClick={onPreview}>
|
||||
{t('datasetPipeline.addDocuments.stepTwo.previewChunks')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{isPending && (
|
||||
<div className='h-full w-full space-y-6 overflow-hidden'>
|
||||
{Array.from({ length: 10 }, (_, i) => (
|
||||
<SkeletonContainer key={i}>
|
||||
<SkeletonRow>
|
||||
<SkeletonRectangle className='w-20' />
|
||||
<SkeletonPoint />
|
||||
<SkeletonRectangle className='w-24' />
|
||||
</SkeletonRow>
|
||||
<SkeletonRectangle className='w-full' />
|
||||
<SkeletonRectangle className='w-full' />
|
||||
<SkeletonRectangle className='w-[422px]' />
|
||||
</SkeletonContainer>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</PreviewContainer>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(ChunkPreview)
|
||||
@@ -0,0 +1,75 @@
|
||||
'use client'
|
||||
import React, { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Loading from './loading'
|
||||
import type { CustomFile as File } from '@/models/datasets'
|
||||
import { RiCloseLine } from '@remixicon/react'
|
||||
import { useFilePreview } from '@/service/use-common'
|
||||
import DocumentFileIcon from '../../../common/document-file-icon'
|
||||
import { formatFileSize, formatNumberAbbreviated } from '@/utils/format'
|
||||
|
||||
type FilePreviewProps = {
|
||||
file: File
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const FilePreview = ({
|
||||
file,
|
||||
hidePreview,
|
||||
}: FilePreviewProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { data: fileData, isFetching } = useFilePreview(file.id || '')
|
||||
|
||||
const fileName = useMemo(() => {
|
||||
if (!file)
|
||||
return ''
|
||||
const arr = file.name.split('.')
|
||||
return arr.slice(0, -1).join()
|
||||
}, [file])
|
||||
|
||||
return (
|
||||
<div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
|
||||
<div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
|
||||
<div className='flex grow flex-col gap-y-1'>
|
||||
<div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{`${fileName}.${file.extension || ''}`}</div>
|
||||
<div className='system-xs-medium flex items-center gap-x-1 text-text-tertiary'>
|
||||
<DocumentFileIcon
|
||||
className='size-3.5 shrink-0'
|
||||
name={file.name}
|
||||
extension={file.extension}
|
||||
/>
|
||||
<span className='uppercase'>{file.extension}</span>
|
||||
<span>·</span>
|
||||
<span>{formatFileSize(file.size)}</span>
|
||||
{fileData && (
|
||||
<>
|
||||
<span>·</span>
|
||||
<span>{`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type='button'
|
||||
className='flex h-8 w-8 shrink-0 items-center justify-center'
|
||||
onClick={hidePreview}
|
||||
>
|
||||
<RiCloseLine className='size-[18px]' />
|
||||
</button>
|
||||
</div>
|
||||
{isFetching && (
|
||||
<div className='grow'>
|
||||
<Loading />
|
||||
</div>
|
||||
)}
|
||||
{!isFetching && fileData && (
|
||||
<div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
|
||||
{fileData.content}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FilePreview
|
||||
@@ -0,0 +1,52 @@
|
||||
import React from 'react'
|
||||
import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton'
|
||||
|
||||
const Loading = () => {
|
||||
return (
|
||||
<div className='flex h-full w-full flex-col gap-y-3 overflow-hidden bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg px-6 py-5'>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-3/5' />
|
||||
</SkeletonContainer>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-[70%]' />
|
||||
</SkeletonContainer>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-[56%]' />
|
||||
</SkeletonContainer>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-3/5' />
|
||||
</SkeletonContainer>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-3/5' />
|
||||
</SkeletonContainer>
|
||||
<SkeletonContainer className='w-full gap-0'>
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-full' />
|
||||
<SkeletonRectangle className='my-1.5 w-1/2' />
|
||||
</SkeletonContainer>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Loading)
|
||||
@@ -0,0 +1,89 @@
|
||||
'use client'
|
||||
import React, { useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { RiCloseLine } from '@remixicon/react'
|
||||
import { formatNumberAbbreviated } from '@/utils/format'
|
||||
import Loading from './loading'
|
||||
import { Notion } from '@/app/components/base/icons/src/public/common'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { usePreviewOnlineDocument } from '@/service/use-pipeline'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { Markdown } from '@/app/components/base/markdown'
|
||||
import { useDataSourceStore } from '../data-source/store'
|
||||
|
||||
type OnlineDocumentPreviewProps = {
|
||||
currentPage: NotionPage
|
||||
datasourceNodeId: string
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const OnlineDocumentPreview = ({
|
||||
currentPage,
|
||||
datasourceNodeId,
|
||||
hidePreview,
|
||||
}: OnlineDocumentPreviewProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [content, setContent] = useState('')
|
||||
const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
|
||||
const { mutateAsync: getOnlineDocumentContent, isPending } = usePreviewOnlineDocument()
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
|
||||
useEffect(() => {
|
||||
const { currentCredentialId } = dataSourceStore.getState()
|
||||
getOnlineDocumentContent({
|
||||
workspaceID: currentPage.workspace_id,
|
||||
pageID: currentPage.page_id,
|
||||
pageType: currentPage.type,
|
||||
pipelineId: pipelineId || '',
|
||||
datasourceNodeId,
|
||||
credentialId: currentCredentialId,
|
||||
}, {
|
||||
onSuccess(data) {
|
||||
setContent(data.content)
|
||||
},
|
||||
onError(error) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: error.message,
|
||||
})
|
||||
},
|
||||
})
|
||||
}, [currentPage.page_id])
|
||||
|
||||
return (
|
||||
<div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
|
||||
<div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
|
||||
<div className='flex grow flex-col gap-y-1'>
|
||||
<div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
|
||||
<div className='system-xs-medium flex items-center gap-x-1 text-text-tertiary'>
|
||||
<Notion className='size-3.5' />
|
||||
<span>{currentPage.type}</span>
|
||||
<span>·</span>
|
||||
<span>{`${formatNumberAbbreviated(content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type='button'
|
||||
className='flex h-8 w-8 shrink-0 items-center justify-center'
|
||||
onClick={hidePreview}
|
||||
>
|
||||
<RiCloseLine className='size-[18px]' />
|
||||
</button>
|
||||
</div>
|
||||
{isPending && (
|
||||
<div className='grow'>
|
||||
<Loading />
|
||||
</div>
|
||||
)}
|
||||
{!isPending && content && (
|
||||
<div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
|
||||
<Markdown content={content} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDocumentPreview
|
||||
@@ -0,0 +1,48 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
|
||||
import { formatNumberAbbreviated } from '@/utils/format'
|
||||
|
||||
type WebsitePreviewProps = {
|
||||
currentWebsite: CrawlResultItem
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const WebsitePreview = ({
|
||||
currentWebsite,
|
||||
hidePreview,
|
||||
}: WebsitePreviewProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
|
||||
<div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
|
||||
<div className='flex grow flex-col gap-y-1'>
|
||||
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{currentWebsite.title}</div>
|
||||
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
|
||||
<RiGlobalLine className='size-3.5' />
|
||||
<span className='uppercase' title={currentWebsite.source_url}>{currentWebsite.source_url}</span>
|
||||
<span>·</span>
|
||||
<span>·</span>
|
||||
<span>{`${formatNumberAbbreviated(currentWebsite.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type='button'
|
||||
className='flex h-8 w-8 shrink-0 items-center justify-center'
|
||||
onClick={hidePreview}
|
||||
>
|
||||
<RiCloseLine className='size-[18px]' />
|
||||
</button>
|
||||
</div>
|
||||
<div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
|
||||
{currentWebsite.content}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default WebsitePreview
|
||||
Reference in New Issue
Block a user