dify

dify/web/app/components/datasets/documents/detail/settings/document-settings.tsx

@@ -0,0 +1,163 @@
+import React, { useMemo } from 'react'
+import { useTranslation } from 'react-i18next'
+import { useBoolean } from 'ahooks'
+import { useContext } from 'use-context-selector'
+import { useRouter } from 'next/navigation'
+import DatasetDetailContext from '@/context/dataset-detail'
+import type {
+  CrawlOptions,
+  CustomFile,
+  DataSourceInfo,
+  DataSourceType,
+  LegacyDataSourceInfo,
+  LocalFileInfo,
+  OnlineDocumentInfo,
+  WebsiteCrawlInfo,
+} from '@/models/datasets'
+import type { DataSourceProvider } from '@/models/common'
+import Loading from '@/app/components/base/loading'
+import StepTwo from '@/app/components/datasets/create/step-two'
+import AccountSetting from '@/app/components/header/account-setting'
+import AppUnavailable from '@/app/components/base/app-unavailable'
+import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { NotionPage } from '@/models/common'
+import { useDocumentDetail, useInvalidDocumentDetail, useInvalidDocumentList } from '@/service/knowledge/use-document'
+
+type DocumentSettingsProps = {
+  datasetId: string
+  documentId: string
+}
+
+const DocumentSettings = ({ datasetId, documentId }: DocumentSettingsProps) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const [isShowSetAPIKey, { setTrue: showSetAPIKey, setFalse: hideSetAPIkey }] = useBoolean()
+  const { indexingTechnique, dataset } = useContext(DatasetDetailContext)
+  const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
+
+  const invalidDocumentList = useInvalidDocumentList(datasetId)
+  const invalidDocumentDetail = useInvalidDocumentDetail()
+  const saveHandler = () => {
+    invalidDocumentList()
+    invalidDocumentDetail()
+    router.push(`/datasets/${datasetId}/documents/${documentId}`)
+  }
+
+  const cancelHandler = () => router.back()
+
+  const { data: documentDetail, error } = useDocumentDetail({
+    datasetId,
+    documentId,
+    params: { metadata: 'without' },
+  })
+
+  const dataSourceInfo = documentDetail?.data_source_info
+
+  const isLegacyDataSourceInfo = (info: DataSourceInfo | undefined): info is LegacyDataSourceInfo => {
+    return !!info && 'upload_file' in info
+  }
+  const isWebsiteCrawlInfo = (info: DataSourceInfo | undefined): info is WebsiteCrawlInfo => {
+    return !!info && 'source_url' in info && 'title' in info
+  }
+  const isOnlineDocumentInfo = (info: DataSourceInfo | undefined): info is OnlineDocumentInfo => {
+    return !!info && 'page' in info
+  }
+  const isLocalFileInfo = (info: DataSourceInfo | undefined): info is LocalFileInfo => {
+    return !!info && 'related_id' in info && 'transfer_method' in info
+  }
+  const legacyInfo = isLegacyDataSourceInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const websiteInfo = isWebsiteCrawlInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const onlineDocumentInfo = isOnlineDocumentInfo(dataSourceInfo) ? dataSourceInfo : undefined
+  const localFileInfo = isLocalFileInfo(dataSourceInfo) ? dataSourceInfo : undefined
+
+  const currentPage = useMemo(() => {
+    if (legacyInfo) {
+      return {
+        workspace_id: legacyInfo.notion_workspace_id ?? '',
+        page_id: legacyInfo.notion_page_id ?? '',
+        page_name: documentDetail?.name,
+        page_icon: legacyInfo.notion_page_icon,
+        type: documentDetail?.data_source_type,
+      }
+    }
+    if (onlineDocumentInfo) {
+      return {
+        workspace_id: onlineDocumentInfo.workspace_id,
+        page_id: onlineDocumentInfo.page.page_id,
+        page_name: onlineDocumentInfo.page.page_name,
+        page_icon: onlineDocumentInfo.page.page_icon,
+        type: onlineDocumentInfo.page.type,
+      }
+    }
+    return undefined
+  }, [documentDetail?.data_source_type, documentDetail?.name, legacyInfo, onlineDocumentInfo])
+
+  const files = useMemo<CustomFile[]>(() => {
+    if (legacyInfo?.upload_file)
+      return [legacyInfo.upload_file as CustomFile]
+    if (localFileInfo) {
+      const { related_id, name, extension } = localFileInfo
+      return [{
+        id: related_id,
+        name,
+        extension,
+      } as unknown as CustomFile]
+    }
+    return []
+  }, [legacyInfo?.upload_file, localFileInfo])
+
+  const websitePages = useMemo(() => {
+    if (!websiteInfo)
+      return []
+    return [{
+      title: websiteInfo.title,
+      source_url: websiteInfo.source_url,
+      content: websiteInfo.content,
+      description: websiteInfo.description,
+    }]
+  }, [websiteInfo])
+
+  const crawlOptions = (dataSourceInfo && typeof dataSourceInfo === 'object' && 'includes' in dataSourceInfo && 'excludes' in dataSourceInfo)
+    ? dataSourceInfo as unknown as CrawlOptions
+    : undefined
+
+  const websiteCrawlProvider = (websiteInfo?.provider ?? legacyInfo?.provider) as DataSourceProvider | undefined
+  const websiteCrawlJobId = websiteInfo?.job_id ?? legacyInfo?.job_id
+
+  if (error)
+    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
+
+  return (
+    <div className='flex' style={{ height: 'calc(100vh - 56px)' }}>
+      <div className='grow'>
+        {!documentDetail && <Loading type='app' />}
+        {dataset && documentDetail && (
+          <StepTwo
+            isAPIKeySet={!!embeddingsDefaultModel}
+            onSetting={showSetAPIKey}
+            datasetId={datasetId}
+            dataSourceType={documentDetail.data_source_type as DataSourceType}
+            notionPages={currentPage ? [currentPage as unknown as NotionPage] : []}
+            notionCredentialId={legacyInfo?.credential_id || onlineDocumentInfo?.credential_id || ''}
+            websitePages={websitePages}
+            websiteCrawlProvider={websiteCrawlProvider}
+            websiteCrawlJobId={websiteCrawlJobId || ''}
+            crawlOptions={crawlOptions}
+            indexingType={indexingTechnique}
+            isSetting
+            documentDetail={documentDetail}
+            files={files}
+            onSave={saveHandler}
+            onCancel={cancelHandler}
+          />
+        )}
+      </div>
+      {isShowSetAPIKey && <AccountSetting activeTab='provider' onCancel={async () => {
+        hideSetAPIkey()
+      }} />}
+    </div>
+  )
+}
+
+export default DocumentSettings

dify/web/app/components/datasets/documents/detail/settings/index.tsx

@@ -0,0 +1,36 @@
+'use client'
+import React from 'react'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import DocumentSettings from './document-settings'
+import PipelineSettings from './pipeline-settings'
+
+type SettingsProps = {
+  datasetId: string
+  documentId: string
+}
+
+const Settings = ({
+  datasetId,
+  documentId,
+}: SettingsProps) => {
+  const runtimeMode = useDatasetDetailContextWithSelector(s => s.dataset?.runtime_mode)
+  const isGeneralDataset = runtimeMode === 'general'
+
+  if (isGeneralDataset) {
+    return (
+      <DocumentSettings
+        datasetId={datasetId}
+        documentId={documentId}
+      />
+    )
+  }
+
+  return (
+    <PipelineSettings
+      datasetId={datasetId}
+      documentId={documentId}
+    />
+  )
+}
+
+export default Settings

dify/web/app/components/datasets/documents/detail/settings/pipeline-settings/index.tsx

@@ -0,0 +1,210 @@
+import { useCallback, useMemo, useRef, useState } from 'react'
+import type { CrawlResultItem, CustomFile, FileIndexingEstimateResponse } from '@/models/datasets'
+import type { NotionPage } from '@/models/common'
+import { useTranslation } from 'react-i18next'
+import AppUnavailable from '@/app/components/base/app-unavailable'
+import ChunkPreview from '../../../create-from-pipeline/preview/chunk-preview'
+import Loading from '@/app/components/base/loading'
+import ProcessDocuments from './process-documents'
+import LeftHeader from './left-header'
+import { usePipelineExecutionLog, useRunPublishedPipeline } from '@/service/use-pipeline'
+import type { OnlineDriveFile, PublishedPipelineRunPreviewResponse } from '@/models/pipeline'
+import { DatasourceType } from '@/models/pipeline'
+import { noop } from 'lodash-es'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { useRouter } from 'next/navigation'
+import { useInvalidDocumentDetail, useInvalidDocumentList } from '@/service/knowledge/use-document'
+
+type PipelineSettingsProps = {
+  datasetId: string
+  documentId: string
+}
+
+const PipelineSettings = ({
+  datasetId,
+  documentId,
+}: PipelineSettingsProps) => {
+  const { t } = useTranslation()
+  const { push } = useRouter()
+  const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
+  const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
+
+  const isPreview = useRef(false)
+  const formRef = useRef<any>(null)
+
+  const { data: lastRunData, isFetching: isFetchingLastRunData, isError } = usePipelineExecutionLog({
+    dataset_id: datasetId,
+    document_id: documentId,
+  })
+
+  const files = useMemo(() => {
+    const files: CustomFile[] = []
+    if (lastRunData?.datasource_type === DatasourceType.localFile) {
+      const { related_id, name, extension } = lastRunData.datasource_info
+      files.push({
+        id: related_id,
+        name,
+        extension,
+      } as CustomFile)
+    }
+    return files
+  }, [lastRunData])
+
+  const websitePages = useMemo(() => {
+    const websitePages: CrawlResultItem[] = []
+    if (lastRunData?.datasource_type === DatasourceType.websiteCrawl) {
+      const { content, description, source_url, title } = lastRunData.datasource_info
+      websitePages.push({
+        content,
+        description,
+        source_url,
+        title,
+      })
+    }
+    return websitePages
+  }, [lastRunData])
+
+  const onlineDocuments = useMemo(() => {
+    const onlineDocuments: NotionPage[] = []
+    if (lastRunData?.datasource_type === DatasourceType.onlineDocument) {
+      const { workspace_id, page } = lastRunData.datasource_info
+      onlineDocuments.push({
+        workspace_id,
+        ...page,
+      })
+    }
+    return onlineDocuments
+  }, [lastRunData])
+
+  const onlineDriveFiles = useMemo(() => {
+    const onlineDriveFiles: OnlineDriveFile[] = []
+    if (lastRunData?.datasource_type === DatasourceType.onlineDrive) {
+      const { id, type, name, size } = lastRunData.datasource_info
+      onlineDriveFiles.push({
+        id,
+        name,
+        type,
+        size,
+      })
+    }
+    return onlineDriveFiles
+  }, [lastRunData])
+
+  const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
+
+  const handlePreviewChunks = useCallback(async (data: Record<string, any>) => {
+    if (!lastRunData)
+      return
+    const datasourceInfoList: Record<string, any>[] = []
+    const documentInfo = lastRunData.datasource_info
+    datasourceInfoList.push(documentInfo)
+    await runPublishedPipeline({
+      pipeline_id: pipelineId!,
+      inputs: data,
+      start_node_id: lastRunData.datasource_node_id,
+      datasource_type: lastRunData.datasource_type,
+      datasource_info_list: datasourceInfoList,
+      is_preview: true,
+    }, {
+      onSuccess: (res) => {
+        setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
+      },
+    })
+  }, [lastRunData, pipelineId, runPublishedPipeline])
+
+  const invalidDocumentList = useInvalidDocumentList(datasetId)
+  const invalidDocumentDetail = useInvalidDocumentDetail()
+  const handleProcess = useCallback(async (data: Record<string, any>) => {
+    if (!lastRunData)
+      return
+    const datasourceInfoList: Record<string, any>[] = []
+    const documentInfo = lastRunData.datasource_info
+    datasourceInfoList.push(documentInfo)
+    await runPublishedPipeline({
+      pipeline_id: pipelineId!,
+      inputs: data,
+      start_node_id: lastRunData.datasource_node_id,
+      datasource_type: lastRunData.datasource_type,
+      datasource_info_list: datasourceInfoList,
+      original_document_id: documentId,
+      is_preview: false,
+    }, {
+      onSuccess: () => {
+        invalidDocumentList()
+        invalidDocumentDetail()
+        push(`/datasets/${datasetId}/documents`)
+      },
+    })
+  }, [datasetId, invalidDocumentDetail, invalidDocumentList, lastRunData, pipelineId, push, runPublishedPipeline])
+
+  const onClickProcess = useCallback(() => {
+    isPreview.current = false
+    formRef.current?.submit()
+  }, [])
+
+  const onClickPreview = useCallback(() => {
+    isPreview.current = true
+    formRef.current?.submit()
+  }, [])
+
+  const handleSubmit = useCallback((data: Record<string, any>) => {
+    if (isPreview.current)
+      handlePreviewChunks(data)
+    else
+      handleProcess(data)
+  }, [handlePreviewChunks, handleProcess])
+
+  if (isFetchingLastRunData) {
+    return (
+      <Loading type='app' />
+    )
+  }
+
+  if (isError)
+    return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
+
+  return (
+    <div
+      className='relative flex h-[calc(100vh-56px)] min-w-[1024px] overflow-x-auto rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
+    >
+      <div className='h-full min-w-0 flex-1'>
+        <div className='flex h-full flex-col px-14'>
+          <LeftHeader title={t('datasetPipeline.documentSettings.title')} />
+          <div className='grow overflow-y-auto'>
+            <ProcessDocuments
+              ref={formRef}
+              lastRunInputData={lastRunData!.input_data}
+              datasourceNodeId={lastRunData!.datasource_node_id}
+              onProcess={onClickProcess}
+              onPreview={onClickPreview}
+              onSubmit={handleSubmit}
+              isRunning={isPending}
+            />
+          </div>
+        </div>
+      </div>
+      {/* Preview */}
+      <div className='h-full min-w-0 flex-1'>
+        <div className='flex h-full flex-col pl-2 pt-2'>
+          <ChunkPreview
+            dataSourceType={lastRunData!.datasource_type}
+            localFiles={files}
+            onlineDocuments={onlineDocuments}
+            websitePages={websitePages}
+            onlineDriveFiles={onlineDriveFiles}
+            isIdle={isIdle}
+            isPending={isPending && isPreview.current}
+            estimateData={estimateData}
+            onPreview={onClickPreview}
+            handlePreviewFileChange={noop}
+            handlePreviewOnlineDocumentChange={noop}
+            handlePreviewWebsitePageChange={noop}
+            handlePreviewOnlineDriveFileChange={noop}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default PipelineSettings

dify/web/app/components/datasets/documents/detail/settings/pipeline-settings/left-header.tsx

@@ -0,0 +1,42 @@
+import React, { useCallback } from 'react'
+import { RiArrowLeftLine } from '@remixicon/react'
+import Button from '@/app/components/base/button'
+import { useRouter } from 'next/navigation'
+import Effect from '@/app/components/base/effect'
+import { useTranslation } from 'react-i18next'
+
+type LeftHeaderProps = {
+  title: string
+}
+
+const LeftHeader = ({
+  title,
+}: LeftHeaderProps) => {
+  const { t } = useTranslation()
+  const { back } = useRouter()
+
+  const navigateBack = useCallback(() => {
+    back()
+  }, [back])
+
+  return (
+    <div className='relative flex flex-col gap-y-0.5 pb-2 pt-4'>
+      <div className='system-2xs-semibold-uppercase bg-pipeline-add-documents-title-bg bg-clip-text text-transparent'>
+        {title}
+      </div>
+      <div className='system-md-semibold text-text-primary'>
+        {t('datasetPipeline.addDocuments.steps.processDocuments')}
+      </div>
+      <Button
+        variant='secondary-accent'
+        className='absolute -left-11 top-3.5 size-9 rounded-full p-0'
+        onClick={navigateBack}
+      >
+        <RiArrowLeftLine className='size-5 ' />
+      </Button>
+      <Effect className='left-8 top-[-34px] opacity-20' />
+    </div>
+  )
+}
+
+export default React.memo(LeftHeader)

dify/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx

@@ -0,0 +1,29 @@
+import React from 'react'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
+
+type ActionsProps = {
+  runDisabled?: boolean
+  onProcess: () => void
+}
+
+const Actions = ({
+  onProcess,
+  runDisabled,
+}: ActionsProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center justify-end'>
+      <Button
+        variant='primary'
+        onClick={onProcess}
+        disabled={runDisabled}
+      >
+        {t('datasetPipeline.operations.saveAndProcess')}
+      </Button>
+    </div>
+  )
+}
+
+export default React.memo(Actions)

dify/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts

@@ -0,0 +1,15 @@
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline'
+
+export const useInputVariables = (datasourceNodeId: string) => {
+  const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
+  const { data: paramsConfig, isFetching: isFetchingParams } = usePublishedPipelineProcessingParams({
+    pipeline_id: pipelineId!,
+    node_id: datasourceNodeId,
+  })
+
+  return {
+    paramsConfig,
+    isFetchingParams,
+  }
+}

dify/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx

@@ -0,0 +1,47 @@
+import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
+import { useInputVariables } from './hooks'
+import Actions from './actions'
+import Form from '../../../../create-from-pipeline/process-documents/form'
+import { useConfigurations, useInitialData } from '@/app/components/rag-pipeline/hooks/use-input-fields'
+
+type ProcessDocumentsProps = {
+  datasourceNodeId: string
+  lastRunInputData: Record<string, any>
+  isRunning: boolean
+  ref: React.RefObject<any>
+  onProcess: () => void
+  onPreview: () => void
+  onSubmit: (data: Record<string, any>) => void
+}
+
+const ProcessDocuments = ({
+  datasourceNodeId,
+  lastRunInputData,
+  isRunning,
+  onProcess,
+  onPreview,
+  onSubmit,
+  ref,
+}: ProcessDocumentsProps) => {
+  const { isFetchingParams, paramsConfig } = useInputVariables(datasourceNodeId)
+  const initialData = useInitialData(paramsConfig?.variables || [], lastRunInputData)
+  const configurations = useConfigurations(paramsConfig?.variables || [])
+  const schema = generateZodSchema(configurations)
+
+  return (
+    <div className='flex flex-col gap-y-4 pt-4'>
+      <Form
+        ref={ref}
+        initialData={initialData}
+        configurations={configurations}
+        schema={schema}
+        onSubmit={onSubmit}
+        onPreview={onPreview}
+        isRunning={isRunning}
+      />
+      <Actions runDisabled={isFetchingParams || isRunning} onProcess={onProcess} />
+    </div>
+  )
+}
+
+export default ProcessDocuments