dify

dify/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx

@@ -0,0 +1,174 @@
+import { useCallback, useEffect, useMemo } from 'react'
+import SearchInput from '@/app/components/base/notion-page-selector/search-input'
+import PageSelector from './page-selector'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
+import Header from '../base/header'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { DatasourceType } from '@/models/pipeline'
+import { ssePost } from '@/service/base'
+import Toast from '@/app/components/base/toast'
+import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
+import { useShallow } from 'zustand/react/shallow'
+import { useModalContextSelector } from '@/context/modal-context'
+import Title from './title'
+import { useGetDataSourceAuth } from '@/service/use-datasource'
+import Loading from '@/app/components/base/loading'
+import { useDocLink } from '@/context/i18n'
+import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
+
+type OnlineDocumentsProps = {
+  isInPipeline?: boolean
+  nodeId: string
+  nodeData: DataSourceNodeType
+  onCredentialChange: (credentialId: string) => void
+}
+
+const OnlineDocuments = ({
+  nodeId,
+  nodeData,
+  isInPipeline = false,
+  onCredentialChange,
+}: OnlineDocumentsProps) => {
+  const docLink = useDocLink()
+  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
+  const {
+    documentsData,
+    searchValue,
+    selectedPagesId,
+    currentCredentialId,
+  } = useDataSourceStoreWithSelector(useShallow(state => ({
+    documentsData: state.documentsData,
+    searchValue: state.searchValue,
+    selectedPagesId: state.selectedPagesId,
+    currentCredentialId: state.currentCredentialId,
+  })))
+
+  const { data: dataSourceAuth } = useGetDataSourceAuth({
+    pluginId: nodeData.plugin_id,
+    provider: nodeData.provider_name,
+  })
+
+  const dataSourceStore = useDataSourceStore()
+
+  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
+    const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return pagesMap
+  }, [documentsData])
+
+  const datasourceNodeRunURL = !isInPipeline
+    ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
+    : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
+
+  const getOnlineDocuments = useCallback(async () => {
+    const { currentCredentialId } = dataSourceStore.getState()
+    ssePost(
+      datasourceNodeRunURL,
+      {
+        body: {
+          inputs: {},
+          credential_id: currentCredentialId,
+          datasource_type: DatasourceType.onlineDocument,
+        },
+      },
+      {
+        onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
+          const { setDocumentsData } = dataSourceStore.getState()
+          setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
+        },
+        onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
+          Toast.notify({
+            type: 'error',
+            message: error.error,
+          })
+        },
+      },
+    )
+  }, [dataSourceStore, datasourceNodeRunURL])
+
+  useEffect(() => {
+    if (!currentCredentialId) return
+    getOnlineDocuments()
+  }, [currentCredentialId])
+
+  const handleSearchValueChange = useCallback((value: string) => {
+    const { setSearchValue } = dataSourceStore.getState()
+    setSearchValue(value)
+  }, [dataSourceStore])
+
+  const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
+    const { setSelectedPagesId, setOnlineDocuments } = dataSourceStore.getState()
+    const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId])
+    setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
+    setOnlineDocuments(selectedPages)
+  }, [dataSourceStore, PagesMapAndSelectedPagesId])
+
+  const handlePreviewPage = useCallback((previewPageId: string) => {
+    const { setCurrentDocument } = dataSourceStore.getState()
+    setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId])
+  }, [PagesMapAndSelectedPagesId, dataSourceStore])
+
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
+    })
+  }, [setShowAccountSettingModal])
+
+  return (
+    <div className='flex flex-col gap-y-2'>
+      <Header
+        docTitle='Docs'
+        docLink={docLink('/guides/knowledge-base/knowledge-pipeline/authorize-data-source')}
+        onClickConfiguration={handleSetting}
+        pluginName={nodeData.datasource_label}
+        currentCredentialId={currentCredentialId}
+        onCredentialChange={onCredentialChange}
+        credentials={dataSourceAuth?.result || []}
+      />
+      <div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
+        <div className='flex items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-1 pl-3'>
+          <div className='flex grow items-center'>
+            <Title name={nodeData.datasource_label} />
+          </div>
+          <SearchInput
+            value={searchValue}
+            onChange={handleSearchValueChange}
+          />
+        </div>
+        <div className='overflow-hidden rounded-b-xl'>
+          {documentsData?.length ? (
+            <PageSelector
+              checkedIds={selectedPagesId}
+              disabledValue={new Set()}
+              searchValue={searchValue}
+              list={documentsData[0].pages || []}
+              pagesMap={PagesMapAndSelectedPagesId}
+              onSelect={handleSelectPages}
+              canPreview={!isInPipeline}
+              onPreview={handlePreviewPage}
+              isMultipleChoice={!isInPipeline}
+              currentCredentialId={currentCredentialId}
+            />
+          ) : (
+            <div className='flex h-[296px] items-center justify-center'>
+              <Loading type='app' />
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default OnlineDocuments

dify/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx

@@ -0,0 +1,190 @@
+import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { FixedSizeList as List } from 'react-window'
+import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
+import Item from './item'
+import { recursivePushInParentDescendants } from './utils'
+
+type PageSelectorProps = {
+  checkedIds: Set<string>
+  disabledValue: Set<string>
+  searchValue: string
+  pagesMap: DataSourceNotionPageMap
+  list: DataSourceNotionPage[]
+  onSelect: (selectedPagesId: Set<string>) => void
+  canPreview?: boolean
+  onPreview?: (selectedPageId: string) => void
+  isMultipleChoice?: boolean
+  currentCredentialId: string
+}
+
+export type NotionPageTreeItem = {
+  children: Set<string>
+  descendants: Set<string>
+  depth: number
+  ancestors: string[]
+} & DataSourceNotionPage
+
+export type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+
+type NotionPageItem = {
+  expand: boolean
+  depth: number
+} & DataSourceNotionPage
+
+const PageSelector = ({
+  checkedIds,
+  disabledValue,
+  searchValue,
+  pagesMap,
+  list,
+  onSelect,
+  canPreview = true,
+  onPreview,
+  isMultipleChoice = true,
+  currentCredentialId,
+}: PageSelectorProps) => {
+  const { t } = useTranslation()
+  const [dataList, setDataList] = useState<NotionPageItem[]>([])
+  const [currentPreviewPageId, setCurrentPreviewPageId] = useState('')
+
+  useEffect(() => {
+    setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
+      return {
+        ...item,
+        expand: false,
+        depth: 0,
+      }
+    }))
+  }, [currentCredentialId])
+
+  const searchDataList = list.filter((item) => {
+    return item.page_name.includes(searchValue)
+  }).map((item) => {
+    return {
+      ...item,
+      expand: false,
+      depth: 0,
+    }
+  })
+  const currentDataList = searchValue ? searchDataList : dataList
+
+  const listMapWithChildrenAndDescendants = useMemo(() => {
+    return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
+      const pageId = next.page_id
+      if (!prev[pageId])
+        prev[pageId] = { ...next, children: new Set(), descendants: new Set(), depth: 0, ancestors: [] }
+
+      recursivePushInParentDescendants(pagesMap, prev, prev[pageId], prev[pageId])
+      return prev
+    }, {})
+  }, [list, pagesMap])
+
+  const handleToggle = useCallback((index: number) => {
+    const current = dataList[index]
+    const pageId = current.page_id
+    const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
+    const descendantsIds = Array.from(currentWithChildrenAndDescendants.descendants)
+    const childrenIds = Array.from(currentWithChildrenAndDescendants.children)
+    let newDataList = []
+
+    if (current.expand) {
+      current.expand = false
+
+      newDataList = dataList.filter(item => !descendantsIds.includes(item.page_id))
+    }
+    else {
+      current.expand = true
+
+      newDataList = [
+        ...dataList.slice(0, index + 1),
+        ...childrenIds.map(item => ({
+          ...pagesMap[item],
+          expand: false,
+          depth: listMapWithChildrenAndDescendants[item].depth,
+        })),
+        ...dataList.slice(index + 1),
+      ]
+    }
+    setDataList(newDataList)
+  }, [dataList, listMapWithChildrenAndDescendants, pagesMap])
+
+  const handleCheck = useCallback((index: number) => {
+    const copyValue = new Set(checkedIds)
+    const current = currentDataList[index]
+    const pageId = current.page_id
+    const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
+
+    if (copyValue.has(pageId)) {
+      if (!searchValue && isMultipleChoice) {
+        for (const item of currentWithChildrenAndDescendants.descendants)
+          copyValue.delete(item)
+      }
+
+      copyValue.delete(pageId)
+    }
+    else {
+      if (!searchValue && isMultipleChoice) {
+        for (const item of currentWithChildrenAndDescendants.descendants)
+          copyValue.add(item)
+      }
+      // Single choice mode, clear previous selection
+      if (!isMultipleChoice && copyValue.size > 0) {
+        copyValue.clear()
+        copyValue.add(pageId)
+      }
+      else {
+        copyValue.add(pageId)
+      }
+    }
+
+    onSelect(new Set(copyValue))
+  }, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, checkedIds])
+
+  const handlePreview = useCallback((index: number) => {
+    const current = currentDataList[index]
+    const pageId = current.page_id
+
+    setCurrentPreviewPageId(pageId)
+
+    if (onPreview)
+      onPreview(pageId)
+  }, [currentDataList, onPreview])
+
+  if (!currentDataList.length) {
+    return (
+      <div className='flex h-[296px] items-center justify-center text-[13px] text-text-tertiary'>
+        {t('common.dataSource.notion.selector.noSearchResult')}
+      </div>
+    )
+  }
+
+  return (
+    <List
+      className='py-2'
+      height={296}
+      itemCount={currentDataList.length}
+      itemSize={28}
+      width='100%'
+      itemKey={(index, data) => data.dataList[index].page_id}
+      itemData={{
+        dataList: currentDataList,
+        handleToggle,
+        checkedIds,
+        disabledCheckedIds: disabledValue,
+        handleCheck,
+        canPreview,
+        handlePreview,
+        listMapWithChildrenAndDescendants,
+        searchValue,
+        previewPageId: currentPreviewPageId,
+        pagesMap,
+        isMultipleChoice,
+      }}
+    >
+      {Item}
+    </List>
+  )
+}
+
+export default PageSelector

dify/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/item.tsx

@@ -0,0 +1,149 @@
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { areEqual } from 'react-window'
+import type { ListChildComponentProps } from 'react-window'
+import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
+import Checkbox from '@/app/components/base/checkbox'
+import NotionIcon from '@/app/components/base/notion-icon'
+import cn from '@/utils/classnames'
+import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
+import Radio from '@/app/components/base/radio/ui'
+
+type NotionPageTreeItem = {
+  children: Set<string>
+  descendants: Set<string>
+  depth: number
+  ancestors: string[]
+} & DataSourceNotionPage
+
+type NotionPageTreeMap = Record<string, NotionPageTreeItem>
+
+type NotionPageItem = {
+  expand: boolean
+  depth: number
+} & DataSourceNotionPage
+
+const Item = ({ index, style, data }: ListChildComponentProps<{
+  dataList: NotionPageItem[]
+  handleToggle: (index: number) => void
+  checkedIds: Set<string>
+  disabledCheckedIds: Set<string>
+  handleCheck: (index: number) => void
+  canPreview?: boolean
+  handlePreview: (index: number) => void
+  listMapWithChildrenAndDescendants: NotionPageTreeMap
+  searchValue: string
+  previewPageId: string
+  pagesMap: DataSourceNotionPageMap
+  isMultipleChoice?: boolean
+}>) => {
+  const { t } = useTranslation()
+  const {
+    dataList,
+    handleToggle,
+    checkedIds,
+    disabledCheckedIds,
+    handleCheck,
+    canPreview,
+    handlePreview,
+    listMapWithChildrenAndDescendants,
+    searchValue,
+    previewPageId,
+    pagesMap,
+    isMultipleChoice,
+  } = data
+  const current = dataList[index]
+  const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
+  const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
+  const ancestors = currentWithChildrenAndDescendants.ancestors
+  const breadCrumbs = ancestors.length ? [...ancestors, current.page_name] : [current.page_name]
+  const disabled = disabledCheckedIds.has(current.page_id)
+
+  const renderArrow = () => {
+    if (hasChild) {
+      return (
+        <div
+          className='mr-1 flex h-5 w-5 shrink-0 items-center justify-center rounded-md hover:bg-components-button-ghost-bg-hover'
+          style={{ marginLeft: current.depth * 8 }}
+          onClick={() => handleToggle(index)}
+        >
+          {
+            current.expand
+              ? <RiArrowDownSLine className='h-4 w-4 text-text-tertiary' />
+              : <RiArrowRightSLine className='h-4 w-4 text-text-tertiary' />
+          }
+        </div>
+      )
+    }
+    if (current.parent_id === 'root' || !pagesMap[current.parent_id]) {
+      return (
+        <div></div>
+      )
+    }
+    return (
+      <div className='mr-1 h-5 w-5 shrink-0' style={{ marginLeft: current.depth * 8 }} />
+    )
+  }
+
+  return (
+    <div
+      className={cn('group flex cursor-pointer items-center rounded-md pl-2 pr-[2px] hover:bg-state-base-hover',
+        previewPageId === current.page_id && 'bg-state-base-hover')}
+      style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }}
+    >
+      {isMultipleChoice ? (
+        <Checkbox
+          className='mr-2 shrink-0'
+          checked={checkedIds.has(current.page_id)}
+          disabled={disabled}
+          onCheck={() => {
+            handleCheck(index)
+          }}
+        />) : (
+        <Radio
+          className='mr-2 shrink-0'
+          isChecked={checkedIds.has(current.page_id)}
+          disabled={disabled}
+          onCheck={() => {
+            handleCheck(index)
+          }}
+        />
+      )}
+      {!searchValue && renderArrow()}
+      <NotionIcon
+        className='mr-1 shrink-0'
+        type='page'
+        src={current.page_icon}
+      />
+      <div
+        className='grow truncate text-[13px] font-medium leading-4 text-text-secondary'
+        title={current.page_name}
+      >
+        {current.page_name}
+      </div>
+      {
+        canPreview && (
+          <div
+            className='ml-1 hidden h-6 shrink-0 cursor-pointer items-center rounded-md border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-2 text-xs
+            font-medium leading-4 text-components-button-secondary-text shadow-xs shadow-shadow-shadow-3 backdrop-blur-[10px]
+            hover:border-components-button-secondary-border-hover hover:bg-components-button-secondary-bg-hover group-hover:flex'
+            onClick={() => handlePreview(index)}>
+            {t('common.dataSource.notion.selector.preview')}
+          </div>
+        )
+      }
+      {
+        searchValue && (
+          <div
+            className='ml-1 max-w-[120px] shrink-0 truncate text-xs text-text-quaternary'
+            title={breadCrumbs.join(' / ')}
+          >
+            {breadCrumbs.join(' / ')}
+          </div>
+        )
+      }
+    </div>
+  )
+}
+
+export default React.memo(Item, areEqual)

dify/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/utils.ts

@@ -0,0 +1,39 @@
+import type { DataSourceNotionPageMap } from '@/models/common'
+import type { NotionPageTreeItem, NotionPageTreeMap } from './index'
+
+export const recursivePushInParentDescendants = (
+  pagesMap: DataSourceNotionPageMap,
+  listTreeMap: NotionPageTreeMap,
+  current: NotionPageTreeItem,
+  leafItem: NotionPageTreeItem,
+) => {
+  const parentId = current.parent_id
+  const pageId = current.page_id
+
+  if (!parentId || !pageId)
+    return
+
+  if (parentId !== 'root' && pagesMap[parentId]) {
+    if (!listTreeMap[parentId]) {
+      const children = new Set([pageId])
+      const descendants = new Set([pageId, leafItem.page_id])
+      listTreeMap[parentId] = {
+        ...pagesMap[parentId],
+        children,
+        descendants,
+        depth: 0,
+        ancestors: [],
+      }
+    }
+    else {
+      listTreeMap[parentId].children.add(pageId)
+      listTreeMap[parentId].descendants.add(pageId)
+      listTreeMap[parentId].descendants.add(leafItem.page_id)
+    }
+    leafItem.depth++
+    leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
+
+    if (listTreeMap[parentId].parent_id !== 'root')
+      recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
+  }
+}

dify/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/title.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+
+type TitleProps = {
+  name: string
+}
+
+const Title = ({
+  name,
+}: TitleProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='system-sm-medium px-[5px] py-1 text-text-secondary'>
+      {t('datasetPipeline.onlineDocument.pageSelectorTitle', { name })}
+    </div>
+  )
+}
+
+export default React.memo(Title)