dify

dify/web/app/components/datasets/metadata/hooks/use-batch-edit-document-metadata.ts

@@ -0,0 +1,149 @@
+import { useBoolean } from 'ahooks'
+import { type MetadataBatchEditToServer, type MetadataItemInBatchEdit, type MetadataItemWithEdit, type MetadataItemWithValue, UpdateType } from '../types'
+import type { SimpleDocumentDetail } from '@/models/datasets'
+import { useMemo } from 'react'
+import { useBatchUpdateDocMetadata } from '@/service/knowledge/use-metadata'
+import Toast from '@/app/components/base/toast'
+import { t } from 'i18next'
+
+type Props = {
+  datasetId: string
+  docList: SimpleDocumentDetail[]
+  selectedDocumentIds?: string[]
+  onUpdate: () => void
+}
+
+const useBatchEditDocumentMetadata = ({
+  datasetId,
+  docList,
+  selectedDocumentIds,
+  onUpdate,
+}: Props) => {
+  const [isShowEditModal, {
+    setTrue: showEditModal,
+    setFalse: hideEditModal,
+  }] = useBoolean(false)
+
+  const metaDataList: MetadataItemWithValue[][] = (() => {
+    const res: MetadataItemWithValue[][] = []
+    docList.forEach((item) => {
+      if (item.doc_metadata) {
+        res.push(item.doc_metadata.filter(item => item.id !== 'built-in'))
+        return
+      }
+      res.push([])
+    })
+    return res
+  })()
+
+  // To check is key has multiple value
+  const originalList: MetadataItemInBatchEdit[] = useMemo(() => {
+    const idNameValue: Record<string, { value: string | number | null, isMultipleValue: boolean }> = {}
+
+    const res: MetadataItemInBatchEdit[] = []
+    metaDataList.forEach((metaData) => {
+      metaData.forEach((item) => {
+        if (idNameValue[item.id]?.isMultipleValue)
+          return
+        const itemInRes = res.find(i => i.id === item.id)
+        if (!idNameValue[item.id]) {
+          idNameValue[item.id] = {
+            value: item.value,
+            isMultipleValue: false,
+          }
+        }
+
+        if (itemInRes && itemInRes.value !== item.value) {
+          idNameValue[item.id].isMultipleValue = true
+          itemInRes.isMultipleValue = true
+          itemInRes.value = null
+          return
+        }
+        if (!itemInRes) {
+          res.push({
+            ...item,
+            isMultipleValue: false,
+          })
+        }
+      })
+    })
+    return res
+  }, [metaDataList])
+
+  const formateToBackendList = (editedList: MetadataItemWithEdit[], addedList: MetadataItemInBatchEdit[], isApplyToAllSelectDocument: boolean) => {
+    const updatedList = editedList.filter((editedItem) => {
+      return editedItem.updateType === UpdateType.changeValue
+    })
+    const removedList = originalList.filter((originalItem) => {
+      const editedItem = editedList.find(i => i.id === originalItem.id)
+      if (!editedItem) // removed item
+        return true
+      return false
+    })
+
+    // Use selectedDocumentIds if available, otherwise fall back to docList
+    const documentIds = selectedDocumentIds || docList.map(doc => doc.id)
+    const res: MetadataBatchEditToServer = documentIds.map((documentId) => {
+      // Find the document in docList to get its metadata
+      const docIndex = docList.findIndex(doc => doc.id === documentId)
+      const oldMetadataList = docIndex >= 0 ? metaDataList[docIndex] : []
+      let newMetadataList: MetadataItemWithValue[] = [...oldMetadataList, ...addedList]
+        .filter((item) => {
+          return !removedList.find(removedItem => removedItem.id === item.id)
+        })
+        .map(item => ({
+          id: item.id,
+          name: item.name,
+          type: item.type,
+          value: item.value,
+        }))
+      if (isApplyToAllSelectDocument) {
+        // add missing metadata item
+        updatedList.forEach((editedItem) => {
+          if (!newMetadataList.find(i => i.id === editedItem.id) && !editedItem.isMultipleValue)
+            newMetadataList.push(editedItem)
+        })
+      }
+
+      newMetadataList = newMetadataList.map((item) => {
+        const editedItem = updatedList.find(i => i.id === item.id)
+        if (editedItem)
+          return editedItem
+        return item
+      })
+
+      return {
+        document_id: documentId,
+        metadata_list: newMetadataList,
+        partial_update: docIndex < 0,
+      }
+    })
+    return res
+  }
+
+  const { mutateAsync } = useBatchUpdateDocMetadata()
+
+  const handleSave = async (editedList: MetadataItemInBatchEdit[], addedList: MetadataItemInBatchEdit[], isApplyToAllSelectDocument: boolean) => {
+    const backendList = formateToBackendList(editedList, addedList, isApplyToAllSelectDocument)
+    await mutateAsync({
+      dataset_id: datasetId,
+      metadata_list: backendList,
+    })
+    onUpdate()
+    hideEditModal()
+    Toast.notify({
+      type: 'success',
+      message: t('common.actionMsg.modifiedSuccessfully'),
+    })
+  }
+
+  return {
+    isShowEditModal,
+    showEditModal,
+    hideEditModal,
+    originalList,
+    handleSave,
+  }
+}
+
+export default useBatchEditDocumentMetadata

dify/web/app/components/datasets/metadata/hooks/use-check-metadata-name.ts

@@ -0,0 +1,34 @@
+import { useTranslation } from 'react-i18next'
+
+const i18nPrefix = 'dataset.metadata.checkName'
+
+const useCheckMetadataName = () => {
+  const { t } = useTranslation()
+  return {
+    checkName: (name: string) => {
+      if (!name) {
+        return {
+          errorMsg: t(`${i18nPrefix}.empty`),
+        }
+      }
+
+      if (!/^[a-z][a-z0-9_]*$/.test(name)) {
+        return {
+          errorMsg: t(`${i18nPrefix}.invalid`),
+        }
+      }
+
+      if (name.length > 255) {
+        return {
+          errorMsg: t(`${i18nPrefix}.tooLong`, { max: 255 }),
+        }
+      }
+
+      return {
+        errorMsg: '',
+      }
+    },
+  }
+}
+
+export default useCheckMetadataName

dify/web/app/components/datasets/metadata/hooks/use-edit-dataset-metadata.ts

@@ -0,0 +1,95 @@
+import { useBoolean } from 'ahooks'
+import { useBuiltInMetaDataFields, useCreateMetaData, useDatasetMetaData, useDeleteMetaData, useRenameMeta, useUpdateBuiltInStatus } from '@/service/knowledge/use-metadata'
+import type { DataSet } from '@/models/datasets'
+import { useCallback, useEffect, useState } from 'react'
+import { type BuiltInMetadataItem, type MetadataItemWithValueLength, isShowManageMetadataLocalStorageKey } from '../types'
+import useCheckMetadataName from './use-check-metadata-name'
+import Toast from '@/app/components/base/toast'
+import { useTranslation } from 'react-i18next'
+
+const useEditDatasetMetadata = ({
+  datasetId,
+  // dataset,
+  onUpdateDocList,
+}: {
+  datasetId: string,
+  dataset?: DataSet,
+  onUpdateDocList: () => void
+}) => {
+  const { t } = useTranslation()
+  const [isShowEditModal, {
+    setTrue: showEditModal,
+    setFalse: hideEditModal,
+  }] = useBoolean(false)
+
+  useEffect(() => {
+    const isShowManageMetadata = localStorage.getItem(isShowManageMetadataLocalStorageKey)
+    if (isShowManageMetadata) {
+      showEditModal()
+      localStorage.removeItem(isShowManageMetadataLocalStorageKey)
+    }
+  }, [])
+
+  const { data: datasetMetaData } = useDatasetMetaData(datasetId)
+  const { mutate: doAddMetaData } = useCreateMetaData(datasetId)
+  const { checkName } = useCheckMetadataName()
+  const handleAddMetaData = useCallback(async (payload: BuiltInMetadataItem) => {
+    const errorMsg = checkName(payload.name).errorMsg
+    if (errorMsg) {
+      Toast.notify({
+        message: errorMsg,
+        type: 'error',
+      })
+      return Promise.reject(new Error(errorMsg))
+    }
+    await doAddMetaData(payload)
+  }, [checkName, doAddMetaData])
+
+  const { mutate: doRenameMetaData } = useRenameMeta(datasetId)
+  const handleRename = useCallback(async (payload: MetadataItemWithValueLength) => {
+    const errorMsg = checkName(payload.name).errorMsg
+    if (errorMsg) {
+      Toast.notify({
+        message: errorMsg,
+        type: 'error',
+      })
+      return Promise.reject(new Error(errorMsg))
+    }
+    await doRenameMetaData(payload)
+    onUpdateDocList()
+  }, [checkName, doRenameMetaData, onUpdateDocList])
+
+  const { mutateAsync: doDeleteMetaData } = useDeleteMetaData(datasetId)
+  const handleDeleteMetaData = useCallback(async (metaDataId: string) => {
+    await doDeleteMetaData(metaDataId)
+    onUpdateDocList()
+  }, [doDeleteMetaData, onUpdateDocList])
+
+  const [builtInEnabled, setBuiltInEnabled] = useState(datasetMetaData?.built_in_field_enabled)
+  useEffect(() => { // wait for api response to set the right value
+    setBuiltInEnabled(datasetMetaData?.built_in_field_enabled)
+  }, [datasetMetaData])
+  const { mutateAsync: toggleBuiltInStatus } = useUpdateBuiltInStatus(datasetId)
+  const { data: builtInMetaData } = useBuiltInMetaDataFields()
+  return {
+    isShowEditModal,
+    showEditModal,
+    hideEditModal,
+    datasetMetaData: datasetMetaData?.doc_metadata,
+    handleAddMetaData,
+    handleRename,
+    handleDeleteMetaData,
+    builtInMetaData: builtInMetaData?.fields,
+    builtInEnabled,
+    setBuiltInEnabled: async (enable: boolean) => {
+      await toggleBuiltInStatus(enable)
+      setBuiltInEnabled(enable)
+      Toast.notify({
+        message: t('common.actionMsg.modifiedSuccessfully'),
+        type: 'success',
+      })
+    },
+  }
+}
+
+export default useEditDatasetMetadata

dify/web/app/components/datasets/metadata/hooks/use-metadata-document.ts

@@ -0,0 +1,159 @@
+import { useBatchUpdateDocMetadata, useDatasetMetaData, useDocumentMetaData } from '@/service/knowledge/use-metadata'
+import { useDatasetDetailContext } from '@/context/dataset-detail'
+import type { BuiltInMetadataItem } from '../types'
+import { DataType, type MetadataItemWithValue } from '../types'
+import { useCallback, useState } from 'react'
+import Toast from '@/app/components/base/toast'
+import type { FullDocumentDetail } from '@/models/datasets'
+import { useTranslation } from 'react-i18next'
+import { useLanguages, useMetadataMap } from '@/hooks/use-metadata'
+import { get } from 'lodash-es'
+import { useCreateMetaData } from '@/service/knowledge/use-metadata'
+import useCheckMetadataName from './use-check-metadata-name'
+
+type Props = {
+  datasetId: string
+  documentId: string
+  docDetail: FullDocumentDetail
+}
+
+const useMetadataDocument = ({
+  datasetId,
+  documentId,
+  docDetail,
+}: Props) => {
+  const { t } = useTranslation()
+
+  const { dataset } = useDatasetDetailContext()
+  const embeddingAvailable = !!dataset?.embedding_available
+
+  const { mutateAsync } = useBatchUpdateDocMetadata()
+  const { checkName } = useCheckMetadataName()
+
+  const [isEdit, setIsEdit] = useState(false)
+  const { data: documentDetail } = useDocumentMetaData({
+    datasetId,
+    documentId,
+  })
+
+  const allList = documentDetail?.doc_metadata || []
+  const list = allList.filter(item => item.id !== 'built-in')
+  const builtList = allList.filter(item => item.id === 'built-in')
+  const [tempList, setTempList] = useState<MetadataItemWithValue[]>(list)
+  const { mutateAsync: doAddMetaData } = useCreateMetaData(datasetId)
+  const handleSelectMetaData = useCallback((metaData: MetadataItemWithValue) => {
+    setTempList((prev) => {
+      const index = prev.findIndex(item => item.id === metaData.id)
+      if (index === -1)
+        return [...prev, metaData]
+
+      return prev
+    })
+  }, [])
+  const handleAddMetaData = useCallback(async (payload: BuiltInMetadataItem) => {
+    const errorMsg = checkName(payload.name).errorMsg
+    if (errorMsg) {
+      Toast.notify({
+        message: errorMsg,
+        type: 'error',
+      })
+      return Promise.reject(new Error(errorMsg))
+    }
+    await doAddMetaData(payload)
+    Toast.notify({
+      type: 'success',
+      message: t('common.api.actionSuccess'),
+    })
+  }, [checkName, doAddMetaData, t])
+
+  const hasData = list.length > 0
+  const handleSave = async () => {
+    await mutateAsync({
+      dataset_id: datasetId,
+      metadata_list: [{
+        document_id: documentId,
+        metadata_list: tempList,
+      }],
+    })
+    setIsEdit(false)
+    Toast.notify({
+      type: 'success',
+      message: t('common.api.actionSuccess'),
+    })
+  }
+
+  const handleCancel = () => {
+    setTempList(list)
+    setIsEdit(false)
+  }
+
+  const startToEdit = () => {
+    setTempList(list)
+    setIsEdit(true)
+  }
+
+  // built in enabled is set in dataset
+  const { data: datasetMetaData } = useDatasetMetaData(datasetId)
+  const builtInEnabled = datasetMetaData?.built_in_field_enabled
+
+  // old metadata and technical params
+  const metadataMap = useMetadataMap()
+  const languageMap = useLanguages()
+
+  const getReadOnlyMetaData = (mainField: 'originInfo' | 'technicalParameters') => {
+    const fieldMap = metadataMap[mainField]?.subFieldsMap
+    const sourceData = docDetail
+    const getTargetMap = (field: string) => {
+      if (field === 'language')
+        return languageMap
+
+      return {} as any
+    }
+
+    const getTargetValue = (field: string) => {
+      const val = get(sourceData, field, '')
+      if (!val && val !== 0)
+        return '-'
+      if (fieldMap[field]?.inputType === 'select')
+        return getTargetMap(field)[val]
+      if (fieldMap[field]?.render)
+        return fieldMap[field]?.render?.(val, field === 'hit_count' ? get(sourceData, 'segment_count', 0) as number : undefined)
+      return val
+    }
+    const fieldList = Object.keys(fieldMap).map((key) => {
+      const field = fieldMap[key]
+      return {
+        id: field?.label,
+        type: DataType.string,
+        name: field?.label,
+        value: getTargetValue(key),
+      }
+    })
+
+    return fieldList
+  }
+
+  const originInfo = getReadOnlyMetaData('originInfo')
+  const technicalParameters = getReadOnlyMetaData('technicalParameters')
+
+  return {
+    embeddingAvailable,
+    isEdit,
+    setIsEdit,
+    list,
+    tempList,
+    setTempList,
+    handleSelectMetaData,
+    handleAddMetaData,
+    hasData,
+    builtList,
+    builtInEnabled,
+    startToEdit,
+    handleSave,
+    handleCancel,
+    originInfo,
+    technicalParameters,
+  }
+}
+
+export default useMetadataDocument