dify

dify/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx

@@ -0,0 +1,161 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
+import Button from '@/app/components/base/button'
+import type { FirecrawlConfig } from '@/models/common'
+import Field from '@/app/components/datasets/create/website/base/field'
+import Toast from '@/app/components/base/toast'
+import { createDataSourceApiKeyBinding } from '@/service/datasets'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+type Props = {
+  onCancel: () => void
+  onSaved: () => void
+}
+
+const I18N_PREFIX = 'datasetCreation.firecrawl'
+
+const DEFAULT_BASE_URL = 'https://api.firecrawl.dev'
+
+const ConfigFirecrawlModal: FC<Props> = ({
+  onCancel,
+  onSaved,
+}) => {
+  const { t } = useTranslation()
+  const [isSaving, setIsSaving] = useState(false)
+  const [config, setConfig] = useState<FirecrawlConfig>({
+    api_key: '',
+    base_url: '',
+  })
+
+  const handleConfigChange = useCallback((key: string) => {
+    return (value: string | number) => {
+      setConfig(prev => ({ ...prev, [key]: value as string }))
+    }
+  }, [])
+
+  const handleSave = useCallback(async () => {
+    if (isSaving)
+      return
+    let errorMsg = ''
+    if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
+      errorMsg = t('common.errorMsg.urlError')
+    if (!errorMsg) {
+      if (!config.api_key) {
+        errorMsg = t('common.errorMsg.fieldRequired', {
+          field: 'API Key',
+        })
+      }
+    }
+
+    if (errorMsg) {
+      Toast.notify({
+        type: 'error',
+        message: errorMsg,
+      })
+      return
+    }
+    const postData = {
+      category: 'website',
+      provider: 'firecrawl',
+      credentials: {
+        auth_type: 'bearer',
+        config: {
+          api_key: config.api_key,
+          base_url: config.base_url || DEFAULT_BASE_URL,
+        },
+      },
+    }
+    try {
+      setIsSaving(true)
+      await createDataSourceApiKeyBinding(postData)
+      Toast.notify({
+        type: 'success',
+        message: t('common.api.success'),
+      })
+    }
+    finally {
+      setIsSaving(false)
+    }
+
+    onSaved()
+  }, [config.api_key, config.base_url, onSaved, t, isSaving])
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='z-[60] h-full w-full'>
+        <div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
+          <div className='mx-2 max-h-[calc(100vh-120px)] w-[640px] overflow-y-auto rounded-2xl bg-components-panel-bg shadow-xl'>
+            <div className='px-8 pt-8'>
+              <div className='mb-4 flex items-center justify-between'>
+                <div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configFirecrawl`)}</div>
+              </div>
+
+              <div className='space-y-4'>
+                <Field
+                  label='API Key'
+                  labelClassName='!text-sm'
+                  isRequired
+                  value={config.api_key}
+                  onChange={handleConfigChange('api_key')}
+                  placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
+                />
+                <Field
+                  label='Base URL'
+                  labelClassName='!text-sm'
+                  value={config.base_url}
+                  onChange={handleConfigChange('base_url')}
+                  placeholder={DEFAULT_BASE_URL}
+                />
+              </div>
+              <div className='my-8 flex h-8 items-center justify-between'>
+                <a className='flex items-center space-x-1 text-xs font-normal leading-[18px] text-text-accent' target='_blank' href='https://www.firecrawl.dev/account'>
+                  <span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
+                  <LinkExternal02 className='h-3 w-3' />
+                </a>
+                <div className='flex'>
+                  <Button
+                    size='large'
+                    className='mr-2'
+                    onClick={onCancel}
+                  >
+                    {t('common.operation.cancel')}
+                  </Button>
+                  <Button
+                    variant='primary'
+                    size='large'
+                    onClick={handleSave}
+                    loading={isSaving}
+                  >
+                    {t('common.operation.save')}
+                  </Button>
+                </div>
+
+              </div>
+            </div>
+            <div className='border-t-[0.5px] border-t-divider-regular'>
+              <div className='flex items-center justify-center bg-background-section-burn py-3 text-xs text-text-tertiary'>
+                <Lock01 className='mr-1 h-3 w-3 text-text-tertiary' />
+                {t('common.modelProvider.encrypted.front')}
+                <a
+                  className='mx-1 text-text-accent'
+                  target='_blank' rel='noopener noreferrer'
+                  href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+                >
+                  PKCS1_OAEP
+                </a>
+                {t('common.modelProvider.encrypted.back')}
+              </div>
+            </div>
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(ConfigFirecrawlModal)

dify/web/app/components/header/account-setting/data-source-page/data-source-website/config-jina-reader-modal.tsx

@@ -0,0 +1,140 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
+import Button from '@/app/components/base/button'
+import { DataSourceProvider } from '@/models/common'
+import Field from '@/app/components/datasets/create/website/base/field'
+import Toast from '@/app/components/base/toast'
+import { createDataSourceApiKeyBinding } from '@/service/datasets'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+type Props = {
+  onCancel: () => void
+  onSaved: () => void
+}
+
+const I18N_PREFIX = 'datasetCreation.jinaReader'
+
+const ConfigJinaReaderModal: FC<Props> = ({
+  onCancel,
+  onSaved,
+}) => {
+  const { t } = useTranslation()
+  const [isSaving, setIsSaving] = useState(false)
+  const [apiKey, setApiKey] = useState('')
+
+  const handleSave = useCallback(async () => {
+    if (isSaving)
+      return
+    let errorMsg = ''
+    if (!errorMsg) {
+      if (!apiKey) {
+        errorMsg = t('common.errorMsg.fieldRequired', {
+          field: 'API Key',
+        })
+      }
+    }
+
+    if (errorMsg) {
+      Toast.notify({
+        type: 'error',
+        message: errorMsg,
+      })
+      return
+    }
+    const postData = {
+      category: 'website',
+      provider: DataSourceProvider.jinaReader,
+      credentials: {
+        auth_type: 'bearer',
+        config: {
+          api_key: apiKey,
+        },
+      },
+    }
+    try {
+      setIsSaving(true)
+      await createDataSourceApiKeyBinding(postData)
+      Toast.notify({
+        type: 'success',
+        message: t('common.api.success'),
+      })
+    }
+    finally {
+      setIsSaving(false)
+    }
+
+    onSaved()
+  }, [apiKey, onSaved, t, isSaving])
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='z-[60] h-full w-full'>
+        <div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
+          <div className='mx-2 max-h-[calc(100vh-120px)] w-[640px] overflow-y-auto rounded-2xl bg-components-panel-bg shadow-xl'>
+            <div className='px-8 pt-8'>
+              <div className='mb-4 flex items-center justify-between'>
+                <div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configJinaReader`)}</div>
+              </div>
+
+              <div className='space-y-4'>
+                <Field
+                  label='API Key'
+                  labelClassName='!text-sm'
+                  isRequired
+                  value={apiKey}
+                  onChange={(value: string | number) => setApiKey(value as string)}
+                  placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
+                />
+              </div>
+              <div className='my-8 flex h-8 items-center justify-between'>
+                <a className='flex items-center space-x-1 text-xs font-normal leading-[18px] text-text-accent' target='_blank' href='https://jina.ai/reader/'>
+                  <span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
+                  <LinkExternal02 className='h-3 w-3' />
+                </a>
+                <div className='flex'>
+                  <Button
+                    size='large'
+                    className='mr-2'
+                    onClick={onCancel}
+                  >
+                    {t('common.operation.cancel')}
+                  </Button>
+                  <Button
+                    variant='primary'
+                    size='large'
+                    onClick={handleSave}
+                    loading={isSaving}
+                  >
+                    {t('common.operation.save')}
+                  </Button>
+                </div>
+
+              </div>
+            </div>
+            <div className='border-t-[0.5px] border-t-divider-regular'>
+              <div className='flex items-center justify-center bg-background-section-burn py-3 text-xs text-text-tertiary'>
+                <Lock01 className='mr-1 h-3 w-3 text-text-tertiary' />
+                {t('common.modelProvider.encrypted.front')}
+                <a
+                  className='mx-1 text-text-accent'
+                  target='_blank' rel='noopener noreferrer'
+                  href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+                >
+                  PKCS1_OAEP
+                </a>
+                {t('common.modelProvider.encrypted.back')}
+              </div>
+            </div>
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(ConfigJinaReaderModal)

dify/web/app/components/header/account-setting/data-source-page/data-source-website/config-watercrawl-modal.tsx

@@ -0,0 +1,161 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
+import Button from '@/app/components/base/button'
+import type { WatercrawlConfig } from '@/models/common'
+import Field from '@/app/components/datasets/create/website/base/field'
+import Toast from '@/app/components/base/toast'
+import { createDataSourceApiKeyBinding } from '@/service/datasets'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+type Props = {
+  onCancel: () => void
+  onSaved: () => void
+}
+
+const I18N_PREFIX = 'datasetCreation.watercrawl'
+
+const DEFAULT_BASE_URL = 'https://app.watercrawl.dev'
+
+const ConfigWatercrawlModal: FC<Props> = ({
+  onCancel,
+  onSaved,
+}) => {
+  const { t } = useTranslation()
+  const [isSaving, setIsSaving] = useState(false)
+  const [config, setConfig] = useState<WatercrawlConfig>({
+    api_key: '',
+    base_url: '',
+  })
+
+  const handleConfigChange = useCallback((key: string) => {
+    return (value: string | number) => {
+      setConfig(prev => ({ ...prev, [key]: value as string }))
+    }
+  }, [])
+
+  const handleSave = useCallback(async () => {
+    if (isSaving)
+      return
+    let errorMsg = ''
+    if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
+      errorMsg = t('common.errorMsg.urlError')
+    if (!errorMsg) {
+      if (!config.api_key) {
+        errorMsg = t('common.errorMsg.fieldRequired', {
+          field: 'API Key',
+        })
+      }
+    }
+
+    if (errorMsg) {
+      Toast.notify({
+        type: 'error',
+        message: errorMsg,
+      })
+      return
+    }
+    const postData = {
+      category: 'website',
+      provider: 'watercrawl',
+      credentials: {
+        auth_type: 'x-api-key',
+        config: {
+          api_key: config.api_key,
+          base_url: config.base_url || DEFAULT_BASE_URL,
+        },
+      },
+    }
+    try {
+      setIsSaving(true)
+      await createDataSourceApiKeyBinding(postData)
+      Toast.notify({
+        type: 'success',
+        message: t('common.api.success'),
+      })
+    }
+    finally {
+      setIsSaving(false)
+    }
+
+    onSaved()
+  }, [config.api_key, config.base_url, onSaved, t, isSaving])
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='z-[60] h-full w-full'>
+        <div className='fixed inset-0 flex items-center justify-center bg-background-overlay'>
+          <div className='mx-2 max-h-[calc(100vh-120px)] w-[640px] overflow-y-auto rounded-2xl bg-components-panel-bg shadow-xl'>
+            <div className='px-8 pt-8'>
+              <div className='mb-4 flex items-center justify-between'>
+                <div className='system-xl-semibold text-text-primary'>{t(`${I18N_PREFIX}.configWatercrawl`)}</div>
+              </div>
+
+              <div className='space-y-4'>
+                <Field
+                  label='API Key'
+                  labelClassName='!text-sm'
+                  isRequired
+                  value={config.api_key}
+                  onChange={handleConfigChange('api_key')}
+                  placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
+                />
+                <Field
+                  label='Base URL'
+                  labelClassName='!text-sm'
+                  value={config.base_url}
+                  onChange={handleConfigChange('base_url')}
+                  placeholder={DEFAULT_BASE_URL}
+                />
+              </div>
+              <div className='my-8 flex h-8 items-center justify-between'>
+                <a className='flex items-center space-x-1 text-xs font-normal leading-[18px] text-text-accent' target='_blank' href='https://app.watercrawl.dev/'>
+                  <span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
+                  <LinkExternal02 className='h-3 w-3' />
+                </a>
+                <div className='flex'>
+                  <Button
+                    size='large'
+                    className='mr-2'
+                    onClick={onCancel}
+                  >
+                    {t('common.operation.cancel')}
+                  </Button>
+                  <Button
+                    variant='primary'
+                    size='large'
+                    onClick={handleSave}
+                    loading={isSaving}
+                  >
+                    {t('common.operation.save')}
+                  </Button>
+                </div>
+
+              </div>
+            </div>
+            <div className='border-t-[0.5px] border-t-divider-regular'>
+              <div className='flex items-center justify-center bg-background-section-burn py-3 text-xs text-text-tertiary'>
+                <Lock01 className='mr-1 h-3 w-3 text-text-tertiary' />
+                {t('common.modelProvider.encrypted.front')}
+                <a
+                  className='mx-1 text-text-accent'
+                  target='_blank' rel='noopener noreferrer'
+                  href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+                >
+                  PKCS1_OAEP
+                </a>
+                {t('common.modelProvider.encrypted.back')}
+              </div>
+            </div>
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(ConfigWatercrawlModal)

dify/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx

@@ -0,0 +1,131 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import Panel from '../panel'
+import { DataSourceType } from '../panel/types'
+import ConfigFirecrawlModal from './config-firecrawl-modal'
+import ConfigWatercrawlModal from './config-watercrawl-modal'
+import ConfigJinaReaderModal from './config-jina-reader-modal'
+import cn from '@/utils/classnames'
+import s from '@/app/components/datasets/create/website/index.module.css'
+import { fetchDataSources, removeDataSourceApiKeyBinding } from '@/service/datasets'
+
+import type { DataSourceItem } from '@/models/common'
+import { DataSourceProvider } from '@/models/common'
+import { useAppContext } from '@/context/app-context'
+import Toast from '@/app/components/base/toast'
+
+type Props = {
+  provider: DataSourceProvider
+}
+
+const DataSourceWebsite: FC<Props> = ({ provider }) => {
+  const { t } = useTranslation()
+  const { isCurrentWorkspaceManager } = useAppContext()
+  const [sources, setSources] = useState<DataSourceItem[]>([])
+  const checkSetApiKey = useCallback(async () => {
+    const res = await fetchDataSources() as any
+    const list = res.sources
+    setSources(list)
+  }, [])
+
+  useEffect(() => {
+    checkSetApiKey()
+  }, [])
+
+  const [configTarget, setConfigTarget] = useState<DataSourceProvider | null>(null)
+  const showConfig = useCallback((provider: DataSourceProvider) => {
+    setConfigTarget(provider)
+  }, [setConfigTarget])
+
+  const hideConfig = useCallback(() => {
+    setConfigTarget(null)
+  }, [setConfigTarget])
+
+  const handleAdded = useCallback(() => {
+    checkSetApiKey()
+    hideConfig()
+  }, [checkSetApiKey, hideConfig])
+
+  const getIdByProvider = (provider: DataSourceProvider): string | undefined => {
+    const source = sources.find(item => item.provider === provider)
+    return source?.id
+  }
+
+  const getProviderName = (provider: DataSourceProvider): string => {
+    if (provider === DataSourceProvider.fireCrawl)
+      return 'Firecrawl'
+
+    if (provider === DataSourceProvider.waterCrawl)
+      return 'WaterCrawl'
+
+    return 'Jina Reader'
+  }
+
+  const handleRemove = useCallback((provider: DataSourceProvider) => {
+    return async () => {
+      const dataSourceId = getIdByProvider(provider)
+      if (dataSourceId) {
+        await removeDataSourceApiKeyBinding(dataSourceId)
+        setSources(sources.filter(item => item.provider !== provider))
+        Toast.notify({
+          type: 'success',
+          message: t('common.api.remove'),
+        })
+      }
+    }
+  }, [sources, t])
+
+  return (
+    <>
+      <Panel
+        type={DataSourceType.website}
+        provider={provider}
+        isConfigured={sources.find(item => item.provider === provider) !== undefined}
+        onConfigure={() => showConfig(provider)}
+        readOnly={!isCurrentWorkspaceManager}
+        configuredList={sources.filter(item => item.provider === provider).map(item => ({
+          id: item.id,
+          logo: ({ className }: { className: string }) => {
+            if (item.provider === DataSourceProvider.fireCrawl) {
+              return (
+                <div
+                  className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>🔥</div>
+              )
+            }
+
+            if (item.provider === DataSourceProvider.waterCrawl) {
+              return (
+                <div
+                  className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>
+                  <span className={s.watercrawlLogo}/>
+                </div>
+              )
+            }
+            return (
+              <div
+                className={cn(className, 'ml-3 flex h-5 w-5 items-center justify-center rounded border border-divider-subtle !bg-background-default text-xs font-medium text-text-tertiary')}>
+                <span className={s.jinaLogo}/>
+              </div>
+            )
+          },
+          name: getProviderName(item.provider),
+          isActive: true,
+        }))}
+        onRemove={handleRemove(provider)}
+      />
+      {configTarget === DataSourceProvider.fireCrawl && (
+        <ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig}/>
+      )}
+      {configTarget === DataSourceProvider.waterCrawl && (
+        <ConfigWatercrawlModal onSaved={handleAdded} onCancel={hideConfig}/>
+      )}
+      {configTarget === DataSourceProvider.jinaReader && (
+        <ConfigJinaReaderModal onSaved={handleAdded} onCancel={hideConfig}/>
+      )}
+    </>
+
+  )
+}
+export default React.memo(DataSourceWebsite)