dify

dify/web/app/components/datasets/create/website/base/checkbox-with-label.tsx

@@ -0,0 +1,40 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from '@/utils/classnames'
+import Checkbox from '@/app/components/base/checkbox'
+import Tooltip from '@/app/components/base/tooltip'
+
+type Props = {
+  className?: string
+  isChecked: boolean
+  onChange: (isChecked: boolean) => void
+  label: string
+  labelClassName?: string
+  tooltip?: string
+}
+
+const CheckboxWithLabel: FC<Props> = ({
+  className = '',
+  isChecked,
+  onChange,
+  label,
+  labelClassName,
+  tooltip,
+}) => {
+  return (
+    <label className={cn(className, 'flex h-7 items-center space-x-2')}>
+      <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
+      <div className={cn('text-sm font-normal text-text-secondary', labelClassName)}>{label}</div>
+      {tooltip && (
+        <Tooltip
+          popupContent={
+            <div className='w-[200px]'>{tooltip}</div>
+          }
+          triggerClassName='ml-0.5 w-4 h-4'
+        />
+      )}
+    </label>
+  )
+}
+export default React.memo(CheckboxWithLabel)

dify/web/app/components/datasets/create/website/base/crawled-result-item.tsx

@@ -0,0 +1,60 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from '@/utils/classnames'
+import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
+import Checkbox from '@/app/components/base/checkbox'
+import Button from '@/app/components/base/button'
+
+type Props = {
+  payload: CrawlResultItemType
+  isChecked: boolean
+  isPreview: boolean
+  onCheckChange: (checked: boolean) => void
+  onPreview: () => void
+}
+
+const CrawledResultItem: FC<Props> = ({
+  isPreview,
+  payload,
+  isChecked,
+  onCheckChange,
+  onPreview,
+}) => {
+  const { t } = useTranslation()
+
+  const handleCheckChange = useCallback(() => {
+    onCheckChange(!isChecked)
+  }, [isChecked, onCheckChange])
+  return (
+    <div className={cn(isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover', 'cursor-pointer rounded-lg p-2')}>
+      <div className='relative flex'>
+        <div className='flex h-5 items-center'>
+          <Checkbox className='mr-2 shrink-0' checked={isChecked} onCheck={handleCheckChange} />
+        </div>
+        <div className='flex min-w-0 grow flex-col'>
+          <div
+            className='truncate text-sm font-medium text-text-secondary'
+            title={payload.title}
+          >
+            {payload.title}
+          </div>
+          <div
+            className='mt-0.5 truncate text-xs text-text-tertiary'
+            title={payload.source_url}
+          >
+            {payload.source_url}
+          </div>
+        </div>
+        <Button
+          onClick={onPreview}
+          className='right-0 top-0 hidden h-6 px-1.5 text-xs font-medium uppercase group-hover:absolute group-hover:block'
+        >
+          {t('datasetCreation.stepOne.website.preview')}
+        </Button>
+      </div>
+    </div>
+  )
+}
+export default React.memo(CrawledResultItem)

dify/web/app/components/datasets/create/website/base/crawled-result.tsx

@@ -0,0 +1,89 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from './checkbox-with-label'
+import CrawledResultItem from './crawled-result-item'
+import cn from '@/utils/classnames'
+import type { CrawlResultItem } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  list: CrawlResultItem[]
+  checkedList: CrawlResultItem[]
+  onSelectedChange: (selected: CrawlResultItem[]) => void
+  onPreview: (payload: CrawlResultItem) => void
+  usedTime: number
+}
+
+const CrawledResult: FC<Props> = ({
+  className = '',
+  list,
+  checkedList,
+  onSelectedChange,
+  onPreview,
+  usedTime,
+}) => {
+  const { t } = useTranslation()
+
+  const isCheckAll = checkedList.length === list.length
+
+  const handleCheckedAll = useCallback(() => {
+    if (!isCheckAll)
+      onSelectedChange(list)
+
+    else
+      onSelectedChange([])
+  }, [isCheckAll, list, onSelectedChange])
+
+  const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
+    return (checked: boolean) => {
+      if (checked)
+        onSelectedChange([...checkedList, item])
+
+      else
+        onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
+    }
+  }, [checkedList, onSelectedChange])
+
+  const [previewIndex, setPreviewIndex] = React.useState<number>(-1)
+  const handlePreview = useCallback((index: number) => {
+    return () => {
+      setPreviewIndex(index)
+      onPreview(list[index])
+    }
+  }, [list, onPreview])
+
+  return (
+    <div className={cn(className, 'border-t-[0.5px] border-divider-regular shadow-xs shadow-shadow-shadow-3')}>
+      <div className='flex h-[34px] items-center justify-between px-4'>
+        <CheckboxWithLabel
+          isChecked={isCheckAll}
+          onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
+          labelClassName='system-[13px] leading-[16px] font-medium text-text-secondary'
+        />
+        <div className='text-xs text-text-tertiary'>
+          {t(`${I18N_PREFIX}.scrapTimeInfo`, {
+            total: list.length,
+            time: usedTime.toFixed(1),
+          })}
+        </div>
+      </div>
+      <div className='p-2'>
+        {list.map((item, index) => (
+          <CrawledResultItem
+            key={item.source_url}
+            isPreview={index === previewIndex}
+            onPreview={handlePreview(index)}
+            payload={item}
+            isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
+            onCheckChange={handleItemCheckChange(item)}
+          />
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(CrawledResult)

dify/web/app/components/datasets/create/website/base/crawling.tsx

@@ -0,0 +1,37 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { RowStruct } from '@/app/components/base/icons/src/public/other'
+
+type Props = {
+  className?: string
+  crawledNum: number
+  totalNum: number
+}
+
+const Crawling: FC<Props> = ({
+  className = '',
+  crawledNum,
+  totalNum,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={className}>
+      <div className='flex h-[34px] items-center border-y-[0.5px] border-divider-regular px-4
+        text-xs text-text-tertiary shadow-xs shadow-shadow-shadow-3'>
+        {t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
+      </div>
+
+      <div className='p-2'>
+        {['', '', '', ''].map((item, index) => (
+          <div className='py-[5px]' key={index}>
+            <RowStruct className='text-text-quaternary' />
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(Crawling)

dify/web/app/components/datasets/create/website/base/error-message.tsx

@@ -0,0 +1,30 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from '@/utils/classnames'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+
+type Props = {
+  className?: string
+  title: string
+  errorMsg?: string
+}
+
+const ErrorMessage: FC<Props> = ({
+  className,
+  title,
+  errorMsg,
+}) => {
+  return (
+    <div className={cn(className, 'border-t border-divider-subtle bg-dataset-warning-message-bg px-4 py-2 opacity-40')}>
+      <div className='flex h-5 items-center'>
+        <AlertTriangle className='mr-2 h-4 w-4 text-text-warning-secondary' />
+        <div className='system-md-medium text-text-warning'>{title}</div>
+      </div>
+      {errorMsg && (
+        <div className='system-xs-regular mt-1 pl-6 text-text-secondary'>{errorMsg}</div>
+      )}
+    </div>
+  )
+}
+export default React.memo(ErrorMessage)

dify/web/app/components/datasets/create/website/base/field.tsx

@@ -0,0 +1,54 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import Input from './input'
+import cn from '@/utils/classnames'
+import Tooltip from '@/app/components/base/tooltip'
+
+type Props = {
+  className?: string
+  label: string
+  labelClassName?: string
+  value: string | number
+  onChange: (value: string | number) => void
+  isRequired?: boolean
+  placeholder?: string
+  isNumber?: boolean
+  tooltip?: string
+}
+
+const Field: FC<Props> = ({
+  className,
+  label,
+  labelClassName,
+  value,
+  onChange,
+  isRequired = false,
+  placeholder = '',
+  isNumber = false,
+  tooltip,
+}) => {
+  return (
+    <div className={cn(className)}>
+      <div className='flex py-[7px]'>
+        <div className={cn(labelClassName, 'flex h-[16px] items-center text-[13px] font-semibold text-text-secondary')}>{label} </div>
+        {isRequired && <span className='ml-0.5 text-xs font-semibold text-text-destructive'>*</span>}
+        {tooltip && (
+          <Tooltip
+            popupContent={
+              <div className='w-[200px]'>{tooltip}</div>
+            }
+            triggerClassName='ml-0.5 w-4 h-4'
+          />
+        )}
+      </div>
+      <Input
+        value={value}
+        onChange={onChange}
+        placeholder={placeholder}
+        isNumber={isNumber}
+      />
+    </div>
+  )
+}
+export default React.memo(Field)

dify/web/app/components/datasets/create/website/base/header.tsx

@@ -0,0 +1,61 @@
+import React from 'react'
+import Divider from '@/app/components/base/divider'
+import Button from '@/app/components/base/button'
+import cn from '@/utils/classnames'
+import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
+
+type HeaderProps = {
+  isInPipeline?: boolean
+  onClickConfiguration?: () => void
+  title: string
+  buttonText?: string
+  docTitle: string
+  docLink: string
+}
+
+const Header = ({
+  isInPipeline = false,
+  onClickConfiguration,
+  title,
+  buttonText,
+  docTitle,
+  docLink,
+}: HeaderProps) => {
+  return (
+    <div className='flex items-center gap-x-2'>
+      <div className='flex shrink-0 grow items-center gap-x-1'>
+        <div className={cn(
+          'text-text-secondary',
+          isInPipeline ? 'system-sm-semibold' : 'system-md-semibold',
+        )}>
+          {title}
+        </div>
+        <Divider type='vertical' className='mx-1 h-3.5' />
+        <Button
+          variant='secondary'
+          size='small'
+          className={cn(isInPipeline ? 'size-6 px-1' : 'gap-x-0.5 px-1.5')}
+          onClick={onClickConfiguration}
+        >
+          <RiEqualizer2Line className='size-4' />
+          {!isInPipeline && (
+            <span className='system-xs-medium'>
+              {buttonText}
+            </span>
+          )}
+        </Button>
+      </div>
+      <a
+        className='system-xs-medium flex items-center gap-x-1 overflow-hidden text-text-accent'
+        href={docLink}
+        target='_blank'
+        rel='noopener noreferrer'
+      >
+        <RiBookOpenLine className='size-3.5 shrink-0' />
+        <span className='grow truncate' title={docTitle}>{docTitle}</span>
+      </a>
+    </div>
+  )
+}
+
+export default React.memo(Header)

dify/web/app/components/datasets/create/website/base/input.tsx

@@ -0,0 +1,63 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+
+type Props = {
+  value: string | number
+  onChange: (value: string | number) => void
+  placeholder?: string
+  isNumber?: boolean
+}
+
+const MIN_VALUE = 0
+
+const Input: FC<Props> = ({
+  value,
+  onChange,
+  placeholder = '',
+  isNumber = false,
+}) => {
+  const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
+    const value = e.target.value
+    if (isNumber) {
+      let numberValue = Number.parseInt(value, 10) // integer only
+      if (Number.isNaN(numberValue)) {
+        onChange('')
+        return
+      }
+      if (numberValue < MIN_VALUE)
+        numberValue = MIN_VALUE
+
+      onChange(numberValue)
+      return
+    }
+    onChange(value)
+  }, [isNumber, onChange])
+
+  const otherOption = (() => {
+    if (isNumber) {
+      return {
+        min: MIN_VALUE,
+      }
+    }
+    return {
+
+    }
+  })()
+  return (
+    <input
+      type={isNumber ? 'number' : 'text'}
+      {...otherOption}
+      value={value}
+      onChange={handleChange}
+      className='system-xs-regular focus:bg-components-inout-border-active flex h-8 w-full rounded-lg border border-transparent
+      bg-components-input-bg-normal p-2 text-components-input-text-filled
+        caret-[#295eff] placeholder:text-components-input-text-placeholder hover:border
+        hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border focus:border-components-input-border-active
+        focus:shadow-xs focus:shadow-shadow-shadow-3
+        focus-visible:outline-none'
+      placeholder={placeholder}
+    />
+  )
+}
+export default React.memo(Input)

dify/web/app/components/datasets/create/website/base/mock-crawl-result.ts

@@ -0,0 +1,24 @@
+import type { CrawlResultItem } from '@/models/datasets'
+
+const result: CrawlResultItem[] = [
+  {
+    title: 'Start the frontend Docker container separately',
+    content: 'Markdown 1',
+    description: 'Description 1',
+    source_url: 'https://example.com/1',
+  },
+  {
+    title: 'Advanced Tool Integration',
+    content: 'Markdown 2',
+    description: 'Description 2',
+    source_url: 'https://example.com/2',
+  },
+  {
+    title: 'Local Source Code Start | English | Dify',
+    content: 'Markdown 3',
+    description: 'Description 3',
+    source_url: 'https://example.com/3',
+  },
+]
+
+export default result

dify/web/app/components/datasets/create/website/base/options-wrap.tsx

@@ -0,0 +1,54 @@
+'use client'
+import { useBoolean } from 'ahooks'
+import type { FC } from 'react'
+import React, { useEffect } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiEqualizer2Line } from '@remixicon/react'
+import cn from '@/utils/classnames'
+import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  children: React.ReactNode
+  controlFoldOptions?: number
+}
+
+const OptionsWrap: FC<Props> = ({
+  className = '',
+  children,
+  controlFoldOptions,
+}) => {
+  const { t } = useTranslation()
+
+  const [fold, {
+    toggle: foldToggle,
+    setTrue: foldHide,
+  }] = useBoolean(false)
+
+  useEffect(() => {
+    if (controlFoldOptions)
+      foldHide()
+  }, [controlFoldOptions])
+  return (
+    <div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
+      <div
+        className='flex h-[26px] cursor-pointer select-none items-center gap-x-1 py-1'
+        onClick={foldToggle}
+      >
+        <div className='flex grow items-center'>
+          <RiEqualizer2Line className='mr-1 h-4 w-4 text-text-secondary' />
+          <span className='text-[13px] font-semibold uppercase leading-[16px] text-text-secondary'>{t(`${I18N_PREFIX}.options`)}</span>
+        </div>
+        <ChevronRight className={cn(!fold && 'rotate-90', 'h-4 w-4 shrink-0 text-text-tertiary')} />
+      </div>
+      {!fold && (
+        <div className='mb-4'>
+          {children}
+        </div>
+      )}
+
+    </div>
+  )
+}
+export default React.memo(OptionsWrap)

dify/web/app/components/datasets/create/website/base/url-input.tsx

@@ -0,0 +1,50 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import Input from './input'
+import Button from '@/app/components/base/button'
+import { useDocLink } from '@/context/i18n'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  isRunning: boolean
+  onRun: (url: string) => void
+}
+
+const UrlInput: FC<Props> = ({
+  isRunning,
+  onRun,
+}) => {
+  const { t } = useTranslation()
+  const docLink = useDocLink()
+  const [url, setUrl] = useState('')
+  const handleUrlChange = useCallback((url: string | number) => {
+    setUrl(url as string)
+  }, [])
+  const handleOnRun = useCallback(() => {
+    if (isRunning)
+      return
+    onRun(url)
+  }, [isRunning, onRun, url])
+
+  return (
+    <div className='flex items-center justify-between gap-x-2'>
+      <Input
+        value={url}
+        onChange={handleUrlChange}
+        placeholder={docLink()}
+      />
+      <Button
+        variant='primary'
+        onClick={handleOnRun}
+        loading={isRunning}
+        spinnerClassName='!ml-0'
+      >
+        {!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
+      </Button>
+    </div>
+  )
+}
+export default React.memo(UrlInput)

dify/web/app/components/datasets/create/website/firecrawl/index.tsx

@@ -0,0 +1,232 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import UrlInput from '../base/url-input'
+import OptionsWrap from '../base/options-wrap'
+import CrawledResult from '../base/crawled-result'
+import Crawling from '../base/crawling'
+import ErrorMessage from '../base/error-message'
+import Options from './options'
+import { useModalContextSelector } from '@/context/modal-context'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import Toast from '@/app/components/base/toast'
+import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
+import { sleep } from '@/utils'
+import Header from '../base/header'
+import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
+
+const ERROR_I18N_PREFIX = 'common.errorMsg'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
+}
+
+enum Step {
+  init = 'init',
+  running = 'running',
+  finished = 'finished',
+}
+
+const FireCrawl: FC<Props> = ({
+  onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
+  onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
+}) => {
+  const { t } = useTranslation()
+  const [step, setStep] = useState<Step>(Step.init)
+  const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
+  useEffect(() => {
+    if (step !== Step.init)
+      setControlFoldOptions(Date.now())
+  }, [step])
+  const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
+    })
+  }, [setShowAccountSettingModal])
+
+  const checkValid = useCallback((url: string) => {
+    let errorMsg = ''
+    if (!url) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: 'url',
+      })
+    }
+
+    if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
+      errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
+
+    if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.limit`),
+      })
+    }
+
+    return {
+      isValid: !errorMsg,
+      errorMsg,
+    }
+  }, [crawlOptions, t])
+
+  const isInit = step === Step.init
+  const isCrawlFinished = step === Step.finished
+  const isRunning = step === Step.running
+  const [crawlResult, setCrawlResult] = useState<{
+    current: number
+    total: number
+    data: CrawlResultItem[]
+    time_consuming: number | string
+  } | undefined>(undefined)
+  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
+  const showError = isCrawlFinished && crawlErrorMessage
+
+  const waitForCrawlFinished = useCallback(async (jobId: string) => {
+    try {
+      const res = await checkFirecrawlTaskStatus(jobId) as any
+      if (res.status === 'completed') {
+        return {
+          isError: false,
+          data: {
+            ...res,
+            total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+          },
+        }
+      }
+      if (res.status === 'error' || !res.status) {
+        // can't get the error message from the firecrawl api
+        return {
+          isError: true,
+          errorMessage: res.message,
+          data: {
+            data: [],
+          },
+        }
+      }
+      res.data = res.data.map((item: any) => ({
+        ...item,
+        content: item.markdown,
+      }))
+      // update the progress
+      setCrawlResult({
+        ...res,
+        total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+      })
+      onCheckedCrawlResultChange(res.data || []) // default select the crawl result
+      await sleep(2500)
+      return await waitForCrawlFinished(jobId)
+    }
+    catch (e: any) {
+      const errorBody = await e.json()
+      return {
+        isError: true,
+        errorMessage: errorBody.message,
+        data: {
+          data: [],
+        },
+      }
+    }
+  }, [crawlOptions.limit, onCheckedCrawlResultChange])
+
+  const handleRun = useCallback(async (url: string) => {
+    const { isValid, errorMsg } = checkValid(url)
+    if (!isValid) {
+      Toast.notify({
+        message: errorMsg!,
+        type: 'error',
+      })
+      return
+    }
+    setStep(Step.running)
+    try {
+      const passToServerCrawlOptions: any = {
+        ...crawlOptions,
+      }
+      if (crawlOptions.max_depth === '')
+        delete passToServerCrawlOptions.max_depth
+
+      const res = await createFirecrawlTask({
+        url,
+        options: passToServerCrawlOptions,
+      }) as any
+      const jobId = res.job_id
+      onJobIdChange(jobId)
+      const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+      if (isError) {
+        setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
+      }
+      else {
+        data.data = data.data.map((item: any) => ({
+          ...item,
+          content: item.markdown,
+        }))
+        setCrawlResult(data)
+        onCheckedCrawlResultChange(data.data || []) // default select the crawl result
+        setCrawlErrorMessage('')
+      }
+    }
+    catch (e) {
+      setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
+      console.log(e)
+    }
+    finally {
+      setStep(Step.finished)
+    }
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
+
+  return (
+    <div>
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.firecrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
+        docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
+        docLink={'https://docs.firecrawl.dev/introduction'}
+      />
+      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
+        <UrlInput onRun={handleRun} isRunning={isRunning} />
+        <OptionsWrap
+          className='mt-4'
+          controlFoldOptions={controlFoldOptions}
+        >
+          <Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />
+        </OptionsWrap>
+
+        {!isInit && (
+          <div className='relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl'>
+            {isRunning
+              && <Crawling
+                className='mt-2'
+                crawledNum={crawlResult?.current || 0}
+                totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
+              />}
+            {showError && (
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={crawlErrorMessage} />
+            )}
+            {isCrawlFinished && !showError
+              && <CrawledResult
+                className='mb-2'
+                list={crawlResult?.data || []}
+                checkedList={checkedCrawlResult}
+                onSelectedChange={onCheckedCrawlResultChange}
+                onPreview={onPreview}
+                usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
+              />
+            }
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+export default React.memo(FireCrawl)

dify/web/app/components/datasets/create/website/firecrawl/options.tsx

@@ -0,0 +1,85 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from '../base/checkbox-with-label'
+import Field from '../base/field'
+import cn from '@/utils/classnames'
+import type { CrawlOptions } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  payload: CrawlOptions
+  onChange: (payload: CrawlOptions) => void
+}
+
+const Options: FC<Props> = ({
+  className = '',
+  payload,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+
+  const handleChange = useCallback((key: keyof CrawlOptions) => {
+    return (value: any) => {
+      onChange({
+        ...payload,
+        [key]: value,
+      })
+    }
+  }, [payload, onChange])
+  return (
+    <div className={cn(className, ' space-y-2')}>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.crawlSubPage`)}
+        isChecked={payload.crawl_sub_pages}
+        onChange={handleChange('crawl_sub_pages')}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.limit`)}
+          value={payload.limit}
+          onChange={handleChange('limit')}
+          isNumber
+          isRequired
+        />
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.maxDepth`)}
+          value={payload.max_depth}
+          onChange={handleChange('max_depth')}
+          isNumber
+          tooltip={t(`${I18N_PREFIX}.maxDepthTooltip`)!}
+        />
+      </div>
+
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.excludePaths`)}
+          value={payload.excludes}
+          onChange={handleChange('excludes')}
+          placeholder='blog/*, /about/*'
+        />
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.includeOnlyPaths`)}
+          value={payload.includes}
+          onChange={handleChange('includes')}
+          placeholder='articles/*'
+        />
+      </div>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.extractOnlyMainContent`)}
+        isChecked={payload.only_main_content}
+        onChange={handleChange('only_main_content')}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+    </div>
+  )
+}
+export default React.memo(Options)

dify/web/app/components/datasets/create/website/index.module.css

@@ -0,0 +1,13 @@
+.jinaLogo {
+  @apply w-4 h-4 bg-center bg-no-repeat inline-block;
+  background-color: #F5FAFF;
+  background-image: url(../assets/jina.png);
+  background-size: 16px;
+}
+
+.watercrawlLogo {
+  @apply w-4 h-4 bg-center bg-no-repeat inline-block;
+  /*background-color: #F5FAFF;*/
+  background-image: url(../assets/watercrawl.svg);
+  background-size: 16px;
+}

dify/web/app/components/datasets/create/website/index.tsx

@@ -0,0 +1,147 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import s from './index.module.css'
+import NoData from './no-data'
+import Firecrawl from './firecrawl'
+import Watercrawl from './watercrawl'
+import JinaReader from './jina-reader'
+import cn from '@/utils/classnames'
+import { useModalContext } from '@/context/modal-context'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import { DataSourceProvider } from '@/models/common'
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'
+import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
+
+type Props = {
+  onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onCrawlProviderChange: (provider: DataSourceProvider) => void
+  onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
+  authedDataSourceList: DataSourceAuth[]
+}
+
+const Website: FC<Props> = ({
+  onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
+  onCrawlProviderChange,
+  onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
+  authedDataSourceList,
+}) => {
+  const { t } = useTranslation()
+  const { setShowAccountSettingModal } = useModalContext()
+  const [selectedProvider, setSelectedProvider] = useState<DataSourceProvider>(DataSourceProvider.jinaReader)
+
+  const availableProviders = useMemo(() => authedDataSourceList.filter((item) => {
+    return [
+      DataSourceProvider.jinaReader,
+      DataSourceProvider.fireCrawl,
+      DataSourceProvider.waterCrawl].includes(item.provider as DataSourceProvider) && item.credentials_list.length > 0
+  }), [authedDataSourceList])
+
+  const handleOnConfig = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
+    })
+  }, [setShowAccountSettingModal])
+
+  const source = availableProviders.find(source => source.provider === selectedProvider)
+
+  return (
+    <div>
+      <div className='mb-4'>
+        <div className='system-md-medium mb-2 text-text-secondary'>
+          {t('datasetCreation.stepOne.website.chooseProvider')}
+        </div>
+        <div className='flex space-x-2'>
+          {ENABLE_WEBSITE_JINAREADER && <button type="button"
+            className={cn('flex items-center justify-center rounded-lg px-4 py-2',
+              selectedProvider === DataSourceProvider.jinaReader
+                ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
+                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
+                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
+            )}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.jinaReader)
+              onCrawlProviderChange(DataSourceProvider.jinaReader)
+            }}
+          >
+            <span className={cn(s.jinaLogo, 'mr-2')} />
+            <span>Jina Reader</span>
+          </button>}
+          {ENABLE_WEBSITE_FIRECRAWL && <button type="button"
+            className={cn('rounded-lg px-4 py-2',
+              selectedProvider === DataSourceProvider.fireCrawl
+                ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
+                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
+                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
+            )}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.fireCrawl)
+              onCrawlProviderChange(DataSourceProvider.fireCrawl)
+            }}
+          >
+            🔥 Firecrawl
+          </button>}
+          {ENABLE_WEBSITE_WATERCRAWL && <button type="button"
+            className={cn('flex items-center justify-center rounded-lg px-4 py-2',
+              selectedProvider === DataSourceProvider.waterCrawl
+                ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
+                : `system-sm-regular border border-components-option-card-option-border bg-components-option-card-option-bg text-text-secondary
+                hover:border-components-option-card-option-border-hover hover:bg-components-option-card-option-bg-hover hover:shadow-xs hover:shadow-shadow-shadow-3`,
+            )}
+            onClick={() => {
+              setSelectedProvider(DataSourceProvider.waterCrawl)
+              onCrawlProviderChange(DataSourceProvider.waterCrawl)
+            }}
+          >
+            <span className={cn(s.watercrawlLogo, 'mr-2')} />
+            <span>WaterCrawl</span>
+          </button>}
+        </div>
+      </div>
+      {source && selectedProvider === DataSourceProvider.fireCrawl && (
+        <Firecrawl
+          onPreview={onPreview}
+          checkedCrawlResult={checkedCrawlResult}
+          onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+          onJobIdChange={onJobIdChange}
+          crawlOptions={crawlOptions}
+          onCrawlOptionsChange={onCrawlOptionsChange}
+        />
+      )}
+      {source && selectedProvider === DataSourceProvider.waterCrawl && (
+        <Watercrawl
+          onPreview={onPreview}
+          checkedCrawlResult={checkedCrawlResult}
+          onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+          onJobIdChange={onJobIdChange}
+          crawlOptions={crawlOptions}
+          onCrawlOptionsChange={onCrawlOptionsChange}
+        />
+      )}
+      {source && selectedProvider === DataSourceProvider.jinaReader && (
+        <JinaReader
+          onPreview={onPreview}
+          checkedCrawlResult={checkedCrawlResult}
+          onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+          onJobIdChange={onJobIdChange}
+          crawlOptions={crawlOptions}
+          onCrawlOptionsChange={onCrawlOptionsChange}
+        />
+      )}
+      {!source && (
+        <NoData onConfig={handleOnConfig} provider={selectedProvider} />
+      )}
+    </div>
+  )
+}
+export default React.memo(Website)

dify/web/app/components/datasets/create/website/jina-reader/base/url-input.tsx

@@ -0,0 +1,50 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import Input from '../../base/input'
+import Button from '@/app/components/base/button'
+import { useDocLink } from '@/context/i18n'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  isRunning: boolean
+  onRun: (url: string) => void
+}
+
+const UrlInput: FC<Props> = ({
+  isRunning,
+  onRun,
+}) => {
+  const { t } = useTranslation()
+  const docLink = useDocLink()
+  const [url, setUrl] = useState('')
+  const handleUrlChange = useCallback((url: string | number) => {
+    setUrl(url as string)
+  }, [])
+  const handleOnRun = useCallback(() => {
+    if (isRunning)
+      return
+    onRun(url)
+  }, [isRunning, onRun, url])
+
+  return (
+    <div className='flex items-center justify-between'>
+      <Input
+        value={url}
+        onChange={handleUrlChange}
+        placeholder={docLink()}
+      />
+      <Button
+        variant='primary'
+        onClick={handleOnRun}
+        className='ml-2'
+        loading={isRunning}
+      >
+        {!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
+      </Button>
+    </div>
+  )
+}
+export default React.memo(UrlInput)

dify/web/app/components/datasets/create/website/jina-reader/index.tsx

@@ -0,0 +1,238 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import UrlInput from '../base/url-input'
+import OptionsWrap from '../base/options-wrap'
+import CrawledResult from '../base/crawled-result'
+import Crawling from '../base/crawling'
+import ErrorMessage from '../base/error-message'
+import Options from './options'
+import { useModalContext } from '@/context/modal-context'
+import Toast from '@/app/components/base/toast'
+import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
+import { sleep } from '@/utils'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import Header from '../base/header'
+import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
+
+const ERROR_I18N_PREFIX = 'common.errorMsg'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
+}
+
+enum Step {
+  init = 'init',
+  running = 'running',
+  finished = 'finished',
+}
+
+const JinaReader: FC<Props> = ({
+  onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
+  onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
+}) => {
+  const { t } = useTranslation()
+  const [step, setStep] = useState<Step>(Step.init)
+  const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
+  useEffect(() => {
+    if (step !== Step.init)
+      setControlFoldOptions(Date.now())
+  }, [step])
+  const { setShowAccountSettingModal } = useModalContext()
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
+    })
+  }, [setShowAccountSettingModal])
+
+  const checkValid = useCallback((url: string) => {
+    let errorMsg = ''
+    if (!url) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: 'url',
+      })
+    }
+
+    if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
+      errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
+
+    if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.limit`),
+      })
+    }
+
+    return {
+      isValid: !errorMsg,
+      errorMsg,
+    }
+  }, [crawlOptions, t])
+
+  const isInit = step === Step.init
+  const isCrawlFinished = step === Step.finished
+  const isRunning = step === Step.running
+  const [crawlResult, setCrawlResult] = useState<{
+    current: number
+    total: number
+    data: CrawlResultItem[]
+    time_consuming: number | string
+  } | undefined>(undefined)
+  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
+  const showError = isCrawlFinished && crawlErrorMessage
+
+  const waitForCrawlFinished = useCallback(async (jobId: string) => {
+    try {
+      const res = await checkJinaReaderTaskStatus(jobId) as any
+      if (res.status === 'completed') {
+        return {
+          isError: false,
+          data: {
+            ...res,
+            total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+          },
+        }
+      }
+      if (res.status === 'failed' || !res.status) {
+        return {
+          isError: true,
+          errorMessage: res.message,
+          data: {
+            data: [],
+          },
+        }
+      }
+      // update the progress
+      setCrawlResult({
+        ...res,
+        total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+      })
+      onCheckedCrawlResultChange(res.data || []) // default select the crawl result
+      await sleep(2500)
+      return await waitForCrawlFinished(jobId)
+    }
+    catch (e: any) {
+      const errorBody = await e.json()
+      return {
+        isError: true,
+        errorMessage: errorBody.message,
+        data: {
+          data: [],
+        },
+      }
+    }
+  }, [crawlOptions.limit, onCheckedCrawlResultChange])
+
+  const handleRun = useCallback(async (url: string) => {
+    const { isValid, errorMsg } = checkValid(url)
+    if (!isValid) {
+      Toast.notify({
+        message: errorMsg!,
+        type: 'error',
+      })
+      return
+    }
+    setStep(Step.running)
+    try {
+      const startTime = Date.now()
+      const res = await createJinaReaderTask({
+        url,
+        options: crawlOptions,
+      }) as any
+
+      if (res.data) {
+        const { title, content, description, url } = res.data
+        const data = {
+          current: 1,
+          total: 1,
+          data: [{
+            title,
+            content,
+            description,
+            source_url: url,
+          }],
+          time_consuming: (Date.now() - startTime) / 1000,
+        }
+        setCrawlResult(data)
+        onCheckedCrawlResultChange(data.data || [])
+        setCrawlErrorMessage('')
+      }
+      else if (res.job_id) {
+        const jobId = res.job_id
+        onJobIdChange(jobId)
+        const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+        if (isError) {
+          setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
+        }
+        else {
+          setCrawlResult(data)
+          onCheckedCrawlResultChange(data.data || []) // default select the crawl result
+          setCrawlErrorMessage('')
+        }
+      }
+    }
+    catch (e) {
+      setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
+      console.log(e)
+    }
+    finally {
+      setStep(Step.finished)
+    }
+  }, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished])
+
+  return (
+    <div>
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.jinaReaderTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureJinaReader`)}
+        docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`)}
+        docLink={'https://jina.ai/reader'}
+      />
+      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
+        <UrlInput onRun={handleRun} isRunning={isRunning} />
+        <OptionsWrap
+          className='mt-4'
+          controlFoldOptions={controlFoldOptions}
+        >
+          <Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />
+        </OptionsWrap>
+
+        {!isInit && (
+          <div className='relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl'>
+            {isRunning
+              && <Crawling
+                className='mt-2'
+                crawledNum={crawlResult?.current || 0}
+                totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
+              />}
+            {showError && (
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={crawlErrorMessage} />
+            )}
+            {isCrawlFinished && !showError
+              && <CrawledResult
+                className='mb-2'
+                list={crawlResult?.data || []}
+                checkedList={checkedCrawlResult}
+                onSelectedChange={onCheckedCrawlResultChange}
+                onPreview={onPreview}
+                usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
+              />
+            }
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+export default React.memo(JinaReader)

dify/web/app/components/datasets/create/website/jina-reader/options.tsx

@@ -0,0 +1,61 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from '../base/checkbox-with-label'
+import Field from '../base/field'
+import cn from '@/utils/classnames'
+import type { CrawlOptions } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  payload: CrawlOptions
+  onChange: (payload: CrawlOptions) => void
+}
+
+const Options: FC<Props> = ({
+  className = '',
+  payload,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+
+  const handleChange = useCallback((key: keyof CrawlOptions) => {
+    return (value: any) => {
+      onChange({
+        ...payload,
+        [key]: value,
+      })
+    }
+  }, [payload, onChange])
+  return (
+    <div className={cn(className, ' space-y-2')}>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.crawlSubPage`)}
+        isChecked={payload.crawl_sub_pages}
+        onChange={handleChange('crawl_sub_pages')}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.useSitemap`)}
+        isChecked={payload.use_sitemap}
+        onChange={handleChange('use_sitemap')}
+        tooltip={t(`${I18N_PREFIX}.useSitemapTooltip`) as string}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.limit`)}
+          value={payload.limit}
+          onChange={handleChange('limit')}
+          isNumber
+          isRequired
+        />
+      </div>
+    </div>
+  )
+}
+export default React.memo(Options)

dify/web/app/components/datasets/create/website/no-data.tsx

@@ -0,0 +1,73 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import s from './index.module.css'
+import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
+import Button from '@/app/components/base/button'
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
+import { DataSourceProvider } from '@/models/common'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onConfig: () => void
+  provider: DataSourceProvider
+}
+
+const NoData: FC<Props> = ({
+  onConfig,
+  provider,
+}) => {
+  const { t } = useTranslation()
+
+  const providerConfig: Record<DataSourceProvider, {
+    emoji: React.ReactNode
+    title: string
+    description: string
+  } | null> = {
+    [DataSourceProvider.jinaReader]: ENABLE_WEBSITE_JINAREADER ? {
+      emoji: <span className={s.jinaLogo} />,
+      title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`),
+      description: t(`${I18N_PREFIX}.jinaReaderNotConfiguredDescription`),
+    } : null,
+    [DataSourceProvider.fireCrawl]: ENABLE_WEBSITE_FIRECRAWL ? {
+      emoji: '🔥',
+      title: t(`${I18N_PREFIX}.fireCrawlNotConfigured`),
+      description: t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`),
+    } : null,
+    [DataSourceProvider.waterCrawl]: ENABLE_WEBSITE_WATERCRAWL ? {
+      emoji: '💧',
+      title: t(`${I18N_PREFIX}.waterCrawlNotConfigured`),
+      description: t(`${I18N_PREFIX}.waterCrawlNotConfiguredDescription`),
+    } : null,
+  }
+
+  const currentProvider = providerConfig[provider] || providerConfig.jinareader
+
+  if (!currentProvider) return null
+
+  return (
+    <>
+      <div className='mt-4 max-w-[640px] rounded-2xl bg-workflow-process-bg p-6'>
+        <div className='flex h-12 w-12 items-center justify-center rounded-[10px] border-[0.5px]
+          border-components-card-border bg-components-card-bg shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'>
+          {currentProvider.emoji}
+        </div>
+        <div className='mb-1 mt-2 flex flex-col gap-y-1 pb-3 pt-1'>
+          <span className='system-md-semibold text-text-secondary'>
+            {currentProvider.title}
+            <Icon3Dots className='relative -left-1.5 -top-2.5 inline' />
+          </span>
+          <div className='system-sm-regular text-text-tertiary'>
+            {currentProvider.description}
+          </div>
+        </div>
+        <Button variant='primary' onClick={onConfig}>
+          {t(`${I18N_PREFIX}.configure`)}
+        </Button>
+      </div>
+    </>
+  )
+}
+export default React.memo(NoData)

dify/web/app/components/datasets/create/website/preview.tsx

@@ -0,0 +1,41 @@
+'use client'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { XMarkIcon } from '@heroicons/react/20/solid'
+import s from '../file-preview/index.module.css'
+import cn from '@/utils/classnames'
+import type { CrawlResultItem } from '@/models/datasets'
+
+type IProps = {
+  payload: CrawlResultItem
+  hidePreview: () => void
+}
+
+const WebsitePreview = ({
+  payload,
+  hidePreview,
+}: IProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={cn(s.filePreview, 'h-full')}>
+      <div className={cn(s.previewHeader)}>
+        <div className={cn(s.title, 'title-md-semi-bold')}>
+          <span>{t('datasetCreation.stepOne.pagePreview')}</span>
+          <div className='flex h-6 w-6 cursor-pointer items-center justify-center' onClick={hidePreview}>
+            <XMarkIcon className='h-4 w-4'></XMarkIcon>
+          </div>
+        </div>
+        <div className='title-sm-semi-bold break-words text-text-primary'>
+          {payload.title}
+        </div>
+        <div className='system-xs-medium truncate text-text-tertiary' title={payload.source_url}>{payload.source_url}</div>
+      </div>
+      <div className={cn(s.previewContent, 'body-md-regular')}>
+        <div className={cn(s.fileContent)}>{payload.content}</div>
+      </div>
+    </div>
+  )
+}
+
+export default WebsitePreview

dify/web/app/components/datasets/create/website/watercrawl/index.tsx

@@ -0,0 +1,224 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import UrlInput from '../base/url-input'
+import OptionsWrap from '../base/options-wrap'
+import CrawledResult from '../base/crawled-result'
+import Crawling from '../base/crawling'
+import ErrorMessage from '../base/error-message'
+import Options from './options'
+import { useModalContext } from '@/context/modal-context'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+import Toast from '@/app/components/base/toast'
+import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets'
+import { sleep } from '@/utils'
+import Header from '../base/header'
+import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
+
+const ERROR_I18N_PREFIX = 'common.errorMsg'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
+}
+
+enum Step {
+  init = 'init',
+  running = 'running',
+  finished = 'finished',
+}
+
+const WaterCrawl: FC<Props> = ({
+  onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
+  onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
+}) => {
+  const { t } = useTranslation()
+  const [step, setStep] = useState<Step>(Step.init)
+  const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
+  useEffect(() => {
+    if (step !== Step.init)
+      setControlFoldOptions(Date.now())
+  }, [step])
+  const { setShowAccountSettingModal } = useModalContext()
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
+    })
+  }, [setShowAccountSettingModal])
+
+  const checkValid = useCallback((url: string) => {
+    let errorMsg = ''
+    if (!url) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: 'url',
+      })
+    }
+
+    if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
+      errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
+
+    if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.limit`),
+      })
+    }
+
+    return {
+      isValid: !errorMsg,
+      errorMsg,
+    }
+  }, [crawlOptions, t])
+
+  const isInit = step === Step.init
+  const isCrawlFinished = step === Step.finished
+  const isRunning = step === Step.running
+  const [crawlResult, setCrawlResult] = useState<{
+    current: number
+    total: number
+    data: CrawlResultItem[]
+    time_consuming: number | string
+  } | undefined>(undefined)
+  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
+  const showError = isCrawlFinished && crawlErrorMessage
+
+  const waitForCrawlFinished = useCallback(async (jobId: string): Promise<any> => {
+    try {
+      const res = await checkWatercrawlTaskStatus(jobId) as any
+      if (res.status === 'completed') {
+        return {
+          isError: false,
+          data: {
+            ...res,
+            total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+          },
+        }
+      }
+      if (res.status === 'error' || !res.status) {
+        // can't get the error message from the watercrawl api
+        return {
+          isError: true,
+          errorMessage: res.message,
+          data: {
+            data: [],
+          },
+        }
+      }
+      // update the progress
+      setCrawlResult({
+        ...res,
+        total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
+      })
+      onCheckedCrawlResultChange(res.data || []) // default select the crawl result
+      await sleep(2500)
+      return await waitForCrawlFinished(jobId)
+    }
+    catch (e: any) {
+      const errorBody = await e.json()
+      return {
+        isError: true,
+        errorMessage: errorBody.message,
+        data: {
+          data: [],
+        },
+      }
+    }
+  }, [crawlOptions.limit])
+
+  const handleRun = useCallback(async (url: string) => {
+    const { isValid, errorMsg } = checkValid(url)
+    if (!isValid) {
+      Toast.notify({
+        message: errorMsg!,
+        type: 'error',
+      })
+      return
+    }
+    setStep(Step.running)
+    try {
+      const passToServerCrawlOptions: any = {
+        ...crawlOptions,
+      }
+      if (crawlOptions.max_depth === '')
+        delete passToServerCrawlOptions.max_depth
+
+      const res = await createWatercrawlTask({
+        url,
+        options: passToServerCrawlOptions,
+      }) as any
+      const jobId = res.job_id
+      onJobIdChange(jobId)
+      const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+      if (isError) {
+        setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
+      }
+      else {
+        setCrawlResult(data)
+        onCheckedCrawlResultChange(data.data || []) // default select the crawl result
+        setCrawlErrorMessage('')
+      }
+    }
+    catch (e) {
+      setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
+      console.log(e)
+    }
+    finally {
+      setStep(Step.finished)
+    }
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
+
+  return (
+    <div>
+      <Header
+        onClickConfiguration={handleSetting}
+        title={t(`${I18N_PREFIX}.watercrawlTitle`)}
+        buttonText={t(`${I18N_PREFIX}.configureWatercrawl`)}
+        docTitle={t(`${I18N_PREFIX}.watercrawlDoc`)}
+        docLink={'https://docs.watercrawl.dev/'}
+      />
+      <div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
+        <UrlInput onRun={handleRun} isRunning={isRunning} />
+        <OptionsWrap
+          className='mt-4'
+          controlFoldOptions={controlFoldOptions}
+        >
+          <Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />
+        </OptionsWrap>
+
+        {!isInit && (
+          <div className='relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl'>
+            {isRunning
+              && <Crawling
+                className='mt-2'
+                crawledNum={crawlResult?.current || 0}
+                totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
+              />}
+            {showError && (
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={crawlErrorMessage} />
+            )}
+            {isCrawlFinished && !showError
+              && <CrawledResult
+                className='mb-2'
+                list={crawlResult?.data || []}
+                checkedList={checkedCrawlResult}
+                onSelectedChange={onCheckedCrawlResultChange}
+                onPreview={onPreview}
+                usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
+              />
+            }
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+export default React.memo(WaterCrawl)

dify/web/app/components/datasets/create/website/watercrawl/options.tsx

@@ -0,0 +1,85 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from '../base/checkbox-with-label'
+import Field from '../base/field'
+import cn from '@/utils/classnames'
+import type { CrawlOptions } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  payload: CrawlOptions
+  onChange: (payload: CrawlOptions) => void
+}
+
+const Options: FC<Props> = ({
+  className = '',
+  payload,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+
+  const handleChange = useCallback((key: keyof CrawlOptions) => {
+    return (value: any) => {
+      onChange({
+        ...payload,
+        [key]: value,
+      })
+    }
+  }, [payload, onChange])
+  return (
+    <div className={cn(className, ' space-y-2')}>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.crawlSubPage`)}
+        isChecked={payload.crawl_sub_pages}
+        onChange={handleChange('crawl_sub_pages')}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.limit`)}
+          value={payload.limit}
+          onChange={handleChange('limit')}
+          isNumber
+          isRequired
+        />
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.maxDepth`)}
+          value={payload.max_depth}
+          onChange={handleChange('max_depth')}
+          isNumber
+          tooltip={t(`${I18N_PREFIX}.maxDepthTooltip`)!}
+        />
+      </div>
+
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.excludePaths`)}
+          value={payload.excludes}
+          onChange={handleChange('excludes')}
+          placeholder='blog/*, /about/*'
+        />
+        <Field
+          className='shrink-0 grow'
+          label={t(`${I18N_PREFIX}.includeOnlyPaths`)}
+          value={payload.includes}
+          onChange={handleChange('includes')}
+          placeholder='articles/*'
+        />
+      </div>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.extractOnlyMainContent`)}
+        isChecked={payload.only_main_content}
+        onChange={handleChange('only_main_content')}
+        labelClassName='text-[13px] leading-[16px] font-medium text-text-secondary'
+      />
+    </div>
+  )
+}
+export default React.memo(Options)