dify

dify/api/controllers/console/datasets/data_source.py

@@ -0,0 +1,323 @@
+import json
+from collections.abc import Generator
+from typing import cast
+
+from flask import request
+from flask_restx import Resource, marshal_with, reqparse
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import NotFound
+
+from controllers.console import console_ns
+from controllers.console.wraps import account_initialization_required, setup_required
+from core.datasource.entities.datasource_entities import DatasourceProviderType, OnlineDocumentPagesMessage
+from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
+from core.indexing_runner import IndexingRunner
+from core.rag.extractor.entity.datasource_type import DatasourceType
+from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo
+from core.rag.extractor.notion_extractor import NotionExtractor
+from extensions.ext_database import db
+from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
+from libs.datetime_utils import naive_utc_now
+from libs.login import current_account_with_tenant, login_required
+from models import DataSourceOauthBinding, Document
+from services.dataset_service import DatasetService, DocumentService
+from services.datasource_provider_service import DatasourceProviderService
+from tasks.document_indexing_sync_task import document_indexing_sync_task
+
+
+@console_ns.route(
+    "/data-source/integrates",
+    "/data-source/integrates/<uuid:binding_id>/<string:action>",
+)
+class DataSourceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(integrate_list_fields)
+    def get(self):
+        _, current_tenant_id = current_account_with_tenant()
+
+        # get workspace data source integrates
+        data_source_integrates = db.session.scalars(
+            select(DataSourceOauthBinding).where(
+                DataSourceOauthBinding.tenant_id == current_tenant_id,
+                DataSourceOauthBinding.disabled == False,
+            )
+        ).all()
+
+        base_url = request.url_root.rstrip("/")
+        data_source_oauth_base_path = "/console/api/oauth/data-source"
+        providers = ["notion"]
+
+        integrate_data = []
+        for provider in providers:
+            # existing_integrate = next((ai for ai in data_source_integrates if ai.provider == provider), None)
+            existing_integrates = filter(lambda item: item.provider == provider, data_source_integrates)
+            if existing_integrates:
+                for existing_integrate in list(existing_integrates):
+                    integrate_data.append(
+                        {
+                            "id": existing_integrate.id,
+                            "provider": provider,
+                            "created_at": existing_integrate.created_at,
+                            "is_bound": True,
+                            "disabled": existing_integrate.disabled,
+                            "source_info": existing_integrate.source_info,
+                            "link": f"{base_url}{data_source_oauth_base_path}/{provider}",
+                        }
+                    )
+            else:
+                integrate_data.append(
+                    {
+                        "id": None,
+                        "provider": provider,
+                        "created_at": None,
+                        "source_info": None,
+                        "is_bound": False,
+                        "disabled": None,
+                        "link": f"{base_url}{data_source_oauth_base_path}/{provider}",
+                    }
+                )
+        return {"data": integrate_data}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, binding_id, action):
+        binding_id = str(binding_id)
+        action = str(action)
+        with Session(db.engine) as session:
+            data_source_binding = session.execute(
+                select(DataSourceOauthBinding).filter_by(id=binding_id)
+            ).scalar_one_or_none()
+        if data_source_binding is None:
+            raise NotFound("Data source binding not found.")
+        # enable binding
+        if action == "enable":
+            if data_source_binding.disabled:
+                data_source_binding.disabled = False
+                data_source_binding.updated_at = naive_utc_now()
+                db.session.add(data_source_binding)
+                db.session.commit()
+            else:
+                raise ValueError("Data source is not disabled.")
+        # disable binding
+        if action == "disable":
+            if not data_source_binding.disabled:
+                data_source_binding.disabled = True
+                data_source_binding.updated_at = naive_utc_now()
+                db.session.add(data_source_binding)
+                db.session.commit()
+            else:
+                raise ValueError("Data source is disabled.")
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/notion/pre-import/pages")
+class DataSourceNotionListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(integrate_notion_info_list_fields)
+    def get(self):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        dataset_id = request.args.get("dataset_id", default=None, type=str)
+        credential_id = request.args.get("credential_id", default=None, type=str)
+        if not credential_id:
+            raise ValueError("Credential id is required.")
+        datasource_provider_service = DatasourceProviderService()
+        credential = datasource_provider_service.get_datasource_credentials(
+            tenant_id=current_tenant_id,
+            credential_id=credential_id,
+            provider="notion_datasource",
+            plugin_id="langgenius/notion_datasource",
+        )
+        if not credential:
+            raise NotFound("Credential not found.")
+        exist_page_ids = []
+        with Session(db.engine) as session:
+            # import notion in the exist dataset
+            if dataset_id:
+                dataset = DatasetService.get_dataset(dataset_id)
+                if not dataset:
+                    raise NotFound("Dataset not found.")
+                if dataset.data_source_type != "notion_import":
+                    raise ValueError("Dataset is not notion type.")
+
+                documents = session.scalars(
+                    select(Document).filter_by(
+                        dataset_id=dataset_id,
+                        tenant_id=current_tenant_id,
+                        data_source_type="notion_import",
+                        enabled=True,
+                    )
+                ).all()
+                if documents:
+                    for document in documents:
+                        data_source_info = json.loads(document.data_source_info)
+                        exist_page_ids.append(data_source_info["notion_page_id"])
+            # get all authorized pages
+            from core.datasource.datasource_manager import DatasourceManager
+
+            datasource_runtime = DatasourceManager.get_datasource_runtime(
+                provider_id="langgenius/notion_datasource/notion_datasource",
+                datasource_name="notion_datasource",
+                tenant_id=current_tenant_id,
+                datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
+            )
+            datasource_provider_service = DatasourceProviderService()
+            if credential:
+                datasource_runtime.runtime.credentials = credential
+            datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
+            online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
+                datasource_runtime.get_online_document_pages(
+                    user_id=current_user.id,
+                    datasource_parameters={},
+                    provider_type=datasource_runtime.datasource_provider_type(),
+                )
+            )
+            try:
+                pages = []
+                workspace_info = {}
+                for message in online_document_result:
+                    result = message.result
+                    for info in result:
+                        workspace_info = {
+                            "workspace_id": info.workspace_id,
+                            "workspace_name": info.workspace_name,
+                            "workspace_icon": info.workspace_icon,
+                        }
+                        for page in info.pages:
+                            page_info = {
+                                "page_id": page.page_id,
+                                "page_name": page.page_name,
+                                "type": page.type,
+                                "parent_id": page.parent_id,
+                                "is_bound": page.page_id in exist_page_ids,
+                                "page_icon": page.page_icon,
+                            }
+                            pages.append(page_info)
+            except Exception as e:
+                raise e
+            return {"notion_info": {**workspace_info, "pages": pages}}, 200
+
+
+@console_ns.route(
+    "/notion/workspaces/<uuid:workspace_id>/pages/<uuid:page_id>/<string:page_type>/preview",
+    "/datasets/notion-indexing-estimate",
+)
+class DataSourceNotionApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, workspace_id, page_id, page_type):
+        _, current_tenant_id = current_account_with_tenant()
+
+        credential_id = request.args.get("credential_id", default=None, type=str)
+        if not credential_id:
+            raise ValueError("Credential id is required.")
+        datasource_provider_service = DatasourceProviderService()
+        credential = datasource_provider_service.get_datasource_credentials(
+            tenant_id=current_tenant_id,
+            credential_id=credential_id,
+            provider="notion_datasource",
+            plugin_id="langgenius/notion_datasource",
+        )
+
+        workspace_id = str(workspace_id)
+        page_id = str(page_id)
+
+        extractor = NotionExtractor(
+            notion_workspace_id=workspace_id,
+            notion_obj_id=page_id,
+            notion_page_type=page_type,
+            notion_access_token=credential.get("integration_secret"),
+            tenant_id=current_tenant_id,
+        )
+
+        text_docs = extractor.extract()
+        return {"content": "\n".join([doc.page_content for doc in text_docs])}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        _, current_tenant_id = current_account_with_tenant()
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("notion_info_list", type=list, required=True, nullable=True, location="json")
+            .add_argument("process_rule", type=dict, required=True, nullable=True, location="json")
+            .add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json")
+            .add_argument("doc_language", type=str, default="English", required=False, nullable=False, location="json")
+        )
+        args = parser.parse_args()
+        # validate args
+        DocumentService.estimate_args_validate(args)
+        notion_info_list = args["notion_info_list"]
+        extract_settings = []
+        for notion_info in notion_info_list:
+            workspace_id = notion_info["workspace_id"]
+            credential_id = notion_info.get("credential_id")
+            for page in notion_info["pages"]:
+                extract_setting = ExtractSetting(
+                    datasource_type=DatasourceType.NOTION,
+                    notion_info=NotionInfo.model_validate(
+                        {
+                            "credential_id": credential_id,
+                            "notion_workspace_id": workspace_id,
+                            "notion_obj_id": page["page_id"],
+                            "notion_page_type": page["type"],
+                            "tenant_id": current_tenant_id,
+                        }
+                    ),
+                    document_model=args["doc_form"],
+                )
+                extract_settings.append(extract_setting)
+        indexing_runner = IndexingRunner()
+        response = indexing_runner.indexing_estimate(
+            current_tenant_id,
+            extract_settings,
+            args["process_rule"],
+            args["doc_form"],
+            args["doc_language"],
+        )
+        return response.model_dump(), 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/notion/sync")
+class DataSourceNotionDatasetSyncApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        documents = DocumentService.get_document_by_dataset_id(dataset_id_str)
+        for document in documents:
+            document_indexing_sync_task.delay(dataset_id_str, document.id)
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/notion/sync")
+class DataSourceNotionDocumentSyncApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id_str = str(dataset_id)
+        document_id_str = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        if document is None:
+            raise NotFound("Document not found.")
+        document_indexing_sync_task.delay(dataset_id_str, document_id_str)
+        return {"result": "success"}, 200

dify/api/controllers/console/datasets/datasets_segments.py

@@ -0,0 +1,723 @@
+import uuid
+
+from flask import request
+from flask_restx import Resource, marshal, reqparse
+from sqlalchemy import select
+from werkzeug.exceptions import Forbidden, NotFound
+
+import services
+from controllers.console import console_ns
+from controllers.console.app.error import ProviderNotInitializeError
+from controllers.console.datasets.error import (
+    ChildChunkDeleteIndexError,
+    ChildChunkIndexingError,
+    InvalidActionError,
+)
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_knowledge_limit_check,
+    cloud_edition_billing_rate_limit_check,
+    cloud_edition_billing_resource_check,
+    setup_required,
+)
+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelType
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from fields.segment_fields import child_chunk_fields, segment_fields
+from libs.login import current_account_with_tenant, login_required
+from models.dataset import ChildChunk, DocumentSegment
+from models.model import UploadFile
+from services.dataset_service import DatasetService, DocumentService, SegmentService
+from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs
+from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
+from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
+from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
+class DatasetDocumentSegmentListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        document = DocumentService.get_document(dataset_id, document_id)
+
+        if not document:
+            raise NotFound("Document not found.")
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("limit", type=int, default=20, location="args")
+            .add_argument("status", type=str, action="append", default=[], location="args")
+            .add_argument("hit_count_gte", type=int, default=None, location="args")
+            .add_argument("enabled", type=str, default="all", location="args")
+            .add_argument("keyword", type=str, default=None, location="args")
+            .add_argument("page", type=int, default=1, location="args")
+        )
+
+        args = parser.parse_args()
+
+        page = args["page"]
+        limit = min(args["limit"], 100)
+        status_list = args["status"]
+        hit_count_gte = args["hit_count_gte"]
+        keyword = args["keyword"]
+
+        query = (
+            select(DocumentSegment)
+            .where(
+                DocumentSegment.document_id == str(document_id),
+                DocumentSegment.tenant_id == current_tenant_id,
+            )
+            .order_by(DocumentSegment.position.asc())
+        )
+
+        if status_list:
+            query = query.where(DocumentSegment.status.in_(status_list))
+
+        if hit_count_gte is not None:
+            query = query.where(DocumentSegment.hit_count >= hit_count_gte)
+
+        if keyword:
+            query = query.where(DocumentSegment.content.ilike(f"%{keyword}%"))
+
+        if args["enabled"].lower() != "all":
+            if args["enabled"].lower() == "true":
+                query = query.where(DocumentSegment.enabled == True)
+            elif args["enabled"].lower() == "false":
+                query = query.where(DocumentSegment.enabled == False)
+
+        segments = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
+
+        response = {
+            "data": marshal(segments.items, segment_fields),
+            "limit": limit,
+            "total": segments.total,
+            "total_pages": segments.pages,
+            "page": page,
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def delete(self, dataset_id, document_id):
+        current_user, _ = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        segment_ids = request.args.getlist("segment_id")
+
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        SegmentService.delete_segments(segment_ids, document, dataset)
+        return {"result": "success"}, 204
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment/<string:action>")
+class DatasetDocumentSegmentApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def patch(self, dataset_id, document_id, action):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        if dataset.indexing_technique == "high_quality":
+            # check embedding model setting
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        segment_ids = request.args.getlist("segment_id")
+
+        document_indexing_cache_key = f"document_{document.id}_indexing"
+        cache_result = redis_client.get(document_indexing_cache_key)
+        if cache_result is not None:
+            raise InvalidActionError("Document is being indexed, please try again later")
+        try:
+            SegmentService.update_segments_status(segment_ids, action, dataset, document)
+        except Exception as e:
+            raise InvalidActionError(str(e))
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segment")
+class DatasetDocumentSegmentAddApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_knowledge_limit_check("add_segment")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self, dataset_id, document_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("content", type=str, required=True, nullable=False, location="json")
+            .add_argument("answer", type=str, required=False, nullable=True, location="json")
+            .add_argument("keywords", type=list, required=False, nullable=True, location="json")
+        )
+        args = parser.parse_args()
+        SegmentService.segment_create_args_validate(args, document)
+        segment = SegmentService.create_segment(args, document, dataset)
+        return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>")
+class DatasetDocumentSegmentUpdateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def patch(self, dataset_id, document_id, segment_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        if dataset.indexing_technique == "high_quality":
+            # check embedding model setting
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+            # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("content", type=str, required=True, nullable=False, location="json")
+            .add_argument("answer", type=str, required=False, nullable=True, location="json")
+            .add_argument("keywords", type=list, required=False, nullable=True, location="json")
+            .add_argument(
+                "regenerate_child_chunks", type=bool, required=False, nullable=True, default=False, location="json"
+            )
+        )
+        args = parser.parse_args()
+        SegmentService.segment_create_args_validate(args, document)
+        segment = SegmentService.update_segment(SegmentUpdateArgs.model_validate(args), segment, document, dataset)
+        return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def delete(self, dataset_id, document_id, segment_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        SegmentService.delete_segment(segment, document, dataset)
+        return {"result": "success"}, 204
+
+
+@console_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/batch_import",
+    "/datasets/batch_import_status/<uuid:job_id>",
+)
+class DatasetDocumentSegmentBatchImportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_knowledge_limit_check("add_segment")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self, dataset_id, document_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+
+        parser = reqparse.RequestParser().add_argument(
+            "upload_file_id", type=str, required=True, nullable=False, location="json"
+        )
+        args = parser.parse_args()
+        upload_file_id = args["upload_file_id"]
+
+        upload_file = db.session.query(UploadFile).where(UploadFile.id == upload_file_id).first()
+        if not upload_file:
+            raise NotFound("UploadFile not found.")
+
+        # check file type
+        if not upload_file.name or not upload_file.name.lower().endswith(".csv"):
+            raise ValueError("Invalid file type. Only CSV files are allowed")
+
+        try:
+            # async job
+            job_id = str(uuid.uuid4())
+            indexing_cache_key = f"segment_batch_import_{str(job_id)}"
+            # send batch add segments task
+            redis_client.setnx(indexing_cache_key, "waiting")
+            batch_create_segment_to_index_task.delay(
+                str(job_id),
+                upload_file_id,
+                dataset_id,
+                document_id,
+                current_tenant_id,
+                current_user.id,
+            )
+        except Exception as e:
+            return {"error": str(e)}, 500
+        return {"job_id": job_id, "job_status": "waiting"}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, job_id=None, dataset_id=None, document_id=None):
+        if job_id is None:
+            raise NotFound("The job does not exist.")
+        job_id = str(job_id)
+        indexing_cache_key = f"segment_batch_import_{job_id}"
+        cache_result = redis_client.get(indexing_cache_key)
+        if cache_result is None:
+            raise ValueError("The job does not exist.")
+
+        return {"job_id": job_id, "job_status": cache_result.decode()}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks")
+class ChildChunkAddApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_knowledge_limit_check("add_segment")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self, dataset_id, document_id, segment_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser().add_argument(
+            "content", type=str, required=True, nullable=False, location="json"
+        )
+        args = parser.parse_args()
+        try:
+            content = args["content"]
+            child_chunk = SegmentService.create_child_chunk(content, segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id, segment_id):
+        _, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("limit", type=int, default=20, location="args")
+            .add_argument("keyword", type=str, default=None, location="args")
+            .add_argument("page", type=int, default=1, location="args")
+        )
+
+        args = parser.parse_args()
+
+        page = args["page"]
+        limit = min(args["limit"], 100)
+        keyword = args["keyword"]
+
+        child_chunks = SegmentService.get_child_chunks(segment_id, document_id, dataset_id, page, limit, keyword)
+        return {
+            "data": marshal(child_chunks.items, child_chunk_fields),
+            "total": child_chunks.total,
+            "total_pages": child_chunks.pages,
+            "page": page,
+            "limit": limit,
+        }, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def patch(self, dataset_id, document_id, segment_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+            # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser().add_argument(
+            "chunks", type=list, required=True, nullable=False, location="json"
+        )
+        args = parser.parse_args()
+        try:
+            chunks_data = args["chunks"]
+            chunks = [ChildChunkUpdateArgs.model_validate(chunk) for chunk in chunks_data]
+            child_chunks = SegmentService.update_child_chunks(chunks, segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunks, child_chunk_fields)}, 200
+
+
+@console_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>"
+)
+class ChildChunkUpdateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def delete(self, dataset_id, document_id, segment_id, child_chunk_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        # check child chunk
+        child_chunk_id = str(child_chunk_id)
+        child_chunk = (
+            db.session.query(ChildChunk)
+            .where(
+                ChildChunk.id == str(child_chunk_id),
+                ChildChunk.tenant_id == current_tenant_id,
+                ChildChunk.segment_id == segment.id,
+                ChildChunk.document_id == document_id,
+            )
+            .first()
+        )
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        try:
+            SegmentService.delete_child_chunk(child_chunk, dataset)
+        except ChildChunkDeleteIndexServiceError as e:
+            raise ChildChunkDeleteIndexError(str(e))
+        return {"result": "success"}, 204
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_resource_check("vector_space")
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def patch(self, dataset_id, document_id, segment_id, child_chunk_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+            # check segment
+        segment_id = str(segment_id)
+        segment = (
+            db.session.query(DocumentSegment)
+            .where(DocumentSegment.id == str(segment_id), DocumentSegment.tenant_id == current_tenant_id)
+            .first()
+        )
+        if not segment:
+            raise NotFound("Segment not found.")
+        # check child chunk
+        child_chunk_id = str(child_chunk_id)
+        child_chunk = (
+            db.session.query(ChildChunk)
+            .where(
+                ChildChunk.id == str(child_chunk_id),
+                ChildChunk.tenant_id == current_tenant_id,
+                ChildChunk.segment_id == segment.id,
+                ChildChunk.document_id == document_id,
+            )
+            .first()
+        )
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        # validate args
+        parser = reqparse.RequestParser().add_argument(
+            "content", type=str, required=True, nullable=False, location="json"
+        )
+        args = parser.parse_args()
+        try:
+            content = args["content"]
+            child_chunk = SegmentService.update_child_chunk(content, child_chunk, segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200

dify/api/controllers/console/datasets/error.py

@@ -0,0 +1,79 @@
+from libs.exception import BaseHTTPException
+
+
+class DatasetNotInitializedError(BaseHTTPException):
+    error_code = "dataset_not_initialized"
+    description = "The dataset is still being initialized or indexing. Please wait a moment."
+    code = 400
+
+
+class ArchivedDocumentImmutableError(BaseHTTPException):
+    error_code = "archived_document_immutable"
+    description = "The archived document is not editable."
+    code = 403
+
+
+class DatasetNameDuplicateError(BaseHTTPException):
+    error_code = "dataset_name_duplicate"
+    description = "The dataset name already exists. Please modify your dataset name."
+    code = 409
+
+
+class InvalidActionError(BaseHTTPException):
+    error_code = "invalid_action"
+    description = "Invalid action."
+    code = 400
+
+
+class DocumentAlreadyFinishedError(BaseHTTPException):
+    error_code = "document_already_finished"
+    description = "The document has been processed. Please refresh the page or go to the document details."
+    code = 400
+
+
+class DocumentIndexingError(BaseHTTPException):
+    error_code = "document_indexing"
+    description = "The document is being processed and cannot be edited."
+    code = 400
+
+
+class InvalidMetadataError(BaseHTTPException):
+    error_code = "invalid_metadata"
+    description = "The metadata content is incorrect. Please check and verify."
+    code = 400
+
+
+class WebsiteCrawlError(BaseHTTPException):
+    error_code = "crawl_failed"
+    description = "{message}"
+    code = 500
+
+
+class DatasetInUseError(BaseHTTPException):
+    error_code = "dataset_in_use"
+    description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
+    code = 409
+
+
+class IndexingEstimateError(BaseHTTPException):
+    error_code = "indexing_estimate_error"
+    description = "Knowledge indexing estimate failed: {message}"
+    code = 500
+
+
+class ChildChunkIndexingError(BaseHTTPException):
+    error_code = "child_chunk_indexing_error"
+    description = "Create child chunk index failed: {message}"
+    code = 500
+
+
+class ChildChunkDeleteIndexError(BaseHTTPException):
+    error_code = "child_chunk_delete_index_error"
+    description = "Delete child chunk index failed: {message}"
+    code = 500
+
+
+class PipelineNotFoundError(BaseHTTPException):
+    error_code = "pipeline_not_found"
+    description = "Pipeline not found."
+    code = 404

dify/api/controllers/console/datasets/external.py

@@ -0,0 +1,388 @@
+from flask import request
+from flask_restx import Resource, fields, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
+from controllers.console import console_ns
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
+from fields.dataset_fields import (
+    dataset_detail_fields,
+    dataset_retrieval_model_fields,
+    doc_metadata_fields,
+    external_knowledge_info_fields,
+    external_retrieval_model_fields,
+    icon_info_fields,
+    keyword_setting_fields,
+    reranking_model_fields,
+    tag_fields,
+    vector_setting_fields,
+    weighted_score_fields,
+)
+from libs.login import current_account_with_tenant, login_required
+from services.dataset_service import DatasetService
+from services.external_knowledge_service import ExternalDatasetService
+from services.hit_testing_service import HitTestingService
+from services.knowledge_service import ExternalDatasetTestService
+
+
+def _get_or_create_model(model_name: str, field_def):
+    existing = console_ns.models.get(model_name)
+    if existing is None:
+        existing = console_ns.model(model_name, field_def)
+    return existing
+
+
+def _build_dataset_detail_model():
+    keyword_setting_model = _get_or_create_model("DatasetKeywordSetting", keyword_setting_fields)
+    vector_setting_model = _get_or_create_model("DatasetVectorSetting", vector_setting_fields)
+
+    weighted_score_fields_copy = weighted_score_fields.copy()
+    weighted_score_fields_copy["keyword_setting"] = fields.Nested(keyword_setting_model)
+    weighted_score_fields_copy["vector_setting"] = fields.Nested(vector_setting_model)
+    weighted_score_model = _get_or_create_model("DatasetWeightedScore", weighted_score_fields_copy)
+
+    reranking_model = _get_or_create_model("DatasetRerankingModel", reranking_model_fields)
+
+    dataset_retrieval_model_fields_copy = dataset_retrieval_model_fields.copy()
+    dataset_retrieval_model_fields_copy["reranking_model"] = fields.Nested(reranking_model)
+    dataset_retrieval_model_fields_copy["weights"] = fields.Nested(weighted_score_model, allow_null=True)
+    dataset_retrieval_model = _get_or_create_model("DatasetRetrievalModel", dataset_retrieval_model_fields_copy)
+
+    tag_model = _get_or_create_model("Tag", tag_fields)
+    doc_metadata_model = _get_or_create_model("DatasetDocMetadata", doc_metadata_fields)
+    external_knowledge_info_model = _get_or_create_model("ExternalKnowledgeInfo", external_knowledge_info_fields)
+    external_retrieval_model = _get_or_create_model("ExternalRetrievalModel", external_retrieval_model_fields)
+    icon_info_model = _get_or_create_model("DatasetIconInfo", icon_info_fields)
+
+    dataset_detail_fields_copy = dataset_detail_fields.copy()
+    dataset_detail_fields_copy["retrieval_model_dict"] = fields.Nested(dataset_retrieval_model)
+    dataset_detail_fields_copy["tags"] = fields.List(fields.Nested(tag_model))
+    dataset_detail_fields_copy["external_knowledge_info"] = fields.Nested(external_knowledge_info_model)
+    dataset_detail_fields_copy["external_retrieval_model"] = fields.Nested(external_retrieval_model, allow_null=True)
+    dataset_detail_fields_copy["doc_metadata"] = fields.List(fields.Nested(doc_metadata_model))
+    dataset_detail_fields_copy["icon_info"] = fields.Nested(icon_info_model)
+    return _get_or_create_model("DatasetDetail", dataset_detail_fields_copy)
+
+
+try:
+    dataset_detail_model = console_ns.models["DatasetDetail"]
+except KeyError:
+    dataset_detail_model = _build_dataset_detail_model()
+
+
+def _validate_name(name: str) -> str:
+    if not name or len(name) < 1 or len(name) > 100:
+        raise ValueError("Name must be between 1 to 100 characters.")
+    return name
+
+
+@console_ns.route("/datasets/external-knowledge-api")
+class ExternalApiTemplateListApi(Resource):
+    @console_ns.doc("get_external_api_templates")
+    @console_ns.doc(description="Get external knowledge API templates")
+    @console_ns.doc(
+        params={
+            "page": "Page number (default: 1)",
+            "limit": "Number of items per page (default: 20)",
+            "keyword": "Search keyword",
+        }
+    )
+    @console_ns.response(200, "External API templates retrieved successfully")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        _, current_tenant_id = current_account_with_tenant()
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        search = request.args.get("keyword", default=None, type=str)
+
+        external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
+            page, limit, current_tenant_id, search
+        )
+        response = {
+            "data": [item.to_dict() for item in external_knowledge_apis],
+            "has_more": len(external_knowledge_apis) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        current_user, current_tenant_id = current_account_with_tenant()
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(
+                "name",
+                nullable=False,
+                required=True,
+                help="Name is required. Name must be between 1 to 100 characters.",
+                type=_validate_name,
+            )
+            .add_argument(
+                "settings",
+                type=dict,
+                location="json",
+                nullable=False,
+                required=True,
+            )
+        )
+        args = parser.parse_args()
+
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            external_knowledge_api = ExternalDatasetService.create_external_knowledge_api(
+                tenant_id=current_tenant_id, user_id=current_user.id, args=args
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return external_knowledge_api.to_dict(), 201
+
+
+@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
+class ExternalApiTemplateApi(Resource):
+    @console_ns.doc("get_external_api_template")
+    @console_ns.doc(description="Get external knowledge API template details")
+    @console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
+    @console_ns.response(200, "External API template retrieved successfully")
+    @console_ns.response(404, "Template not found")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+        external_knowledge_api = ExternalDatasetService.get_external_knowledge_api(external_knowledge_api_id)
+        if external_knowledge_api is None:
+            raise NotFound("API template not found.")
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, external_knowledge_api_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(
+                "name",
+                nullable=False,
+                required=True,
+                help="type is required. Name must be between 1 to 100 characters.",
+                type=_validate_name,
+            )
+            .add_argument(
+                "settings",
+                type=dict,
+                location="json",
+                nullable=False,
+                required=True,
+            )
+        )
+        args = parser.parse_args()
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        external_knowledge_api = ExternalDatasetService.update_external_knowledge_api(
+            tenant_id=current_tenant_id,
+            user_id=current_user.id,
+            external_knowledge_api_id=external_knowledge_api_id,
+            args=args,
+        )
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, external_knowledge_api_id):
+        current_user, current_tenant_id = current_account_with_tenant()
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        if not (current_user.has_edit_permission or current_user.is_dataset_operator):
+            raise Forbidden()
+
+        ExternalDatasetService.delete_external_knowledge_api(current_tenant_id, external_knowledge_api_id)
+        return {"result": "success"}, 204
+
+
+@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")
+class ExternalApiUseCheckApi(Resource):
+    @console_ns.doc("check_external_api_usage")
+    @console_ns.doc(description="Check if external knowledge API is being used")
+    @console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
+    @console_ns.response(200, "Usage check completed successfully")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
+            external_knowledge_api_id
+        )
+        return {"is_using": external_knowledge_api_is_using, "count": count}, 200
+
+
+@console_ns.route("/datasets/external")
+class ExternalDatasetCreateApi(Resource):
+    @console_ns.doc("create_external_dataset")
+    @console_ns.doc(description="Create external knowledge dataset")
+    @console_ns.expect(
+        console_ns.model(
+            "CreateExternalDatasetRequest",
+            {
+                "external_knowledge_api_id": fields.String(required=True, description="External knowledge API ID"),
+                "external_knowledge_id": fields.String(required=True, description="External knowledge ID"),
+                "name": fields.String(required=True, description="Dataset name"),
+                "description": fields.String(description="Dataset description"),
+            },
+        )
+    )
+    @console_ns.response(201, "External dataset created successfully", dataset_detail_model)
+    @console_ns.response(400, "Invalid parameters")
+    @console_ns.response(403, "Permission denied")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        current_user, current_tenant_id = current_account_with_tenant()
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("external_knowledge_api_id", type=str, required=True, nullable=False, location="json")
+            .add_argument("external_knowledge_id", type=str, required=True, nullable=False, location="json")
+            .add_argument(
+                "name",
+                nullable=False,
+                required=True,
+                help="name is required. Name must be between 1 to 100 characters.",
+                type=_validate_name,
+            )
+            .add_argument("description", type=str, required=False, nullable=True, location="json")
+            .add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        )
+
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            dataset = ExternalDatasetService.create_external_dataset(
+                tenant_id=current_tenant_id,
+                user_id=current_user.id,
+                args=args,
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 201
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/external-hit-testing")
+class ExternalKnowledgeHitTestingApi(Resource):
+    @console_ns.doc("test_external_knowledge_retrieval")
+    @console_ns.doc(description="Test external knowledge retrieval for dataset")
+    @console_ns.doc(params={"dataset_id": "Dataset ID"})
+    @console_ns.expect(
+        console_ns.model(
+            "ExternalHitTestingRequest",
+            {
+                "query": fields.String(required=True, description="Query text for testing"),
+                "retrieval_model": fields.Raw(description="Retrieval model configuration"),
+                "external_retrieval_model": fields.Raw(description="External retrieval model configuration"),
+            },
+        )
+    )
+    @console_ns.response(200, "External hit testing completed successfully")
+    @console_ns.response(404, "Dataset not found")
+    @console_ns.response(400, "Invalid parameters")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id):
+        current_user, _ = current_account_with_tenant()
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("query", type=str, location="json")
+            .add_argument("external_retrieval_model", type=dict, required=False, location="json")
+            .add_argument("metadata_filtering_conditions", type=dict, required=False, location="json")
+        )
+        args = parser.parse_args()
+
+        HitTestingService.hit_testing_args_check(args)
+
+        try:
+            response = HitTestingService.external_retrieve(
+                dataset=dataset,
+                query=args["query"],
+                account=current_user,
+                external_retrieval_model=args["external_retrieval_model"],
+                metadata_filtering_conditions=args["metadata_filtering_conditions"],
+            )
+
+            return response
+        except Exception as e:
+            raise InternalServerError(str(e))
+
+
+@console_ns.route("/test/retrieval")
+class BedrockRetrievalApi(Resource):
+    # this api is only for internal testing
+    @console_ns.doc("bedrock_retrieval_test")
+    @console_ns.doc(description="Bedrock retrieval test (internal use only)")
+    @console_ns.expect(
+        console_ns.model(
+            "BedrockRetrievalTestRequest",
+            {
+                "retrieval_setting": fields.Raw(required=True, description="Retrieval settings"),
+                "query": fields.String(required=True, description="Query text"),
+                "knowledge_id": fields.String(required=True, description="Knowledge ID"),
+            },
+        )
+    )
+    @console_ns.response(200, "Bedrock retrieval test completed")
+    def post(self):
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("retrieval_setting", nullable=False, required=True, type=dict, location="json")
+            .add_argument(
+                "query",
+                nullable=False,
+                required=True,
+                type=str,
+            )
+            .add_argument("knowledge_id", nullable=False, required=True, type=str)
+        )
+        args = parser.parse_args()
+
+        # Call the knowledge retrieval service
+        result = ExternalDatasetTestService.knowledge_retrieval(
+            args["retrieval_setting"], args["query"], args["knowledge_id"]
+        )
+        return result, 200

dify/api/controllers/console/datasets/hit_testing.py

@@ -0,0 +1,43 @@
+from flask_restx import Resource, fields
+
+from controllers.console import console_ns
+from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_rate_limit_check,
+    setup_required,
+)
+from libs.login import login_required
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/hit-testing")
+class HitTestingApi(Resource, DatasetsHitTestingBase):
+    @console_ns.doc("test_dataset_retrieval")
+    @console_ns.doc(description="Test dataset knowledge retrieval")
+    @console_ns.doc(params={"dataset_id": "Dataset ID"})
+    @console_ns.expect(
+        console_ns.model(
+            "HitTestingRequest",
+            {
+                "query": fields.String(required=True, description="Query text for testing"),
+                "retrieval_model": fields.Raw(description="Retrieval model configuration"),
+                "top_k": fields.Integer(description="Number of top results to return"),
+                "score_threshold": fields.Float(description="Score threshold for filtering results"),
+            },
+        )
+    )
+    @console_ns.response(200, "Hit testing completed successfully")
+    @console_ns.response(404, "Dataset not found")
+    @console_ns.response(400, "Invalid parameters")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        dataset = self.get_and_validate_dataset(dataset_id_str)
+        args = self.parse_args()
+        self.hit_testing_args_check(args)
+
+        return self.perform_hit_testing(dataset, args)

dify/api/controllers/console/datasets/hit_testing_base.py

@@ -0,0 +1,91 @@
+import logging
+
+from flask_restx import marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.datasets.error import DatasetNotInitializedError
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
+from core.model_runtime.errors.invoke import InvokeError
+from fields.hit_testing_fields import hit_testing_record_fields
+from libs.login import current_user
+from models.account import Account
+from services.dataset_service import DatasetService
+from services.hit_testing_service import HitTestingService
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetsHitTestingBase:
+    @staticmethod
+    def get_and_validate_dataset(dataset_id: str):
+        assert isinstance(current_user, Account)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        return dataset
+
+    @staticmethod
+    def hit_testing_args_check(args):
+        HitTestingService.hit_testing_args_check(args)
+
+    @staticmethod
+    def parse_args():
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("query", type=str, location="json")
+            .add_argument("retrieval_model", type=dict, required=False, location="json")
+            .add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        )
+        return parser.parse_args()
+
+    @staticmethod
+    def perform_hit_testing(dataset, args):
+        assert isinstance(current_user, Account)
+        try:
+            response = HitTestingService.retrieve(
+                dataset=dataset,
+                query=args["query"],
+                account=current_user,
+                retrieval_model=args["retrieval_model"],
+                external_retrieval_model=args["external_retrieval_model"],
+                limit=10,
+            )
+            return {"query": response["query"], "records": marshal(response["records"], hit_testing_record_fields)}
+        except services.errors.index.IndexNotInitializedError:
+            raise DatasetNotInitializedError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except LLMBadRequestError:
+            raise ProviderNotInitializeError(
+                "No Embedding Model or Reranking Model available. Please configure a valid provider "
+                "in the Settings -> Model Provider."
+            )
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise ValueError(str(e))
+        except Exception as e:
+            logger.exception("Hit testing failed.")
+            raise InternalServerError(str(e))

dify/api/controllers/console/datasets/metadata.py

@@ -0,0 +1,150 @@
+from typing import Literal
+
+from flask_restx import Resource, marshal_with, reqparse
+from werkzeug.exceptions import NotFound
+
+from controllers.console import console_ns
+from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
+from fields.dataset_fields import dataset_metadata_fields
+from libs.login import current_account_with_tenant, login_required
+from services.dataset_service import DatasetService
+from services.entities.knowledge_entities.knowledge_entities import (
+    MetadataArgs,
+    MetadataOperationData,
+)
+from services.metadata_service import MetadataService
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/metadata")
+class DatasetMetadataCreateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    @marshal_with(dataset_metadata_fields)
+    def post(self, dataset_id):
+        current_user, _ = current_account_with_tenant()
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("type", type=str, required=True, nullable=False, location="json")
+            .add_argument("name", type=str, required=True, nullable=False, location="json")
+        )
+        args = parser.parse_args()
+        metadata_args = MetadataArgs.model_validate(args)
+
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        metadata = MetadataService.create_metadata(dataset_id_str, metadata_args)
+        return metadata, 201
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        return MetadataService.get_dataset_metadatas(dataset), 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
+class DatasetMetadataApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    @marshal_with(dataset_metadata_fields)
+    def patch(self, dataset_id, metadata_id):
+        current_user, _ = current_account_with_tenant()
+        parser = reqparse.RequestParser().add_argument("name", type=str, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+        name = args["name"]
+
+        dataset_id_str = str(dataset_id)
+        metadata_id_str = str(metadata_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        metadata = MetadataService.update_metadata_name(dataset_id_str, metadata_id_str, name)
+        return metadata, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def delete(self, dataset_id, metadata_id):
+        current_user, _ = current_account_with_tenant()
+        dataset_id_str = str(dataset_id)
+        metadata_id_str = str(metadata_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
+        return {"result": "success"}, 204
+
+
+@console_ns.route("/datasets/metadata/built-in")
+class DatasetMetadataBuiltInFieldApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def get(self):
+        built_in_fields = MetadataService.get_built_in_fields()
+        return {"fields": built_in_fields}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in/<string:action>")
+class DatasetMetadataBuiltInFieldActionApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def post(self, dataset_id, action: Literal["enable", "disable"]):
+        current_user, _ = current_account_with_tenant()
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        if action == "enable":
+            MetadataService.enable_built_in_field(dataset)
+        elif action == "disable":
+            MetadataService.disable_built_in_field(dataset)
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/metadata")
+class DocumentMetadataEditApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def post(self, dataset_id):
+        current_user, _ = current_account_with_tenant()
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        parser = reqparse.RequestParser().add_argument(
+            "operation_data", type=list, required=True, nullable=False, location="json"
+        )
+        args = parser.parse_args()
+        metadata_args = MetadataOperationData.model_validate(args)
+
+        MetadataService.update_documents_metadata(dataset, metadata_args)
+
+        return {"result": "success"}, 200

dify/api/controllers/console/datasets/rag_pipeline/datasource_auth.py

@@ -0,0 +1,354 @@
+from flask import make_response, redirect, request
+from flask_restx import Resource, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
+from configs import dify_config
+from controllers.console import console_ns
+from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.utils.encoders import jsonable_encoder
+from core.plugin.impl.oauth import OAuthHandler
+from libs.helper import StrLen
+from libs.login import current_account_with_tenant, login_required
+from models.provider_ids import DatasourceProviderID
+from services.datasource_provider_service import DatasourceProviderService
+from services.plugin.oauth_service import OAuthProxyService
+
+
+@console_ns.route("/oauth/plugin/<path:provider_id>/datasource/get-authorization-url")
+class DatasourcePluginOAuthAuthorizationUrl(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def get(self, provider_id: str):
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        tenant_id = current_tenant_id
+
+        credential_id = request.args.get("credential_id")
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        provider_name = datasource_provider_id.provider_name
+        plugin_id = datasource_provider_id.plugin_id
+        oauth_config = DatasourceProviderService().get_oauth_client(
+            tenant_id=tenant_id,
+            datasource_provider_id=datasource_provider_id,
+        )
+        if not oauth_config:
+            raise ValueError(f"No OAuth Client Config for {provider_id}")
+
+        context_id = OAuthProxyService.create_proxy_context(
+            user_id=current_user.id,
+            tenant_id=tenant_id,
+            plugin_id=plugin_id,
+            provider=provider_name,
+            credential_id=credential_id,
+        )
+        oauth_handler = OAuthHandler()
+        redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_id}/datasource/callback"
+        authorization_url_response = oauth_handler.get_authorization_url(
+            tenant_id=tenant_id,
+            user_id=current_user.id,
+            plugin_id=plugin_id,
+            provider=provider_name,
+            redirect_uri=redirect_uri,
+            system_credentials=oauth_config,
+        )
+        response = make_response(jsonable_encoder(authorization_url_response))
+        response.set_cookie(
+            "context_id",
+            context_id,
+            httponly=True,
+            samesite="Lax",
+            max_age=OAuthProxyService.__MAX_AGE__,
+        )
+        return response
+
+
+@console_ns.route("/oauth/plugin/<path:provider_id>/datasource/callback")
+class DatasourceOAuthCallback(Resource):
+    @setup_required
+    def get(self, provider_id: str):
+        context_id = request.cookies.get("context_id") or request.args.get("context_id")
+        if not context_id:
+            raise Forbidden("context_id not found")
+
+        context = OAuthProxyService.use_proxy_context(context_id)
+        if context is None:
+            raise Forbidden("Invalid context_id")
+
+        user_id, tenant_id = context.get("user_id"), context.get("tenant_id")
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        plugin_id = datasource_provider_id.plugin_id
+        datasource_provider_service = DatasourceProviderService()
+        oauth_client_params = datasource_provider_service.get_oauth_client(
+            tenant_id=tenant_id,
+            datasource_provider_id=datasource_provider_id,
+        )
+        if not oauth_client_params:
+            raise NotFound()
+        redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_id}/datasource/callback"
+        oauth_handler = OAuthHandler()
+        oauth_response = oauth_handler.get_credentials(
+            tenant_id=tenant_id,
+            user_id=user_id,
+            plugin_id=plugin_id,
+            provider=datasource_provider_id.provider_name,
+            redirect_uri=redirect_uri,
+            system_credentials=oauth_client_params,
+            request=request,
+        )
+        credential_id = context.get("credential_id")
+        if credential_id:
+            datasource_provider_service.reauthorize_datasource_oauth_provider(
+                tenant_id=tenant_id,
+                provider_id=datasource_provider_id,
+                avatar_url=oauth_response.metadata.get("avatar_url") or None,
+                name=oauth_response.metadata.get("name") or None,
+                expire_at=oauth_response.expires_at,
+                credentials=dict(oauth_response.credentials),
+                credential_id=context.get("credential_id"),
+            )
+        else:
+            datasource_provider_service.add_datasource_oauth_provider(
+                tenant_id=tenant_id,
+                provider_id=datasource_provider_id,
+                avatar_url=oauth_response.metadata.get("avatar_url") or None,
+                name=oauth_response.metadata.get("name") or None,
+                expire_at=oauth_response.expires_at,
+                credentials=dict(oauth_response.credentials),
+            )
+        return redirect(f"{dify_config.CONSOLE_WEB_URL}/oauth-callback")
+
+
+parser_datasource = (
+    reqparse.RequestParser()
+    .add_argument("name", type=StrLen(max_length=100), required=False, nullable=True, location="json", default=None)
+    .add_argument("credentials", type=dict, required=True, nullable=False, location="json")
+)
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>")
+class DatasourceAuth(Resource):
+    @console_ns.expect(parser_datasource)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        args = parser_datasource.parse_args()
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+
+        try:
+            datasource_provider_service.add_datasource_api_key_provider(
+                tenant_id=current_tenant_id,
+                provider_id=datasource_provider_id,
+                credentials=args["credentials"],
+                name=args["name"],
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ValueError(str(ex))
+        return {"result": "success"}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, provider_id: str):
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasources = datasource_provider_service.list_datasource_credentials(
+            tenant_id=current_tenant_id,
+            provider=datasource_provider_id.provider_name,
+            plugin_id=datasource_provider_id.plugin_id,
+        )
+        return {"result": datasources}, 200
+
+
+parser_datasource_delete = reqparse.RequestParser().add_argument(
+    "credential_id", type=str, required=True, nullable=False, location="json"
+)
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>/delete")
+class DatasourceAuthDeleteApi(Resource):
+    @console_ns.expect(parser_datasource_delete)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        plugin_id = datasource_provider_id.plugin_id
+        provider_name = datasource_provider_id.provider_name
+
+        args = parser_datasource_delete.parse_args()
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.remove_datasource_credentials(
+            tenant_id=current_tenant_id,
+            auth_id=args["credential_id"],
+            provider=provider_name,
+            plugin_id=plugin_id,
+        )
+        return {"result": "success"}, 200
+
+
+parser_datasource_update = (
+    reqparse.RequestParser()
+    .add_argument("credentials", type=dict, required=False, nullable=True, location="json")
+    .add_argument("name", type=StrLen(max_length=100), required=False, nullable=True, location="json")
+    .add_argument("credential_id", type=str, required=True, nullable=False, location="json")
+)
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>/update")
+class DatasourceAuthUpdateApi(Resource):
+    @console_ns.expect(parser_datasource_update)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        args = parser_datasource_update.parse_args()
+
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.update_datasource_credentials(
+            tenant_id=current_tenant_id,
+            auth_id=args["credential_id"],
+            provider=datasource_provider_id.provider_name,
+            plugin_id=datasource_provider_id.plugin_id,
+            credentials=args.get("credentials", {}),
+            name=args.get("name", None),
+        )
+        return {"result": "success"}, 201
+
+
+@console_ns.route("/auth/plugin/datasource/list")
+class DatasourceAuthListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasource_provider_service = DatasourceProviderService()
+        datasources = datasource_provider_service.get_all_datasource_credentials(tenant_id=current_tenant_id)
+        return {"result": jsonable_encoder(datasources)}, 200
+
+
+@console_ns.route("/auth/plugin/datasource/default-list")
+class DatasourceHardCodeAuthListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasource_provider_service = DatasourceProviderService()
+        datasources = datasource_provider_service.get_hard_code_datasource_credentials(tenant_id=current_tenant_id)
+        return {"result": jsonable_encoder(datasources)}, 200
+
+
+parser_datasource_custom = (
+    reqparse.RequestParser()
+    .add_argument("client_params", type=dict, required=False, nullable=True, location="json")
+    .add_argument("enable_oauth_custom_client", type=bool, required=False, nullable=True, location="json")
+)
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>/custom-client")
+class DatasourceAuthOauthCustomClient(Resource):
+    @console_ns.expect(parser_datasource_custom)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        args = parser_datasource_custom.parse_args()
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.setup_oauth_custom_client_params(
+            tenant_id=current_tenant_id,
+            datasource_provider_id=datasource_provider_id,
+            client_params=args.get("client_params", {}),
+            enabled=args.get("enable_oauth_custom_client", False),
+        )
+        return {"result": "success"}, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.remove_oauth_custom_client_params(
+            tenant_id=current_tenant_id,
+            datasource_provider_id=datasource_provider_id,
+        )
+        return {"result": "success"}, 200
+
+
+parser_default = reqparse.RequestParser().add_argument("id", type=str, required=True, nullable=False, location="json")
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>/default")
+class DatasourceAuthDefaultApi(Resource):
+    @console_ns.expect(parser_default)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        args = parser_default.parse_args()
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.set_default_datasource_provider(
+            tenant_id=current_tenant_id,
+            datasource_provider_id=datasource_provider_id,
+            credential_id=args["id"],
+        )
+        return {"result": "success"}, 200
+
+
+parser_update_name = (
+    reqparse.RequestParser()
+    .add_argument("name", type=StrLen(max_length=100), required=True, nullable=False, location="json")
+    .add_argument("credential_id", type=str, required=True, nullable=False, location="json")
+)
+
+
+@console_ns.route("/auth/plugin/datasource/<path:provider_id>/update-name")
+class DatasourceUpdateProviderNameApi(Resource):
+    @console_ns.expect(parser_update_name)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def post(self, provider_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+
+        args = parser_update_name.parse_args()
+        datasource_provider_id = DatasourceProviderID(provider_id)
+        datasource_provider_service = DatasourceProviderService()
+        datasource_provider_service.update_datasource_provider_name(
+            tenant_id=current_tenant_id,
+            datasource_provider_id=datasource_provider_id,
+            name=args["name"],
+            credential_id=args["credential_id"],
+        )
+        return {"result": "success"}, 200

dify/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py

@@ -0,0 +1,55 @@
+from flask_restx import (  # type: ignore
+    Resource,  # type: ignore
+)
+from pydantic import BaseModel
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import console_ns
+from controllers.console.datasets.wraps import get_rag_pipeline
+from controllers.console.wraps import account_initialization_required, setup_required
+from libs.login import current_user, login_required
+from models import Account
+from models.dataset import Pipeline
+from services.rag_pipeline.rag_pipeline import RagPipelineService
+
+DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
+
+
+class Parser(BaseModel):
+    inputs: dict
+    datasource_type: str
+    credential_id: str | None = None
+
+
+console_ns.schema_model(Parser.__name__, Parser.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/published/datasource/nodes/<string:node_id>/preview")
+class DataSourceContentPreviewApi(Resource):
+    @console_ns.expect(console_ns.models[Parser.__name__], validate=True)
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_rag_pipeline
+    def post(self, pipeline: Pipeline, node_id: str):
+        """
+        Run datasource content preview
+        """
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        args = Parser.model_validate(console_ns.payload)
+
+        inputs = args.inputs
+        datasource_type = args.datasource_type
+        rag_pipeline_service = RagPipelineService()
+        preview_content = rag_pipeline_service.run_datasource_node_preview(
+            pipeline=pipeline,
+            node_id=node_id,
+            user_inputs=inputs,
+            account=current_user,
+            datasource_type=datasource_type,
+            is_published=True,
+            credential_id=args.credential_id,
+        )
+        return preview_content, 200

dify/api/controllers/console/datasets/rag_pipeline/rag_pipeline.py

@@ -0,0 +1,154 @@
+import logging
+
+from flask import request
+from flask_restx import Resource, reqparse
+from sqlalchemy.orm import Session
+
+from controllers.console import console_ns
+from controllers.console.wraps import (
+    account_initialization_required,
+    enterprise_license_required,
+    knowledge_pipeline_publish_enabled,
+    setup_required,
+)
+from extensions.ext_database import db
+from libs.login import login_required
+from models.dataset import PipelineCustomizedTemplate
+from services.entities.knowledge_entities.rag_pipeline_entities import PipelineTemplateInfoEntity
+from services.rag_pipeline.rag_pipeline import RagPipelineService
+
+logger = logging.getLogger(__name__)
+
+
+def _validate_name(name: str) -> str:
+    if not name or len(name) < 1 or len(name) > 40:
+        raise ValueError("Name must be between 1 to 40 characters.")
+    return name
+
+
+def _validate_description_length(description: str) -> str:
+    if len(description) > 400:
+        raise ValueError("Description cannot exceed 400 characters.")
+    return description
+
+
+@console_ns.route("/rag/pipeline/templates")
+class PipelineTemplateListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def get(self):
+        type = request.args.get("type", default="built-in", type=str)
+        language = request.args.get("language", default="en-US", type=str)
+        # get pipeline templates
+        pipeline_templates = RagPipelineService.get_pipeline_templates(type, language)
+        return pipeline_templates, 200
+
+
+@console_ns.route("/rag/pipeline/templates/<string:template_id>")
+class PipelineTemplateDetailApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def get(self, template_id: str):
+        type = request.args.get("type", default="built-in", type=str)
+        rag_pipeline_service = RagPipelineService()
+        pipeline_template = rag_pipeline_service.get_pipeline_template_detail(template_id, type)
+        return pipeline_template, 200
+
+
+@console_ns.route("/rag/pipeline/customized/templates/<string:template_id>")
+class CustomizedPipelineTemplateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def patch(self, template_id: str):
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(
+                "name",
+                nullable=False,
+                required=True,
+                help="Name must be between 1 to 40 characters.",
+                type=_validate_name,
+            )
+            .add_argument(
+                "description",
+                type=_validate_description_length,
+                nullable=True,
+                required=False,
+                default="",
+            )
+            .add_argument(
+                "icon_info",
+                type=dict,
+                location="json",
+                nullable=True,
+            )
+        )
+        args = parser.parse_args()
+        pipeline_template_info = PipelineTemplateInfoEntity.model_validate(args)
+        RagPipelineService.update_customized_pipeline_template(template_id, pipeline_template_info)
+        return 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def delete(self, template_id: str):
+        RagPipelineService.delete_customized_pipeline_template(template_id)
+        return 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    def post(self, template_id: str):
+        with Session(db.engine) as session:
+            template = (
+                session.query(PipelineCustomizedTemplate).where(PipelineCustomizedTemplate.id == template_id).first()
+            )
+            if not template:
+                raise ValueError("Customized pipeline template not found.")
+
+        return {"data": template.yaml_content}, 200
+
+
+@console_ns.route("/rag/pipelines/<string:pipeline_id>/customized/publish")
+class PublishCustomizedPipelineTemplateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @enterprise_license_required
+    @knowledge_pipeline_publish_enabled
+    def post(self, pipeline_id: str):
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(
+                "name",
+                nullable=False,
+                required=True,
+                help="Name must be between 1 to 40 characters.",
+                type=_validate_name,
+            )
+            .add_argument(
+                "description",
+                type=_validate_description_length,
+                nullable=True,
+                required=False,
+                default="",
+            )
+            .add_argument(
+                "icon_info",
+                type=dict,
+                location="json",
+                nullable=True,
+            )
+        )
+        args = parser.parse_args()
+        rag_pipeline_service = RagPipelineService()
+        rag_pipeline_service.publish_customized_pipeline_template(pipeline_id, args)
+        return {"result": "success"}

dify/api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py

@@ -0,0 +1,99 @@
+from flask_restx import Resource, marshal, reqparse
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Forbidden
+
+import services
+from controllers.console import console_ns
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.wraps import (
+    account_initialization_required,
+    cloud_edition_billing_rate_limit_check,
+    setup_required,
+)
+from extensions.ext_database import db
+from fields.dataset_fields import dataset_detail_fields
+from libs.login import current_account_with_tenant, login_required
+from models.dataset import DatasetPermissionEnum
+from services.dataset_service import DatasetPermissionService, DatasetService
+from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, RagPipelineDatasetCreateEntity
+from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService
+
+
+@console_ns.route("/rag/pipeline/dataset")
+class CreateRagPipelineDatasetApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self):
+        parser = reqparse.RequestParser().add_argument(
+            "yaml_content",
+            type=str,
+            nullable=False,
+            required=True,
+            help="yaml_content is required.",
+        )
+
+        args = parser.parse_args()
+        current_user, current_tenant_id = current_account_with_tenant()
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        rag_pipeline_dataset_create_entity = RagPipelineDatasetCreateEntity(
+            name="",
+            description="",
+            icon_info=IconInfo(
+                icon="📙",
+                icon_background="#FFF4ED",
+                icon_type="emoji",
+            ),
+            permission=DatasetPermissionEnum.ONLY_ME,
+            partial_member_list=None,
+            yaml_content=args["yaml_content"],
+        )
+        try:
+            with Session(db.engine) as session:
+                rag_pipeline_dsl_service = RagPipelineDslService(session)
+                import_info = rag_pipeline_dsl_service.create_rag_pipeline_dataset(
+                    tenant_id=current_tenant_id,
+                    rag_pipeline_dataset_create_entity=rag_pipeline_dataset_create_entity,
+                )
+            if rag_pipeline_dataset_create_entity.permission == "partial_members":
+                DatasetPermissionService.update_partial_member_list(
+                    current_tenant_id,
+                    import_info["dataset_id"],
+                    rag_pipeline_dataset_create_entity.partial_member_list,
+                )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return import_info, 201
+
+
+@console_ns.route("/rag/pipeline/empty-dataset")
+class CreateEmptyRagPipelineDatasetApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def post(self):
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        current_user, current_tenant_id = current_account_with_tenant()
+
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        dataset = DatasetService.create_empty_rag_pipeline_dataset(
+            tenant_id=current_tenant_id,
+            rag_pipeline_dataset_create_entity=RagPipelineDatasetCreateEntity(
+                name="",
+                description="",
+                icon_info=IconInfo(
+                    icon="📙",
+                    icon_background="#FFF4ED",
+                    icon_type="emoji",
+                ),
+                permission=DatasetPermissionEnum.ONLY_ME,
+                partial_member_list=None,
+            ),
+        )
+        return marshal(dataset, dataset_detail_fields), 201

dify/api/controllers/console/datasets/rag_pipeline/rag_pipeline_draft_variable.py

@@ -0,0 +1,347 @@
+import logging
+from typing import NoReturn
+
+from flask import Response
+from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqparse
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import console_ns
+from controllers.console.app.error import (
+    DraftWorkflowNotExist,
+)
+from controllers.console.app.workflow_draft_variable import (
+    _WORKFLOW_DRAFT_VARIABLE_FIELDS,  # type: ignore[private-usage]
+    _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS,  # type: ignore[private-usage]
+)
+from controllers.console.datasets.wraps import get_rag_pipeline
+from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.web.error import InvalidArgumentError, NotFoundError
+from core.variables.types import SegmentType
+from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
+from extensions.ext_database import db
+from factories.file_factory import build_from_mapping, build_from_mappings
+from factories.variable_factory import build_segment_with_type
+from libs.login import current_user, login_required
+from models import Account
+from models.dataset import Pipeline
+from models.workflow import WorkflowDraftVariable
+from services.rag_pipeline.rag_pipeline import RagPipelineService
+from services.workflow_draft_variable_service import WorkflowDraftVariableList, WorkflowDraftVariableService
+
+logger = logging.getLogger(__name__)
+
+
+def _create_pagination_parser():
+    parser = (
+        reqparse.RequestParser()
+        .add_argument(
+            "page",
+            type=inputs.int_range(1, 100_000),
+            required=False,
+            default=1,
+            location="args",
+            help="the page of data requested",
+        )
+        .add_argument("limit", type=inputs.int_range(1, 100), required=False, default=20, location="args")
+    )
+    return parser
+
+
+def _get_items(var_list: WorkflowDraftVariableList) -> list[WorkflowDraftVariable]:
+    return var_list.variables
+
+
+_WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS = {
+    "items": fields.List(fields.Nested(_WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS), attribute=_get_items),
+    "total": fields.Raw(),
+}
+
+_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS = {
+    "items": fields.List(fields.Nested(_WORKFLOW_DRAFT_VARIABLE_FIELDS), attribute=_get_items),
+}
+
+
+def _api_prerequisite(f):
+    """Common prerequisites for all draft workflow variable APIs.
+
+    It ensures the following conditions are satisfied:
+
+    - Dify has been property setup.
+    - The request user has logged in and initialized.
+    - The requested app is a workflow or a chat flow.
+    - The request user has the edit permission for the app.
+    """
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_rag_pipeline
+    def wrapper(*args, **kwargs):
+        if not isinstance(current_user, Account) or not current_user.has_edit_permission:
+            raise Forbidden()
+        return f(*args, **kwargs)
+
+    return wrapper
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/variables")
+class RagPipelineVariableCollectionApi(Resource):
+    @_api_prerequisite
+    @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS)
+    def get(self, pipeline: Pipeline):
+        """
+        Get draft workflow
+        """
+        parser = _create_pagination_parser()
+        args = parser.parse_args()
+
+        # fetch draft workflow by app_model
+        rag_pipeline_service = RagPipelineService()
+        workflow_exist = rag_pipeline_service.is_workflow_exist(pipeline=pipeline)
+        if not workflow_exist:
+            raise DraftWorkflowNotExist()
+
+        # fetch draft workflow by app_model
+        with Session(bind=db.engine, expire_on_commit=False) as session:
+            draft_var_srv = WorkflowDraftVariableService(
+                session=session,
+            )
+        workflow_vars = draft_var_srv.list_variables_without_values(
+            app_id=pipeline.id,
+            page=args.page,
+            limit=args.limit,
+        )
+
+        return workflow_vars
+
+    @_api_prerequisite
+    def delete(self, pipeline: Pipeline):
+        draft_var_srv = WorkflowDraftVariableService(
+            session=db.session(),
+        )
+        draft_var_srv.delete_workflow_variables(pipeline.id)
+        db.session.commit()
+        return Response("", 204)
+
+
+def validate_node_id(node_id: str) -> NoReturn | None:
+    if node_id in [
+        CONVERSATION_VARIABLE_NODE_ID,
+        SYSTEM_VARIABLE_NODE_ID,
+    ]:
+        # NOTE(QuantumGhost): While we store the system and conversation variables as node variables
+        # with specific `node_id` in database, we still want to make the API separated. By disallowing
+        # accessing system and conversation variables in `WorkflowDraftNodeVariableListApi`,
+        # we mitigate the risk that user of the API depending on the implementation detail of the API.
+        #
+        # ref: [Hyrum's Law](https://www.hyrumslaw.com/)
+
+        raise InvalidArgumentError(
+            f"invalid node_id, please use correspond api for conversation and system variables, node_id={node_id}",
+        )
+    return None
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/nodes/<string:node_id>/variables")
+class RagPipelineNodeVariableCollectionApi(Resource):
+    @_api_prerequisite
+    @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
+    def get(self, pipeline: Pipeline, node_id: str):
+        validate_node_id(node_id)
+        with Session(bind=db.engine, expire_on_commit=False) as session:
+            draft_var_srv = WorkflowDraftVariableService(
+                session=session,
+            )
+            node_vars = draft_var_srv.list_node_variables(pipeline.id, node_id)
+
+        return node_vars
+
+    @_api_prerequisite
+    def delete(self, pipeline: Pipeline, node_id: str):
+        validate_node_id(node_id)
+        srv = WorkflowDraftVariableService(db.session())
+        srv.delete_node_variables(pipeline.id, node_id)
+        db.session.commit()
+        return Response("", 204)
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/variables/<uuid:variable_id>")
+class RagPipelineVariableApi(Resource):
+    _PATCH_NAME_FIELD = "name"
+    _PATCH_VALUE_FIELD = "value"
+
+    @_api_prerequisite
+    @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS)
+    def get(self, pipeline: Pipeline, variable_id: str):
+        draft_var_srv = WorkflowDraftVariableService(
+            session=db.session(),
+        )
+        variable = draft_var_srv.get_variable(variable_id=variable_id)
+        if variable is None:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        if variable.app_id != pipeline.id:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        return variable
+
+    @_api_prerequisite
+    @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS)
+    def patch(self, pipeline: Pipeline, variable_id: str):
+        # Request payload for file types:
+        #
+        # Local File:
+        #
+        #     {
+        #         "type": "image",
+        #         "transfer_method": "local_file",
+        #         "url": "",
+        #         "upload_file_id": "daded54f-72c7-4f8e-9d18-9b0abdd9f190"
+        #     }
+        #
+        # Remote File:
+        #
+        #
+        #     {
+        #         "type": "image",
+        #         "transfer_method": "remote_url",
+        #         "url": "http://127.0.0.1:5001/files/1602650a-4fe4-423c-85a2-af76c083e3c4/file-preview?timestamp=1750041099&nonce=...&sign=...=",
+        #         "upload_file_id": "1602650a-4fe4-423c-85a2-af76c083e3c4"
+        #     }
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(self._PATCH_NAME_FIELD, type=str, required=False, nullable=True, location="json")
+            .add_argument(self._PATCH_VALUE_FIELD, type=lambda x: x, required=False, nullable=True, location="json")
+        )
+
+        draft_var_srv = WorkflowDraftVariableService(
+            session=db.session(),
+        )
+        args = parser.parse_args(strict=True)
+
+        variable = draft_var_srv.get_variable(variable_id=variable_id)
+        if variable is None:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        if variable.app_id != pipeline.id:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+
+        new_name = args.get(self._PATCH_NAME_FIELD, None)
+        raw_value = args.get(self._PATCH_VALUE_FIELD, None)
+        if new_name is None and raw_value is None:
+            return variable
+
+        new_value = None
+        if raw_value is not None:
+            if variable.value_type == SegmentType.FILE:
+                if not isinstance(raw_value, dict):
+                    raise InvalidArgumentError(description=f"expected dict for file, got {type(raw_value)}")
+                raw_value = build_from_mapping(mapping=raw_value, tenant_id=pipeline.tenant_id)
+            elif variable.value_type == SegmentType.ARRAY_FILE:
+                if not isinstance(raw_value, list):
+                    raise InvalidArgumentError(description=f"expected list for files, got {type(raw_value)}")
+                if len(raw_value) > 0 and not isinstance(raw_value[0], dict):
+                    raise InvalidArgumentError(description=f"expected dict for files[0], got {type(raw_value)}")
+                raw_value = build_from_mappings(mappings=raw_value, tenant_id=pipeline.tenant_id)
+            new_value = build_segment_with_type(variable.value_type, raw_value)
+        draft_var_srv.update_variable(variable, name=new_name, value=new_value)
+        db.session.commit()
+        return variable
+
+    @_api_prerequisite
+    def delete(self, pipeline: Pipeline, variable_id: str):
+        draft_var_srv = WorkflowDraftVariableService(
+            session=db.session(),
+        )
+        variable = draft_var_srv.get_variable(variable_id=variable_id)
+        if variable is None:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        if variable.app_id != pipeline.id:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        draft_var_srv.delete_variable(variable)
+        db.session.commit()
+        return Response("", 204)
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/variables/<uuid:variable_id>/reset")
+class RagPipelineVariableResetApi(Resource):
+    @_api_prerequisite
+    def put(self, pipeline: Pipeline, variable_id: str):
+        draft_var_srv = WorkflowDraftVariableService(
+            session=db.session(),
+        )
+
+        rag_pipeline_service = RagPipelineService()
+        draft_workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline)
+        if draft_workflow is None:
+            raise NotFoundError(
+                f"Draft workflow not found, pipeline_id={pipeline.id}",
+            )
+        variable = draft_var_srv.get_variable(variable_id=variable_id)
+        if variable is None:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+        if variable.app_id != pipeline.id:
+            raise NotFoundError(description=f"variable not found, id={variable_id}")
+
+        resetted = draft_var_srv.reset_variable(draft_workflow, variable)
+        db.session.commit()
+        if resetted is None:
+            return Response("", 204)
+        else:
+            return marshal(resetted, _WORKFLOW_DRAFT_VARIABLE_FIELDS)
+
+
+def _get_variable_list(pipeline: Pipeline, node_id) -> WorkflowDraftVariableList:
+    with Session(bind=db.engine, expire_on_commit=False) as session:
+        draft_var_srv = WorkflowDraftVariableService(
+            session=session,
+        )
+        if node_id == CONVERSATION_VARIABLE_NODE_ID:
+            draft_vars = draft_var_srv.list_conversation_variables(pipeline.id)
+        elif node_id == SYSTEM_VARIABLE_NODE_ID:
+            draft_vars = draft_var_srv.list_system_variables(pipeline.id)
+        else:
+            draft_vars = draft_var_srv.list_node_variables(app_id=pipeline.id, node_id=node_id)
+    return draft_vars
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/system-variables")
+class RagPipelineSystemVariableCollectionApi(Resource):
+    @_api_prerequisite
+    @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
+    def get(self, pipeline: Pipeline):
+        return _get_variable_list(pipeline, SYSTEM_VARIABLE_NODE_ID)
+
+
+@console_ns.route("/rag/pipelines/<uuid:pipeline_id>/workflows/draft/environment-variables")
+class RagPipelineEnvironmentVariableCollectionApi(Resource):
+    @_api_prerequisite
+    def get(self, pipeline: Pipeline):
+        """
+        Get draft workflow
+        """
+        # fetch draft workflow by app_model
+        rag_pipeline_service = RagPipelineService()
+        workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline)
+        if workflow is None:
+            raise DraftWorkflowNotExist()
+
+        env_vars = workflow.environment_variables
+        env_vars_list = []
+        for v in env_vars:
+            env_vars_list.append(
+                {
+                    "id": v.id,
+                    "type": "env",
+                    "name": v.name,
+                    "description": v.description,
+                    "selector": v.selector,
+                    "value_type": v.value_type.value,
+                    "value": v.value,
+                    # Do not track edited for env vars.
+                    "edited": False,
+                    "visible": True,
+                    "editable": True,
+                }
+            )
+
+        return {"items": env_vars_list}

dify/api/controllers/console/datasets/rag_pipeline/rag_pipeline_import.py

@@ -0,0 +1,126 @@
+from flask_restx import Resource, marshal_with, reqparse  # type: ignore
+from sqlalchemy.orm import Session
+
+from controllers.console import console_ns
+from controllers.console.datasets.wraps import get_rag_pipeline
+from controllers.console.wraps import (
+    account_initialization_required,
+    edit_permission_required,
+    setup_required,
+)
+from extensions.ext_database import db
+from fields.rag_pipeline_fields import pipeline_import_check_dependencies_fields, pipeline_import_fields
+from libs.login import current_account_with_tenant, login_required
+from models.dataset import Pipeline
+from services.app_dsl_service import ImportStatus
+from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService
+
+
+@console_ns.route("/rag/pipelines/imports")
+class RagPipelineImportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @marshal_with(pipeline_import_fields)
+    def post(self):
+        # Check user role first
+        current_user, _ = current_account_with_tenant()
+
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("mode", type=str, required=True, location="json")
+            .add_argument("yaml_content", type=str, location="json")
+            .add_argument("yaml_url", type=str, location="json")
+            .add_argument("name", type=str, location="json")
+            .add_argument("description", type=str, location="json")
+            .add_argument("icon_type", type=str, location="json")
+            .add_argument("icon", type=str, location="json")
+            .add_argument("icon_background", type=str, location="json")
+            .add_argument("pipeline_id", type=str, location="json")
+        )
+        args = parser.parse_args()
+
+        # Create service with session
+        with Session(db.engine) as session:
+            import_service = RagPipelineDslService(session)
+            # Import app
+            account = current_user
+            result = import_service.import_rag_pipeline(
+                account=account,
+                import_mode=args["mode"],
+                yaml_content=args.get("yaml_content"),
+                yaml_url=args.get("yaml_url"),
+                pipeline_id=args.get("pipeline_id"),
+                dataset_name=args.get("name"),
+            )
+            session.commit()
+
+        # Return appropriate status code based on result
+        status = result.status
+        if status == ImportStatus.FAILED:
+            return result.model_dump(mode="json"), 400
+        elif status == ImportStatus.PENDING:
+            return result.model_dump(mode="json"), 202
+        return result.model_dump(mode="json"), 200
+
+
+@console_ns.route("/rag/pipelines/imports/<string:import_id>/confirm")
+class RagPipelineImportConfirmApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @marshal_with(pipeline_import_fields)
+    def post(self, import_id):
+        current_user, _ = current_account_with_tenant()
+
+        # Create service with session
+        with Session(db.engine) as session:
+            import_service = RagPipelineDslService(session)
+            # Confirm import
+            account = current_user
+            result = import_service.confirm_import(import_id=import_id, account=account)
+            session.commit()
+
+        # Return appropriate status code based on result
+        if result.status == ImportStatus.FAILED:
+            return result.model_dump(mode="json"), 400
+        return result.model_dump(mode="json"), 200
+
+
+@console_ns.route("/rag/pipelines/imports/<string:pipeline_id>/check-dependencies")
+class RagPipelineImportCheckDependenciesApi(Resource):
+    @setup_required
+    @login_required
+    @get_rag_pipeline
+    @account_initialization_required
+    @edit_permission_required
+    @marshal_with(pipeline_import_check_dependencies_fields)
+    def get(self, pipeline: Pipeline):
+        with Session(db.engine) as session:
+            import_service = RagPipelineDslService(session)
+            result = import_service.check_dependencies(pipeline=pipeline)
+
+        return result.model_dump(mode="json"), 200
+
+
+@console_ns.route("/rag/pipelines/<string:pipeline_id>/exports")
+class RagPipelineExportApi(Resource):
+    @setup_required
+    @login_required
+    @get_rag_pipeline
+    @account_initialization_required
+    @edit_permission_required
+    def get(self, pipeline: Pipeline):
+        # Add include_secret params
+        parser = reqparse.RequestParser().add_argument("include_secret", type=str, default="false", location="args")
+        args = parser.parse_args()
+
+        with Session(db.engine) as session:
+            export_service = RagPipelineDslService(session)
+            result = export_service.export_rag_pipeline_dsl(
+                pipeline=pipeline, include_secret=args["include_secret"] == "true"
+            )
+
+        return {"data": result}, 200

dify/api/controllers/console/datasets/website.py

@@ -0,0 +1,91 @@
+from flask_restx import Resource, fields, reqparse
+
+from controllers.console import console_ns
+from controllers.console.datasets.error import WebsiteCrawlError
+from controllers.console.wraps import account_initialization_required, setup_required
+from libs.login import login_required
+from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService
+
+
+@console_ns.route("/website/crawl")
+class WebsiteCrawlApi(Resource):
+    @console_ns.doc("crawl_website")
+    @console_ns.doc(description="Crawl website content")
+    @console_ns.expect(
+        console_ns.model(
+            "WebsiteCrawlRequest",
+            {
+                "provider": fields.String(
+                    required=True,
+                    description="Crawl provider (firecrawl/watercrawl/jinareader)",
+                    enum=["firecrawl", "watercrawl", "jinareader"],
+                ),
+                "url": fields.String(required=True, description="URL to crawl"),
+                "options": fields.Raw(required=True, description="Crawl options"),
+            },
+        )
+    )
+    @console_ns.response(200, "Website crawl initiated successfully")
+    @console_ns.response(400, "Invalid crawl parameters")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = (
+            reqparse.RequestParser()
+            .add_argument(
+                "provider",
+                type=str,
+                choices=["firecrawl", "watercrawl", "jinareader"],
+                required=True,
+                nullable=True,
+                location="json",
+            )
+            .add_argument("url", type=str, required=True, nullable=True, location="json")
+            .add_argument("options", type=dict, required=True, nullable=True, location="json")
+        )
+        args = parser.parse_args()
+
+        # Create typed request and validate
+        try:
+            api_request = WebsiteCrawlApiRequest.from_args(args)
+        except ValueError as e:
+            raise WebsiteCrawlError(str(e))
+
+        # Crawl URL using typed request
+        try:
+            result = WebsiteService.crawl_url(api_request)
+        except Exception as e:
+            raise WebsiteCrawlError(str(e))
+        return result, 200
+
+
+@console_ns.route("/website/crawl/status/<string:job_id>")
+class WebsiteCrawlStatusApi(Resource):
+    @console_ns.doc("get_crawl_status")
+    @console_ns.doc(description="Get website crawl status")
+    @console_ns.doc(params={"job_id": "Crawl job ID", "provider": "Crawl provider (firecrawl/watercrawl/jinareader)"})
+    @console_ns.response(200, "Crawl status retrieved successfully")
+    @console_ns.response(404, "Crawl job not found")
+    @console_ns.response(400, "Invalid provider")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, job_id: str):
+        parser = reqparse.RequestParser().add_argument(
+            "provider", type=str, choices=["firecrawl", "watercrawl", "jinareader"], required=True, location="args"
+        )
+        args = parser.parse_args()
+
+        # Create typed request and validate
+        try:
+            api_request = WebsiteCrawlStatusApiRequest.from_args(args, job_id)
+        except ValueError as e:
+            raise WebsiteCrawlError(str(e))
+
+        # Get crawl status using typed request
+        try:
+            result = WebsiteService.get_crawl_status_typed(api_request)
+        except Exception as e:
+            raise WebsiteCrawlError(str(e))
+        return result, 200

dify/api/controllers/console/datasets/wraps.py

@@ -0,0 +1,40 @@
+from collections.abc import Callable
+from functools import wraps
+from typing import ParamSpec, TypeVar
+
+from controllers.console.datasets.error import PipelineNotFoundError
+from extensions.ext_database import db
+from libs.login import current_account_with_tenant
+from models.dataset import Pipeline
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def get_rag_pipeline(view_func: Callable[P, R]):
+    @wraps(view_func)
+    def decorated_view(*args: P.args, **kwargs: P.kwargs):
+        if not kwargs.get("pipeline_id"):
+            raise ValueError("missing pipeline_id in path parameters")
+
+        _, current_tenant_id = current_account_with_tenant()
+
+        pipeline_id = kwargs.get("pipeline_id")
+        pipeline_id = str(pipeline_id)
+
+        del kwargs["pipeline_id"]
+
+        pipeline = (
+            db.session.query(Pipeline)
+            .where(Pipeline.id == pipeline_id, Pipeline.tenant_id == current_tenant_id)
+            .first()
+        )
+
+        if not pipeline:
+            raise PipelineNotFoundError()
+
+        kwargs["pipeline"] = pipeline
+
+        return view_func(*args, **kwargs)
+
+    return decorated_view