dify

dify/api/controllers/service_api/dataset/dataset.py

@@ -0,0 +1,704 @@
+from typing import Any, Literal, cast
+
+from flask import request
+from flask_restx import marshal, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
+import services
+from controllers.console.wraps import edit_permission_required
+from controllers.service_api import service_api_ns
+from controllers.service_api.dataset.error import DatasetInUseError, DatasetNameDuplicateError, InvalidActionError
+from controllers.service_api.wraps import (
+    DatasetApiResource,
+    cloud_edition_billing_rate_limit_check,
+    validate_dataset_token,
+)
+from core.model_runtime.entities.model_entities import ModelType
+from core.provider_manager import ProviderManager
+from fields.dataset_fields import dataset_detail_fields
+from fields.tag_fields import build_dataset_tag_fields
+from libs.login import current_user
+from libs.validators import validate_description_length
+from models.account import Account
+from models.dataset import Dataset, DatasetPermissionEnum
+from models.provider_ids import ModelProviderID
+from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
+from services.entities.knowledge_entities.knowledge_entities import RetrievalModel
+from services.tag_service import TagService
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 40:
+        raise ValueError("Name must be between 1 to 40 characters.")
+    return name
+
+
+# Define parsers for dataset operations
+dataset_create_parser = (
+    reqparse.RequestParser()
+    .add_argument(
+        "name",
+        nullable=False,
+        required=True,
+        help="type is required. Name must be between 1 to 40 characters.",
+        type=_validate_name,
+    )
+    .add_argument(
+        "description",
+        type=validate_description_length,
+        nullable=True,
+        required=False,
+        default="",
+    )
+    .add_argument(
+        "indexing_technique",
+        type=str,
+        location="json",
+        choices=Dataset.INDEXING_TECHNIQUE_LIST,
+        help="Invalid indexing technique.",
+    )
+    .add_argument(
+        "permission",
+        type=str,
+        location="json",
+        choices=(DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM),
+        help="Invalid permission.",
+        required=False,
+        nullable=False,
+    )
+    .add_argument(
+        "external_knowledge_api_id",
+        type=str,
+        nullable=True,
+        required=False,
+        default="_validate_name",
+    )
+    .add_argument(
+        "provider",
+        type=str,
+        nullable=True,
+        required=False,
+        default="vendor",
+    )
+    .add_argument(
+        "external_knowledge_id",
+        type=str,
+        nullable=True,
+        required=False,
+    )
+    .add_argument("retrieval_model", type=dict, required=False, nullable=True, location="json")
+    .add_argument("embedding_model", type=str, required=False, nullable=True, location="json")
+    .add_argument("embedding_model_provider", type=str, required=False, nullable=True, location="json")
+)
+
+dataset_update_parser = (
+    reqparse.RequestParser()
+    .add_argument(
+        "name",
+        nullable=False,
+        help="type is required. Name must be between 1 to 40 characters.",
+        type=_validate_name,
+    )
+    .add_argument("description", location="json", store_missing=False, type=validate_description_length)
+    .add_argument(
+        "indexing_technique",
+        type=str,
+        location="json",
+        choices=Dataset.INDEXING_TECHNIQUE_LIST,
+        nullable=True,
+        help="Invalid indexing technique.",
+    )
+    .add_argument(
+        "permission",
+        type=str,
+        location="json",
+        choices=(DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM, DatasetPermissionEnum.PARTIAL_TEAM),
+        help="Invalid permission.",
+    )
+    .add_argument("embedding_model", type=str, location="json", help="Invalid embedding model.")
+    .add_argument("embedding_model_provider", type=str, location="json", help="Invalid embedding model provider.")
+    .add_argument("retrieval_model", type=dict, location="json", help="Invalid retrieval model.")
+    .add_argument("partial_member_list", type=list, location="json", help="Invalid parent user list.")
+    .add_argument(
+        "external_retrieval_model",
+        type=dict,
+        required=False,
+        nullable=True,
+        location="json",
+        help="Invalid external retrieval model.",
+    )
+    .add_argument(
+        "external_knowledge_id",
+        type=str,
+        required=False,
+        nullable=True,
+        location="json",
+        help="Invalid external knowledge id.",
+    )
+    .add_argument(
+        "external_knowledge_api_id",
+        type=str,
+        required=False,
+        nullable=True,
+        location="json",
+        help="Invalid external knowledge api id.",
+    )
+)
+
+tag_create_parser = reqparse.RequestParser().add_argument(
+    "name",
+    nullable=False,
+    required=True,
+    help="Name must be between 1 to 50 characters.",
+    type=lambda x: x
+    if x and 1 <= len(x) <= 50
+    else (_ for _ in ()).throw(ValueError("Name must be between 1 to 50 characters.")),
+)
+
+tag_update_parser = (
+    reqparse.RequestParser()
+    .add_argument(
+        "name",
+        nullable=False,
+        required=True,
+        help="Name must be between 1 to 50 characters.",
+        type=lambda x: x
+        if x and 1 <= len(x) <= 50
+        else (_ for _ in ()).throw(ValueError("Name must be between 1 to 50 characters.")),
+    )
+    .add_argument("tag_id", nullable=False, required=True, help="Id of a tag.", type=str)
+)
+
+tag_delete_parser = reqparse.RequestParser().add_argument(
+    "tag_id", nullable=False, required=True, help="Id of a tag.", type=str
+)
+
+tag_binding_parser = (
+    reqparse.RequestParser()
+    .add_argument("tag_ids", type=list, nullable=False, required=True, location="json", help="Tag IDs is required.")
+    .add_argument(
+        "target_id", type=str, nullable=False, required=True, location="json", help="Target Dataset ID is required."
+    )
+)
+
+tag_unbinding_parser = (
+    reqparse.RequestParser()
+    .add_argument("tag_id", type=str, nullable=False, required=True, help="Tag ID is required.")
+    .add_argument("target_id", type=str, nullable=False, required=True, help="Target ID is required.")
+)
+
+
+@service_api_ns.route("/datasets")
+class DatasetListApi(DatasetApiResource):
+    """Resource for datasets."""
+
+    @service_api_ns.doc("list_datasets")
+    @service_api_ns.doc(description="List all datasets")
+    @service_api_ns.doc(
+        responses={
+            200: "Datasets retrieved successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    def get(self, tenant_id):
+        """Resource for getting datasets."""
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        # provider = request.args.get("provider", default="vendor")
+        search = request.args.get("keyword", default=None, type=str)
+        tag_ids = request.args.getlist("tag_ids")
+        include_all = request.args.get("include_all", default="false").lower() == "true"
+
+        datasets, total = DatasetService.get_datasets(
+            page, limit, tenant_id, current_user, search, tag_ids, include_all
+        )
+        # check embedding setting
+        provider_manager = ProviderManager()
+        assert isinstance(current_user, Account)
+        cid = current_user.current_tenant_id
+        assert cid is not None
+        configurations = provider_manager.get_configurations(tenant_id=cid)
+
+        embedding_models = configurations.get_models(model_type=ModelType.TEXT_EMBEDDING, only_active=True)
+
+        model_names = []
+        for embedding_model in embedding_models:
+            model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")
+
+        data = marshal(datasets, dataset_detail_fields)
+        for item in data:
+            if item["indexing_technique"] == "high_quality" and item["embedding_model_provider"]:
+                item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
+                item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"
+                if item_model in model_names:
+                    item["embedding_available"] = True
+                else:
+                    item["embedding_available"] = False
+            else:
+                item["embedding_available"] = True
+        response = {"data": data, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page}
+        return response, 200
+
+    @service_api_ns.expect(dataset_create_parser)
+    @service_api_ns.doc("create_dataset")
+    @service_api_ns.doc(description="Create a new dataset")
+    @service_api_ns.doc(
+        responses={
+            200: "Dataset created successfully",
+            401: "Unauthorized - invalid API token",
+            400: "Bad request - invalid parameters",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id):
+        """Resource for creating datasets."""
+        args = dataset_create_parser.parse_args()
+
+        embedding_model_provider = args.get("embedding_model_provider")
+        embedding_model = args.get("embedding_model")
+        if embedding_model_provider and embedding_model:
+            DatasetService.check_embedding_model_setting(tenant_id, embedding_model_provider, embedding_model)
+
+        retrieval_model = args.get("retrieval_model")
+        if (
+            retrieval_model
+            and retrieval_model.get("reranking_model")
+            and retrieval_model.get("reranking_model").get("reranking_provider_name")
+        ):
+            DatasetService.check_reranking_model_setting(
+                tenant_id,
+                retrieval_model.get("reranking_model").get("reranking_provider_name"),
+                retrieval_model.get("reranking_model").get("reranking_model_name"),
+            )
+
+        try:
+            assert isinstance(current_user, Account)
+            dataset = DatasetService.create_empty_dataset(
+                tenant_id=tenant_id,
+                name=args["name"],
+                description=args["description"],
+                indexing_technique=args["indexing_technique"],
+                account=current_user,
+                permission=args["permission"],
+                provider=args["provider"],
+                external_knowledge_api_id=args["external_knowledge_api_id"],
+                external_knowledge_id=args["external_knowledge_id"],
+                embedding_model_provider=args["embedding_model_provider"],
+                embedding_model_name=args["embedding_model"],
+                retrieval_model=RetrievalModel.model_validate(args["retrieval_model"])
+                if args["retrieval_model"] is not None
+                else None,
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>")
+class DatasetApi(DatasetApiResource):
+    """Resource for dataset."""
+
+    @service_api_ns.doc("get_dataset")
+    @service_api_ns.doc(description="Get a specific dataset by ID")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Dataset retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Dataset not found",
+        }
+    )
+    def get(self, _, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+        data = cast(dict[str, Any], marshal(dataset, dataset_detail_fields))
+        # check embedding setting
+        provider_manager = ProviderManager()
+        assert isinstance(current_user, Account)
+        cid = current_user.current_tenant_id
+        assert cid is not None
+        configurations = provider_manager.get_configurations(tenant_id=cid)
+
+        embedding_models = configurations.get_models(model_type=ModelType.TEXT_EMBEDDING, only_active=True)
+
+        model_names = []
+        for embedding_model in embedding_models:
+            model_names.append(f"{embedding_model.model}:{embedding_model.provider.provider}")
+
+        if data.get("indexing_technique") == "high_quality":
+            item_model = f"{data.get('embedding_model')}:{data.get('embedding_model_provider')}"
+            if item_model in model_names:
+                data["embedding_available"] = True
+            else:
+                data["embedding_available"] = False
+        else:
+            data["embedding_available"] = True
+
+            # force update search method to keyword_search if indexing_technique is economic
+            retrieval_model_dict = data.get("retrieval_model_dict")
+            if retrieval_model_dict:
+                retrieval_model_dict["search_method"] = "keyword_search"
+
+        if data.get("permission") == "partial_members":
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            data.update({"partial_member_list": part_users_list})
+
+        return data, 200
+
+    @service_api_ns.expect(dataset_update_parser)
+    @service_api_ns.doc("update_dataset")
+    @service_api_ns.doc(description="Update an existing dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Dataset updated successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def patch(self, _, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        args = dataset_update_parser.parse_args()
+        data = request.get_json()
+
+        # check embedding model setting
+        embedding_model_provider = data.get("embedding_model_provider")
+        embedding_model = data.get("embedding_model")
+        if data.get("indexing_technique") == "high_quality" or embedding_model_provider:
+            if embedding_model_provider and embedding_model:
+                DatasetService.check_embedding_model_setting(
+                    dataset.tenant_id, embedding_model_provider, embedding_model
+                )
+
+        retrieval_model = data.get("retrieval_model")
+        if (
+            retrieval_model
+            and retrieval_model.get("reranking_model")
+            and retrieval_model.get("reranking_model").get("reranking_provider_name")
+        ):
+            DatasetService.check_reranking_model_setting(
+                dataset.tenant_id,
+                retrieval_model.get("reranking_model").get("reranking_provider_name"),
+                retrieval_model.get("reranking_model").get("reranking_model_name"),
+            )
+
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        DatasetPermissionService.check_permission(
+            current_user, dataset, data.get("permission"), data.get("partial_member_list")
+        )
+
+        dataset = DatasetService.update_dataset(dataset_id_str, args, current_user)
+
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        result_data = cast(dict[str, Any], marshal(dataset, dataset_detail_fields))
+        assert isinstance(current_user, Account)
+        tenant_id = current_user.current_tenant_id
+
+        if data.get("partial_member_list") and data.get("permission") == "partial_members":
+            DatasetPermissionService.update_partial_member_list(
+                tenant_id, dataset_id_str, data.get("partial_member_list")
+            )
+        # clear partial member list when permission is only_me or all_team_members
+        elif (
+            data.get("permission") == DatasetPermissionEnum.ONLY_ME
+            or data.get("permission") == DatasetPermissionEnum.ALL_TEAM
+        ):
+            DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+
+        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+        result_data.update({"partial_member_list": partial_member_list})
+
+        return result_data, 200
+
+    @service_api_ns.doc("delete_dataset")
+    @service_api_ns.doc(description="Delete a dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            204: "Dataset deleted successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+            409: "Conflict - dataset is in use",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, _, dataset_id):
+        """
+        Deletes a dataset given its ID.
+
+        Args:
+            _: ignore
+            dataset_id (UUID): The ID of the dataset to be deleted.
+
+        Returns:
+            dict: A dictionary with a key 'result' and a value 'success'
+                  if the dataset was successfully deleted. Omitted in HTTP response.
+            int: HTTP status code 204 indicating that the operation was successful.
+
+        Raises:
+            NotFound: If the dataset with the given ID does not exist.
+        """
+
+        dataset_id_str = str(dataset_id)
+
+        try:
+            if DatasetService.delete_dataset(dataset_id_str, current_user):
+                DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+                return 204
+            else:
+                raise NotFound("Dataset not found.")
+        except services.errors.dataset.DatasetInUseError:
+            raise DatasetInUseError()
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/status/<string:action>")
+class DocumentStatusApi(DatasetApiResource):
+    """Resource for batch document status operations."""
+
+    @service_api_ns.doc("update_document_status")
+    @service_api_ns.doc(description="Batch update document status")
+    @service_api_ns.doc(
+        params={
+            "dataset_id": "Dataset ID",
+            "action": "Action to perform: 'enable', 'disable', 'archive', or 'un_archive'",
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Document status updated successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Dataset not found",
+            400: "Bad request - invalid action",
+        }
+    )
+    def patch(self, tenant_id, dataset_id, action: Literal["enable", "disable", "archive", "un_archive"]):
+        """
+        Batch update document status.
+
+        Args:
+            tenant_id: tenant id
+            dataset_id: dataset id
+            action: action to perform (Literal["enable", "disable", "archive", "un_archive"])
+
+        Returns:
+            dict: A dictionary with a key 'result' and a value 'success'
+            int: HTTP status code 200 indicating that the operation was successful.
+
+        Raises:
+            NotFound: If the dataset with the given ID does not exist.
+            Forbidden: If the user does not have permission.
+            InvalidActionError: If the action is invalid or cannot be performed.
+        """
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        # Check user's permission
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        # Check dataset model setting
+        DatasetService.check_dataset_model_setting(dataset)
+
+        # Get document IDs from request body
+        data = request.get_json()
+        document_ids = data.get("document_ids", [])
+
+        try:
+            DocumentService.batch_update_document_status(dataset, document_ids, action, current_user)
+        except services.errors.document.DocumentIndexingError as e:
+            raise InvalidActionError(str(e))
+        except ValueError as e:
+            raise InvalidActionError(str(e))
+
+        return {"result": "success"}, 200
+
+
+@service_api_ns.route("/datasets/tags")
+class DatasetTagsApi(DatasetApiResource):
+    @service_api_ns.doc("list_dataset_tags")
+    @service_api_ns.doc(description="Get all knowledge type tags")
+    @service_api_ns.doc(
+        responses={
+            200: "Tags retrieved successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    @validate_dataset_token
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
+    def get(self, _, dataset_id):
+        """Get all knowledge type tags."""
+        assert isinstance(current_user, Account)
+        cid = current_user.current_tenant_id
+        assert cid is not None
+        tags = TagService.get_tags("knowledge", cid)
+
+        return tags, 200
+
+    @service_api_ns.expect(tag_create_parser)
+    @service_api_ns.doc("create_dataset_tag")
+    @service_api_ns.doc(description="Add a knowledge type tag")
+    @service_api_ns.doc(
+        responses={
+            200: "Tag created successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+        }
+    )
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
+    @validate_dataset_token
+    def post(self, _, dataset_id):
+        """Add a knowledge type tag."""
+        assert isinstance(current_user, Account)
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
+
+        args = tag_create_parser.parse_args()
+        args["type"] = "knowledge"
+        tag = TagService.save_tags(args)
+
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
+        return response, 200
+
+    @service_api_ns.expect(tag_update_parser)
+    @service_api_ns.doc("update_dataset_tag")
+    @service_api_ns.doc(description="Update a knowledge type tag")
+    @service_api_ns.doc(
+        responses={
+            200: "Tag updated successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+        }
+    )
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
+    @validate_dataset_token
+    def patch(self, _, dataset_id):
+        assert isinstance(current_user, Account)
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
+
+        args = tag_update_parser.parse_args()
+        args["type"] = "knowledge"
+        tag_id = args["tag_id"]
+        tag = TagService.update_tags(args, tag_id)
+
+        binding_count = TagService.get_tag_binding_count(tag_id)
+
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
+
+        return response, 200
+
+    @service_api_ns.expect(tag_delete_parser)
+    @service_api_ns.doc("delete_dataset_tag")
+    @service_api_ns.doc(description="Delete a knowledge type tag")
+    @service_api_ns.doc(
+        responses={
+            204: "Tag deleted successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+        }
+    )
+    @validate_dataset_token
+    @edit_permission_required
+    def delete(self, _, dataset_id):
+        """Delete a knowledge type tag."""
+        args = tag_delete_parser.parse_args()
+        TagService.delete_tag(args["tag_id"])
+
+        return 204
+
+
+@service_api_ns.route("/datasets/tags/binding")
+class DatasetTagBindingApi(DatasetApiResource):
+    @service_api_ns.expect(tag_binding_parser)
+    @service_api_ns.doc("bind_dataset_tags")
+    @service_api_ns.doc(description="Bind tags to a dataset")
+    @service_api_ns.doc(
+        responses={
+            204: "Tags bound successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+        }
+    )
+    @validate_dataset_token
+    def post(self, _, dataset_id):
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        assert isinstance(current_user, Account)
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
+
+        args = tag_binding_parser.parse_args()
+        args["type"] = "knowledge"
+        TagService.save_tag_binding(args)
+
+        return 204
+
+
+@service_api_ns.route("/datasets/tags/unbinding")
+class DatasetTagUnbindingApi(DatasetApiResource):
+    @service_api_ns.expect(tag_unbinding_parser)
+    @service_api_ns.doc("unbind_dataset_tag")
+    @service_api_ns.doc(description="Unbind a tag from a dataset")
+    @service_api_ns.doc(
+        responses={
+            204: "Tag unbound successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+        }
+    )
+    @validate_dataset_token
+    def post(self, _, dataset_id):
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        assert isinstance(current_user, Account)
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
+
+        args = tag_unbinding_parser.parse_args()
+        args["type"] = "knowledge"
+        TagService.delete_tag_binding(args)
+
+        return 204
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/tags")
+class DatasetTagsBindingStatusApi(DatasetApiResource):
+    @service_api_ns.doc("get_dataset_tags_binding_status")
+    @service_api_ns.doc(description="Get tags bound to a specific dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Tags retrieved successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    @validate_dataset_token
+    def get(self, _, *args, **kwargs):
+        """Get all knowledge type tags."""
+        dataset_id = kwargs.get("dataset_id")
+        assert isinstance(current_user, Account)
+        assert current_user.current_tenant_id is not None
+        tags = TagService.get_tags_by_target_id("knowledge", current_user.current_tenant_id, str(dataset_id))
+        tags_list = [{"id": tag.id, "name": tag.name} for tag in tags]
+        response = {"data": tags_list, "total": len(tags)}
+        return response, 200

dify/api/controllers/service_api/dataset/document.py

@@ -0,0 +1,705 @@
+import json
+from typing import Self
+from uuid import UUID
+
+from flask import request
+from flask_restx import marshal, reqparse
+from pydantic import BaseModel, model_validator
+from sqlalchemy import desc, select
+from werkzeug.exceptions import Forbidden, NotFound
+
+import services
+from controllers.common.errors import (
+    FilenameNotExistsError,
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
+from controllers.service_api import service_api_ns
+from controllers.service_api.app.error import ProviderNotInitializeError
+from controllers.service_api.dataset.error import (
+    ArchivedDocumentImmutableError,
+    DocumentIndexingError,
+    InvalidMetadataError,
+)
+from controllers.service_api.wraps import (
+    DatasetApiResource,
+    cloud_edition_billing_rate_limit_check,
+    cloud_edition_billing_resource_check,
+)
+from core.errors.error import ProviderTokenNotInitError
+from extensions.ext_database import db
+from fields.document_fields import document_fields, document_status_fields
+from libs.login import current_user
+from models.dataset import Dataset, Document, DocumentSegment
+from services.dataset_service import DatasetService, DocumentService
+from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel
+from services.file_service import FileService
+
+# Define parsers for document operations
+document_text_create_parser = (
+    reqparse.RequestParser()
+    .add_argument("name", type=str, required=True, nullable=False, location="json")
+    .add_argument("text", type=str, required=True, nullable=False, location="json")
+    .add_argument("process_rule", type=dict, required=False, nullable=True, location="json")
+    .add_argument("original_document_id", type=str, required=False, location="json")
+    .add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json")
+    .add_argument("doc_language", type=str, default="English", required=False, nullable=False, location="json")
+    .add_argument(
+        "indexing_technique", type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False, location="json"
+    )
+    .add_argument("retrieval_model", type=dict, required=False, nullable=True, location="json")
+    .add_argument("embedding_model", type=str, required=False, nullable=True, location="json")
+    .add_argument("embedding_model_provider", type=str, required=False, nullable=True, location="json")
+)
+
+DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
+
+
+class DocumentTextUpdate(BaseModel):
+    name: str | None = None
+    text: str | None = None
+    process_rule: ProcessRule | None = None
+    doc_form: str = "text_model"
+    doc_language: str = "English"
+    retrieval_model: RetrievalModel | None = None
+
+    @model_validator(mode="after")
+    def check_text_and_name(self) -> Self:
+        if self.text is not None and self.name is None:
+            raise ValueError("name is required when text is provided")
+        return self
+
+
+for m in [ProcessRule, RetrievalModel, DocumentTextUpdate]:
+    service_api_ns.schema_model(m.__name__, m.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))  # type: ignore
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/document/create_by_text",
+    "/datasets/<uuid:dataset_id>/document/create-by-text",
+)
+class DocumentAddByTextApi(DatasetApiResource):
+    """Resource for documents."""
+
+    @service_api_ns.expect(document_text_create_parser)
+    @service_api_ns.doc("create_document_by_text")
+    @service_api_ns.doc(description="Create a new document by providing text content")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Document created successfully",
+            401: "Unauthorized - invalid API token",
+            400: "Bad request - invalid parameters",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_resource_check("documents", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        """Create document by text."""
+        args = document_text_create_parser.parse_args()
+
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        if not dataset.indexing_technique and not args["indexing_technique"]:
+            raise ValueError("indexing_technique is required.")
+
+        text = args.get("text")
+        name = args.get("name")
+        if text is None or name is None:
+            raise ValueError("Both 'text' and 'name' must be non-null values.")
+
+        embedding_model_provider = args.get("embedding_model_provider")
+        embedding_model = args.get("embedding_model")
+        if embedding_model_provider and embedding_model:
+            DatasetService.check_embedding_model_setting(tenant_id, embedding_model_provider, embedding_model)
+
+        retrieval_model = args.get("retrieval_model")
+        if (
+            retrieval_model
+            and retrieval_model.get("reranking_model")
+            and retrieval_model.get("reranking_model").get("reranking_provider_name")
+        ):
+            DatasetService.check_reranking_model_setting(
+                tenant_id,
+                retrieval_model.get("reranking_model").get("reranking_provider_name"),
+                retrieval_model.get("reranking_model").get("reranking_model_name"),
+            )
+
+        if not current_user:
+            raise ValueError("current_user is required")
+
+        upload_file = FileService(db.engine).upload_text(
+            text=str(text), text_name=str(name), user_id=current_user.id, tenant_id=tenant_id
+        )
+        data_source = {
+            "type": "upload_file",
+            "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
+        }
+        args["data_source"] = data_source
+        knowledge_config = KnowledgeConfig.model_validate(args)
+        # validate args
+        DocumentService.document_create_args_validate(knowledge_config)
+
+        if not current_user:
+            raise ValueError("current_user is required")
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                knowledge_config=knowledge_config,
+                account=current_user,
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
+        return documents_and_batch_fields, 200
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text",
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-text",
+)
+class DocumentUpdateByTextApi(DatasetApiResource):
+    """Resource for update documents."""
+
+    @service_api_ns.expect(service_api_ns.models[DocumentTextUpdate.__name__], validate=True)
+    @service_api_ns.doc("update_document_by_text")
+    @service_api_ns.doc(description="Update an existing document by providing text content")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Document updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Document not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id: str, dataset_id: UUID, document_id: UUID):
+        """Update document by text."""
+        args = DocumentTextUpdate.model_validate(service_api_ns.payload).model_dump(exclude_unset=True)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == str(dataset_id)).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        retrieval_model = args.get("retrieval_model")
+        if (
+            retrieval_model
+            and retrieval_model.get("reranking_model")
+            and retrieval_model.get("reranking_model").get("reranking_provider_name")
+        ):
+            DatasetService.check_reranking_model_setting(
+                tenant_id,
+                retrieval_model.get("reranking_model").get("reranking_provider_name"),
+                retrieval_model.get("reranking_model").get("reranking_model_name"),
+            )
+
+        # indexing_technique is already set in dataset since this is an update
+        args["indexing_technique"] = dataset.indexing_technique
+
+        if args.get("text"):
+            text = args.get("text")
+            name = args.get("name")
+            if not current_user:
+                raise ValueError("current_user is required")
+            upload_file = FileService(db.engine).upload_text(
+                text=str(text), text_name=str(name), user_id=current_user.id, tenant_id=tenant_id
+            )
+            data_source = {
+                "type": "upload_file",
+                "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
+            }
+            args["data_source"] = data_source
+        # validate args
+        args["original_document_id"] = str(document_id)
+        knowledge_config = KnowledgeConfig.model_validate(args)
+        DocumentService.document_create_args_validate(knowledge_config)
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                knowledge_config=knowledge_config,
+                account=current_user,
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
+        return documents_and_batch_fields, 200
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/document/create_by_file",
+    "/datasets/<uuid:dataset_id>/document/create-by-file",
+)
+class DocumentAddByFileApi(DatasetApiResource):
+    """Resource for documents."""
+
+    @service_api_ns.doc("create_document_by_file")
+    @service_api_ns.doc(description="Create a new document by uploading a file")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Document created successfully",
+            401: "Unauthorized - invalid API token",
+            400: "Bad request - invalid file or parameters",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_resource_check("documents", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        """Create document by upload file."""
+        args = {}
+        if "data" in request.form:
+            args = json.loads(request.form["data"])
+        if "doc_form" not in args:
+            args["doc_form"] = "text_model"
+        if "doc_language" not in args:
+            args["doc_language"] = "English"
+
+        # get dataset info
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        if dataset.provider == "external":
+            raise ValueError("External datasets are not supported.")
+
+        indexing_technique = args.get("indexing_technique") or dataset.indexing_technique
+        if not indexing_technique:
+            raise ValueError("indexing_technique is required.")
+        args["indexing_technique"] = indexing_technique
+
+        if "embedding_model_provider" in args:
+            DatasetService.check_embedding_model_setting(
+                tenant_id, args["embedding_model_provider"], args["embedding_model"]
+            )
+        if (
+            "retrieval_model" in args
+            and args["retrieval_model"].get("reranking_model")
+            and args["retrieval_model"].get("reranking_model").get("reranking_provider_name")
+        ):
+            DatasetService.check_reranking_model_setting(
+                tenant_id,
+                args["retrieval_model"].get("reranking_model").get("reranking_provider_name"),
+                args["retrieval_model"].get("reranking_model").get("reranking_model_name"),
+            )
+
+        # check file
+        if "file" not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+
+        # save file info
+        file = request.files["file"]
+        if not file.filename:
+            raise FilenameNotExistsError
+
+        if not current_user:
+            raise ValueError("current_user is required")
+        upload_file = FileService(db.engine).upload_file(
+            filename=file.filename,
+            content=file.read(),
+            mimetype=file.mimetype,
+            user=current_user,
+            source="datasets",
+        )
+        data_source = {
+            "type": "upload_file",
+            "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
+        }
+        args["data_source"] = data_source
+        # validate args
+        knowledge_config = KnowledgeConfig.model_validate(args)
+        DocumentService.document_create_args_validate(knowledge_config)
+
+        dataset_process_rule = dataset.latest_process_rule if "process_rule" not in args else None
+        if not knowledge_config.original_document_id and not dataset_process_rule and not knowledge_config.process_rule:
+            raise ValueError("process_rule is required.")
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                knowledge_config=knowledge_config,
+                account=dataset.created_by_account,
+                dataset_process_rule=dataset_process_rule,
+                created_from="api",
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": batch}
+        return documents_and_batch_fields, 200
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file",
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-file",
+)
+class DocumentUpdateByFileApi(DatasetApiResource):
+    """Resource for update documents."""
+
+    @service_api_ns.doc("update_document_by_file")
+    @service_api_ns.doc(description="Update an existing document by uploading a file")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Document updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Document not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id, document_id):
+        """Update document by upload file."""
+        args = {}
+        if "data" in request.form:
+            args = json.loads(request.form["data"])
+        if "doc_form" not in args:
+            args["doc_form"] = "text_model"
+        if "doc_language" not in args:
+            args["doc_language"] = "English"
+
+        # get dataset info
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        if dataset.provider == "external":
+            raise ValueError("External datasets are not supported.")
+
+        # indexing_technique is already set in dataset since this is an update
+        args["indexing_technique"] = dataset.indexing_technique
+
+        if "file" in request.files:
+            # save file info
+            file = request.files["file"]
+
+            if len(request.files) > 1:
+                raise TooManyFilesError()
+
+            if not file.filename:
+                raise FilenameNotExistsError
+
+            if not current_user:
+                raise ValueError("current_user is required")
+
+            try:
+                upload_file = FileService(db.engine).upload_file(
+                    filename=file.filename,
+                    content=file.read(),
+                    mimetype=file.mimetype,
+                    user=current_user,
+                    source="datasets",
+                )
+            except services.errors.file.FileTooLargeError as file_too_large_error:
+                raise FileTooLargeError(file_too_large_error.description)
+            except services.errors.file.UnsupportedFileTypeError:
+                raise UnsupportedFileTypeError()
+            data_source = {
+                "type": "upload_file",
+                "info_list": {"data_source_type": "upload_file", "file_info_list": {"file_ids": [upload_file.id]}},
+            }
+            args["data_source"] = data_source
+        # validate args
+        args["original_document_id"] = str(document_id)
+
+        knowledge_config = KnowledgeConfig.model_validate(args)
+        DocumentService.document_create_args_validate(knowledge_config)
+
+        try:
+            documents, _ = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                knowledge_config=knowledge_config,
+                account=dataset.created_by_account,
+                dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
+                created_from="api",
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+        documents_and_batch_fields = {"document": marshal(document, document_fields), "batch": document.batch}
+        return documents_and_batch_fields, 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents")
+class DocumentListApi(DatasetApiResource):
+    @service_api_ns.doc("list_documents")
+    @service_api_ns.doc(description="List all documents in a dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Documents retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    def get(self, tenant_id, dataset_id):
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        search = request.args.get("keyword", default=None, type=str)
+        status = request.args.get("status", default=None, type=str)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        query = select(Document).filter_by(dataset_id=str(dataset_id), tenant_id=tenant_id)
+
+        if status:
+            query = DocumentService.apply_display_status_filter(query, status)
+
+        if search:
+            search = f"%{search}%"
+            query = query.where(Document.name.like(search))
+
+        query = query.order_by(desc(Document.created_at), desc(Document.position))
+
+        paginated_documents = db.paginate(select=query, page=page, per_page=limit, max_per_page=100, error_out=False)
+        documents = paginated_documents.items
+
+        response = {
+            "data": marshal(documents, document_fields),
+            "has_more": len(documents) == limit,
+            "limit": limit,
+            "total": paginated_documents.total,
+            "page": page,
+        }
+
+        return response
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")
+class DocumentIndexingStatusApi(DatasetApiResource):
+    @service_api_ns.doc("get_document_indexing_status")
+    @service_api_ns.doc(description="Get indexing status for documents in a batch")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "batch": "Batch ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Indexing status retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset or documents not found",
+        }
+    )
+    def get(self, tenant_id, dataset_id, batch):
+        dataset_id = str(dataset_id)
+        batch = str(batch)
+        tenant_id = str(tenant_id)
+        # get dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # get documents
+        documents = DocumentService.get_batch_documents(dataset_id, batch)
+        if not documents:
+            raise NotFound("Documents not found.")
+        documents_status = []
+        for document in documents:
+            completed_segments = (
+                db.session.query(DocumentSegment)
+                .where(
+                    DocumentSegment.completed_at.isnot(None),
+                    DocumentSegment.document_id == str(document.id),
+                    DocumentSegment.status != "re_segment",
+                )
+                .count()
+            )
+            total_segments = (
+                db.session.query(DocumentSegment)
+                .where(DocumentSegment.document_id == str(document.id), DocumentSegment.status != "re_segment")
+                .count()
+            )
+            # Create a dictionary with document attributes and additional fields
+            document_dict = {
+                "id": document.id,
+                "indexing_status": "paused" if document.is_paused else document.indexing_status,
+                "processing_started_at": document.processing_started_at,
+                "parsing_completed_at": document.parsing_completed_at,
+                "cleaning_completed_at": document.cleaning_completed_at,
+                "splitting_completed_at": document.splitting_completed_at,
+                "completed_at": document.completed_at,
+                "paused_at": document.paused_at,
+                "error": document.error,
+                "stopped_at": document.stopped_at,
+                "completed_segments": completed_segments,
+                "total_segments": total_segments,
+            }
+            documents_status.append(marshal(document_dict, document_status_fields))
+        data = {"data": documents_status}
+        return data
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
+class DocumentApi(DatasetApiResource):
+    METADATA_CHOICES = {"all", "only", "without"}
+
+    @service_api_ns.doc("get_document")
+    @service_api_ns.doc(description="Get a specific document by ID")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Document retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Document not found",
+        }
+    )
+    def get(self, tenant_id, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+
+        dataset = self.get_dataset(dataset_id, tenant_id)
+
+        document = DocumentService.get_document(dataset.id, document_id)
+
+        if not document:
+            raise NotFound("Document not found.")
+
+        if document.tenant_id != str(tenant_id):
+            raise Forbidden("No permission.")
+
+        metadata = request.args.get("metadata", "all")
+        if metadata not in self.METADATA_CHOICES:
+            raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
+
+        if metadata == "only":
+            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
+        elif metadata == "without":
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id)
+            document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
+            data_source_info = document.data_source_detail_dict
+            response = {
+                "id": document.id,
+                "position": document.position,
+                "data_source_type": document.data_source_type,
+                "data_source_info": data_source_info,
+                "dataset_process_rule_id": document.dataset_process_rule_id,
+                "dataset_process_rule": dataset_process_rules,
+                "document_process_rule": document_process_rules,
+                "name": document.name,
+                "created_from": document.created_from,
+                "created_by": document.created_by,
+                "created_at": int(document.created_at.timestamp()),
+                "tokens": document.tokens,
+                "indexing_status": document.indexing_status,
+                "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
+                "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
+                "indexing_latency": document.indexing_latency,
+                "error": document.error,
+                "enabled": document.enabled,
+                "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
+                "disabled_by": document.disabled_by,
+                "archived": document.archived,
+                "segment_count": document.segment_count,
+                "average_segment_length": document.average_segment_length,
+                "hit_count": document.hit_count,
+                "display_status": document.display_status,
+                "doc_form": document.doc_form,
+                "doc_language": document.doc_language,
+            }
+        else:
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id)
+            document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
+            data_source_info = document.data_source_detail_dict
+            response = {
+                "id": document.id,
+                "position": document.position,
+                "data_source_type": document.data_source_type,
+                "data_source_info": data_source_info,
+                "dataset_process_rule_id": document.dataset_process_rule_id,
+                "dataset_process_rule": dataset_process_rules,
+                "document_process_rule": document_process_rules,
+                "name": document.name,
+                "created_from": document.created_from,
+                "created_by": document.created_by,
+                "created_at": int(document.created_at.timestamp()),
+                "tokens": document.tokens,
+                "indexing_status": document.indexing_status,
+                "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
+                "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
+                "indexing_latency": document.indexing_latency,
+                "error": document.error,
+                "enabled": document.enabled,
+                "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
+                "disabled_by": document.disabled_by,
+                "archived": document.archived,
+                "doc_type": document.doc_type,
+                "doc_metadata": document.doc_metadata_details,
+                "segment_count": document.segment_count,
+                "average_segment_length": document.average_segment_length,
+                "hit_count": document.hit_count,
+                "display_status": document.display_status,
+                "doc_form": document.doc_form,
+                "doc_language": document.doc_language,
+            }
+
+        return response
+
+    @service_api_ns.doc("delete_document")
+    @service_api_ns.doc(description="Delete a document")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            204: "Document deleted successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - document is archived",
+            404: "Document not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, tenant_id, dataset_id, document_id):
+        """Delete document."""
+        document_id = str(document_id)
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+
+        # get dataset info
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        document = DocumentService.get_document(dataset.id, document_id)
+
+        # 404 if document not found
+        if document is None:
+            raise NotFound("Document Not Exists.")
+
+        # 403 if document is archived
+        if DocumentService.check_archived(document):
+            raise ArchivedDocumentImmutableError()
+
+        try:
+            # delete document
+            DocumentService.delete_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError("Cannot delete document during indexing.")
+
+        return 204

dify/api/controllers/service_api/dataset/error.py

@@ -0,0 +1,55 @@
+from libs.exception import BaseHTTPException
+
+
+class DatasetNotInitializedError(BaseHTTPException):
+    error_code = "dataset_not_initialized"
+    description = "The dataset is still being initialized or indexing. Please wait a moment."
+    code = 400
+
+
+class ArchivedDocumentImmutableError(BaseHTTPException):
+    error_code = "archived_document_immutable"
+    description = "The archived document is not editable."
+    code = 403
+
+
+class DatasetNameDuplicateError(BaseHTTPException):
+    error_code = "dataset_name_duplicate"
+    description = "The dataset name already exists. Please modify your dataset name."
+    code = 409
+
+
+class InvalidActionError(BaseHTTPException):
+    error_code = "invalid_action"
+    description = "Invalid action."
+    code = 400
+
+
+class DocumentAlreadyFinishedError(BaseHTTPException):
+    error_code = "document_already_finished"
+    description = "The document has been processed. Please refresh the page or go to the document details."
+    code = 400
+
+
+class DocumentIndexingError(BaseHTTPException):
+    error_code = "document_indexing"
+    description = "The document is being processed and cannot be edited."
+    code = 400
+
+
+class InvalidMetadataError(BaseHTTPException):
+    error_code = "invalid_metadata"
+    description = "The metadata content is incorrect. Please check and verify."
+    code = 400
+
+
+class DatasetInUseError(BaseHTTPException):
+    error_code = "dataset_in_use"
+    description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
+    code = 409
+
+
+class PipelineRunError(BaseHTTPException):
+    error_code = "pipeline_run_error"
+    description = "An error occurred while running the pipeline."
+    code = 500

dify/api/controllers/service_api/dataset/hit_testing.py

@@ -0,0 +1,30 @@
+from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase
+from controllers.service_api import service_api_ns
+from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/hit-testing", "/datasets/<uuid:dataset_id>/retrieve")
+class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase):
+    @service_api_ns.doc("dataset_hit_testing")
+    @service_api_ns.doc(description="Perform hit testing on a dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Hit testing results",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        """Perform hit testing on a dataset.
+
+        Tests retrieval performance for the specified dataset.
+        """
+        dataset_id_str = str(dataset_id)
+
+        dataset = self.get_and_validate_dataset(dataset_id_str)
+        args = self.parse_args()
+        self.hit_testing_args_check(args)
+
+        return self.perform_hit_testing(dataset, args)

dify/api/controllers/service_api/dataset/metadata.py

@@ -0,0 +1,203 @@
+from typing import Literal
+
+from flask_login import current_user
+from flask_restx import marshal, reqparse
+from werkzeug.exceptions import NotFound
+
+from controllers.service_api import service_api_ns
+from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check
+from fields.dataset_fields import dataset_metadata_fields
+from services.dataset_service import DatasetService
+from services.entities.knowledge_entities.knowledge_entities import (
+    MetadataArgs,
+    MetadataOperationData,
+)
+from services.metadata_service import MetadataService
+
+# Define parsers for metadata APIs
+metadata_create_parser = (
+    reqparse.RequestParser()
+    .add_argument("type", type=str, required=True, nullable=False, location="json", help="Metadata type")
+    .add_argument("name", type=str, required=True, nullable=False, location="json", help="Metadata name")
+)
+
+metadata_update_parser = reqparse.RequestParser().add_argument(
+    "name", type=str, required=True, nullable=False, location="json", help="New metadata name"
+)
+
+document_metadata_parser = reqparse.RequestParser().add_argument(
+    "operation_data", type=list, required=True, nullable=False, location="json", help="Metadata operation data"
+)
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata")
+class DatasetMetadataCreateServiceApi(DatasetApiResource):
+    @service_api_ns.expect(metadata_create_parser)
+    @service_api_ns.doc("create_dataset_metadata")
+    @service_api_ns.doc(description="Create metadata for a dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            201: "Metadata created successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        """Create metadata for a dataset."""
+        args = metadata_create_parser.parse_args()
+        metadata_args = MetadataArgs.model_validate(args)
+
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        metadata = MetadataService.create_metadata(dataset_id_str, metadata_args)
+        return marshal(metadata, dataset_metadata_fields), 201
+
+    @service_api_ns.doc("get_dataset_metadata")
+    @service_api_ns.doc(description="Get all metadata for a dataset")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Metadata retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    def get(self, tenant_id, dataset_id):
+        """Get all metadata for a dataset."""
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        return MetadataService.get_dataset_metadatas(dataset), 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
+class DatasetMetadataServiceApi(DatasetApiResource):
+    @service_api_ns.expect(metadata_update_parser)
+    @service_api_ns.doc("update_dataset_metadata")
+    @service_api_ns.doc(description="Update metadata name")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "metadata_id": "Metadata ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Metadata updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset or metadata not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def patch(self, tenant_id, dataset_id, metadata_id):
+        """Update metadata name."""
+        args = metadata_update_parser.parse_args()
+
+        dataset_id_str = str(dataset_id)
+        metadata_id_str = str(metadata_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        metadata = MetadataService.update_metadata_name(dataset_id_str, metadata_id_str, args["name"])
+        return marshal(metadata, dataset_metadata_fields), 200
+
+    @service_api_ns.doc("delete_dataset_metadata")
+    @service_api_ns.doc(description="Delete metadata")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "metadata_id": "Metadata ID"})
+    @service_api_ns.doc(
+        responses={
+            204: "Metadata deleted successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset or metadata not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, tenant_id, dataset_id, metadata_id):
+        """Delete metadata."""
+        dataset_id_str = str(dataset_id)
+        metadata_id_str = str(metadata_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
+        return 204
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in")
+class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource):
+    @service_api_ns.doc("get_built_in_fields")
+    @service_api_ns.doc(description="Get all built-in metadata fields")
+    @service_api_ns.doc(
+        responses={
+            200: "Built-in fields retrieved successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    def get(self, tenant_id, dataset_id):
+        """Get all built-in metadata fields."""
+        built_in_fields = MetadataService.get_built_in_fields()
+        return {"fields": built_in_fields}, 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in/<string:action>")
+class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
+    @service_api_ns.doc("toggle_built_in_field")
+    @service_api_ns.doc(description="Enable or disable built-in metadata field")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "action": "Action to perform: 'enable' or 'disable'"})
+    @service_api_ns.doc(
+        responses={
+            200: "Action completed successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id, action: Literal["enable", "disable"]):
+        """Enable or disable built-in metadata field."""
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        if action == "enable":
+            MetadataService.enable_built_in_field(dataset)
+        elif action == "disable":
+            MetadataService.disable_built_in_field(dataset)
+        return {"result": "success"}, 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/metadata")
+class DocumentMetadataEditServiceApi(DatasetApiResource):
+    @service_api_ns.expect(document_metadata_parser)
+    @service_api_ns.doc("update_documents_metadata")
+    @service_api_ns.doc(description="Update metadata for multiple documents")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Documents metadata updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        """Update metadata for multiple documents."""
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        DatasetService.check_dataset_permission(dataset, current_user)
+
+        args = document_metadata_parser.parse_args()
+        metadata_args = MetadataOperationData.model_validate(args)
+
+        MetadataService.update_documents_metadata(dataset, metadata_args)
+
+        return {"result": "success"}, 200

dify/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py

@@ -0,0 +1,246 @@
+import string
+import uuid
+from collections.abc import Generator
+from typing import Any
+
+from flask import request
+from flask_restx import reqparse
+from flask_restx.reqparse import ParseResult, RequestParser
+from werkzeug.exceptions import Forbidden
+
+import services
+from controllers.common.errors import FilenameNotExistsError, NoFileUploadedError, TooManyFilesError
+from controllers.service_api import service_api_ns
+from controllers.service_api.dataset.error import PipelineRunError
+from controllers.service_api.wraps import DatasetApiResource
+from core.app.apps.pipeline.pipeline_generator import PipelineGenerator
+from core.app.entities.app_invoke_entities import InvokeFrom
+from libs import helper
+from libs.login import current_user
+from models import Account
+from models.dataset import Pipeline
+from models.engine import db
+from services.errors.file import FileTooLargeError, UnsupportedFileTypeError
+from services.file_service import FileService
+from services.rag_pipeline.entity.pipeline_service_api_entities import DatasourceNodeRunApiEntity
+from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService
+from services.rag_pipeline.rag_pipeline import RagPipelineService
+
+
+@service_api_ns.route(f"/datasets/{uuid:dataset_id}/pipeline/datasource-plugins")
+class DatasourcePluginsApi(DatasetApiResource):
+    """Resource for datasource plugins."""
+
+    @service_api_ns.doc(shortcut="list_rag_pipeline_datasource_plugins")
+    @service_api_ns.doc(description="List all datasource plugins for a rag pipeline")
+    @service_api_ns.doc(
+        path={
+            "dataset_id": "Dataset ID",
+        }
+    )
+    @service_api_ns.doc(
+        params={
+            "is_published": "Whether to get published or draft datasource plugins "
+            "(true for published, false for draft, default: true)"
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Datasource plugins retrieved successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    def get(self, tenant_id: str, dataset_id: str):
+        """Resource for getting datasource plugins."""
+        # Get query parameter to determine published or draft
+        is_published: bool = request.args.get("is_published", default=True, type=bool)
+
+        rag_pipeline_service: RagPipelineService = RagPipelineService()
+        datasource_plugins: list[dict[Any, Any]] = rag_pipeline_service.get_datasource_plugins(
+            tenant_id=tenant_id, dataset_id=dataset_id, is_published=is_published
+        )
+        return datasource_plugins, 200
+
+
+@service_api_ns.route(f"/datasets/{uuid:dataset_id}/pipeline/datasource/nodes/{string:node_id}/run")
+class DatasourceNodeRunApi(DatasetApiResource):
+    """Resource for datasource node run."""
+
+    @service_api_ns.doc(shortcut="pipeline_datasource_node_run")
+    @service_api_ns.doc(description="Run a datasource node for a rag pipeline")
+    @service_api_ns.doc(
+        path={
+            "dataset_id": "Dataset ID",
+        }
+    )
+    @service_api_ns.doc(
+        body={
+            "inputs": "User input variables",
+            "datasource_type": "Datasource type, e.g. online_document",
+            "credential_id": "Credential ID",
+            "is_published": "Whether to get published or draft datasource plugins "
+            "(true for published, false for draft, default: true)",
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Datasource node run successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    def post(self, tenant_id: str, dataset_id: str, node_id: str):
+        """Resource for getting datasource plugins."""
+        # Get query parameter to determine published or draft
+        parser: RequestParser = (
+            reqparse.RequestParser()
+            .add_argument("inputs", type=dict, required=True, nullable=False, location="json")
+            .add_argument("datasource_type", type=str, required=True, location="json")
+            .add_argument("credential_id", type=str, required=False, location="json")
+            .add_argument("is_published", type=bool, required=True, location="json")
+        )
+        args: ParseResult = parser.parse_args()
+
+        datasource_node_run_api_entity = DatasourceNodeRunApiEntity.model_validate(args)
+        assert isinstance(current_user, Account)
+        rag_pipeline_service: RagPipelineService = RagPipelineService()
+        pipeline: Pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id)
+        return helper.compact_generate_response(
+            PipelineGenerator.convert_to_event_stream(
+                rag_pipeline_service.run_datasource_workflow_node(
+                    pipeline=pipeline,
+                    node_id=node_id,
+                    user_inputs=datasource_node_run_api_entity.inputs,
+                    account=current_user,
+                    datasource_type=datasource_node_run_api_entity.datasource_type,
+                    is_published=datasource_node_run_api_entity.is_published,
+                    credential_id=datasource_node_run_api_entity.credential_id,
+                )
+            )
+        )
+
+
+@service_api_ns.route(f"/datasets/{uuid:dataset_id}/pipeline/run")
+class PipelineRunApi(DatasetApiResource):
+    """Resource for datasource node run."""
+
+    @service_api_ns.doc(shortcut="pipeline_datasource_node_run")
+    @service_api_ns.doc(description="Run a datasource node for a rag pipeline")
+    @service_api_ns.doc(
+        path={
+            "dataset_id": "Dataset ID",
+        }
+    )
+    @service_api_ns.doc(
+        body={
+            "inputs": "User input variables",
+            "datasource_type": "Datasource type, e.g. online_document",
+            "datasource_info_list": "Datasource info list",
+            "start_node_id": "Start node ID",
+            "is_published": "Whether to get published or draft datasource plugins "
+            "(true for published, false for draft, default: true)",
+            "streaming": "Whether to stream the response(streaming or blocking), default: streaming",
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Pipeline run successfully",
+            401: "Unauthorized - invalid API token",
+        }
+    )
+    def post(self, tenant_id: str, dataset_id: str):
+        """Resource for running a rag pipeline."""
+        parser: RequestParser = (
+            reqparse.RequestParser()
+            .add_argument("inputs", type=dict, required=True, nullable=False, location="json")
+            .add_argument("datasource_type", type=str, required=True, location="json")
+            .add_argument("datasource_info_list", type=list, required=True, location="json")
+            .add_argument("start_node_id", type=str, required=True, location="json")
+            .add_argument("is_published", type=bool, required=True, default=True, location="json")
+            .add_argument(
+                "response_mode",
+                type=str,
+                required=True,
+                choices=["streaming", "blocking"],
+                default="blocking",
+                location="json",
+            )
+        )
+        args: ParseResult = parser.parse_args()
+
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        rag_pipeline_service: RagPipelineService = RagPipelineService()
+        pipeline: Pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id)
+        try:
+            response: dict[Any, Any] | Generator[str, Any, None] = PipelineGenerateService.generate(
+                pipeline=pipeline,
+                user=current_user,
+                args=args,
+                invoke_from=InvokeFrom.PUBLISHED if args.get("is_published") else InvokeFrom.DEBUGGER,
+                streaming=args.get("response_mode") == "streaming",
+            )
+
+            return helper.compact_generate_response(response)
+        except Exception as ex:
+            raise PipelineRunError(description=str(ex))
+
+
+@service_api_ns.route("/datasets/pipeline/file-upload")
+class KnowledgebasePipelineFileUploadApi(DatasetApiResource):
+    """Resource for uploading a file to a knowledgebase pipeline."""
+
+    @service_api_ns.doc(shortcut="knowledgebase_pipeline_file_upload")
+    @service_api_ns.doc(description="Upload a file to a knowledgebase pipeline")
+    @service_api_ns.doc(
+        responses={
+            201: "File uploaded successfully",
+            400: "Bad request - no file or invalid file",
+            401: "Unauthorized - invalid API token",
+            413: "File too large",
+            415: "Unsupported file type",
+        }
+    )
+    def post(self, tenant_id: str):
+        """Upload a file for use in conversations.
+
+        Accepts a single file upload via multipart/form-data.
+        """
+        # check file
+        if "file" not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+
+        file = request.files["file"]
+        if not file.mimetype:
+            raise UnsupportedFileTypeError()
+
+        if not file.filename:
+            raise FilenameNotExistsError
+
+        if not current_user:
+            raise ValueError("Invalid user account")
+
+        try:
+            upload_file = FileService(db.engine).upload_file(
+                filename=file.filename,
+                content=file.read(),
+                mimetype=file.mimetype,
+                user=current_user,
+            )
+        except services.errors.file.FileTooLargeError as file_too_large_error:
+            raise FileTooLargeError(file_too_large_error.description)
+        except services.errors.file.UnsupportedFileTypeError:
+            raise UnsupportedFileTypeError()
+
+        return {
+            "id": upload_file.id,
+            "name": upload_file.name,
+            "size": upload_file.size,
+            "extension": upload_file.extension,
+            "mime_type": upload_file.mime_type,
+            "created_by": upload_file.created_by,
+            "created_at": upload_file.created_at,
+        }, 201

dify/api/controllers/service_api/dataset/segment.py

@@ -0,0 +1,543 @@
+from flask import request
+from flask_restx import marshal, reqparse
+from werkzeug.exceptions import NotFound
+
+from configs import dify_config
+from controllers.service_api import service_api_ns
+from controllers.service_api.app.error import ProviderNotInitializeError
+from controllers.service_api.wraps import (
+    DatasetApiResource,
+    cloud_edition_billing_knowledge_limit_check,
+    cloud_edition_billing_rate_limit_check,
+    cloud_edition_billing_resource_check,
+)
+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelType
+from extensions.ext_database import db
+from fields.segment_fields import child_chunk_fields, segment_fields
+from libs.login import current_account_with_tenant
+from models.dataset import Dataset
+from services.dataset_service import DatasetService, DocumentService, SegmentService
+from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs
+from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
+from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
+from services.errors.chunk import ChildChunkIndexingError as ChildChunkIndexingServiceError
+
+# Define parsers for segment operations
+segment_create_parser = reqparse.RequestParser().add_argument(
+    "segments", type=list, required=False, nullable=True, location="json"
+)
+
+segment_list_parser = (
+    reqparse.RequestParser()
+    .add_argument("status", type=str, action="append", default=[], location="args")
+    .add_argument("keyword", type=str, default=None, location="args")
+)
+
+segment_update_parser = reqparse.RequestParser().add_argument(
+    "segment", type=dict, required=False, nullable=True, location="json"
+)
+
+child_chunk_create_parser = reqparse.RequestParser().add_argument(
+    "content", type=str, required=True, nullable=False, location="json"
+)
+
+child_chunk_list_parser = (
+    reqparse.RequestParser()
+    .add_argument("limit", type=int, default=20, location="args")
+    .add_argument("keyword", type=str, default=None, location="args")
+    .add_argument("page", type=int, default=1, location="args")
+)
+
+child_chunk_update_parser = reqparse.RequestParser().add_argument(
+    "content", type=str, required=True, nullable=False, location="json"
+)
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
+class SegmentApi(DatasetApiResource):
+    """Resource for segments."""
+
+    @service_api_ns.expect(segment_create_parser)
+    @service_api_ns.doc("create_segments")
+    @service_api_ns.doc(description="Create segments in a document")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Segments created successfully",
+            400: "Bad request - segments data is missing",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset or document not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id: str, dataset_id: str, document_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Create single segment."""
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        if document.indexing_status != "completed":
+            raise NotFound("Document is not completed.")
+        if not document.enabled:
+            raise NotFound("Document is disabled.")
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        # validate args
+        args = segment_create_parser.parse_args()
+        if args["segments"] is not None:
+            segments_limit = dify_config.DATASET_MAX_SEGMENTS_PER_REQUEST
+            if segments_limit > 0 and len(args["segments"]) > segments_limit:
+                raise ValueError(f"Exceeded maximum segments limit of {segments_limit}.")
+
+            for args_item in args["segments"]:
+                SegmentService.segment_create_args_validate(args_item, document)
+            segments = SegmentService.multi_create_segment(args["segments"], document, dataset)
+            return {"data": marshal(segments, segment_fields), "doc_form": document.doc_form}, 200
+        else:
+            return {"error": "Segments is required"}, 400
+
+    @service_api_ns.expect(segment_list_parser)
+    @service_api_ns.doc("list_segments")
+    @service_api_ns.doc(description="List segments in a document")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Segments retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset or document not found",
+        }
+    )
+    def get(self, tenant_id: str, dataset_id: str, document_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Get segments."""
+        # check dataset
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+
+        args = segment_list_parser.parse_args()
+
+        segments, total = SegmentService.get_segments(
+            document_id=document_id,
+            tenant_id=current_tenant_id,
+            status_list=args["status"],
+            keyword=args["keyword"],
+            page=page,
+            limit=limit,
+        )
+
+        response = {
+            "data": marshal(segments, segment_fields),
+            "doc_form": document.doc_form,
+            "total": total,
+            "has_more": len(segments) == limit,
+            "limit": limit,
+            "page": page,
+        }
+
+        return response, 200
+
+
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>")
+class DatasetSegmentApi(DatasetApiResource):
+    @service_api_ns.doc("delete_segment")
+    @service_api_ns.doc(description="Delete a specific segment")
+    @service_api_ns.doc(
+        params={"dataset_id": "Dataset ID", "document_id": "Document ID", "segment_id": "Segment ID to delete"}
+    )
+    @service_api_ns.doc(
+        responses={
+            204: "Segment deleted successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, or segment not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+        SegmentService.delete_segment(segment, document, dataset)
+        return 204
+
+    @service_api_ns.expect(segment_update_parser)
+    @service_api_ns.doc("update_segment")
+    @service_api_ns.doc(description="Update a specific segment")
+    @service_api_ns.doc(
+        params={"dataset_id": "Dataset ID", "document_id": "Document ID", "segment_id": "Segment ID to update"}
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Segment updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, or segment not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        if dataset.indexing_technique == "high_quality":
+            # check embedding model setting
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+            # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        # validate args
+        args = segment_update_parser.parse_args()
+
+        updated_segment = SegmentService.update_segment(
+            SegmentUpdateArgs.model_validate(args["segment"]), segment, document, dataset
+        )
+        return {"data": marshal(updated_segment, segment_fields), "doc_form": document.doc_form}, 200
+
+    @service_api_ns.doc("get_segment")
+    @service_api_ns.doc(description="Get a specific segment by ID")
+    @service_api_ns.doc(
+        responses={
+            200: "Segment retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, or segment not found",
+        }
+    )
+    def get(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+        # check document
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        return {"data": marshal(segment, segment_fields), "doc_form": document.doc_form}, 200
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks"
+)
+class ChildChunkApi(DatasetApiResource):
+    """Resource for child chunks."""
+
+    @service_api_ns.expect(child_chunk_create_parser)
+    @service_api_ns.doc("create_child_chunk")
+    @service_api_ns.doc(description="Create a new child chunk for a segment")
+    @service_api_ns.doc(
+        params={"dataset_id": "Dataset ID", "document_id": "Document ID", "segment_id": "Parent segment ID"}
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Child chunk created successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, or segment not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Create child chunk."""
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        # check document
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+
+        # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        # check embedding model setting
+        if dataset.indexing_technique == "high_quality":
+            try:
+                model_manager = ModelManager()
+                model_manager.get_model_instance(
+                    tenant_id=current_tenant_id,
+                    provider=dataset.embedding_model_provider,
+                    model_type=ModelType.TEXT_EMBEDDING,
+                    model=dataset.embedding_model,
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                )
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+
+        # validate args
+        args = child_chunk_create_parser.parse_args()
+
+        try:
+            child_chunk = SegmentService.create_child_chunk(args["content"], segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200
+
+    @service_api_ns.expect(child_chunk_list_parser)
+    @service_api_ns.doc("list_child_chunks")
+    @service_api_ns.doc(description="List child chunks for a segment")
+    @service_api_ns.doc(
+        params={"dataset_id": "Dataset ID", "document_id": "Document ID", "segment_id": "Parent segment ID"}
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Child chunks retrieved successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, or segment not found",
+        }
+    )
+    def get(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Get child chunks."""
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        # check document
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+
+        # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        args = child_chunk_list_parser.parse_args()
+
+        page = args["page"]
+        limit = min(args["limit"], 100)
+        keyword = args["keyword"]
+
+        child_chunks = SegmentService.get_child_chunks(segment_id, document_id, dataset_id, page, limit, keyword)
+
+        return {
+            "data": marshal(child_chunks.items, child_chunk_fields),
+            "total": child_chunks.total,
+            "total_pages": child_chunks.pages,
+            "page": page,
+            "limit": limit,
+        }, 200
+
+
+@service_api_ns.route(
+    "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments/<uuid:segment_id>/child_chunks/<uuid:child_chunk_id>"
+)
+class DatasetChildChunkApi(DatasetApiResource):
+    """Resource for updating child chunks."""
+
+    @service_api_ns.doc("delete_child_chunk")
+    @service_api_ns.doc(description="Delete a specific child chunk")
+    @service_api_ns.doc(
+        params={
+            "dataset_id": "Dataset ID",
+            "document_id": "Document ID",
+            "segment_id": "Parent segment ID",
+            "child_chunk_id": "Child chunk ID to delete",
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            204: "Child chunk deleted successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, segment, or child chunk not found",
+        }
+    )
+    @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str, child_chunk_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Delete child chunk."""
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        # check document
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+
+        # check segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        # validate segment belongs to the specified document
+        if str(segment.document_id) != str(document_id):
+            raise NotFound("Document not found.")
+
+        # check child chunk
+        child_chunk = SegmentService.get_child_chunk_by_id(child_chunk_id=child_chunk_id, tenant_id=current_tenant_id)
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+
+        # validate child chunk belongs to the specified segment
+        if str(child_chunk.segment_id) != str(segment.id):
+            raise NotFound("Child chunk not found.")
+
+        try:
+            SegmentService.delete_child_chunk(child_chunk, dataset)
+        except ChildChunkDeleteIndexServiceError as e:
+            raise ChildChunkDeleteIndexError(str(e))
+
+        return 204
+
+    @service_api_ns.expect(child_chunk_update_parser)
+    @service_api_ns.doc("update_child_chunk")
+    @service_api_ns.doc(description="Update a specific child chunk")
+    @service_api_ns.doc(
+        params={
+            "dataset_id": "Dataset ID",
+            "document_id": "Document ID",
+            "segment_id": "Parent segment ID",
+            "child_chunk_id": "Child chunk ID to update",
+        }
+    )
+    @service_api_ns.doc(
+        responses={
+            200: "Child chunk updated successfully",
+            401: "Unauthorized - invalid API token",
+            404: "Dataset, document, segment, or child chunk not found",
+        }
+    )
+    @cloud_edition_billing_resource_check("vector_space", "dataset")
+    @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def patch(self, tenant_id: str, dataset_id: str, document_id: str, segment_id: str, child_chunk_id: str):
+        _, current_tenant_id = current_account_with_tenant()
+        """Update child chunk."""
+        # check dataset
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+        if not dataset:
+            raise NotFound("Dataset not found.")
+
+        # get document
+        document = DocumentService.get_document(dataset_id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+
+        # get segment
+        segment = SegmentService.get_segment_by_id(segment_id=segment_id, tenant_id=current_tenant_id)
+        if not segment:
+            raise NotFound("Segment not found.")
+
+        # validate segment belongs to the specified document
+        if str(segment.document_id) != str(document_id):
+            raise NotFound("Segment not found.")
+
+        # get child chunk
+        child_chunk = SegmentService.get_child_chunk_by_id(child_chunk_id=child_chunk_id, tenant_id=current_tenant_id)
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
+
+        # validate child chunk belongs to the specified segment
+        if str(child_chunk.segment_id) != str(segment.id):
+            raise NotFound("Child chunk not found.")
+
+        # validate args
+        args = child_chunk_update_parser.parse_args()
+
+        try:
+            child_chunk = SegmentService.update_child_chunk(args["content"], child_chunk, segment, document, dataset)
+        except ChildChunkIndexingServiceError as e:
+            raise ChildChunkIndexingError(str(e))
+
+        return {"data": marshal(child_chunk, child_chunk_fields)}, 200