dify

dify/api/configs/__init__.py

@@ -0,0 +1,3 @@
+from .app_config import DifyConfig
+
+dify_config = DifyConfig()  # type: ignore

dify/api/configs/app_config.py

@@ -0,0 +1,113 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource
+
+from libs.file_utils import search_file_upwards
+
+from .deploy import DeploymentConfig
+from .enterprise import EnterpriseFeatureConfig
+from .extra import ExtraServiceConfig
+from .feature import FeatureConfig
+from .middleware import MiddlewareConfig
+from .observability import ObservabilityConfig
+from .packaging import PackagingInfo
+from .remote_settings_sources import RemoteSettingsSource, RemoteSettingsSourceConfig, RemoteSettingsSourceName
+from .remote_settings_sources.apollo import ApolloSettingsSource
+from .remote_settings_sources.nacos import NacosSettingsSource
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteSettingsSourceFactory(PydanticBaseSettingsSource):
+    def __init__(self, settings_cls: type[BaseSettings]):
+        super().__init__(settings_cls)
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        raise NotImplementedError
+
+    def __call__(self) -> dict[str, Any]:
+        current_state = self.current_state
+        remote_source_name = current_state.get("REMOTE_SETTINGS_SOURCE_NAME")
+        if not remote_source_name:
+            return {}
+
+        remote_source: RemoteSettingsSource | None = None
+        match remote_source_name:
+            case RemoteSettingsSourceName.APOLLO:
+                remote_source = ApolloSettingsSource(current_state)
+            case RemoteSettingsSourceName.NACOS:
+                remote_source = NacosSettingsSource(current_state)
+            case _:
+                logger.warning("Unsupported remote source: %s", remote_source_name)
+                return {}
+
+        d: dict[str, Any] = {}
+
+        for field_name, field in self.settings_cls.model_fields.items():
+            field_value, field_key, value_is_complex = remote_source.get_field_value(field, field_name)
+            field_value = remote_source.prepare_field_value(field_name, field, field_value, value_is_complex)
+            if field_value is not None:
+                d[field_key] = field_value
+
+        return d
+
+
+class DifyConfig(
+    # Packaging info
+    PackagingInfo,
+    # Deployment configs
+    DeploymentConfig,
+    # Feature configs
+    FeatureConfig,
+    # Middleware configs
+    MiddlewareConfig,
+    # Extra service configs
+    ExtraServiceConfig,
+    # Observability configs
+    ObservabilityConfig,
+    # Remote source configs
+    RemoteSettingsSourceConfig,
+    # Enterprise feature configs
+    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    EnterpriseFeatureConfig,
+):
+    model_config = SettingsConfigDict(
+        # read from dotenv format config file
+        env_file=".env",
+        env_file_encoding="utf-8",
+        # ignore extra attributes
+        extra="ignore",
+    )
+
+    # Before adding any config,
+    # please consider to arrange it in the proper config group of existed or added
+    # for better readability and maintainability.
+    # Thanks for your concentration and consideration.
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            env_settings,
+            RemoteSettingsSourceFactory(settings_cls),
+            dotenv_settings,
+            file_secret_settings,
+            TomlConfigSettingsSource(
+                settings_cls=settings_cls,
+                toml_file=search_file_upwards(
+                    base_dir_path=Path(__file__).parent,
+                    target_file_name="pyproject.toml",
+                    max_search_parent_depth=2,
+                ),
+            ),
+        )

dify/api/configs/deploy/__init__.py

@@ -0,0 +1,34 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class DeploymentConfig(BaseSettings):
+    """
+    Configuration settings for application deployment
+    """
+
+    APPLICATION_NAME: str = Field(
+        description="Name of the application, used for identification and logging purposes",
+        default="langgenius/dify",
+    )
+
+    DEBUG: bool = Field(
+        description="Enable debug mode for additional logging and development features",
+        default=False,
+    )
+
+    # Request logging configuration
+    ENABLE_REQUEST_LOGGING: bool = Field(
+        description="Enable request and response body logging",
+        default=False,
+    )
+
+    EDITION: str = Field(
+        description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
+        default="SELF_HOSTED",
+    )
+
+    DEPLOY_ENV: str = Field(
+        description="Deployment environment (e.g., 'PRODUCTION', 'DEVELOPMENT'), default to PRODUCTION",
+        default="PRODUCTION",
+    )

dify/api/configs/enterprise/__init__.py

@@ -0,0 +1,20 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class EnterpriseFeatureConfig(BaseSettings):
+    """
+    Configuration for enterprise-level features.
+    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    """
+
+    ENTERPRISE_ENABLED: bool = Field(
+        description="Enable or disable enterprise-level features."
+        "Before using, please contact business@dify.ai by email to inquire about licensing matters.",
+        default=False,
+    )
+
+    CAN_REPLACE_LOGO: bool = Field(
+        description="Allow customization of the enterprise logo.",
+        default=False,
+    )

dify/api/configs/extra/__init__.py

@@ -0,0 +1,10 @@
+from configs.extra.notion_config import NotionConfig
+from configs.extra.sentry_config import SentryConfig
+
+
+class ExtraServiceConfig(
+    # place the configs in alphabet order
+    NotionConfig,
+    SentryConfig,
+):
+    pass

dify/api/configs/extra/notion_config.py

@@ -0,0 +1,34 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class NotionConfig(BaseSettings):
+    """
+    Configuration settings for Notion integration
+    """
+
+    NOTION_CLIENT_ID: str | None = Field(
+        description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
+        default=None,
+    )
+
+    NOTION_CLIENT_SECRET: str | None = Field(
+        description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TYPE: str | None = Field(
+        description="Type of Notion integration."
+        " Set to 'internal' for internal integrations, or None for public integrations.",
+        default=None,
+    )
+
+    NOTION_INTERNAL_SECRET: str | None = Field(
+        description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TOKEN: str | None = Field(
+        description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
+        default=None,
+    )

dify/api/configs/extra/sentry_config.py

@@ -0,0 +1,26 @@
+from pydantic import Field, NonNegativeFloat
+from pydantic_settings import BaseSettings
+
+
+class SentryConfig(BaseSettings):
+    """
+    Configuration settings for Sentry error tracking and performance monitoring
+    """
+
+    SENTRY_DSN: str | None = Field(
+        description="Sentry Data Source Name (DSN)."
+        " This is the unique identifier of your Sentry project, used to send events to the correct project.",
+        default=None,
+    )
+
+    SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description="Sample rate for Sentry performance monitoring traces."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of traces are sent to Sentry.",
+        default=1.0,
+    )
+
+    SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description="Sample rate for Sentry profiling."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of profiles are sent to Sentry.",
+        default=1.0,
+    )

dify/api/configs/feature/hosted_service/__init__.py

@@ -0,0 +1,254 @@
+from pydantic import Field, NonNegativeInt
+from pydantic_settings import BaseSettings
+
+
+class HostedCreditConfig(BaseSettings):
+    HOSTED_MODEL_CREDIT_CONFIG: str = Field(
+        description="Model credit configuration in format 'model:credits,model:credits', e.g., 'gpt-4:20,gpt-4o:10'",
+        default="",
+    )
+
+    def get_model_credits(self, model_name: str) -> int:
+        """
+        Get credit value for a specific model name.
+        Returns 1 if model is not found in configuration (default credit).
+
+        :param model_name: The name of the model to search for
+        :return: The credit value for the model
+        """
+        if not self.HOSTED_MODEL_CREDIT_CONFIG:
+            return 1
+
+        try:
+            credit_map = dict(
+                item.strip().split(":", 1) for item in self.HOSTED_MODEL_CREDIT_CONFIG.split(",") if ":" in item
+            )
+
+            # Search for matching model pattern
+            for pattern, credit in credit_map.items():
+                if pattern.strip() == model_name:
+                    return int(credit)
+            return 1  # Default quota if no match found
+        except (ValueError, AttributeError):
+            return 1  # Return default quota if parsing fails
+
+
+class HostedOpenAiConfig(BaseSettings):
+    """
+    Configuration for hosted OpenAI service
+    """
+
+    HOSTED_OPENAI_API_KEY: str | None = Field(
+        description="API key for hosted OpenAI service",
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_BASE: str | None = Field(
+        description="Base URL for hosted OpenAI API",
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_ORGANIZATION: str | None = Field(
+        description="Organization ID for hosted OpenAI service",
+        default=None,
+    )
+
+    HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted OpenAI service",
+        default=False,
+    )
+
+    HOSTED_OPENAI_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for trial access",
+        default="gpt-3.5-turbo,"
+        "gpt-3.5-turbo-1106,"
+        "gpt-3.5-turbo-instruct,"
+        "gpt-3.5-turbo-16k,"
+        "gpt-3.5-turbo-16k-0613,"
+        "gpt-3.5-turbo-0613,"
+        "gpt-3.5-turbo-0125,"
+        "text-davinci-003",
+    )
+
+    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted OpenAI service usage",
+        default=200,
+    )
+
+    HOSTED_OPENAI_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted OpenAI service",
+        default=False,
+    )
+
+    HOSTED_OPENAI_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="gpt-4,"
+        "gpt-4-turbo-preview,"
+        "gpt-4-turbo-2024-04-09,"
+        "gpt-4-1106-preview,"
+        "gpt-4-0125-preview,"
+        "gpt-3.5-turbo,"
+        "gpt-3.5-turbo-16k,"
+        "gpt-3.5-turbo-16k-0613,"
+        "gpt-3.5-turbo-1106,"
+        "gpt-3.5-turbo-0613,"
+        "gpt-3.5-turbo-0125,"
+        "gpt-3.5-turbo-instruct,"
+        "text-davinci-003",
+    )
+
+
+class HostedAzureOpenAiConfig(BaseSettings):
+    """
+    Configuration for hosted Azure OpenAI service
+    """
+
+    HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
+        description="Enable hosted Azure OpenAI service",
+        default=False,
+    )
+
+    HOSTED_AZURE_OPENAI_API_KEY: str | None = Field(
+        description="API key for hosted Azure OpenAI service",
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_API_BASE: str | None = Field(
+        description="Base URL for hosted Azure OpenAI API",
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted Azure OpenAI service usage",
+        default=200,
+    )
+
+
+class HostedAnthropicConfig(BaseSettings):
+    """
+    Configuration for hosted Anthropic service
+    """
+
+    HOSTED_ANTHROPIC_API_BASE: str | None = Field(
+        description="Base URL for hosted Anthropic API",
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_API_KEY: str | None = Field(
+        description="API key for hosted Anthropic service",
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted Anthropic service",
+        default=False,
+    )
+
+    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted Anthropic service usage",
+        default=600000,
+    )
+
+    HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted Anthropic service",
+        default=False,
+    )
+
+
+class HostedMinmaxConfig(BaseSettings):
+    """
+    Configuration for hosted Minmax service
+    """
+
+    HOSTED_MINIMAX_ENABLED: bool = Field(
+        description="Enable hosted Minmax service",
+        default=False,
+    )
+
+
+class HostedSparkConfig(BaseSettings):
+    """
+    Configuration for hosted Spark service
+    """
+
+    HOSTED_SPARK_ENABLED: bool = Field(
+        description="Enable hosted Spark service",
+        default=False,
+    )
+
+
+class HostedZhipuAIConfig(BaseSettings):
+    """
+    Configuration for hosted ZhipuAI service
+    """
+
+    HOSTED_ZHIPUAI_ENABLED: bool = Field(
+        description="Enable hosted ZhipuAI service",
+        default=False,
+    )
+
+
+class HostedModerationConfig(BaseSettings):
+    """
+    Configuration for hosted Moderation service
+    """
+
+    HOSTED_MODERATION_ENABLED: bool = Field(
+        description="Enable hosted Moderation service",
+        default=False,
+    )
+
+    HOSTED_MODERATION_PROVIDERS: str = Field(
+        description="Comma-separated list of moderation providers",
+        default="",
+    )
+
+
+class HostedFetchAppTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching app templates
+    """
+
+    HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching app templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote app templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
+class HostedFetchPipelineTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching pipeline templates
+    """
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching pipeline templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote pipeline templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
+class HostedServiceConfig(
+    # place the configs in alphabet order
+    HostedAnthropicConfig,
+    HostedAzureOpenAiConfig,
+    HostedFetchAppTemplateConfig,
+    HostedFetchPipelineTemplateConfig,
+    HostedMinmaxConfig,
+    HostedOpenAiConfig,
+    HostedSparkConfig,
+    HostedZhipuAIConfig,
+    # moderation
+    HostedModerationConfig,
+    # credit config
+    HostedCreditConfig,
+):
+    pass

dify/api/configs/middleware/__init__.py

@@ -0,0 +1,366 @@
+import os
+from typing import Any, Literal
+from urllib.parse import parse_qsl, quote_plus
+
+from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
+from pydantic_settings import BaseSettings
+
+from .cache.redis_config import RedisConfig
+from .storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from .storage.amazon_s3_storage_config import S3StorageConfig
+from .storage.azure_blob_storage_config import AzureBlobStorageConfig
+from .storage.baidu_obs_storage_config import BaiduOBSStorageConfig
+from .storage.clickzetta_volume_storage_config import ClickZettaVolumeStorageConfig
+from .storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from .storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
+from .storage.oci_storage_config import OCIStorageConfig
+from .storage.opendal_storage_config import OpenDALStorageConfig
+from .storage.supabase_storage_config import SupabaseStorageConfig
+from .storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from .storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
+from .vdb.alibabacloud_mysql_config import AlibabaCloudMySQLConfig
+from .vdb.analyticdb_config import AnalyticdbConfig
+from .vdb.baidu_vector_config import BaiduVectorDBConfig
+from .vdb.chroma_config import ChromaConfig
+from .vdb.clickzetta_config import ClickzettaConfig
+from .vdb.couchbase_config import CouchbaseConfig
+from .vdb.elasticsearch_config import ElasticsearchConfig
+from .vdb.huawei_cloud_config import HuaweiCloudConfig
+from .vdb.lindorm_config import LindormConfig
+from .vdb.matrixone_config import MatrixoneConfig
+from .vdb.milvus_config import MilvusConfig
+from .vdb.myscale_config import MyScaleConfig
+from .vdb.oceanbase_config import OceanBaseVectorConfig
+from .vdb.opengauss_config import OpenGaussConfig
+from .vdb.opensearch_config import OpenSearchConfig
+from .vdb.oracle_config import OracleConfig
+from .vdb.pgvector_config import PGVectorConfig
+from .vdb.pgvectors_config import PGVectoRSConfig
+from .vdb.qdrant_config import QdrantConfig
+from .vdb.relyt_config import RelytConfig
+from .vdb.tablestore_config import TableStoreConfig
+from .vdb.tencent_vector_config import TencentVectorDBConfig
+from .vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
+from .vdb.tidb_vector_config import TiDBVectorConfig
+from .vdb.upstash_config import UpstashConfig
+from .vdb.vastbase_vector_config import VastbaseVectorConfig
+from .vdb.vikingdb_config import VikingDBConfig
+from .vdb.weaviate_config import WeaviateConfig
+
+
+class StorageConfig(BaseSettings):
+    STORAGE_TYPE: Literal[
+        "opendal",
+        "s3",
+        "aliyun-oss",
+        "azure-blob",
+        "baidu-obs",
+        "clickzetta-volume",
+        "google-storage",
+        "huawei-obs",
+        "oci-storage",
+        "tencent-cos",
+        "volcengine-tos",
+        "supabase",
+        "local",
+    ] = Field(
+        description="Type of storage to use."
+        " Options: 'opendal', '(deprecated) local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', "
+        "'clickzetta-volume', 'google-storage', 'huawei-obs', 'oci-storage', 'tencent-cos', "
+        "'volcengine-tos', 'supabase'. Default is 'opendal'.",
+        default="opendal",
+    )
+
+    STORAGE_LOCAL_PATH: str = Field(
+        description="Path for local storage when STORAGE_TYPE is set to 'local'.",
+        default="storage",
+        deprecated=True,
+    )
+
+
+class VectorStoreConfig(BaseSettings):
+    VECTOR_STORE: str | None = Field(
+        description="Type of vector store to use for efficient similarity search."
+        " Set to None if not using a vector store.",
+        default=None,
+    )
+
+    VECTOR_STORE_WHITELIST_ENABLE: bool | None = Field(
+        description="Enable whitelist for vector store.",
+        default=False,
+    )
+
+    VECTOR_INDEX_NAME_PREFIX: str | None = Field(
+        description="Prefix used to create collection name in vector database",
+        default="Vector_index",
+    )
+
+
+class KeywordStoreConfig(BaseSettings):
+    KEYWORD_STORE: str = Field(
+        description="Method for keyword extraction and storage."
+        " Default is 'jieba', a Chinese text segmentation library.",
+        default="jieba",
+    )
+
+
+class DatabaseConfig(BaseSettings):
+    # Database type selector
+    DB_TYPE: Literal["postgresql", "mysql", "oceanbase"] = Field(
+        description="Database type to use. OceanBase is MySQL-compatible.",
+        default="postgresql",
+    )
+
+    DB_HOST: str = Field(
+        description="Hostname or IP address of the database server.",
+        default="localhost",
+    )
+
+    DB_PORT: PositiveInt = Field(
+        description="Port number for database connection.",
+        default=5432,
+    )
+
+    DB_USERNAME: str = Field(
+        description="Username for database authentication.",
+        default="postgres",
+    )
+
+    DB_PASSWORD: str = Field(
+        description="Password for database authentication.",
+        default="",
+    )
+
+    DB_DATABASE: str = Field(
+        description="Name of the database to connect to.",
+        default="dify",
+    )
+
+    DB_CHARSET: str = Field(
+        description="Character set for database connection.",
+        default="",
+    )
+
+    DB_EXTRAS: str = Field(
+        description="Additional database connection parameters. Example: 'keepalives_idle=60&keepalives=1'",
+        default="",
+    )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_DATABASE_URI_SCHEME(self) -> str:
+        return "postgresql" if self.DB_TYPE == "postgresql" else "mysql+pymysql"
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_DATABASE_URI(self) -> str:
+        db_extras = (
+            f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else self.DB_EXTRAS
+        ).strip("&")
+        db_extras = f"?{db_extras}" if db_extras else ""
+        return (
+            f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
+            f"{quote_plus(self.DB_USERNAME)}:{quote_plus(self.DB_PASSWORD)}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
+            f"{db_extras}"
+        )
+
+    SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
+        description="Maximum number of database connections in the pool.",
+        default=30,
+    )
+
+    SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
+        description="Maximum number of connections that can be created beyond the pool_size.",
+        default=10,
+    )
+
+    SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
+        description="Number of seconds after which a connection is automatically recycled.",
+        default=3600,
+    )
+
+    SQLALCHEMY_POOL_USE_LIFO: bool = Field(
+        description="If True, SQLAlchemy will use last-in-first-out way to retrieve connections from pool.",
+        default=False,
+    )
+
+    SQLALCHEMY_POOL_PRE_PING: bool = Field(
+        description="If True, enables connection pool pre-ping feature to check connections.",
+        default=False,
+    )
+
+    SQLALCHEMY_ECHO: bool | str = Field(
+        description="If True, SQLAlchemy will log all SQL statements.",
+        default=False,
+    )
+
+    SQLALCHEMY_POOL_TIMEOUT: NonNegativeInt = Field(
+        description="Number of seconds to wait for a connection from the pool before raising a timeout error.",
+        default=30,
+    )
+
+    RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
+        description="Number of processes for the retrieval service, default to CPU cores.",
+        default=os.cpu_count() or 1,
+    )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
+        # Parse DB_EXTRAS for 'options'
+        db_extras_dict = dict(parse_qsl(self.DB_EXTRAS))
+        options = db_extras_dict.get("options", "")
+        connect_args = {}
+        # Use the dynamic SQLALCHEMY_DATABASE_URI_SCHEME property
+        if self.SQLALCHEMY_DATABASE_URI_SCHEME.startswith("postgresql"):
+            timezone_opt = "-c timezone=UTC"
+            if options:
+                merged_options = f"{options} {timezone_opt}"
+            else:
+                merged_options = timezone_opt
+            connect_args = {"options": merged_options}
+
+        return {
+            "pool_size": self.SQLALCHEMY_POOL_SIZE,
+            "max_overflow": self.SQLALCHEMY_MAX_OVERFLOW,
+            "pool_recycle": self.SQLALCHEMY_POOL_RECYCLE,
+            "pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
+            "connect_args": connect_args,
+            "pool_use_lifo": self.SQLALCHEMY_POOL_USE_LIFO,
+            "pool_reset_on_return": None,
+            "pool_timeout": self.SQLALCHEMY_POOL_TIMEOUT,
+        }
+
+
+class CeleryConfig(DatabaseConfig):
+    CELERY_BACKEND: str = Field(
+        description="Backend for Celery task results. Options: 'database', 'redis', 'rabbitmq'.",
+        default="redis",
+    )
+
+    CELERY_BROKER_URL: str | None = Field(
+        description="URL of the message broker for Celery tasks.",
+        default=None,
+    )
+
+    CELERY_USE_SENTINEL: bool | None = Field(
+        description="Whether to use Redis Sentinel for high availability.",
+        default=False,
+    )
+
+    CELERY_SENTINEL_MASTER_NAME: str | None = Field(
+        description="Name of the Redis Sentinel master.",
+        default=None,
+    )
+
+    CELERY_SENTINEL_PASSWORD: str | None = Field(
+        description="Password of the Redis Sentinel master.",
+        default=None,
+    )
+    CELERY_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
+        description="Timeout for Redis Sentinel socket operations in seconds.",
+        default=0.1,
+    )
+
+    @computed_field
+    def CELERY_RESULT_BACKEND(self) -> str | None:
+        if self.CELERY_BACKEND in ("database", "rabbitmq"):
+            return f"db+{self.SQLALCHEMY_DATABASE_URI}"
+        elif self.CELERY_BACKEND == "redis":
+            return self.CELERY_BROKER_URL
+        else:
+            return None
+
+    @property
+    def BROKER_USE_SSL(self) -> bool:
+        return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False
+
+
+class InternalTestConfig(BaseSettings):
+    """
+    Configuration settings for Internal Test
+    """
+
+    AWS_SECRET_ACCESS_KEY: str | None = Field(
+        description="Internal test AWS secret access key",
+        default=None,
+    )
+
+    AWS_ACCESS_KEY_ID: str | None = Field(
+        description="Internal test AWS access key ID",
+        default=None,
+    )
+
+
+class DatasetQueueMonitorConfig(BaseSettings):
+    """
+    Configuration settings for Dataset Queue Monitor
+    """
+
+    QUEUE_MONITOR_THRESHOLD: NonNegativeInt | None = Field(
+        description="Threshold for dataset queue monitor",
+        default=200,
+    )
+    QUEUE_MONITOR_ALERT_EMAILS: str | None = Field(
+        description="Emails for dataset queue monitor alert, separated by commas",
+        default=None,
+    )
+    QUEUE_MONITOR_INTERVAL: NonNegativeFloat | None = Field(
+        description="Interval for dataset queue monitor in minutes",
+        default=30,
+    )
+
+
+class MiddlewareConfig(
+    # place the configs in alphabet order
+    CeleryConfig,  # Note: CeleryConfig already inherits from DatabaseConfig
+    KeywordStoreConfig,
+    RedisConfig,
+    # configs of storage and storage providers
+    StorageConfig,
+    AliyunOSSStorageConfig,
+    AzureBlobStorageConfig,
+    BaiduOBSStorageConfig,
+    ClickZettaVolumeStorageConfig,
+    GoogleCloudStorageConfig,
+    HuaweiCloudOBSStorageConfig,
+    OCIStorageConfig,
+    OpenDALStorageConfig,
+    S3StorageConfig,
+    SupabaseStorageConfig,
+    TencentCloudCOSStorageConfig,
+    VolcengineTOSStorageConfig,
+    # configs of vdb and vdb providers
+    VectorStoreConfig,
+    AnalyticdbConfig,
+    ChromaConfig,
+    ClickzettaConfig,
+    HuaweiCloudConfig,
+    MilvusConfig,
+    AlibabaCloudMySQLConfig,
+    MyScaleConfig,
+    OpenSearchConfig,
+    OracleConfig,
+    PGVectorConfig,
+    VastbaseVectorConfig,
+    PGVectoRSConfig,
+    QdrantConfig,
+    RelytConfig,
+    TencentVectorDBConfig,
+    TiDBVectorConfig,
+    WeaviateConfig,
+    ElasticsearchConfig,
+    CouchbaseConfig,
+    InternalTestConfig,
+    VikingDBConfig,
+    UpstashConfig,
+    TidbOnQdrantConfig,
+    LindormConfig,
+    OceanBaseVectorConfig,
+    BaiduVectorDBConfig,
+    OpenGaussConfig,
+    TableStoreConfig,
+    DatasetQueueMonitorConfig,
+    MatrixoneConfig,
+):
+    pass

dify/api/configs/middleware/cache/redis_config.py

@@ -0,0 +1,113 @@
+from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RedisConfig(BaseSettings):
+    """
+    Configuration settings for Redis connection
+    """
+
+    REDIS_HOST: str = Field(
+        description="Hostname or IP address of the Redis server",
+        default="localhost",
+    )
+
+    REDIS_PORT: PositiveInt = Field(
+        description="Port number on which the Redis server is listening",
+        default=6379,
+    )
+
+    REDIS_USERNAME: str | None = Field(
+        description="Username for Redis authentication (if required)",
+        default=None,
+    )
+
+    REDIS_PASSWORD: str | None = Field(
+        description="Password for Redis authentication (if required)",
+        default=None,
+    )
+
+    REDIS_DB: NonNegativeInt = Field(
+        description="Redis database number to use (0-15)",
+        default=0,
+    )
+
+    REDIS_USE_SSL: bool = Field(
+        description="Enable SSL/TLS for the Redis connection",
+        default=False,
+    )
+
+    REDIS_SSL_CERT_REQS: str = Field(
+        description="SSL certificate requirements (CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED)",
+        default="CERT_NONE",
+    )
+
+    REDIS_SSL_CA_CERTS: str | None = Field(
+        description="Path to the CA certificate file for SSL verification",
+        default=None,
+    )
+
+    REDIS_SSL_CERTFILE: str | None = Field(
+        description="Path to the client certificate file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_SSL_KEYFILE: str | None = Field(
+        description="Path to the client private key file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_USE_SENTINEL: bool | None = Field(
+        description="Enable Redis Sentinel mode for high availability",
+        default=False,
+    )
+
+    REDIS_SENTINELS: str | None = Field(
+        description="Comma-separated list of Redis Sentinel nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SERVICE_NAME: str | None = Field(
+        description="Name of the Redis Sentinel service to monitor",
+        default=None,
+    )
+
+    REDIS_SENTINEL_USERNAME: str | None = Field(
+        description="Username for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_PASSWORD: str | None = Field(
+        description="Password for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
+        description="Socket timeout in seconds for Redis Sentinel connections",
+        default=0.1,
+    )
+
+    REDIS_USE_CLUSTERS: bool = Field(
+        description="Enable Redis Clusters mode for high availability",
+        default=False,
+    )
+
+    REDIS_CLUSTERS: str | None = Field(
+        description="Comma-separated list of Redis Clusters nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_CLUSTERS_PASSWORD: str | None = Field(
+        description="Password for Redis Clusters authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SERIALIZATION_PROTOCOL: int = Field(
+        description="Redis serialization protocol (RESP) version",
+        default=3,
+    )
+
+    REDIS_ENABLE_CLIENT_SIDE_CACHE: bool = Field(
+        description="Enable client side cache in redis",
+        default=False,
+    )

dify/api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -0,0 +1,43 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AliyunOSSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Aliyun Object Storage Service (OSS)
+    """
+
+    ALIYUN_OSS_BUCKET_NAME: str | None = Field(
+        description="Name of the Aliyun OSS bucket to store and retrieve objects",
+        default=None,
+    )
+
+    ALIYUN_OSS_ACCESS_KEY: str | None = Field(
+        description="Access key ID for authenticating with Aliyun OSS",
+        default=None,
+    )
+
+    ALIYUN_OSS_SECRET_KEY: str | None = Field(
+        description="Secret access key for authenticating with Aliyun OSS",
+        default=None,
+    )
+
+    ALIYUN_OSS_ENDPOINT: str | None = Field(
+        description="URL of the Aliyun OSS endpoint for your chosen region",
+        default=None,
+    )
+
+    ALIYUN_OSS_REGION: str | None = Field(
+        description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
+        default=None,
+    )
+
+    ALIYUN_OSS_AUTH_VERSION: str | None = Field(
+        description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
+        default=None,
+    )
+
+    ALIYUN_OSS_PATH: str | None = Field(
+        description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
+        default=None,
+    )

dify/api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -0,0 +1,45 @@
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class S3StorageConfig(BaseSettings):
+    """
+    Configuration settings for S3-compatible object storage
+    """
+
+    S3_ENDPOINT: str | None = Field(
+        description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
+        default=None,
+    )
+
+    S3_REGION: str | None = Field(
+        description="Region where the S3 bucket is located (e.g., 'us-east-1')",
+        default=None,
+    )
+
+    S3_BUCKET_NAME: str | None = Field(
+        description="Name of the S3 bucket to store and retrieve objects",
+        default=None,
+    )
+
+    S3_ACCESS_KEY: str | None = Field(
+        description="Access key ID for authenticating with the S3 service",
+        default=None,
+    )
+
+    S3_SECRET_KEY: str | None = Field(
+        description="Secret access key for authenticating with the S3 service",
+        default=None,
+    )
+
+    S3_ADDRESS_STYLE: Literal["auto", "virtual", "path"] = Field(
+        description="S3 addressing style: 'auto', 'path', or 'virtual'",
+        default="auto",
+    )
+
+    S3_USE_AWS_MANAGED_IAM: bool = Field(
+        description="Use AWS managed IAM roles for authentication instead of access/secret keys",
+        default=False,
+    )

dify/api/configs/middleware/storage/azure_blob_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AzureBlobStorageConfig(BaseSettings):
+    """
+    Configuration settings for Azure Blob Storage
+    """
+
+    AZURE_BLOB_ACCOUNT_NAME: str | None = Field(
+        description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_KEY: str | None = Field(
+        description="Access key for authenticating with the Azure Storage account",
+        default=None,
+    )
+
+    AZURE_BLOB_CONTAINER_NAME: str | None = Field(
+        description="Name of the Azure Blob container to store and retrieve objects",
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_URL: str | None = Field(
+        description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
+        default=None,
+    )

dify/api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class BaiduOBSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Baidu Object Storage Service (OBS)
+    """
+
+    BAIDU_OBS_BUCKET_NAME: str | None = Field(
+        description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
+        default=None,
+    )
+
+    BAIDU_OBS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Baidu OBS",
+        default=None,
+    )
+
+    BAIDU_OBS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Baidu OBS",
+        default=None,
+    )
+
+    BAIDU_OBS_ENDPOINT: str | None = Field(
+        description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
+        default=None,
+    )

dify/api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -0,0 +1,63 @@
+"""ClickZetta Volume Storage Configuration"""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ClickZettaVolumeStorageConfig(BaseSettings):
+    """Configuration for ClickZetta Volume storage."""
+
+    CLICKZETTA_VOLUME_USERNAME: str | None = Field(
+        description="Username for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_PASSWORD: str | None = Field(
+        description="Password for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_INSTANCE: str | None = Field(
+        description="ClickZetta instance identifier",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_SERVICE: str = Field(
+        description="ClickZetta service endpoint",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_VOLUME_WORKSPACE: str = Field(
+        description="ClickZetta workspace name",
+        default="quick_start",
+    )
+
+    CLICKZETTA_VOLUME_VCLUSTER: str = Field(
+        description="ClickZetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_VOLUME_SCHEMA: str = Field(
+        description="ClickZetta schema name",
+        default="dify",
+    )
+
+    CLICKZETTA_VOLUME_TYPE: str = Field(
+        description="ClickZetta volume type (table|user|external)",
+        default="user",
+    )
+
+    CLICKZETTA_VOLUME_NAME: str | None = Field(
+        description="ClickZetta volume name for external volumes",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_TABLE_PREFIX: str = Field(
+        description="Prefix for ClickZetta volume table names",
+        default="dataset_",
+    )
+
+    CLICKZETTA_VOLUME_DIFY_PREFIX: str = Field(
+        description="Directory prefix for User Volume to organize Dify files",
+        default="dify_km",
+    )

dify/api/configs/middleware/storage/google_cloud_storage_config.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class GoogleCloudStorageConfig(BaseSettings):
+    """
+    Configuration settings for Google Cloud Storage
+    """
+
+    GOOGLE_STORAGE_BUCKET_NAME: str | None = Field(
+        description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
+        default=None,
+    )
+
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: str | None = Field(
+        description="Base64-encoded JSON key file for Google Cloud service account authentication",
+        default=None,
+    )

dify/api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class HuaweiCloudOBSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Huawei Cloud Object Storage Service (OBS)
+    """
+
+    HUAWEI_OBS_BUCKET_NAME: str | None = Field(
+        description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
+        default=None,
+    )
+
+    HUAWEI_OBS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SERVER: str | None = Field(
+        description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
+        default=None,
+    )

dify/api/configs/middleware/storage/oci_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OCIStorageConfig(BaseSettings):
+    """
+    Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
+    """
+
+    OCI_ENDPOINT: str | None = Field(
+        description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
+        default=None,
+    )
+
+    OCI_REGION: str | None = Field(
+        description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
+        default=None,
+    )
+
+    OCI_BUCKET_NAME: str | None = Field(
+        description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
+        default=None,
+    )
+
+    OCI_ACCESS_KEY: str | None = Field(
+        description="Access key (also known as API key) for authenticating with OCI Object Storage",
+        default=None,
+    )
+
+    OCI_SECRET_KEY: str | None = Field(
+        description="Secret key associated with the access key for authenticating with OCI Object Storage",
+        default=None,
+    )

dify/api/configs/middleware/storage/opendal_storage_config.py

@@ -0,0 +1,9 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OpenDALStorageConfig(BaseSettings):
+    OPENDAL_SCHEME: str = Field(
+        default="fs",
+        description="OpenDAL scheme.",
+    )

dify/api/configs/middleware/storage/supabase_storage_config.py

@@ -0,0 +1,23 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class SupabaseStorageConfig(BaseSettings):
+    """
+    Configuration settings for Supabase Object Storage Service
+    """
+
+    SUPABASE_BUCKET_NAME: str | None = Field(
+        description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
+        default=None,
+    )
+
+    SUPABASE_API_KEY: str | None = Field(
+        description="API KEY for authenticating with Supabase",
+        default=None,
+    )
+
+    SUPABASE_URL: str | None = Field(
+        description="URL of the Supabase",
+        default=None,
+    )

dify/api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class TencentCloudCOSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Tencent Cloud Object Storage (COS)
+    """
+
+    TENCENT_COS_BUCKET_NAME: str | None = Field(
+        description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
+        default=None,
+    )
+
+    TENCENT_COS_REGION: str | None = Field(
+        description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_ID: str | None = Field(
+        description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_KEY: str | None = Field(
+        description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
+        default=None,
+    )
+
+    TENCENT_COS_SCHEME: str | None = Field(
+        description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
+        default=None,
+    )

dify/api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class VolcengineTOSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Volcengine Tinder Object Storage (TOS)
+    """
+
+    VOLCENGINE_TOS_BUCKET_NAME: str | None = Field(
+        description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ENDPOINT: str | None = Field(
+        description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_REGION: str | None = Field(
+        description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
+        default=None,
+    )

dify/api/configs/middleware/vdb/alibabacloud_mysql_config.py

@@ -0,0 +1,54 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AlibabaCloudMySQLConfig(BaseSettings):
+    """
+    Configuration settings for AlibabaCloud MySQL vector database
+    """
+
+    ALIBABACLOUD_MYSQL_HOST: str = Field(
+        description="Hostname or IP address of the AlibabaCloud MySQL server (e.g., 'localhost' or 'mysql.aliyun.com')",
+        default="localhost",
+    )
+
+    ALIBABACLOUD_MYSQL_PORT: PositiveInt = Field(
+        description="Port number on which the AlibabaCloud MySQL server is listening (default is 3306)",
+        default=3306,
+    )
+
+    ALIBABACLOUD_MYSQL_USER: str = Field(
+        description="Username for authenticating with AlibabaCloud MySQL (default is 'root')",
+        default="root",
+    )
+
+    ALIBABACLOUD_MYSQL_PASSWORD: str = Field(
+        description="Password for authenticating with AlibabaCloud MySQL (default is an empty string)",
+        default="",
+    )
+
+    ALIBABACLOUD_MYSQL_DATABASE: str = Field(
+        description="Name of the AlibabaCloud MySQL database to connect to (default is 'dify')",
+        default="dify",
+    )
+
+    ALIBABACLOUD_MYSQL_MAX_CONNECTION: PositiveInt = Field(
+        description="Maximum number of connections in the connection pool",
+        default=5,
+    )
+
+    ALIBABACLOUD_MYSQL_CHARSET: str = Field(
+        description="Character set for AlibabaCloud MySQL connection (default is 'utf8mb4')",
+        default="utf8mb4",
+    )
+
+    ALIBABACLOUD_MYSQL_DISTANCE_FUNCTION: str = Field(
+        description="Distance function used for vector similarity search in AlibabaCloud MySQL "
+        "(e.g., 'cosine', 'euclidean')",
+        default="cosine",
+    )
+
+    ALIBABACLOUD_MYSQL_HNSW_M: PositiveInt = Field(
+        description="Maximum number of connections per layer for HNSW vector index (default is 6, range: 3-200)",
+        default=6,
+    )

dify/api/configs/middleware/vdb/analyticdb_config.py

@@ -0,0 +1,49 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AnalyticdbConfig(BaseSettings):
+    """
+    Configuration for connecting to Alibaba Cloud AnalyticDB for PostgreSQL.
+    Refer to the following documentation for details on obtaining credentials:
+    https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
+    """
+
+    ANALYTICDB_KEY_ID: str | None = Field(
+        default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
+    )
+    ANALYTICDB_KEY_SECRET: str | None = Field(
+        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
+    )
+    ANALYTICDB_REGION_ID: str | None = Field(
+        default=None,
+        description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
+    )
+    ANALYTICDB_INSTANCE_ID: str | None = Field(
+        default=None,
+        description="The unique identifier of the AnalyticDB instance you want to connect to.",
+    )
+    ANALYTICDB_ACCOUNT: str | None = Field(
+        default=None,
+        description="The account name used to log in to the AnalyticDB instance"
+        " (usually the initial account created with the instance).",
+    )
+    ANALYTICDB_PASSWORD: str | None = Field(
+        default=None, description="The password associated with the AnalyticDB account for database authentication."
+    )
+    ANALYTICDB_NAMESPACE: str | None = Field(
+        default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
+    )
+    ANALYTICDB_NAMESPACE_PASSWORD: str | None = Field(
+        default=None,
+        description="The password for accessing the specified namespace within the AnalyticDB instance"
+        " (if namespace feature is enabled).",
+    )
+    ANALYTICDB_HOST: str | None = Field(
+        default=None, description="The host of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_PORT: PositiveInt = Field(
+        default=5432, description="The port of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_MIN_CONNECTION: PositiveInt = Field(default=1, description="Min connection of the AnalyticDB database.")
+    ANALYTICDB_MAX_CONNECTION: PositiveInt = Field(default=5, description="Max connection of the AnalyticDB database.")

dify/api/configs/middleware/vdb/baidu_vector_config.py

@@ -0,0 +1,53 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class BaiduVectorDBConfig(BaseSettings):
+    """
+    Configuration settings for Baidu Vector Database
+    """
+
+    BAIDU_VECTOR_DB_ENDPOINT: str | None = Field(
+        description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: PositiveInt = Field(
+        description="Timeout in milliseconds for Baidu Vector Database operations (default is 30000 milliseconds)",
+        default=30000,
+    )
+
+    BAIDU_VECTOR_DB_ACCOUNT: str | None = Field(
+        description="Account for authenticating with the Baidu Vector Database",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_API_KEY: str | None = Field(
+        description="API key for authenticating with the Baidu Vector Database service",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_DATABASE: str | None = Field(
+        description="Name of the specific Baidu Vector Database to connect to",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_SHARD: PositiveInt = Field(
+        description="Number of shards for the Baidu Vector Database (default is 1)",
+        default=1,
+    )
+
+    BAIDU_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+        description="Number of replicas for the Baidu Vector Database (default is 3)",
+        default=3,
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER: str = Field(
+        description="Analyzer type for inverted index in Baidu Vector Database (default is DEFAULT_ANALYZER)",
+        default="DEFAULT_ANALYZER",
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE: str = Field(
+        description="Parser mode for inverted index in Baidu Vector Database (default is COARSE_MODE)",
+        default="COARSE_MODE",
+    )

dify/api/configs/middleware/vdb/chroma_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class ChromaConfig(BaseSettings):
+    """
+    Configuration settings for Chroma vector database
+    """
+
+    CHROMA_HOST: str | None = Field(
+        description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
+        default=None,
+    )
+
+    CHROMA_PORT: PositiveInt = Field(
+        description="Port number on which the Chroma server is listening (default is 8000)",
+        default=8000,
+    )
+
+    CHROMA_TENANT: str | None = Field(
+        description="Tenant identifier for multi-tenancy support in Chroma",
+        default=None,
+    )
+
+    CHROMA_DATABASE: str | None = Field(
+        description="Name of the Chroma database to connect to",
+        default=None,
+    )
+
+    CHROMA_AUTH_PROVIDER: str | None = Field(
+        description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
+        default=None,
+    )
+
+    CHROMA_AUTH_CREDENTIALS: str | None = Field(
+        description="Authentication credentials for Chroma (format depends on the auth provider)",
+        default=None,
+    )

dify/api/configs/middleware/vdb/clickzetta_config.py

@@ -0,0 +1,68 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ClickzettaConfig(BaseSettings):
+    """
+    Clickzetta Lakehouse vector database configuration
+    """
+
+    CLICKZETTA_USERNAME: str | None = Field(
+        description="Username for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_PASSWORD: str | None = Field(
+        description="Password for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_INSTANCE: str | None = Field(
+        description="Clickzetta Lakehouse instance ID",
+        default=None,
+    )
+
+    CLICKZETTA_SERVICE: str | None = Field(
+        description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_WORKSPACE: str | None = Field(
+        description="Clickzetta workspace name",
+        default="default",
+    )
+
+    CLICKZETTA_VCLUSTER: str | None = Field(
+        description="Clickzetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_SCHEMA: str | None = Field(
+        description="Database schema name in Clickzetta",
+        default="public",
+    )
+
+    CLICKZETTA_BATCH_SIZE: int | None = Field(
+        description="Batch size for bulk insert operations",
+        default=100,
+    )
+
+    CLICKZETTA_ENABLE_INVERTED_INDEX: bool | None = Field(
+        description="Enable inverted index for full-text search capabilities",
+        default=True,
+    )
+
+    CLICKZETTA_ANALYZER_TYPE: str | None = Field(
+        description="Analyzer type for full-text search: keyword, english, chinese, unicode",
+        default="chinese",
+    )
+
+    CLICKZETTA_ANALYZER_MODE: str | None = Field(
+        description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
+        default="smart",
+    )
+
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: str | None = Field(
+        description="Distance function for vector similarity: l2_distance or cosine_distance",
+        default="cosine_distance",
+    )

dify/api/configs/middleware/vdb/couchbase_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class CouchbaseConfig(BaseSettings):
+    """
+    Couchbase configs
+    """
+
+    COUCHBASE_CONNECTION_STRING: str | None = Field(
+        description="COUCHBASE connection string",
+        default=None,
+    )
+
+    COUCHBASE_USER: str | None = Field(
+        description="COUCHBASE user",
+        default=None,
+    )
+
+    COUCHBASE_PASSWORD: str | None = Field(
+        description="COUCHBASE password",
+        default=None,
+    )
+
+    COUCHBASE_BUCKET_NAME: str | None = Field(
+        description="COUCHBASE bucket name",
+        default=None,
+    )
+
+    COUCHBASE_SCOPE_NAME: str | None = Field(
+        description="COUCHBASE scope name",
+        default=None,
+    )

dify/api/configs/middleware/vdb/elasticsearch_config.py

@@ -0,0 +1,74 @@
+from pydantic import Field, PositiveInt, model_validator
+from pydantic_settings import BaseSettings
+
+
+class ElasticsearchConfig(BaseSettings):
+    """
+    Configuration settings for both self-managed and Elastic Cloud deployments.
+    Can load from environment variables or .env files.
+    """
+
+    ELASTICSEARCH_HOST: str | None = Field(
+        description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
+        default="127.0.0.1",
+    )
+
+    ELASTICSEARCH_PORT: PositiveInt = Field(
+        description="Port number on which the Elasticsearch server is listening (default is 9200)",
+        default=9200,
+    )
+
+    ELASTICSEARCH_USERNAME: str | None = Field(
+        description="Username for authenticating with Elasticsearch (default is 'elastic')",
+        default="elastic",
+    )
+
+    ELASTICSEARCH_PASSWORD: str | None = Field(
+        description="Password for authenticating with Elasticsearch (default is 'elastic')",
+        default="elastic",
+    )
+
+    # Elastic Cloud (optional)
+    ELASTICSEARCH_USE_CLOUD: bool | None = Field(
+        description="Set to True to use Elastic Cloud instead of self-hosted Elasticsearch", default=False
+    )
+    ELASTICSEARCH_CLOUD_URL: str | None = Field(
+        description="Full URL for Elastic Cloud deployment (e.g., 'https://example.es.region.aws.found.io:443')",
+        default=None,
+    )
+    ELASTICSEARCH_API_KEY: str | None = Field(description="API key for authenticating with Elastic Cloud", default=None)
+
+    # Common options
+    ELASTICSEARCH_CA_CERTS: str | None = Field(
+        description="Path to CA certificate file for SSL verification", default=None
+    )
+    ELASTICSEARCH_VERIFY_CERTS: bool = Field(
+        description="Whether to verify SSL certificates (default is False)", default=False
+    )
+    ELASTICSEARCH_REQUEST_TIMEOUT: int = Field(
+        description="Request timeout in milliseconds (default is 100000)", default=100000
+    )
+    ELASTICSEARCH_RETRY_ON_TIMEOUT: bool = Field(
+        description="Whether to retry requests on timeout (default is True)", default=True
+    )
+    ELASTICSEARCH_MAX_RETRIES: int = Field(
+        description="Maximum number of retry attempts (default is 10000)", default=10000
+    )
+
+    @model_validator(mode="after")
+    def validate_elasticsearch_config(self):
+        """Validate Elasticsearch configuration based on deployment type."""
+        if self.ELASTICSEARCH_USE_CLOUD:
+            if not self.ELASTICSEARCH_CLOUD_URL:
+                raise ValueError("ELASTICSEARCH_CLOUD_URL is required when using Elastic Cloud")
+            if not self.ELASTICSEARCH_API_KEY:
+                raise ValueError("ELASTICSEARCH_API_KEY is required when using Elastic Cloud")
+        else:
+            if not self.ELASTICSEARCH_HOST:
+                raise ValueError("ELASTICSEARCH_HOST is required for self-hosted Elasticsearch")
+            if not self.ELASTICSEARCH_USERNAME:
+                raise ValueError("ELASTICSEARCH_USERNAME is required for self-hosted Elasticsearch")
+            if not self.ELASTICSEARCH_PASSWORD:
+                raise ValueError("ELASTICSEARCH_PASSWORD is required for self-hosted Elasticsearch")
+
+        return self

dify/api/configs/middleware/vdb/huawei_cloud_config.py

@@ -0,0 +1,23 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class HuaweiCloudConfig(BaseSettings):
+    """
+    Configuration settings for Huawei cloud search service
+    """
+
+    HUAWEI_CLOUD_HOSTS: str | None = Field(
+        description="Hostname or IP address of the Huawei cloud search service instance",
+        default=None,
+    )
+
+    HUAWEI_CLOUD_USER: str | None = Field(
+        description="Username for authenticating with Huawei cloud search service",
+        default=None,
+    )
+
+    HUAWEI_CLOUD_PASSWORD: str | None = Field(
+        description="Password for authenticating with Huawei cloud search service",
+        default=None,
+    )

dify/api/configs/middleware/vdb/lindorm_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class LindormConfig(BaseSettings):
+    """
+    Lindorm configs
+    """
+
+    LINDORM_URL: str | None = Field(
+        description="Lindorm url",
+        default=None,
+    )
+    LINDORM_USERNAME: str | None = Field(
+        description="Lindorm user",
+        default=None,
+    )
+    LINDORM_PASSWORD: str | None = Field(
+        description="Lindorm password",
+        default=None,
+    )
+    LINDORM_INDEX_TYPE: str | None = Field(
+        description="Lindorm Vector Index Type, hnsw or flat is available in dify",
+        default="hnsw",
+    )
+    LINDORM_DISTANCE_TYPE: str | None = Field(
+        description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
+    )
+    LINDORM_USING_UGC: bool | None = Field(
+        description="Using UGC index will store indexes with the same IndexType/Dimension in a single big index.",
+        default=True,
+    )
+    LINDORM_QUERY_TIMEOUT: float | None = Field(description="The lindorm search request timeout (s)", default=2.0)

dify/api/configs/middleware/vdb/matrixone_config.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class MatrixoneConfig(BaseSettings):
+    """Matrixone vector database configuration."""
+
+    MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server")
+    MATRIXONE_PORT: int = Field(default=6001, description="Port number of the Matrixone server")
+    MATRIXONE_USER: str = Field(default="dump", description="Username for authenticating with Matrixone")
+    MATRIXONE_PASSWORD: str = Field(default="111", description="Password for authenticating with Matrixone")
+    MATRIXONE_DATABASE: str = Field(default="dify", description="Name of the Matrixone database to connect to")
+    MATRIXONE_METRIC: str = Field(
+        default="l2", description="Distance metric type for vector similarity search (cosine or l2)"
+    )

dify/api/configs/middleware/vdb/milvus_config.py

@@ -0,0 +1,44 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class MilvusConfig(BaseSettings):
+    """
+    Configuration settings for Milvus vector database
+    """
+
+    MILVUS_URI: str | None = Field(
+        description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
+        default="http://127.0.0.1:19530",
+    )
+
+    MILVUS_TOKEN: str | None = Field(
+        description="Authentication token for Milvus, if token-based authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_USER: str | None = Field(
+        description="Username for authenticating with Milvus, if username/password authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_PASSWORD: str | None = Field(
+        description="Password for authenticating with Milvus, if username/password authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_DATABASE: str = Field(
+        description="Name of the Milvus database to connect to (default is 'default')",
+        default="default",
+    )
+
+    MILVUS_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features (requires Milvus >= 2.5.0). Set to false for compatibility with "
+        "older versions",
+        default=True,
+    )
+
+    MILVUS_ANALYZER_PARAMS: str | None = Field(
+        description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
+        default=None,
+    )

dify/api/configs/middleware/vdb/myscale_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class MyScaleConfig(BaseSettings):
+    """
+    Configuration settings for MyScale vector database
+    """
+
+    MYSCALE_HOST: str = Field(
+        description="Hostname or IP address of the MyScale server (e.g., 'localhost' or 'myscale.example.com')",
+        default="localhost",
+    )
+
+    MYSCALE_PORT: PositiveInt = Field(
+        description="Port number on which the MyScale server is listening (default is 8123)",
+        default=8123,
+    )
+
+    MYSCALE_USER: str = Field(
+        description="Username for authenticating with MyScale (default is 'default')",
+        default="default",
+    )
+
+    MYSCALE_PASSWORD: str = Field(
+        description="Password for authenticating with MyScale (default is an empty string)",
+        default="",
+    )
+
+    MYSCALE_DATABASE: str = Field(
+        description="Name of the MyScale database to connect to (default is 'default')",
+        default="default",
+    )
+
+    MYSCALE_FTS_PARAMS: str = Field(
+        description="Additional parameters for MyScale Full Text Search index)",
+        default="",
+    )

dify/api/configs/middleware/vdb/oceanbase_config.py

@@ -0,0 +1,51 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OceanBaseVectorConfig(BaseSettings):
+    """
+    Configuration settings for OceanBase Vector database
+    """
+
+    OCEANBASE_VECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_PORT: PositiveInt | None = Field(
+        description="Port number on which the OceanBase Vector server is listening (default is 2881)",
+        default=2881,
+    )
+
+    OCEANBASE_VECTOR_USER: str | None = Field(
+        description="Username for authenticating with the OceanBase Vector database",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the OceanBase Vector database",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_DATABASE: str | None = Field(
+        description="Name of the OceanBase Vector database to connect to",
+        default=None,
+    )
+
+    OCEANBASE_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features (requires OceanBase >= 4.3.5.1). Set to false for compatibility "
+        "with older versions",
+        default=False,
+    )
+
+    OCEANBASE_FULLTEXT_PARSER: str | None = Field(
+        description=(
+            "Fulltext parser to use for text indexing. "
+            "Built-in options: 'ngram' (N-gram tokenizer for English/numbers), "
+            "'beng' (Basic English tokenizer), 'space' (Space-based tokenizer), "
+            "'ngram2' (Improved N-gram tokenizer), 'ik' (Chinese tokenizer). "
+            "External plugins (require installation): 'japanese_ftparser' (Japanese tokenizer), "
+            "'thai_ftparser' (Thai tokenizer). Default is 'ik'"
+        ),
+        default="ik",
+    )

dify/api/configs/middleware/vdb/opengauss_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OpenGaussConfig(BaseSettings):
+    """
+    Configuration settings for OpenGauss
+    """
+
+    OPENGAUSS_HOST: str | None = Field(
+        description="Hostname or IP address of the OpenGauss server(e.g., 'localhost')",
+        default=None,
+    )
+
+    OPENGAUSS_PORT: PositiveInt = Field(
+        description="Port number on which the OpenGauss server is listening (default is 6600)",
+        default=6600,
+    )
+
+    OPENGAUSS_USER: str | None = Field(
+        description="Username for authenticating with the OpenGauss database",
+        default=None,
+    )
+
+    OPENGAUSS_PASSWORD: str | None = Field(
+        description="Password for authenticating with the OpenGauss database",
+        default=None,
+    )
+
+    OPENGAUSS_DATABASE: str | None = Field(
+        description="Name of the OpenGauss database to connect to",
+        default=None,
+    )
+
+    OPENGAUSS_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the OpenGauss database",
+        default=1,
+    )
+
+    OPENGAUSS_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the OpenGauss database",
+        default=5,
+    )
+
+    OPENGAUSS_ENABLE_PQ: bool = Field(
+        description="Enable openGauss PQ acceleration feature",
+        default=False,
+    )

dify/api/configs/middleware/vdb/opensearch_config.py

@@ -0,0 +1,64 @@
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AuthMethod(StrEnum):
+    """
+    Authentication method for OpenSearch
+    """
+
+    BASIC = "basic"
+    AWS_MANAGED_IAM = "aws_managed_iam"
+
+
+class OpenSearchConfig(BaseSettings):
+    """
+    Configuration settings for OpenSearch
+    """
+
+    OPENSEARCH_HOST: str | None = Field(
+        description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
+        default=None,
+    )
+
+    OPENSEARCH_PORT: PositiveInt = Field(
+        description="Port number on which the OpenSearch server is listening (default is 9200)",
+        default=9200,
+    )
+
+    OPENSEARCH_SECURE: bool = Field(
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+        default=False,
+    )
+
+    OPENSEARCH_VERIFY_CERTS: bool = Field(
+        description="Whether to verify SSL certificates for HTTPS connections (recommended to set True in production)",
+        default=True,
+    )
+
+    OPENSEARCH_AUTH_METHOD: AuthMethod = Field(
+        description="Authentication method for OpenSearch connection (default is 'basic')",
+        default=AuthMethod.BASIC,
+    )
+
+    OPENSEARCH_USER: str | None = Field(
+        description="Username for authenticating with OpenSearch",
+        default=None,
+    )
+
+    OPENSEARCH_PASSWORD: str | None = Field(
+        description="Password for authenticating with OpenSearch",
+        default=None,
+    )
+
+    OPENSEARCH_AWS_REGION: str | None = Field(
+        description="AWS region for OpenSearch (e.g. 'us-west-2')",
+        default=None,
+    )
+
+    OPENSEARCH_AWS_SERVICE: Literal["es", "aoss"] | None = Field(
+        description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
+    )

dify/api/configs/middleware/vdb/oracle_config.py

@@ -0,0 +1,44 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OracleConfig(BaseSettings):
+    """
+    Configuration settings for Oracle database
+    """
+
+    ORACLE_USER: str | None = Field(
+        description="Username for authenticating with the Oracle database",
+        default=None,
+    )
+
+    ORACLE_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Oracle database",
+        default=None,
+    )
+
+    ORACLE_DSN: str | None = Field(
+        description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
+        "For autonomous database, use the service name from tnsnames.ora in the wallet",
+        default=None,
+    )
+
+    ORACLE_CONFIG_DIR: str | None = Field(
+        description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_LOCATION: str | None = Field(
+        description="Oracle wallet directory path containing the wallet files for secure connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_PASSWORD: str | None = Field(
+        description="Password to decrypt the Oracle wallet, if it is encrypted",
+        default=None,
+    )
+
+    ORACLE_IS_AUTONOMOUS: bool = Field(
+        description="Flag indicating whether connecting to Oracle Autonomous Database",
+        default=False,
+    )

dify/api/configs/middleware/vdb/pgvector_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectorConfig(BaseSettings):
+    """
+    Configuration settings for PGVector (PostgreSQL with vector extension)
+    """
+
+    PGVECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    PGVECTOR_PORT: PositiveInt = Field(
+        description="Port number on which the PostgreSQL server is listening (default is 5433)",
+        default=5433,
+    )
+
+    PGVECTOR_USER: str | None = Field(
+        description="Username for authenticating with the PostgreSQL database",
+        default=None,
+    )
+
+    PGVECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the PostgreSQL database",
+        default=None,
+    )
+
+    PGVECTOR_DATABASE: str | None = Field(
+        description="Name of the PostgreSQL database to connect to",
+        default=None,
+    )
+
+    PGVECTOR_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the PostgreSQL database",
+        default=1,
+    )
+
+    PGVECTOR_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the PostgreSQL database",
+        default=5,
+    )
+
+    PGVECTOR_PG_BIGM: bool = Field(
+        description="Whether to use pg_bigm module for full text search",
+        default=False,
+    )

dify/api/configs/middleware/vdb/pgvectors_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectoRSConfig(BaseSettings):
+    """
+    Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
+    """
+
+    PGVECTO_RS_HOST: str | None = Field(
+        description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    PGVECTO_RS_PORT: PositiveInt = Field(
+        description="Port number on which the PostgreSQL server with PGVecto.RS is listening (default is 5431)",
+        default=5431,
+    )
+
+    PGVECTO_RS_USER: str | None = Field(
+        description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
+        default=None,
+    )
+
+    PGVECTO_RS_PASSWORD: str | None = Field(
+        description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
+        default=None,
+    )
+
+    PGVECTO_RS_DATABASE: str | None = Field(
+        description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
+        default=None,
+    )

dify/api/configs/middleware/vdb/qdrant_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class QdrantConfig(BaseSettings):
+    """
+    Configuration settings for Qdrant vector database
+    """
+
+    QDRANT_URL: str | None = Field(
+        description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
+        default=None,
+    )
+
+    QDRANT_API_KEY: str | None = Field(
+        description="API key for authenticating with the Qdrant server",
+        default=None,
+    )
+
+    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description="Timeout in seconds for Qdrant client operations (default is 20 seconds)",
+        default=20,
+    )
+
+    QDRANT_GRPC_ENABLED: bool = Field(
+        description="Whether to enable gRPC support for Qdrant connection (True for gRPC, False for HTTP)",
+        default=False,
+    )
+
+    QDRANT_GRPC_PORT: PositiveInt = Field(
+        description="Port number for gRPC connection to Qdrant server (default is 6334)",
+        default=6334,
+    )
+
+    QDRANT_REPLICATION_FACTOR: PositiveInt = Field(
+        description="Replication factor for Qdrant collections (default is 1)",
+        default=1,
+    )

dify/api/configs/middleware/vdb/relyt_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RelytConfig(BaseSettings):
+    """
+    Configuration settings for Relyt database
+    """
+
+    RELYT_HOST: str | None = Field(
+        description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
+        default=None,
+    )
+
+    RELYT_PORT: PositiveInt = Field(
+        description="Port number on which the Relyt server is listening (default is 9200)",
+        default=9200,
+    )
+
+    RELYT_USER: str | None = Field(
+        description="Username for authenticating with the Relyt database",
+        default=None,
+    )
+
+    RELYT_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Relyt database",
+        default=None,
+    )
+
+    RELYT_DATABASE: str | None = Field(
+        description="Name of the Relyt database to connect to (default is 'default')",
+        default="default",
+    )

dify/api/configs/middleware/vdb/tablestore_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class TableStoreConfig(BaseSettings):
+    """
+    Configuration settings for TableStore.
+    """
+
+    TABLESTORE_ENDPOINT: str | None = Field(
+        description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')",
+        default=None,
+    )
+
+    TABLESTORE_INSTANCE_NAME: str | None = Field(
+        description="Instance name to access TableStore server (eg. 'instance-name')",
+        default=None,
+    )
+
+    TABLESTORE_ACCESS_KEY_ID: str | None = Field(
+        description="AccessKey id for the instance name",
+        default=None,
+    )
+
+    TABLESTORE_ACCESS_KEY_SECRET: str | None = Field(
+        description="AccessKey secret for the instance name",
+        default=None,
+    )
+
+    TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: bool = Field(
+        description="Whether to normalize full-text search scores to [0, 1]",
+        default=False,
+    )

dify/api/configs/middleware/vdb/tencent_vector_config.py

@@ -0,0 +1,53 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TencentVectorDBConfig(BaseSettings):
+    """
+    Configuration settings for Tencent Vector Database
+    """
+
+    TENCENT_VECTOR_DB_URL: str | None = Field(
+        description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_API_KEY: str | None = Field(
+        description="API key for authenticating with the Tencent Vector Database service",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
+        description="Timeout in seconds for Tencent Vector Database operations (default is 30 seconds)",
+        default=30,
+    )
+
+    TENCENT_VECTOR_DB_USERNAME: str | None = Field(
+        description="Username for authenticating with the Tencent Vector Database (if required)",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Tencent Vector Database (if required)",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
+        description="Number of shards for the Tencent Vector Database (default is 1)",
+        default=1,
+    )
+
+    TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+        description="Number of replicas for the Tencent Vector Database (default is 2)",
+        default=2,
+    )
+
+    TENCENT_VECTOR_DB_DATABASE: str | None = Field(
+        description="Name of the specific Tencent Vector Database to connect to",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features",
+        default=False,
+    )

dify/api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -0,0 +1,68 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TidbOnQdrantConfig(BaseSettings):
+    """
+    Tidb on Qdrant configs
+    """
+
+    TIDB_ON_QDRANT_URL: str | None = Field(
+        description="Tidb on Qdrant url",
+        default=None,
+    )
+
+    TIDB_ON_QDRANT_API_KEY: str | None = Field(
+        description="Tidb on Qdrant api key",
+        default=None,
+    )
+
+    TIDB_ON_QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description="Tidb on Qdrant client timeout in seconds",
+        default=20,
+    )
+
+    TIDB_ON_QDRANT_GRPC_ENABLED: bool = Field(
+        description="whether enable grpc support for Tidb on Qdrant connection",
+        default=False,
+    )
+
+    TIDB_ON_QDRANT_GRPC_PORT: PositiveInt = Field(
+        description="Tidb on Qdrant grpc port",
+        default=6334,
+    )
+
+    TIDB_PUBLIC_KEY: str | None = Field(
+        description="Tidb account public key",
+        default=None,
+    )
+
+    TIDB_PRIVATE_KEY: str | None = Field(
+        description="Tidb account private key",
+        default=None,
+    )
+
+    TIDB_API_URL: str | None = Field(
+        description="Tidb API url",
+        default=None,
+    )
+
+    TIDB_IAM_API_URL: str | None = Field(
+        description="Tidb IAM API url",
+        default=None,
+    )
+
+    TIDB_REGION: str | None = Field(
+        description="Tidb serverless region",
+        default="regions/aws-us-east-1",
+    )
+
+    TIDB_PROJECT_ID: str | None = Field(
+        description="Tidb project id",
+        default=None,
+    )
+
+    TIDB_SPEND_LIMIT: int | None = Field(
+        description="Tidb spend limit",
+        default=100,
+    )

dify/api/configs/middleware/vdb/tidb_vector_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TiDBVectorConfig(BaseSettings):
+    """
+    Configuration settings for TiDB Vector database
+    """
+
+    TIDB_VECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
+        default=None,
+    )
+
+    TIDB_VECTOR_PORT: PositiveInt | None = Field(
+        description="Port number on which the TiDB Vector server is listening (default is 4000)",
+        default=4000,
+    )
+
+    TIDB_VECTOR_USER: str | None = Field(
+        description="Username for authenticating with the TiDB Vector database",
+        default=None,
+    )
+
+    TIDB_VECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the TiDB Vector database",
+        default=None,
+    )
+
+    TIDB_VECTOR_DATABASE: str | None = Field(
+        description="Name of the TiDB Vector database to connect to",
+        default=None,
+    )

dify/api/configs/middleware/vdb/upstash_config.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class UpstashConfig(BaseSettings):
+    """
+    Configuration settings for Upstash vector database
+    """
+
+    UPSTASH_VECTOR_URL: str | None = Field(
+        description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
+        default=None,
+    )
+
+    UPSTASH_VECTOR_TOKEN: str | None = Field(
+        description="Token for authenticating with the upstash server",
+        default=None,
+    )

dify/api/configs/middleware/vdb/vastbase_vector_config.py

@@ -0,0 +1,43 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class VastbaseVectorConfig(BaseSettings):
+    """
+    Configuration settings for Vector (Vastbase with vector extension)
+    """
+
+    VASTBASE_HOST: str | None = Field(
+        description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    VASTBASE_PORT: PositiveInt = Field(
+        description="Port number on which the Vastbase server is listening (default is 5432)",
+        default=5432,
+    )
+
+    VASTBASE_USER: str | None = Field(
+        description="Username for authenticating with the Vastbase database",
+        default=None,
+    )
+
+    VASTBASE_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Vastbase database",
+        default=None,
+    )
+
+    VASTBASE_DATABASE: str | None = Field(
+        description="Name of the Vastbase database to connect to",
+        default=None,
+    )
+
+    VASTBASE_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the Vastbase database",
+        default=1,
+    )
+
+    VASTBASE_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the Vastbase database",
+        default=5,
+    )

dify/api/configs/middleware/vdb/vikingdb_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class VikingDBConfig(BaseSettings):
+    """
+    Configuration for connecting to Volcengine VikingDB.
+    Refer to the following documentation for details on obtaining credentials:
+    https://www.volcengine.com/docs/6291/65568
+    """
+
+    VIKINGDB_ACCESS_KEY: str | None = Field(
+        description="The Access Key provided by Volcengine VikingDB for API authentication."
+        "Refer to the following documentation for details on obtaining credentials:"
+        "https://www.volcengine.com/docs/6291/65568",
+        default=None,
+    )
+
+    VIKINGDB_SECRET_KEY: str | None = Field(
+        description="The Secret Key provided by Volcengine VikingDB for API authentication.",
+        default=None,
+    )
+
+    VIKINGDB_REGION: str = Field(
+        description="The region of the Volcengine VikingDB service.(e.g., 'cn-shanghai', 'cn-beijing').",
+        default="cn-shanghai",
+    )
+
+    VIKINGDB_HOST: str = Field(
+        description="The host of the Volcengine VikingDB service.(e.g., 'api-vikingdb.volces.com', \
+            'api-vikingdb.mlp.cn-shanghai.volces.com')",
+        default="api-vikingdb.mlp.cn-shanghai.volces.com",
+    )
+
+    VIKINGDB_SCHEME: str = Field(
+        description="The scheme of the Volcengine VikingDB service.(e.g., 'http', 'https').",
+        default="http",
+    )
+
+    VIKINGDB_CONNECTION_TIMEOUT: int = Field(
+        description="The connection timeout of the Volcengine VikingDB service.",
+        default=30,
+    )
+
+    VIKINGDB_SOCKET_TIMEOUT: int = Field(
+        description="The socket timeout of the Volcengine VikingDB service.",
+        default=30,
+    )

dify/api/configs/middleware/vdb/weaviate_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class WeaviateConfig(BaseSettings):
+    """
+    Configuration settings for Weaviate vector database
+    """
+
+    WEAVIATE_ENDPOINT: str | None = Field(
+        description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
+        default=None,
+    )
+
+    WEAVIATE_API_KEY: str | None = Field(
+        description="API key for authenticating with the Weaviate server",
+        default=None,
+    )
+
+    WEAVIATE_GRPC_ENABLED: bool = Field(
+        description="Whether to enable gRPC for Weaviate connection (True for gRPC, False for HTTP)",
+        default=True,
+    )
+
+    WEAVIATE_GRPC_ENDPOINT: str | None = Field(
+        description="URL of the Weaviate gRPC server (e.g., 'grpc://localhost:50051' or 'grpcs://weaviate.example.com:443')",
+        default=None,
+    )
+
+    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
+        description="Number of objects to be processed in a single batch operation (default is 100)",
+        default=100,
+    )
+
+    WEAVIATE_TOKENIZATION: str | None = Field(
+        description="Tokenization for Weaviate (default is word)",
+        default="word",
+    )

dify/api/configs/observability/__init__.py

@@ -0,0 +1,9 @@
+from configs.observability.otel.otel_config import OTelConfig
+
+
+class ObservabilityConfig(OTelConfig):
+    """
+    Observability configuration settings
+    """
+
+    pass

dify/api/configs/observability/otel/otel_config.py

@@ -0,0 +1,59 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OTelConfig(BaseSettings):
+    """
+    OpenTelemetry configuration settings
+    """
+
+    ENABLE_OTEL: bool = Field(
+        description="Whether to enable OpenTelemetry",
+        default=False,
+    )
+
+    OTLP_TRACE_ENDPOINT: str = Field(
+        description="OTLP trace endpoint",
+        default="",
+    )
+
+    OTLP_METRIC_ENDPOINT: str = Field(
+        description="OTLP metric endpoint",
+        default="",
+    )
+
+    OTLP_BASE_ENDPOINT: str = Field(
+        description="OTLP base endpoint",
+        default="http://localhost:4318",
+    )
+
+    OTLP_API_KEY: str = Field(
+        description="OTLP API key",
+        default="",
+    )
+
+    OTEL_EXPORTER_TYPE: str = Field(
+        description="OTEL exporter type",
+        default="otlp",
+    )
+
+    OTEL_EXPORTER_OTLP_PROTOCOL: str = Field(
+        description="OTLP exporter protocol ('grpc' or 'http')",
+        default="http",
+    )
+
+    OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
+
+    OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(
+        default=5000, description="Batch export schedule delay in milliseconds"
+    )
+
+    OTEL_MAX_QUEUE_SIZE: int = Field(default=2048, description="Maximum queue size for the batch span processor")
+
+    OTEL_MAX_EXPORT_BATCH_SIZE: int = Field(default=512, description="Maximum export batch size")
+
+    OTEL_METRIC_EXPORT_INTERVAL: int = Field(default=60000, description="Metric export interval in milliseconds")
+
+    OTEL_BATCH_EXPORT_TIMEOUT: int = Field(default=10000, description="Batch export timeout in milliseconds")
+
+    OTEL_METRIC_EXPORT_TIMEOUT: int = Field(default=30000, description="Metric export timeout in milliseconds")

dify/api/configs/packaging/__init__.py

@@ -0,0 +1,14 @@
+from pydantic import Field
+
+from configs.packaging.pyproject import PyProjectTomlConfig
+
+
+class PackagingInfo(PyProjectTomlConfig):
+    """
+    Packaging build information
+    """
+
+    COMMIT_SHA: str = Field(
+        description="SHA-1 checksum of the git commit used to build the app",
+        default="",
+    )

dify/api/configs/packaging/pyproject.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+from pydantic_settings import BaseSettings
+
+
+class PyProjectConfig(BaseModel):
+    version: str = Field(description="Dify version", default="")
+
+
+class PyProjectTomlConfig(BaseSettings):
+    """
+    configs in api/pyproject.toml
+    """
+
+    project: PyProjectConfig = Field(
+        description="configs in the project section of pyproject.toml",
+        default=PyProjectConfig(),
+    )

dify/api/configs/remote_settings_sources/__init__.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+
+from .apollo import ApolloSettingsSourceInfo
+from .base import RemoteSettingsSource
+from .enums import RemoteSettingsSourceName
+
+
+class RemoteSettingsSourceConfig(ApolloSettingsSourceInfo):
+    REMOTE_SETTINGS_SOURCE_NAME: RemoteSettingsSourceName | str = Field(
+        description="name of remote config source",
+        default="",
+    )
+
+
+__all__ = ["RemoteSettingsSource", "RemoteSettingsSourceConfig", "RemoteSettingsSourceName"]

dify/api/configs/remote_settings_sources/apollo/__init__.py

@@ -0,0 +1,55 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import Field
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings
+
+from configs.remote_settings_sources.base import RemoteSettingsSource
+
+from .client import ApolloClient
+
+
+class ApolloSettingsSourceInfo(BaseSettings):
+    """
+    Packaging build information
+    """
+
+    APOLLO_APP_ID: str | None = Field(
+        description="apollo app_id",
+        default=None,
+    )
+
+    APOLLO_CLUSTER: str | None = Field(
+        description="apollo cluster",
+        default=None,
+    )
+
+    APOLLO_CONFIG_URL: str | None = Field(
+        description="apollo config url",
+        default=None,
+    )
+
+    APOLLO_NAMESPACE: str | None = Field(
+        description="apollo namespace",
+        default=None,
+    )
+
+
+class ApolloSettingsSource(RemoteSettingsSource):
+    def __init__(self, configs: Mapping[str, Any]):
+        self.client = ApolloClient(
+            app_id=configs["APOLLO_APP_ID"],
+            cluster=configs["APOLLO_CLUSTER"],
+            config_url=configs["APOLLO_CONFIG_URL"],
+            start_hot_update=False,
+            _notification_map={configs["APOLLO_NAMESPACE"]: -1},
+        )
+        self.namespace = configs["APOLLO_NAMESPACE"]
+        self.remote_configs = self.client.get_all_dicts(self.namespace)
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        if not isinstance(self.remote_configs, dict):
+            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
+        field_value = self.remote_configs.get(field_name)
+        return field_value, field_name, False

dify/api/configs/remote_settings_sources/apollo/client.py

@@ -0,0 +1,306 @@
+import hashlib
+import json
+import logging
+import os
+import threading
+import time
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any
+
+from .python_3x import http_request, makedirs_wrapper
+from .utils import (
+    CONFIGURATIONS,
+    NAMESPACE_NAME,
+    NOTIFICATION_ID,
+    get_value_from_dict,
+    init_ip,
+    no_key_cache_key,
+    signature,
+    url_encode_wrapper,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ApolloClient:
+    def __init__(
+        self,
+        config_url: str,
+        app_id: str,
+        cluster: str = "default",
+        secret: str = "",
+        start_hot_update: bool = True,
+        change_listener: Callable[[str, str, str, Any], None] | None = None,
+        _notification_map: dict[str, int] | None = None,
+    ):
+        # Core routing parameters
+        self.config_url = config_url
+        self.cluster = cluster
+        self.app_id = app_id
+
+        # Non-core parameters
+        self.ip = init_ip()
+        self.secret = secret
+
+        # Check the parameter variables
+
+        # Private control variables
+        self._cycle_time = 5
+        self._stopping = False
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._no_key: dict[str, str] = {}
+        self._hash: dict[str, str] = {}
+        self._pull_timeout = 75
+        self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
+        self._long_poll_thread: threading.Thread | None = None
+        self._change_listener = change_listener  # "add" "delete" "update"
+        if _notification_map is None:
+            _notification_map = {"application": -1}
+        self._notification_map = _notification_map
+        self.last_release_key: str | None = None
+        # Private startup method
+        self._path_checker()
+        if start_hot_update:
+            self._start_hot_update()
+
+        # start the heartbeat thread
+        heartbeat = threading.Thread(target=self._heart_beat)
+        heartbeat.daemon = True
+        heartbeat.start()
+
+    def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None:
+        url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
+            self.config_url, self.app_id, self.cluster, namespace, "", self.ip
+        )
+        try:
+            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
+            if code == 200:
+                if not body:
+                    logger.error("get_json_from_net load configs failed, body is %s", body)
+                    return None
+                data = json.loads(body)
+                data = data["configurations"]
+                return_data = {CONFIGURATIONS: data}
+                return return_data
+            else:
+                return None
+        except Exception:
+            logger.exception("an error occurred in get_json_from_net")
+            return None
+
+    def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any:
+        try:
+            # read memory configuration
+            namespace_cache = self._cache.get(namespace)
+            val = get_value_from_dict(namespace_cache, key)
+            if val is not None:
+                return val
+
+            no_key = no_key_cache_key(namespace, key)
+            if no_key in self._no_key:
+                return default_val
+
+            # read the network configuration
+            namespace_data = self.get_json_from_net(namespace)
+            val = get_value_from_dict(namespace_data, key)
+            if val is not None:
+                if namespace_data is not None:
+                    self._update_cache_and_file(namespace_data, namespace)
+                return val
+
+            # read the file configuration
+            namespace_cache = self._get_local_cache(namespace)
+            val = get_value_from_dict(namespace_cache, key)
+            if val is not None:
+                self._update_cache_and_file(namespace_cache, namespace)
+                return val
+
+            # If all of them are not obtained, the default value is returned
+            # and the local cache is set to None
+            self._set_local_cache_none(namespace, key)
+            return default_val
+        except Exception:
+            logger.exception("get_value has error, [key is %s], [namespace is %s]", key, namespace)
+            return default_val
+
+    # Set the key of a namespace to none, and do not set default val
+    # to ensure the real-time correctness of the function call.
+    # If the user does not have the same default val twice
+    # and the default val is used here, there may be a problem.
+    def _set_local_cache_none(self, namespace: str, key: str) -> None:
+        no_key = no_key_cache_key(namespace, key)
+        self._no_key[no_key] = key
+
+    def _start_hot_update(self) -> None:
+        self._long_poll_thread = threading.Thread(target=self._listener)
+        # When the asynchronous thread is started, the daemon thread will automatically exit
+        # when the main thread is launched.
+        self._long_poll_thread.daemon = True
+        self._long_poll_thread.start()
+
+    def stop(self) -> None:
+        self._stopping = True
+        logger.info("Stopping listener...")
+
+    # Call the set callback function, and if it is abnormal, try it out
+    def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None:
+        if self._change_listener is None:
+            return
+        if old_kv is None:
+            old_kv = {}
+        if new_kv is None:
+            new_kv = {}
+        try:
+            for key in old_kv:
+                new_value = new_kv.get(key)
+                old_value = old_kv.get(key)
+                if new_value is None:
+                    # If newValue is empty, it means key, and the value is deleted.
+                    self._change_listener("delete", namespace, key, old_value)
+                    continue
+                if new_value != old_value:
+                    self._change_listener("update", namespace, key, new_value)
+                    continue
+            for key in new_kv:
+                new_value = new_kv.get(key)
+                old_value = old_kv.get(key)
+                if old_value is None:
+                    self._change_listener("add", namespace, key, new_value)
+        except BaseException as e:
+            logger.warning(str(e))
+
+    def _path_checker(self) -> None:
+        if not os.path.isdir(self._cache_file_path):
+            makedirs_wrapper(self._cache_file_path)
+
+    # update the local cache and file cache
+    def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None:
+        # update the local cache
+        self._cache[namespace] = namespace_data
+        # update the file cache
+        new_string = json.dumps(namespace_data)
+        new_hash = hashlib.md5(new_string.encode("utf-8")).hexdigest()
+        if self._hash.get(namespace) == new_hash:
+            pass
+        else:
+            file_path = Path(self._cache_file_path) / f"{self.app_id}_configuration_{namespace}.txt"
+            file_path.write_text(new_string)
+            self._hash[namespace] = new_hash
+
+    # get the configuration from the local file
+    def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]:
+        cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
+        if os.path.isfile(cache_file_path):
+            with open(cache_file_path) as f:
+                result = json.loads(f.readline())
+            return result
+        return {}
+
+    def _long_poll(self) -> None:
+        notifications: list[dict[str, Any]] = []
+        for key in self._cache:
+            namespace_data = self._cache[key]
+            notification_id = -1
+            if NOTIFICATION_ID in namespace_data:
+                notification_id = self._cache[key][NOTIFICATION_ID]
+            notifications.append({NAMESPACE_NAME: key, NOTIFICATION_ID: notification_id})
+        try:
+            # if the length is 0 it is returned directly
+            if len(notifications) == 0:
+                return
+            url = f"{self.config_url}/notifications/v2"
+            params = {
+                "appId": self.app_id,
+                "cluster": self.cluster,
+                "notifications": json.dumps(notifications, ensure_ascii=False),
+            }
+            param_str = url_encode_wrapper(params)
+            url = url + "?" + param_str
+            code, body = http_request(url, self._pull_timeout, headers=self._sign_headers(url))
+            http_code = code
+            if http_code == 304:
+                logger.debug("No change, loop...")
+                return
+            if http_code == 200:
+                if not body:
+                    logger.error("_long_poll load configs failed,body is %s", body)
+                    return
+                data = json.loads(body)
+                for entry in data:
+                    namespace = entry[NAMESPACE_NAME]
+                    n_id = entry[NOTIFICATION_ID]
+                    logger.info("%s has changes: notificationId=%d", namespace, n_id)
+                    self._get_net_and_set_local(namespace, n_id, call_change=True)
+                    return
+            else:
+                logger.warning("Sleep...")
+        except Exception as e:
+            logger.warning(str(e))
+
+    def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None:
+        namespace_data = self.get_json_from_net(namespace)
+        if not namespace_data:
+            return
+        namespace_data[NOTIFICATION_ID] = n_id
+        old_namespace = self._cache.get(namespace)
+        self._update_cache_and_file(namespace_data, namespace)
+        if self._change_listener is not None and call_change and old_namespace:
+            old_kv = old_namespace.get(CONFIGURATIONS)
+            new_kv = namespace_data.get(CONFIGURATIONS)
+            self._call_listener(namespace, old_kv, new_kv)
+
+    def _listener(self) -> None:
+        logger.info("start long_poll")
+        while not self._stopping:
+            self._long_poll()
+            time.sleep(self._cycle_time)
+        logger.info("stopped, long_poll")
+
+    # add the need for endorsement to the header
+    def _sign_headers(self, url: str) -> Mapping[str, str]:
+        headers: dict[str, str] = {}
+        if self.secret == "":
+            return headers
+        uri = url[len(self.config_url) : len(url)]
+        time_unix_now = str(int(round(time.time() * 1000)))
+        headers["Authorization"] = "Apollo " + self.app_id + ":" + signature(time_unix_now, uri, self.secret)
+        headers["Timestamp"] = time_unix_now
+        return headers
+
+    def _heart_beat(self) -> None:
+        while not self._stopping:
+            for namespace in self._notification_map:
+                self._do_heart_beat(namespace)
+            time.sleep(60 * 10)  # 10 minutes
+
+    def _do_heart_beat(self, namespace: str) -> None:
+        url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}"
+        try:
+            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
+            if code == 200:
+                if not body:
+                    logger.error("_do_heart_beat load configs failed,body is %s", body)
+                    return None
+                data = json.loads(body)
+                if self.last_release_key == data["releaseKey"]:
+                    return None
+                self.last_release_key = data["releaseKey"]
+                data = data["configurations"]
+                self._update_cache_and_file(data, namespace)
+            else:
+                return None
+        except Exception:
+            logger.exception("an error occurred in _do_heart_beat")
+            return None
+
+    def get_all_dicts(self, namespace: str) -> dict[str, Any] | None:
+        namespace_data = self._cache.get(namespace)
+        if namespace_data is None:
+            net_namespace_data = self.get_json_from_net(namespace)
+            if not net_namespace_data:
+                return namespace_data
+            namespace_data = net_namespace_data.get(CONFIGURATIONS)
+            if namespace_data:
+                self._update_cache_and_file(namespace_data, namespace)
+        return namespace_data

dify/api/configs/remote_settings_sources/apollo/python_3x.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import ssl
+import urllib.request
+from collections.abc import Mapping
+from typing import Any
+from urllib import parse
+from urllib.error import HTTPError
+
+# Create an SSL context that allows for a lower level of security
+ssl_context = ssl.create_default_context()
+ssl_context.set_ciphers("HIGH:!DH:!aNULL")
+ssl_context.check_hostname = False
+ssl_context.verify_mode = ssl.CERT_NONE
+
+# Create an opener object and pass in a custom SSL context
+opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ssl_context))
+
+urllib.request.install_opener(opener)
+
+logger = logging.getLogger(__name__)
+
+
+def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]:
+    try:
+        request = urllib.request.Request(url, headers=dict(headers))
+        res = urllib.request.urlopen(request, timeout=timeout)
+        body = res.read().decode("utf-8")
+        return res.code, body
+    except HTTPError as e:
+        if e.code == 304:
+            logger.warning("http_request error,code is 304, maybe you should check secret")
+            return 304, None
+        logger.warning("http_request error,code is %d, msg is %s", e.code, e.msg)
+        raise e
+
+
+def url_encode(params: dict[str, Any]) -> str:
+    return parse.urlencode(params)
+
+
+def makedirs_wrapper(path: str) -> None:
+    os.makedirs(path, exist_ok=True)

dify/api/configs/remote_settings_sources/apollo/utils.py

@@ -0,0 +1,52 @@
+import hashlib
+import socket
+from typing import Any
+
+from .python_3x import url_encode
+
+# define constants
+CONFIGURATIONS = "configurations"
+NOTIFICATION_ID = "notificationId"
+NAMESPACE_NAME = "namespaceName"
+
+
+# add timestamps uris and keys
+def signature(timestamp: str, uri: str, secret: str) -> str:
+    import base64
+    import hmac
+
+    string_to_sign = "" + timestamp + "\n" + uri
+    hmac_code = hmac.new(secret.encode(), string_to_sign.encode(), hashlib.sha1).digest()
+    return base64.b64encode(hmac_code).decode()
+
+
+def url_encode_wrapper(params: dict[str, Any]) -> str:
+    return url_encode(params)
+
+
+def no_key_cache_key(namespace: str, key: str) -> str:
+    return f"{namespace}{len(namespace)}{key}"
+
+
+# Returns whether the obtained value is obtained, and None if it does not
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any:
+    if namespace_cache:
+        kv_data = namespace_cache.get(CONFIGURATIONS)
+        if kv_data is None:
+            return None
+        if key in kv_data:
+            return kv_data[key]
+    return None
+
+
+def init_ip() -> str:
+    ip = ""
+    s = None
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 53))
+        ip = s.getsockname()[0]
+    finally:
+        if s:
+            s.close()
+    return ip

dify/api/configs/remote_settings_sources/base.py

@@ -0,0 +1,15 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic.fields import FieldInfo
+
+
+class RemoteSettingsSource:
+    def __init__(self, configs: Mapping[str, Any]):
+        pass
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        raise NotImplementedError
+
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool):
+        return value

dify/api/configs/remote_settings_sources/enums.py

@@ -0,0 +1,6 @@
+from enum import StrEnum
+
+
+class RemoteSettingsSourceName(StrEnum):
+    APOLLO = "apollo"
+    NACOS = "nacos"

dify/api/configs/remote_settings_sources/nacos/__init__.py

@@ -0,0 +1,49 @@
+import logging
+import os
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic.fields import FieldInfo
+
+from .http_request import NacosHttpClient
+
+logger = logging.getLogger(__name__)
+
+from configs.remote_settings_sources.base import RemoteSettingsSource
+
+from .utils import parse_config
+
+
+class NacosSettingsSource(RemoteSettingsSource):
+    def __init__(self, configs: Mapping[str, Any]):
+        self.configs = configs
+        self.remote_configs: dict[str, str] = {}
+        self.async_init()
+
+    def async_init(self) -> None:
+        data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
+        group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
+        tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
+
+        params = {"dataId": data_id, "group": group, "tenant": tenant}
+        try:
+            content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
+            self.remote_configs = self._parse_config(content)
+        except Exception:
+            logger.exception("[get-access-token] exception occurred")
+            raise
+
+    def _parse_config(self, content: str) -> dict[str, str]:
+        if not content:
+            return {}
+        try:
+            return parse_config(content)
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse config: {e}")
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        field_value = self.remote_configs.get(field_name)
+        if field_value is None:
+            return None, field_name, False
+
+        return field_value, field_name, False

dify/api/configs/remote_settings_sources/nacos/http_request.py

@@ -0,0 +1,90 @@
+import base64
+import hashlib
+import hmac
+import logging
+import os
+import time
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class NacosHttpClient:
+    def __init__(self):
+        self.username = os.getenv("DIFY_ENV_NACOS_USERNAME")
+        self.password = os.getenv("DIFY_ENV_NACOS_PASSWORD")
+        self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
+        self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
+        self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
+        self.token: str | None = None
+        self.token_ttl = 18000
+        self.token_expire_time: float = 0
+
+    def http_request(
+        self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None
+    ) -> str:
+        if headers is None:
+            headers = {}
+        if params is None:
+            params = {}
+        try:
+            self._inject_auth_info(headers, params)
+            response = httpx.request(method, url="http://" + self.server + url, headers=headers, params=params)
+            response.raise_for_status()
+            return response.text
+        except httpx.RequestError as e:
+            return f"Request to Nacos failed: {e}"
+
+    def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
+        headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
+
+        if module == "login":
+            return
+
+        ts = str(int(time.time() * 1000))
+
+        if self.ak and self.sk:
+            sign_str = self.get_sign_str(params["group"], params["tenant"], ts)
+            headers["Spas-AccessKey"] = self.ak
+            headers["Spas-Signature"] = self.__do_sign(sign_str, self.sk)
+            headers["timeStamp"] = ts
+        if self.username and self.password:
+            self.get_access_token(force_refresh=False)
+            if self.token is not None:
+                params["accessToken"] = self.token
+
+    def __do_sign(self, sign_str: str, sk: str) -> str:
+        return (
+            base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
+            .decode()
+            .strip()
+        )
+
+    def get_sign_str(self, group: str, tenant: str, ts: str) -> str:
+        sign_str = ""
+        if tenant:
+            sign_str = tenant + "+"
+        if group:
+            sign_str = sign_str + group + "+"
+        sign_str += ts  # Directly concatenate ts without conditional checks, because the nacos auth header forced it.
+        return sign_str
+
+    def get_access_token(self, force_refresh: bool = False) -> str | None:
+        current_time = time.time()
+        if self.token and not force_refresh and self.token_expire_time > current_time:
+            return self.token
+
+        params = {"username": self.username, "password": self.password}
+        url = "http://" + self.server + "/nacos/v1/auth/login"
+        try:
+            resp = httpx.request("POST", url, headers=None, params=params)
+            resp.raise_for_status()
+            response_data = resp.json()
+            self.token = response_data.get("accessToken")
+            self.token_ttl = response_data.get("tokenTtl", 18000)
+            self.token_expire_time = current_time + self.token_ttl - 10
+            return self.token
+        except Exception:
+            logger.exception("[get-access-token] exception occur")
+            raise

dify/api/configs/remote_settings_sources/nacos/utils.py

@@ -0,0 +1,31 @@
+def parse_config(content: str) -> dict[str, str]:
+    config: dict[str, str] = {}
+    if not content:
+        return config
+
+    for line in content.splitlines():
+        cleaned_line = line.strip()
+        if not cleaned_line or cleaned_line.startswith(("#", "!")):
+            continue
+
+        separator_index = -1
+        for i, c in enumerate(cleaned_line):
+            if c in ("=", ":") and (i == 0 or cleaned_line[i - 1] != "\\"):
+                separator_index = i
+                break
+
+        if separator_index == -1:
+            continue
+
+        key = cleaned_line[:separator_index].strip()
+        raw_value = cleaned_line[separator_index + 1 :].strip()
+
+        try:
+            decoded_value = bytes(raw_value, "utf-8").decode("unicode_escape")
+            decoded_value = decoded_value.replace(r"\=", "=").replace(r"\:", ":")
+        except UnicodeDecodeError:
+            decoded_value = raw_value
+
+        config[key] = decoded_value
+
+    return config