dify

dify/api/.dockerignore

@@ -0,0 +1,19 @@
+.env
+*.env.*
+
+storage/generate_files/*
+storage/privkeys/*
+storage/tools/*
+storage/upload_files/*
+
+# Logs
+logs
+*.log*
+
+# jetbrains
+.idea
+.mypy_cache
+.ruff_cache
+
+# venv
+.venv

dify/api/.env.example

@@ -0,0 +1,639 @@
+# Your App secret key will be used for securely signing the session cookie
+# Make sure you are changing this key for your deployment with a strong key.
+# You can generate a strong key using `openssl rand -base64 42`.
+# Alternatively you can set it with `SECRET_KEY` environment variable.
+SECRET_KEY=
+
+# Ensure UTF-8 encoding
+LANG=en_US.UTF-8
+LC_ALL=en_US.UTF-8
+PYTHONIOENCODING=utf-8
+
+# Console API base URL
+CONSOLE_API_URL=http://localhost:5001
+CONSOLE_WEB_URL=http://localhost:3000
+
+# Service API base URL
+SERVICE_API_URL=http://localhost:5001
+
+# Web APP base URL
+APP_WEB_URL=http://localhost:3000
+
+# Files URL
+FILES_URL=http://localhost:5001
+
+# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network.
+# Set this to the internal Docker service URL for proper plugin file access.
+# Example: INTERNAL_FILES_URL=http://api:5001
+INTERNAL_FILES_URL=http://127.0.0.1:5001
+
+# TRIGGER URL
+TRIGGER_URL=http://localhost:5001
+
+# The time in seconds after the signature is rejected
+FILES_ACCESS_TIMEOUT=300
+
+# Access token expiration time in minutes
+ACCESS_TOKEN_EXPIRE_MINUTES=60
+
+# Refresh token expiration time in days
+REFRESH_TOKEN_EXPIRE_DAYS=30
+
+# redis configuration
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_USERNAME=
+REDIS_PASSWORD=difyai123456
+REDIS_USE_SSL=false
+# SSL configuration for Redis (when REDIS_USE_SSL=true)
+REDIS_SSL_CERT_REQS=CERT_NONE
+# Options: CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
+REDIS_SSL_CA_CERTS=
+# Path to CA certificate file for SSL verification
+REDIS_SSL_CERTFILE=
+# Path to client certificate file for SSL authentication
+REDIS_SSL_KEYFILE=
+# Path to client private key file for SSL authentication
+REDIS_DB=0
+
+# redis Sentinel configuration.
+REDIS_USE_SENTINEL=false
+REDIS_SENTINELS=
+REDIS_SENTINEL_SERVICE_NAME=
+REDIS_SENTINEL_USERNAME=
+REDIS_SENTINEL_PASSWORD=
+REDIS_SENTINEL_SOCKET_TIMEOUT=0.1
+
+# redis Cluster configuration.
+REDIS_USE_CLUSTERS=false
+REDIS_CLUSTERS=
+REDIS_CLUSTERS_PASSWORD=
+
+# celery configuration
+CELERY_BROKER_URL=redis://:difyai123456@localhost:${REDIS_PORT}/1
+CELERY_BACKEND=redis
+
+# Database configuration
+DB_TYPE=postgresql
+DB_USERNAME=postgres
+DB_PASSWORD=difyai123456
+DB_HOST=localhost
+DB_PORT=5432
+DB_DATABASE=dify
+
+SQLALCHEMY_POOL_PRE_PING=true
+SQLALCHEMY_POOL_TIMEOUT=30
+
+# Storage configuration
+# use for store upload files, private keys...
+# storage type: opendal, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
+STORAGE_TYPE=opendal
+
+# Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal
+OPENDAL_SCHEME=fs
+OPENDAL_FS_ROOT=storage
+
+# S3 Storage configuration
+S3_USE_AWS_MANAGED_IAM=false
+S3_ENDPOINT=https://your-bucket-name.storage.s3.cloudflare.com
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY=your-access-key
+S3_SECRET_KEY=your-secret-key
+S3_REGION=your-region
+
+# Azure Blob Storage configuration
+AZURE_BLOB_ACCOUNT_NAME=your-account-name
+AZURE_BLOB_ACCOUNT_KEY=your-account-key
+AZURE_BLOB_CONTAINER_NAME=your-container-name
+AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
+
+# Aliyun oss Storage configuration
+ALIYUN_OSS_BUCKET_NAME=your-bucket-name
+ALIYUN_OSS_ACCESS_KEY=your-access-key
+ALIYUN_OSS_SECRET_KEY=your-secret-key
+ALIYUN_OSS_ENDPOINT=your-endpoint
+ALIYUN_OSS_AUTH_VERSION=v1
+ALIYUN_OSS_REGION=your-region
+# Don't start with '/'. OSS doesn't support leading slash in object names.
+ALIYUN_OSS_PATH=your-path
+
+# Google Storage configuration
+GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
+
+# Tencent COS Storage configuration
+TENCENT_COS_BUCKET_NAME=your-bucket-name
+TENCENT_COS_SECRET_KEY=your-secret-key
+TENCENT_COS_SECRET_ID=your-secret-id
+TENCENT_COS_REGION=your-region
+TENCENT_COS_SCHEME=your-scheme
+
+# Huawei OBS Storage Configuration
+HUAWEI_OBS_BUCKET_NAME=your-bucket-name
+HUAWEI_OBS_SECRET_KEY=your-secret-key
+HUAWEI_OBS_ACCESS_KEY=your-access-key
+HUAWEI_OBS_SERVER=your-server-url
+
+# Baidu OBS Storage Configuration
+BAIDU_OBS_BUCKET_NAME=your-bucket-name
+BAIDU_OBS_SECRET_KEY=your-secret-key
+BAIDU_OBS_ACCESS_KEY=your-access-key
+BAIDU_OBS_ENDPOINT=your-server-url
+
+# OCI Storage configuration
+OCI_ENDPOINT=your-endpoint
+OCI_BUCKET_NAME=your-bucket-name
+OCI_ACCESS_KEY=your-access-key
+OCI_SECRET_KEY=your-secret-key
+OCI_REGION=your-region
+
+# Volcengine tos Storage configuration
+VOLCENGINE_TOS_ENDPOINT=your-endpoint
+VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name
+VOLCENGINE_TOS_ACCESS_KEY=your-access-key
+VOLCENGINE_TOS_SECRET_KEY=your-secret-key
+VOLCENGINE_TOS_REGION=your-region
+
+# Supabase Storage Configuration
+SUPABASE_BUCKET_NAME=your-bucket-name
+SUPABASE_API_KEY=your-access-key
+SUPABASE_URL=your-server-url
+
+# CORS configuration
+WEB_API_CORS_ALLOW_ORIGINS=http://localhost:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://localhost:3000,*
+# When the frontend and backend run on different subdomains, set COOKIE_DOMAIN to the site’s top-level domain (e.g., `example.com`). Leading dots are optional.
+COOKIE_DOMAIN=
+
+# Vector database configuration
+# Supported values are `weaviate`, `oceanbase`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`,  `opengauss`, `tablestore`,`vastbase`,`tidb`,`tidb_on_qdrant`,`baidu`,`lindorm`,`huawei_cloud`,`upstash`, `matrixone`.
+VECTOR_STORE=weaviate
+# Prefix used to create collection name in vector database
+VECTOR_INDEX_NAME_PREFIX=Vector_index
+
+# Weaviate configuration
+WEAVIATE_ENDPOINT=http://localhost:8080
+WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+WEAVIATE_GRPC_ENABLED=false
+WEAVIATE_BATCH_SIZE=100
+WEAVIATE_TOKENIZATION=word
+
+# OceanBase Vector configuration
+OCEANBASE_VECTOR_HOST=127.0.0.1
+OCEANBASE_VECTOR_PORT=2881
+OCEANBASE_VECTOR_USER=root@test
+OCEANBASE_VECTOR_PASSWORD=difyai123456
+OCEANBASE_VECTOR_DATABASE=test
+OCEANBASE_MEMORY_LIMIT=6G
+OCEANBASE_ENABLE_HYBRID_SEARCH=false
+OCEANBASE_FULLTEXT_PARSER=ik
+SEEKDB_MEMORY_LIMIT=2G
+
+# Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=difyai123456
+QDRANT_CLIENT_TIMEOUT=20
+QDRANT_GRPC_ENABLED=false
+QDRANT_GRPC_PORT=6334
+QDRANT_REPLICATION_FACTOR=1
+
+#Couchbase configuration
+COUCHBASE_CONNECTION_STRING=127.0.0.1
+COUCHBASE_USER=Administrator
+COUCHBASE_PASSWORD=password
+COUCHBASE_BUCKET_NAME=Embeddings
+COUCHBASE_SCOPE_NAME=_default
+
+# Milvus configuration
+MILVUS_URI=http://127.0.0.1:19530
+MILVUS_TOKEN=
+MILVUS_USER=root
+MILVUS_PASSWORD=Milvus
+MILVUS_ANALYZER_PARAMS=
+
+# MyScale configuration
+MYSCALE_HOST=127.0.0.1
+MYSCALE_PORT=8123
+MYSCALE_USER=default
+MYSCALE_PASSWORD=
+MYSCALE_DATABASE=default
+MYSCALE_FTS_PARAMS=
+
+# Relyt configuration
+RELYT_HOST=127.0.0.1
+RELYT_PORT=5432
+RELYT_USER=postgres
+RELYT_PASSWORD=postgres
+RELYT_DATABASE=postgres
+
+# Tencent configuration
+TENCENT_VECTOR_DB_URL=http://127.0.0.1
+TENCENT_VECTOR_DB_API_KEY=dify
+TENCENT_VECTOR_DB_TIMEOUT=30
+TENCENT_VECTOR_DB_USERNAME=dify
+TENCENT_VECTOR_DB_DATABASE=dify
+TENCENT_VECTOR_DB_SHARD=1
+TENCENT_VECTOR_DB_REPLICAS=2
+TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH=false
+
+# ElasticSearch configuration
+ELASTICSEARCH_HOST=127.0.0.1
+ELASTICSEARCH_PORT=9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=elastic
+
+# PGVECTO_RS configuration
+PGVECTO_RS_HOST=localhost
+PGVECTO_RS_PORT=5431
+PGVECTO_RS_USER=postgres
+PGVECTO_RS_PASSWORD=difyai123456
+PGVECTO_RS_DATABASE=postgres
+
+# PGVector configuration
+PGVECTOR_HOST=127.0.0.1
+PGVECTOR_PORT=5433
+PGVECTOR_USER=postgres
+PGVECTOR_PASSWORD=postgres
+PGVECTOR_DATABASE=postgres
+PGVECTOR_MIN_CONNECTION=1
+PGVECTOR_MAX_CONNECTION=5
+
+# TableStore Vector configuration
+TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com
+TABLESTORE_INSTANCE_NAME=instance-name
+TABLESTORE_ACCESS_KEY_ID=xxx
+TABLESTORE_ACCESS_KEY_SECRET=xxx
+TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE=false
+
+# Tidb Vector configuration
+TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com
+TIDB_VECTOR_PORT=4000
+TIDB_VECTOR_USER=xxx.root
+TIDB_VECTOR_PASSWORD=xxxxxx
+TIDB_VECTOR_DATABASE=dify
+
+# Tidb on qdrant configuration
+TIDB_ON_QDRANT_URL=http://127.0.0.1
+TIDB_ON_QDRANT_API_KEY=dify
+TIDB_ON_QDRANT_CLIENT_TIMEOUT=20
+TIDB_ON_QDRANT_GRPC_ENABLED=false
+TIDB_ON_QDRANT_GRPC_PORT=6334
+TIDB_PUBLIC_KEY=dify
+TIDB_PRIVATE_KEY=dify
+TIDB_API_URL=http://127.0.0.1
+TIDB_IAM_API_URL=http://127.0.0.1
+TIDB_REGION=regions/aws-us-east-1
+TIDB_PROJECT_ID=dify
+TIDB_SPEND_LIMIT=100
+
+# Chroma configuration
+CHROMA_HOST=127.0.0.1
+CHROMA_PORT=8000
+CHROMA_TENANT=default_tenant
+CHROMA_DATABASE=default_database
+CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
+CHROMA_AUTH_CREDENTIALS=difyai123456
+
+# AnalyticDB configuration
+ANALYTICDB_KEY_ID=your-ak
+ANALYTICDB_KEY_SECRET=your-sk
+ANALYTICDB_REGION_ID=cn-hangzhou
+ANALYTICDB_INSTANCE_ID=gp-ab123456
+ANALYTICDB_ACCOUNT=testaccount
+ANALYTICDB_PASSWORD=testpassword
+ANALYTICDB_NAMESPACE=dify
+ANALYTICDB_NAMESPACE_PASSWORD=difypassword
+ANALYTICDB_HOST=gp-test.aliyuncs.com
+ANALYTICDB_PORT=5432
+ANALYTICDB_MIN_CONNECTION=1
+ANALYTICDB_MAX_CONNECTION=5
+
+# OpenSearch configuration
+OPENSEARCH_HOST=127.0.0.1
+OPENSEARCH_PORT=9200
+OPENSEARCH_USER=admin
+OPENSEARCH_PASSWORD=admin
+OPENSEARCH_SECURE=true
+OPENSEARCH_VERIFY_CERTS=true
+
+# Baidu configuration
+BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287
+BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000
+BAIDU_VECTOR_DB_ACCOUNT=root
+BAIDU_VECTOR_DB_API_KEY=dify
+BAIDU_VECTOR_DB_DATABASE=dify
+BAIDU_VECTOR_DB_SHARD=1
+BAIDU_VECTOR_DB_REPLICAS=3
+BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER=DEFAULT_ANALYZER
+BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE=COARSE_MODE
+
+# Upstash configuration
+UPSTASH_VECTOR_URL=your-server-url
+UPSTASH_VECTOR_TOKEN=your-access-token
+
+# ViKingDB configuration
+VIKINGDB_ACCESS_KEY=your-ak
+VIKINGDB_SECRET_KEY=your-sk
+VIKINGDB_REGION=cn-shanghai
+VIKINGDB_HOST=api-vikingdb.xxx.volces.com
+VIKINGDB_SCHEMA=http
+VIKINGDB_CONNECTION_TIMEOUT=30
+VIKINGDB_SOCKET_TIMEOUT=30
+
+# Matrixone configration
+MATRIXONE_HOST=127.0.0.1
+MATRIXONE_PORT=6001
+MATRIXONE_USER=dump
+MATRIXONE_PASSWORD=111
+MATRIXONE_DATABASE=dify
+
+# Lindorm configuration
+LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
+LINDORM_USERNAME=admin
+LINDORM_PASSWORD=admin
+LINDORM_USING_UGC=True
+LINDORM_QUERY_TIMEOUT=1
+
+# AlibabaCloud MySQL Vector configuration
+ALIBABACLOUD_MYSQL_HOST=127.0.0.1
+ALIBABACLOUD_MYSQL_PORT=3306
+ALIBABACLOUD_MYSQL_USER=root
+ALIBABACLOUD_MYSQL_PASSWORD=root
+ALIBABACLOUD_MYSQL_DATABASE=dify
+ALIBABACLOUD_MYSQL_MAX_CONNECTION=5
+ALIBABACLOUD_MYSQL_HNSW_M=6
+
+# openGauss configuration
+OPENGAUSS_HOST=127.0.0.1
+OPENGAUSS_PORT=6600
+OPENGAUSS_USER=postgres
+OPENGAUSS_PASSWORD=Dify@123
+OPENGAUSS_DATABASE=dify
+OPENGAUSS_MIN_CONNECTION=1
+OPENGAUSS_MAX_CONNECTION=5
+
+# Upload configuration
+UPLOAD_FILE_SIZE_LIMIT=15
+UPLOAD_FILE_BATCH_LIMIT=5
+UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
+UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
+UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
+
+# Comma-separated list of file extensions blocked from upload for security reasons.
+# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll).
+# Empty by default to allow all file types.
+# Recommended: exe,bat,cmd,com,scr,vbs,ps1,msi,dll
+UPLOAD_FILE_EXTENSION_BLACKLIST=
+
+# Model configuration
+MULTIMODAL_SEND_FORMAT=base64
+PROMPT_GENERATION_MAX_TOKENS=512
+CODE_GENERATION_MAX_TOKENS=1024
+PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false
+
+# Mail configuration, support: resend, smtp, sendgrid
+MAIL_TYPE=
+# If using SendGrid, use the 'from' field for authentication if necessary.
+MAIL_DEFAULT_SEND_FROM=no-reply <no-reply@dify.ai>
+# resend configuration
+RESEND_API_KEY=
+RESEND_API_URL=https://api.resend.com
+# smtp configuration
+SMTP_SERVER=smtp.gmail.com
+SMTP_PORT=465
+SMTP_USERNAME=123
+SMTP_PASSWORD=abc
+SMTP_USE_TLS=true
+SMTP_OPPORTUNISTIC_TLS=false
+# Sendgid configuration
+SENDGRID_API_KEY=
+# Sentry configuration
+SENTRY_DSN=
+
+# DEBUG
+DEBUG=false
+ENABLE_REQUEST_LOGGING=False
+SQLALCHEMY_ECHO=false
+
+# Notion import configuration, support public and internal
+NOTION_INTEGRATION_TYPE=public
+NOTION_CLIENT_SECRET=you-client-secret
+NOTION_CLIENT_ID=you-client-id
+NOTION_INTERNAL_SECRET=you-internal-secret
+
+ETL_TYPE=dify
+UNSTRUCTURED_API_URL=
+UNSTRUCTURED_API_KEY=
+SCARF_NO_ANALYTICS=true
+
+#ssrf
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=
+SSRF_DEFAULT_MAX_RETRIES=3
+SSRF_DEFAULT_TIME_OUT=5
+SSRF_DEFAULT_CONNECT_TIME_OUT=5
+SSRF_DEFAULT_READ_TIME_OUT=5
+SSRF_DEFAULT_WRITE_TIME_OUT=5
+SSRF_POOL_MAX_CONNECTIONS=100
+SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+SSRF_POOL_KEEPALIVE_EXPIRY=5.0
+
+BATCH_UPLOAD_LIMIT=10
+KEYWORD_DATA_SOURCE_TYPE=database
+
+# Workflow file upload limit
+WORKFLOW_FILE_UPLOAD_LIMIT=10
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
+CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_EXECUTION_SSL_VERIFY=True
+CODE_EXECUTION_POOL_MAX_CONNECTIONS=100
+CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY=5.0
+CODE_EXECUTION_CONNECT_TIMEOUT=10
+CODE_EXECUTION_READ_TIMEOUT=60
+CODE_EXECUTION_WRITE_TIMEOUT=10
+CODE_MAX_NUMBER=9223372036854775807
+CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_STRING_LENGTH=400000
+TEMPLATE_TRANSFORM_MAX_LENGTH=400000
+CODE_MAX_STRING_ARRAY_LENGTH=30
+CODE_MAX_OBJECT_ARRAY_LENGTH=30
+CODE_MAX_NUMBER_ARRAY_LENGTH=1000
+
+# API Tool configuration
+API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
+API_TOOL_DEFAULT_READ_TIMEOUT=60
+
+# HTTP Node configuration
+HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300
+HTTP_REQUEST_MAX_READ_TIMEOUT=600
+HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
+HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
+HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
+HTTP_REQUEST_NODE_SSL_VERIFY=True
+
+# Webhook request configuration
+WEBHOOK_REQUEST_BODY_MAX_SIZE=10485760
+
+# Respect X-* headers to redirect clients
+RESPECT_XFORWARD_HEADERS_ENABLED=false
+
+# Log file path
+LOG_FILE=
+# Log file max size, the unit is MB
+LOG_FILE_MAX_SIZE=20
+# Log file max backup count
+LOG_FILE_BACKUP_COUNT=5
+# Log dateformat
+LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
+# Log Timezone
+LOG_TZ=UTC
+# Log format
+LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s
+
+# Indexing configuration
+INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
+
+# Workflow runtime configuration
+WORKFLOW_MAX_EXECUTION_STEPS=500
+WORKFLOW_MAX_EXECUTION_TIME=1200
+WORKFLOW_CALL_MAX_DEPTH=5
+MAX_VARIABLE_SIZE=204800
+
+# GraphEngine Worker Pool Configuration
+# Minimum number of workers per GraphEngine instance (default: 1)
+GRAPH_ENGINE_MIN_WORKERS=1
+# Maximum number of workers per GraphEngine instance (default: 10)
+GRAPH_ENGINE_MAX_WORKERS=10
+# Queue depth threshold that triggers worker scale up (default: 3)
+GRAPH_ENGINE_SCALE_UP_THRESHOLD=3
+# Seconds of idle time before scaling down workers (default: 5.0)
+GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0
+
+# Workflow storage configuration
+# Options: rdbms, hybrid
+# rdbms: Use only the relational database (default)
+# hybrid: Save new data to object storage, read from both object storage and RDBMS
+WORKFLOW_NODE_EXECUTION_STORAGE=rdbms
+
+# Repository configuration
+# Core workflow execution repository implementation
+CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository
+
+# Core workflow node execution repository implementation
+CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository
+
+# API workflow node execution repository implementation
+API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository
+
+# API workflow run repository implementation
+API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository
+# Workflow log cleanup configuration
+# Enable automatic cleanup of workflow run logs to manage database size
+WORKFLOW_LOG_CLEANUP_ENABLED=false
+# Number of days to retain workflow run logs (default: 30 days)
+WORKFLOW_LOG_RETENTION_DAYS=30
+# Batch size for workflow log cleanup operations (default: 100)
+WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
+
+# App configuration
+APP_MAX_EXECUTION_TIME=1200
+APP_MAX_ACTIVE_REQUESTS=0
+
+# Celery beat configuration
+CELERY_BEAT_SCHEDULER_TIME=1
+
+# Celery schedule tasks configuration
+ENABLE_CLEAN_EMBEDDING_CACHE_TASK=false
+ENABLE_CLEAN_UNUSED_DATASETS_TASK=false
+ENABLE_CREATE_TIDB_SERVERLESS_TASK=false
+ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false
+ENABLE_CLEAN_MESSAGES=false
+ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false
+ENABLE_DATASETS_QUEUE_MONITOR=false
+ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true
+ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK=true
+# Interval time in minutes for polling scheduled workflows(default: 1 min)
+WORKFLOW_SCHEDULE_POLLER_INTERVAL=1
+WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE=100
+# Maximum number of scheduled workflows to dispatch per tick (0 for unlimited)
+WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK=0
+
+# Position configuration
+POSITION_TOOL_PINS=
+POSITION_TOOL_INCLUDES=
+POSITION_TOOL_EXCLUDES=
+
+POSITION_PROVIDER_PINS=
+POSITION_PROVIDER_INCLUDES=
+POSITION_PROVIDER_EXCLUDES=
+
+# Plugin configuration
+PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
+PLUGIN_DAEMON_URL=http://127.0.0.1:5002
+PLUGIN_REMOTE_INSTALL_PORT=5003
+PLUGIN_REMOTE_INSTALL_HOST=localhost
+PLUGIN_MAX_PACKAGE_SIZE=15728640
+INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+
+# Marketplace configuration
+MARKETPLACE_ENABLED=true
+MARKETPLACE_API_URL=https://marketplace.dify.ai
+
+# Endpoint configuration
+ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
+
+# Reset password token expiry minutes
+RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5
+CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
+OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
+
+CREATE_TIDB_SERVICE_JOB_ENABLED=false
+
+# Maximum number of submitted thread count in a ThreadPool for parallel node execution
+MAX_SUBMIT_COUNT=100
+# Lockout duration in seconds
+LOGIN_LOCKOUT_DURATION=86400
+
+# Enable OpenTelemetry
+ENABLE_OTEL=false
+OTLP_TRACE_ENDPOINT=
+OTLP_METRIC_ENDPOINT=
+OTLP_BASE_ENDPOINT=http://localhost:4318
+OTLP_API_KEY=
+OTEL_EXPORTER_OTLP_PROTOCOL=
+OTEL_EXPORTER_TYPE=otlp
+OTEL_SAMPLING_RATE=0.1
+OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000
+OTEL_MAX_QUEUE_SIZE=2048
+OTEL_MAX_EXPORT_BATCH_SIZE=512
+OTEL_METRIC_EXPORT_INTERVAL=60000
+OTEL_BATCH_EXPORT_TIMEOUT=10000
+OTEL_METRIC_EXPORT_TIMEOUT=30000
+
+# Prevent Clickjacking
+ALLOW_EMBED=false
+
+# Dataset queue monitor configuration
+QUEUE_MONITOR_THRESHOLD=200
+# You can configure multiple ones, separated by commas. eg: test1@dify.ai,test2@dify.ai
+QUEUE_MONITOR_ALERT_EMAILS=
+# Monitor interval in minutes, default is 30 minutes
+QUEUE_MONITOR_INTERVAL=30
+
+# Swagger UI configuration
+SWAGGER_UI_ENABLED=true
+SWAGGER_UI_PATH=/swagger-ui.html
+
+# Whether to encrypt dataset IDs when exporting DSL files (default: true)
+# Set to false to export dataset IDs as plain text for easier cross-environment import
+DSL_EXPORT_ENCRYPT_DATASET_ID=true
+
+# Tenant isolated task queue configuration
+TENANT_ISOLATED_TASK_CONCURRENCY=1
+
+# Maximum number of segments for dataset segments API (0 for unlimited)
+DATASET_MAX_SEGMENTS_PER_REQUEST=0

dify/api/.importlinter

@@ -0,0 +1,105 @@
+[importlinter]
+root_packages =
+    core
+    configs
+    controllers
+    models
+    tasks
+    services
+
+[importlinter:contract:workflow]
+name = Workflow
+type=layers
+layers =
+    graph_engine
+    graph_events
+    graph
+    nodes
+    node_events
+    entities
+containers =
+    core.workflow
+ignore_imports =
+    core.workflow.nodes.base.node -> core.workflow.graph_events
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_events
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_events
+
+    core.workflow.nodes.node_factory -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine.command_channels
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels
+
+[importlinter:contract:rsc]
+name = RSC
+type = layers
+layers =
+    graph_engine
+    response_coordinator
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:worker]
+name = Worker
+type = layers
+layers =
+    graph_engine
+    worker
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:graph-engine-architecture]
+name = Graph Engine Architecture
+type = layers
+layers =
+    graph_engine
+    orchestration
+    command_processing
+    event_management
+    error_handler
+    graph_traversal
+    graph_state_manager
+    worker_management
+    domain
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:domain-isolation]
+name = Domain Model Isolation
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.domain
+forbidden_modules =
+    core.workflow.graph_engine.worker_management
+    core.workflow.graph_engine.command_channels
+    core.workflow.graph_engine.layers
+    core.workflow.graph_engine.protocols
+
+[importlinter:contract:worker-management]
+name = Worker Management
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.worker_management
+forbidden_modules =
+    core.workflow.graph_engine.orchestration
+    core.workflow.graph_engine.command_processing
+    core.workflow.graph_engine.event_management
+
+
+[importlinter:contract:graph-traversal-components]
+name = Graph Traversal Components
+type = layers
+layers =
+    edge_processor
+    skip_propagator
+containers =
+    core.workflow.graph_engine.graph_traversal
+
+[importlinter:contract:command-channels]
+name = Command Channels Independence
+type = independence
+modules =
+    core.workflow.graph_engine.command_channels.in_memory_channel
+    core.workflow.graph_engine.command_channels.redis_channel

dify/api/.ruff.toml

@@ -0,0 +1,113 @@
+exclude = ["migrations/*"]
+line-length = 120
+
+[format]
+quote-style = "double"
+
+[lint]
+preview = true
+select = [
+    "B",       # flake8-bugbear rules
+    "C4",      # flake8-comprehensions
+    "E",       # pycodestyle E rules
+    "F",       # pyflakes rules
+    "FURB",    # refurb rules
+    "I",       # isort rules
+    "N",       # pep8-naming
+    "PT",      # flake8-pytest-style rules
+    "PLC0208", # iteration-over-set
+    "PLC0414", # useless-import-alias
+    "PLE0604", # invalid-all-object
+    "PLE0605", # invalid-all-format
+    "PLR0402", # manual-from-import
+    "PLR1711", # useless-return
+    "PLR1714", # repeated-equality-comparison
+    "RUF013",  # implicit-optional
+    "RUF019",  # unnecessary-key-check
+    "RUF100",  # unused-noqa
+    "RUF101",  # redirected-noqa
+    "RUF200",  # invalid-pyproject-toml
+    "RUF022",  # unsorted-dunder-all
+    "S506",    # unsafe-yaml-load
+    "SIM",     # flake8-simplify rules
+    "T201",    # print-found
+    "TRY400",  # error-instead-of-exception
+    "TRY401",  # verbose-log-message
+    "UP",      # pyupgrade rules
+    "W191",    # tab-indentation
+    "W605",    # invalid-escape-sequence
+    # security related linting rules
+    # RCE proctection (sort of)
+    "S102", # exec-builtin, disallow use of `exec`
+    "S307", # suspicious-eval-usage, disallow use of `eval` and `ast.literal_eval`
+    "S301", # suspicious-pickle-usage, disallow use of `pickle` and its wrappers.
+    "S302", # suspicious-marshal-usage, disallow use of `marshal` module
+    "S311", # suspicious-non-cryptographic-random-usage
+    "G001", # don't use str format to logging messages
+    "G003", # don't use + in logging messages
+    "G004", # don't use f-strings to format logging messages
+    "UP042", # use StrEnum
+]
+
+ignore = [
+    "E402",    # module-import-not-at-top-of-file
+    "E711",    # none-comparison
+    "E712",    # true-false-comparison
+    "E721",    # type-comparison
+    "E722",    # bare-except
+    "F821",    # undefined-name
+    "F841",    # unused-variable
+    "FURB113", # repeated-append
+    "FURB152", # math-constant
+    "UP007",   # non-pep604-annotation
+    "UP032",   # f-string
+    "UP045",   # non-pep604-annotation-optional
+    "B005",    # strip-with-multi-characters
+    "B006",    # mutable-argument-default
+    "B007",    # unused-loop-control-variable
+    "B026",    # star-arg-unpacking-after-keyword-arg
+    "B901",    # allow return in yield
+    "B903",    # class-as-data-structure
+    "B904",    # raise-without-from-inside-except
+    "B905",    # zip-without-explicit-strict
+    "N806",    # non-lowercase-variable-in-function
+    "N815",    # mixed-case-variable-in-class-scope
+    "PT011",   # pytest-raises-too-broad
+    "SIM102",  # collapsible-if
+    "SIM103",  # needless-bool
+    "SIM105",  # suppressible-exception
+    "SIM107",  # return-in-try-except-finally
+    "SIM108",  # if-else-block-instead-of-if-exp
+    "SIM113",  # enumerate-for-loop
+    "SIM117",  # multiple-with-statements
+    "SIM210",  # if-expr-with-true-false
+]
+
+[lint.per-file-ignores]
+"__init__.py" = [
+    "F401", # unused-import
+    "F811", # redefined-while-unused
+]
+"configs/*" = [
+    "N802", # invalid-function-name
+]
+"core/model_runtime/callbacks/base_callback.py" = [
+    "T201",
+]
+"core/workflow/callbacks/workflow_logging_callback.py" = [
+    "T201",
+]
+"libs/gmpy2_pkcs10aep_cipher.py" = [
+    "N803", # invalid-argument-name
+]
+"tests/*" = [
+    "F811", # redefined-while-unused
+    "T201", # allow print in tests
+]
+
+[lint.pyflakes]
+allowed-unused-imports = [
+    "_pytest.monkeypatch",
+    "tests.integration_tests",
+    "tests.unit_tests",
+]

dify/api/AGENTS.md

@@ -0,0 +1,62 @@
+# Agent Skill Index
+
+Start with the section that best matches your need. Each entry lists the problems it solves plus key files/concepts so you know what to expect before opening it.
+
+______________________________________________________________________
+
+## Platform Foundations
+
+- **[Infrastructure Overview](agent_skills/infra.md)**\
+  When to read this:
+
+  - You need to understand where a feature belongs in the architecture.
+  - You’re wiring storage, Redis, vector stores, or OTEL.
+  - You’re about to add CLI commands or async jobs.\
+    What it covers: configuration stack (`configs/app_config.py`, remote settings), storage entry points (`extensions/ext_storage.py`, `core/file/file_manager.py`), Redis conventions (`extensions/ext_redis.py`), plugin runtime topology, vector-store factory (`core/rag/datasource/vdb/*`), observability hooks, SSRF proxy usage, and core CLI commands.
+
+- **[Coding Style](agent_skills/coding_style.md)**\
+  When to read this:
+
+  - You’re writing or reviewing backend code and need the authoritative checklist.
+  - You’re unsure about Pydantic validators, SQLAlchemy session usage, or logging patterns.
+  - You want the exact lint/type/test commands used in PRs.\
+    Includes: Ruff & BasedPyright commands, no-annotation policy, session examples (`with Session(db.engine, ...)`), `@field_validator` usage, logging expectations, and the rule set for file size, helpers, and package management.
+
+______________________________________________________________________
+
+## Plugin & Extension Development
+
+- **[Plugin Systems](agent_skills/plugin.md)**\
+  When to read this:
+
+  - You’re building or debugging a marketplace plugin.
+  - You need to know how manifests, providers, daemons, and migrations fit together.\
+    What it covers: plugin manifests (`core/plugin/entities/plugin.py`), installation/upgrade flows (`services/plugin/plugin_service.py`, CLI commands), runtime adapters (`core/plugin/impl/*` for tool/model/datasource/trigger/endpoint/agent), daemon coordination (`core/plugin/entities/plugin_daemon.py`), and how provider registries surface capabilities to the rest of the platform.
+
+- **[Plugin OAuth](agent_skills/plugin_oauth.md)**\
+  When to read this:
+
+  - You must integrate OAuth for a plugin or datasource.
+  - You’re handling credential encryption or refresh flows.\
+    Topics: credential storage, encryption helpers (`core/helper/provider_encryption.py`), OAuth client bootstrap (`services/plugin/oauth_service.py`, `services/plugin/plugin_parameter_service.py`), and how console/API layers expose the flows.
+
+______________________________________________________________________
+
+## Workflow Entry & Execution
+
+- **[Trigger Concepts](agent_skills/trigger.md)**\
+  When to read this:
+  - You’re debugging why a workflow didn’t start.
+  - You’re adding a new trigger type or hook.
+  - You need to trace async execution, draft debugging, or webhook/schedule pipelines.\
+    Details: Start-node taxonomy, webhook & schedule internals (`core/workflow/nodes/trigger_*`, `services/trigger/*`), async orchestration (`services/async_workflow_service.py`, Celery queues), debug event bus, and storage/logging interactions.
+
+______________________________________________________________________
+
+## Additional Notes for Agents
+
+- All skill docs assume you follow the coding style guide—run Ruff/BasedPyright/tests listed there before submitting changes.
+- When you cannot find an answer in these briefs, search the codebase using the paths referenced (e.g., `core/plugin/impl/tool.py`, `services/dataset_service.py`).
+- If you run into cross-cutting concerns (tenancy, configuration, storage), check the infrastructure guide first; it links to most supporting modules.
+- Keep multi-tenancy and configuration central: everything flows through `configs.dify_config` and `tenant_id`.
+- When touching plugins or triggers, consult both the system overview and the specialised doc to ensure you adjust lifecycle, storage, and observability consistently.

dify/api/Dockerfile

@@ -0,0 +1,101 @@
+# base image
+FROM python:3.12-slim-bookworm AS base
+
+WORKDIR /app/api
+
+# Install uv
+ENV UV_VERSION=0.8.9
+
+RUN pip install --no-cache-dir uv==${UV_VERSION}
+
+
+FROM base AS packages
+
+# if you located in China, you can use aliyun mirror to speed up
+# RUN sed -i 's@deb.debian.org@mirrors.aliyun.com@g' /etc/apt/sources.list.d/debian.sources
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+          # basic environment
+          g++ \
+          # for building gmpy2
+          libmpfr-dev libmpc-dev
+
+# Install Python dependencies
+COPY pyproject.toml uv.lock ./
+RUN uv sync --locked --no-dev
+
+# production stage
+FROM base AS production
+
+ENV FLASK_APP=app.py
+ENV EDITION=SELF_HOSTED
+ENV DEPLOY_ENV=PRODUCTION
+ENV CONSOLE_API_URL=http://127.0.0.1:5001
+ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
+ENV SERVICE_API_URL=http://127.0.0.1:5001
+ENV APP_WEB_URL=http://127.0.0.1:3000
+
+EXPOSE 5001
+
+# set timezone
+ENV TZ=UTC
+
+# Set UTF-8 locale
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+ENV PYTHONIOENCODING=utf-8
+
+WORKDIR /app/api
+
+RUN \
+    apt-get update \
+    # Install dependencies
+    && apt-get install -y --no-install-recommends \
+        # basic environment
+        curl nodejs \
+        # for gmpy2 \
+        libgmp-dev libmpfr-dev libmpc-dev \
+        # For Security
+        expat libldap-2.5-0=2.5.13+dfsg-5 perl libsqlite3-0=3.40.1-2+deb12u2 zlib1g=1:1.2.13.dfsg-1 \
+        # install fonts to support the use of tools like pypdfium2
+        fonts-noto-cjk \
+        # install a package to improve the accuracy of guessing mime type and file extension
+        media-types \
+        # install libmagic to support the use of python-magic guess MIMETYPE
+        libmagic1 \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy Python environment and packages
+ENV VIRTUAL_ENV=/app/api/.venv
+COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# Download nltk data
+RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords')" \
+    && chmod -R 755 /usr/local/share/nltk_data
+
+ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache
+
+RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')"
+
+# Copy source code
+COPY . /app/api/
+
+# Copy entrypoint
+COPY docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Create non-root user and set permissions
+RUN groupadd -r -g 1001 dify && \
+    useradd -r -u 1001 -g 1001 -s /bin/bash dify && \
+    mkdir -p /home/dify && \
+    chown -R 1001:1001 /app /home/dify ${TIKTOKEN_CACHE_DIR} /entrypoint.sh
+
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+ENV NLTK_DATA=/usr/local/share/nltk_data
+USER 1001
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

dify/api/README.md

@@ -0,0 +1,116 @@
+# Dify Backend API
+
+## Usage
+
+> [!IMPORTANT]
+>
+> In the v1.3.0 release, `poetry` has been replaced with
+> [`uv`](https://docs.astral.sh/uv/) as the package manager
+> for Dify API backend service.
+
+1. Start the docker-compose stack
+
+   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
+
+   ```bash
+   cd ../docker
+   cp middleware.env.example middleware.env
+   # change the profile to mysql if you are not using postgres,change the profile to other vector database if you are not using weaviate
+   docker compose -f docker-compose.middleware.yaml --profile postgresql --profile weaviate -p dify up -d
+   cd ../api
+   ```
+
+1. Copy `.env.example` to `.env`
+
+   ```cli
+   cp .env.example .env
+   ```
+
+> [!IMPORTANT]
+>
+> When the frontend and backend run on different subdomains, set COOKIE_DOMAIN to the site’s top-level domain (e.g., `example.com`). The frontend and backend must be under the same top-level domain in order to share authentication cookies.
+
+1. Generate a `SECRET_KEY` in the `.env` file.
+
+   bash for Linux
+
+   ```bash for Linux
+   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
+   ```
+
+   bash for Mac
+
+   ```bash for Mac
+   secret_key=$(openssl rand -base64 42)
+   sed -i '' "/^SECRET_KEY=/c\\
+   SECRET_KEY=${secret_key}" .env
+   ```
+
+1. Create environment.
+
+   Dify API service uses [UV](https://docs.astral.sh/uv/) to manage dependencies.
+   First, you need to add the uv package manager, if you don't have it already.
+
+   ```bash
+   pip install uv
+   # Or on macOS
+   brew install uv
+   ```
+
+1. Install dependencies
+
+   ```bash
+   uv sync --dev
+   ```
+
+1. Run migrate
+
+   Before the first launch, migrate the database to the latest version.
+
+   ```bash
+   uv run flask db upgrade
+   ```
+
+1. Start backend
+
+   ```bash
+   uv run flask run --host 0.0.0.0 --port=5001 --debug
+   ```
+
+1. Start Dify [web](../web) service.
+
+1. Setup your application by visiting `http://localhost:3000`.
+
+1. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
+
+```bash
+uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor
+```
+
+Additionally, if you want to debug the celery scheduled tasks, you can run the following command in another terminal to start the beat service:
+
+```bash
+uv run celery -A app.celery beat
+```
+
+## Testing
+
+1. Install dependencies for both the backend and the test environment
+
+   ```bash
+   uv sync --dev
+   ```
+
+1. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`, more can check [Claude.md](../CLAUDE.md)
+
+   ```bash
+   uv run pytest                           # Run all tests
+   uv run pytest tests/unit_tests/         # Unit tests only
+   uv run pytest tests/integration_tests/  # Integration tests
+
+   # Code quality
+   ../dev/reformat               # Run all formatters and linters
+   uv run ruff check --fix ./    # Fix linting issues
+   uv run ruff format ./         # Format code
+   uv run basedpyright .         # Type checking
+   ```

dify/api/agent_skills/coding_style.md

@@ -0,0 +1,115 @@
+## Linter
+
+- Always follow `.ruff.toml`.
+- Run `uv run ruff check --fix --unsafe-fixes`.
+- Keep each line under 100 characters (including spaces).
+
+## Code Style
+
+- `snake_case` for variables and functions.
+- `PascalCase` for classes.
+- `UPPER_CASE` for constants.
+
+## Rules
+
+- Use Pydantic v2 standard.
+- Use `uv` for package management.
+- Do not override dunder methods like `__init__`, `__iadd__`, etc.
+- Never launch services (`uv run app.py`, `flask run`, etc.); running tests under `tests/` is allowed.
+- Prefer simple functions over classes for lightweight helpers.
+- Keep files below 800 lines; split when necessary.
+- Keep code readable—no clever hacks.
+- Never use `print`; log with `logger = logging.getLogger(__name__)`.
+
+## Guiding Principles
+
+- Mirror the project’s layered architecture: controller → service → core/domain.
+- Reuse existing helpers in `core/`, `services/`, and `libs/` before creating new abstractions.
+- Optimise for observability: deterministic control flow, clear logging, actionable errors.
+
+## SQLAlchemy Patterns
+
+- Models inherit from `models.base.Base`; never create ad-hoc metadata or engines.
+
+- Open sessions with context managers:
+
+  ```python
+  from sqlalchemy.orm import Session
+
+  with Session(db.engine, expire_on_commit=False) as session:
+      stmt = select(Workflow).where(
+          Workflow.id == workflow_id,
+          Workflow.tenant_id == tenant_id,
+      )
+      workflow = session.execute(stmt).scalar_one_or_none()
+  ```
+
+- Use SQLAlchemy expressions; avoid raw SQL unless necessary.
+
+- Introduce repository abstractions only for very large tables (e.g., workflow executions) to support alternative storage strategies.
+
+- Always scope queries by `tenant_id` and protect write paths with safeguards (`FOR UPDATE`, row counts, etc.).
+
+## Storage & External IO
+
+- Access storage via `extensions.ext_storage.storage`.
+- Use `core.helper.ssrf_proxy` for outbound HTTP fetches.
+- Background tasks that touch storage must be idempotent and log the relevant object identifiers.
+
+## Pydantic Usage
+
+- Define DTOs with Pydantic v2 models and forbid extras by default.
+
+- Use `@field_validator` / `@model_validator` for domain rules.
+
+- Example:
+
+  ```python
+  from pydantic import BaseModel, ConfigDict, HttpUrl, field_validator
+
+  class TriggerConfig(BaseModel):
+      endpoint: HttpUrl
+      secret: str
+
+      model_config = ConfigDict(extra="forbid")
+
+      @field_validator("secret")
+      def ensure_secret_prefix(cls, value: str) -> str:
+          if not value.startswith("dify_"):
+              raise ValueError("secret must start with dify_")
+          return value
+  ```
+
+## Generics & Protocols
+
+- Use `typing.Protocol` to define behavioural contracts (e.g., cache interfaces).
+- Apply generics (`TypeVar`, `Generic`) for reusable utilities like caches or providers.
+- Validate dynamic inputs at runtime when generics cannot enforce safety alone.
+
+## Error Handling & Logging
+
+- Raise domain-specific exceptions (`services/errors`, `core/errors`) and translate to HTTP responses in controllers.
+- Declare `logger = logging.getLogger(__name__)` at module top.
+- Include tenant/app/workflow identifiers in log context.
+- Log retryable events at `warning`, terminal failures at `error`.
+
+## Tooling & Checks
+
+- Format/lint: `uv run --project api --dev ruff format ./api` and `uv run --project api --dev ruff check --fix --unsafe-fixes ./api`.
+- Type checks: `uv run --directory api --dev basedpyright`.
+- Tests: `uv run --project api --dev dev/pytest/pytest_unit_tests.sh`.
+- Run all of the above before submitting your work.
+
+## Controllers & Services
+
+- Controllers: parse input via Pydantic, invoke services, return serialised responses; no business logic.
+- Services: coordinate repositories, providers, background tasks; keep side effects explicit.
+- Avoid repositories unless necessary; direct SQLAlchemy usage is preferred for typical tables.
+- Document non-obvious behaviour with concise comments.
+
+## Miscellaneous
+
+- Use `configs.dify_config` for configuration—never read environment variables directly.
+- Maintain tenant awareness end-to-end; `tenant_id` must flow through every layer touching shared resources.
+- Queue async work through `services/async_workflow_service`; implement tasks under `tasks/` with explicit queue selection.
+- Keep experimental scripts under `dev/`; do not ship them in production builds.

dify/api/agent_skills/infra.md

@@ -0,0 +1,96 @@
+## Configuration
+
+- Import `configs.dify_config` for every runtime toggle. Do not read environment variables directly.
+- Add new settings to the proper mixin inside `configs/` (deployment, feature, middleware, etc.) so they load through `DifyConfig`.
+- Remote overrides come from the optional providers in `configs/remote_settings_sources`; keep defaults in code safe when the value is missing.
+- Example: logging pulls targets from `extensions/ext_logging.py`, and model provider URLs are assembled in `services/entities/model_provider_entities.py`.
+
+## Dependencies
+
+- Runtime dependencies live in `[project].dependencies` inside `pyproject.toml`. Optional clients go into the `storage`, `tools`, or `vdb` groups under `[dependency-groups]`.
+- Always pin versions and keep the list alphabetised. Shared tooling (lint, typing, pytest) belongs in the `dev` group.
+- When code needs a new package, explain why in the PR and run `uv lock` so the lockfile stays current.
+
+## Storage & Files
+
+- Use `extensions.ext_storage.storage` for all blob IO; it already respects the configured backend.
+- Convert files for workflows with helpers in `core/file/file_manager.py`; they handle signed URLs and multimodal payloads.
+- When writing controller logic, delegate upload quotas and metadata to `services/file_service.py` instead of touching storage directly.
+- All outbound HTTP fetches (webhooks, remote files) must go through the SSRF-safe client in `core/helper/ssrf_proxy.py`; it wraps `httpx` with the allow/deny rules configured for the platform.
+
+## Redis & Shared State
+
+- Access Redis through `extensions.ext_redis.redis_client`. For locking, reuse `redis_client.lock`.
+- Prefer higher-level helpers when available: rate limits use `libs.helper.RateLimiter`, provider metadata uses caches in `core/helper/provider_cache.py`.
+
+## Models
+
+- SQLAlchemy models sit in `models/` and inherit from the shared declarative `Base` defined in `models/base.py` (metadata configured via `models/engine.py`).
+- `models/__init__.py` exposes grouped aggregates: account/tenant models, app and conversation tables, datasets, providers, workflow runs, triggers, etc. Import from there to avoid deep path churn.
+- Follow the DDD boundary: persistence objects live in `models/`, repositories under `repositories/` translate them into domain entities, and services consume those repositories.
+- When adding a table, create the model class, register it in `models/__init__.py`, wire a repository if needed, and generate an Alembic migration as described below.
+
+## Vector Stores
+
+- Vector client implementations live in `core/rag/datasource/vdb/<provider>`, with a common factory in `core/rag/datasource/vdb/vector_factory.py` and enums in `core/rag/datasource/vdb/vector_type.py`.
+- Retrieval pipelines call these providers through `core/rag/datasource/retrieval_service.py` and dataset ingestion flows in `services/dataset_service.py`.
+- The CLI helper `flask vdb-migrate` orchestrates bulk migrations using routines in `commands.py`; reuse that pattern when adding new backend transitions.
+- To add another store, mirror the provider layout, register it with the factory, and include any schema changes in Alembic migrations.
+
+## Observability & OTEL
+
+- OpenTelemetry settings live under the observability mixin in `configs/observability`. Toggle exporters and sampling via `dify_config`, not ad-hoc env reads.
+- HTTP, Celery, Redis, SQLAlchemy, and httpx instrumentation is initialised in `extensions/ext_app_metrics.py` and `extensions/ext_request_logging.py`; reuse these hooks when adding new workers or entrypoints.
+- When creating background tasks or external calls, propagate tracing context with helpers in the existing instrumented clients (e.g. use the shared `httpx` session from `core/helper/http_client_pooling.py`).
+- If you add a new external integration, ensure spans and metrics are emitted by wiring the appropriate OTEL instrumentation package in `pyproject.toml` and configuring it in `extensions/`.
+
+## Ops Integrations
+
+- Langfuse support and other tracing bridges live under `core/ops/opik_trace`. Config toggles sit in `configs/observability`, while exporters are initialised in the OTEL extensions mentioned above.
+- External monitoring services should follow this pattern: keep client code in `core/ops`, expose switches via `dify_config`, and hook initialisation in `extensions/ext_app_metrics.py` or sibling modules.
+- Before instrumenting new code paths, check whether existing context helpers (e.g. `extensions/ext_request_logging.py`) already capture the necessary metadata.
+
+## Controllers, Services, Core
+
+- Controllers only parse HTTP input and call a service method. Keep business rules in `services/`.
+- Services enforce tenant rules, quotas, and orchestration, then call into `core/` engines (workflow execution, tools, LLMs).
+- When adding a new endpoint, search for an existing service to extend before introducing a new layer. Example: workflow APIs pipe through `services/workflow_service.py` into `core/workflow`.
+
+## Plugins, Tools, Providers
+
+- In Dify a plugin is a tenant-installable bundle that declares one or more providers (tool, model, datasource, trigger, endpoint, agent strategy) plus its resource needs and version metadata. The manifest (`core/plugin/entities/plugin.py`) mirrors what you see in the marketplace documentation.
+- Installation, upgrades, and migrations are orchestrated by `services/plugin/plugin_service.py` together with helpers such as `services/plugin/plugin_migration.py`.
+- Runtime loading happens through the implementations under `core/plugin/impl/*` (tool/model/datasource/trigger/endpoint/agent). These modules normalise plugin providers so that downstream systems (`core/tools/tool_manager.py`, `services/model_provider_service.py`, `services/trigger/*`) can treat builtin and plugin capabilities the same way.
+- For remote execution, plugin daemons (`core/plugin/entities/plugin_daemon.py`, `core/plugin/impl/plugin.py`) manage lifecycle hooks, credential forwarding, and background workers that keep plugin processes in sync with the main application.
+- Acquire tool implementations through `core/tools/tool_manager.py`; it resolves builtin, plugin, and workflow-as-tool providers uniformly, injecting the right context (tenant, credentials, runtime config).
+- To add a new plugin capability, extend the relevant `core/plugin/entities` schema and register the implementation in the matching `core/plugin/impl` module rather than importing the provider directly.
+
+## Async Workloads
+
+see `agent_skills/trigger.md` for more detailed documentation.
+
+- Enqueue background work through `services/async_workflow_service.py`. It routes jobs to the tiered Celery queues defined in `tasks/`.
+- Workers boot from `celery_entrypoint.py` and execute functions in `tasks/workflow_execution_tasks.py`, `tasks/trigger_processing_tasks.py`, etc.
+- Scheduled workflows poll from `schedule/workflow_schedule_tasks.py`. Follow the same pattern if you need new periodic jobs.
+
+## Database & Migrations
+
+- SQLAlchemy models live under `models/` and map directly to migration files in `migrations/versions`.
+- Generate migrations with `uv run --project api flask db revision --autogenerate -m "<summary>"`, then review the diff; never hand-edit the database outside Alembic.
+- Apply migrations locally using `uv run --project api flask db upgrade`; production deploys expect the same history.
+- If you add tenant-scoped data, confirm the upgrade includes tenant filters or defaults consistent with the service logic touching those tables.
+
+## CLI Commands
+
+- Maintenance commands from `commands.py` are registered on the Flask CLI. Run them via `uv run --project api flask <command>`.
+- Use the built-in `db` commands from Flask-Migrate for schema operations (`flask db upgrade`, `flask db stamp`, etc.). Only fall back to custom helpers if you need their extra behaviour.
+- Custom entries such as `flask reset-password`, `flask reset-email`, and `flask vdb-migrate` handle self-hosted account recovery and vector database migrations.
+- Before adding a new command, check whether an existing service can be reused and ensure the command guards edition-specific behaviour (many enforce `SELF_HOSTED`). Document any additions in the PR.
+- Ruff helpers are run directly with `uv`: `uv run --project api --dev ruff format ./api` for formatting and `uv run --project api --dev ruff check ./api` (add `--fix` if you want automatic fixes).
+
+## When You Add Features
+
+- Check for an existing helper or service before writing a new util.
+- Uphold tenancy: every service method should receive the tenant ID from controller wrappers such as `controllers/console/wraps.py`.
+- Update or create tests alongside behaviour changes (`tests/unit_tests` for fast coverage, `tests/integration_tests` when touching orchestrations).
+- Run `uv run --project api --dev ruff check ./api`, `uv run --directory api --dev basedpyright`, and `uv run --project api --dev dev/pytest/pytest_unit_tests.sh` before submitting changes.

dify/api/agent_skills/plugin.md

@@ -0,0 +1 @@
+// TBD

dify/api/agent_skills/plugin_oauth.md

@@ -0,0 +1 @@
+// TBD

dify/api/agent_skills/trigger.md

@@ -0,0 +1,53 @@
+## Overview
+
+Trigger is a collection of nodes that we called `Start` nodes, also, the concept of `Start` is the same as `RootNode` in the workflow engine `core/workflow/graph_engine`, On the other hand, `Start` node is the entry point of workflows, every workflow run always starts from a `Start` node.
+
+## Trigger nodes
+
+- `UserInput`
+- `Trigger Webhook`
+- `Trigger Schedule`
+- `Trigger Plugin`
+
+### UserInput
+
+Before `Trigger` concept is introduced, it's what we called `Start` node, but now, to avoid confusion, it was renamed to `UserInput` node, has a strong relation with `ServiceAPI` in `controllers/service_api/app`
+
+1. `UserInput` node introduces a list of arguments that need to be provided by the user, finally it will be converted into variables in the workflow variable pool.
+1. `ServiceAPI` accept those arguments, and pass through them into `UserInput` node.
+1. For its detailed implementation, please refer to `core/workflow/nodes/start`
+
+### Trigger Webhook
+
+Inside Webhook Node, Dify provided a UI panel that allows user define a HTTP manifest `core/workflow/nodes/trigger_webhook/entities.py`.`WebhookData`, also, Dify generates a random webhook id for each `Trigger Webhook` node, the implementation was implemented in `core/trigger/utils/endpoint.py`, as you can see, `webhook-debug` is a debug mode for webhook, you may find it in `controllers/trigger/webhook.py`.
+
+Finally, requests to `webhook` endpoint will be converted into variables in workflow variable pool during workflow execution.
+
+### Trigger Schedule
+
+`Trigger Schedule` node is a node that allows user define a schedule to trigger the workflow, detailed manifest is here `core/workflow/nodes/trigger_schedule/entities.py`, we have a poller and executor to handle millions of schedules, see `docker/entrypoint.sh` / `schedule/workflow_schedule_task.py` for help.
+
+To Achieve this, a `WorkflowSchedulePlan` model was introduced in `models/trigger.py`, and a `events/event_handlers/sync_workflow_schedule_when_app_published.py` was used to sync workflow schedule plans when app is published.
+
+### Trigger Plugin
+
+`Trigger Plugin` node allows user define there own distributed trigger plugin, whenever a request was received, Dify forwards it to the plugin and wait for parsed variables from it.
+
+1. Requests were saved in storage by `services/trigger/trigger_request_service.py`, referenced by `services/trigger/trigger_service.py`.`TriggerService`.`process_endpoint`
+1. Plugins accept those requests and parse variables from it, see `core/plugin/impl/trigger.py` for details.
+
+A `subscription` concept was out here by Dify, it means an endpoint address from Dify was bound to thirdparty webhook service like `Github` `Slack` `Linear` `GoogleDrive` `Gmail` etc. Once a subscription was created, Dify continually receives requests from the platforms and handle them one by one.
+
+## Worker Pool / Async Task
+
+All the events that triggered a new workflow run is always in async mode, a unified entrypoint can be found here `services/async_workflow_service.py`.`AsyncWorkflowService`.`trigger_workflow_async`.
+
+The infrastructure we used is `celery`, we've already configured it in `docker/entrypoint.sh`, and the consumers are in `tasks/async_workflow_tasks.py`, 3 queues were used to handle different tiers of users, `PROFESSIONAL_QUEUE` `TEAM_QUEUE` `SANDBOX_QUEUE`.
+
+## Debug Strategy
+
+Dify divided users into 2 groups: builders / end users.
+
+Builders are the users who create workflows, in this stage, debugging a workflow becomes a critical part of the workflow development process, as the start node in workflows, trigger nodes can `listen` to the events from `WebhookDebug` `Schedule` `Plugin`, debugging process was created in `controllers/console/app/workflow.py`.`DraftWorkflowTriggerNodeApi`.
+
+A polling process can be considered as combine of few single `poll` operations, each `poll` operation fetches events cached in `Redis`, returns `None` if no event was found, more detailed implemented: `core/trigger/debug/event_bus.py` was used to handle the polling process, and `core/trigger/debug/event_selectors.py` was used to select the event poller based on the trigger type.

dify/api/app.py

@@ -0,0 +1,29 @@
+import sys
+
+
+def is_db_command() -> bool:
+    if len(sys.argv) > 1 and sys.argv[0].endswith("flask") and sys.argv[1] == "db":
+        return True
+    return False
+
+
+# create app
+if is_db_command():
+    from app_factory import create_migrations_app
+
+    app = create_migrations_app()
+else:
+    # Gunicorn and Celery handle monkey patching automatically in production by
+    # specifying the `gevent` worker class. Manual monkey patching is not required here.
+    #
+    # See `api/docker/entrypoint.sh` (lines 33 and 47) for details.
+    #
+    # For third-party library patching, refer to `gunicorn.conf.py` and `celery_entrypoint.py`.
+
+    from app_factory import create_app
+
+    app = create_app()
+    celery = app.extensions["celery"]
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5001)

dify/api/app_factory.py

@@ -0,0 +1,120 @@
+import logging
+import time
+
+from configs import dify_config
+from contexts.wrapper import RecyclableContextVar
+from dify_app import DifyApp
+
+logger = logging.getLogger(__name__)
+
+
+# ----------------------------
+# Application Factory Function
+# ----------------------------
+def create_flask_app_with_configs() -> DifyApp:
+    """
+    create a raw flask app
+    with configs loaded from .env file
+    """
+    dify_app = DifyApp(__name__)
+    dify_app.config.from_mapping(dify_config.model_dump())
+    dify_app.config["RESTX_INCLUDE_ALL_MODELS"] = True
+
+    # add before request hook
+    @dify_app.before_request
+    def before_request():
+        # add an unique identifier to each request
+        RecyclableContextVar.increment_thread_recycles()
+
+    # Capture the decorator's return value to avoid pyright reportUnusedFunction
+    _ = before_request
+
+    return dify_app
+
+
+def create_app() -> DifyApp:
+    start_time = time.perf_counter()
+    app = create_flask_app_with_configs()
+    initialize_extensions(app)
+    end_time = time.perf_counter()
+    if dify_config.DEBUG:
+        logger.info("Finished create_app (%s ms)", round((end_time - start_time) * 1000, 2))
+    return app
+
+
+def initialize_extensions(app: DifyApp):
+    from extensions import (
+        ext_app_metrics,
+        ext_blueprints,
+        ext_celery,
+        ext_code_based_extension,
+        ext_commands,
+        ext_compress,
+        ext_database,
+        ext_hosting_provider,
+        ext_import_modules,
+        ext_logging,
+        ext_login,
+        ext_mail,
+        ext_migrate,
+        ext_orjson,
+        ext_otel,
+        ext_proxy_fix,
+        ext_redis,
+        ext_request_logging,
+        ext_sentry,
+        ext_set_secretkey,
+        ext_storage,
+        ext_timezone,
+        ext_warnings,
+    )
+
+    extensions = [
+        ext_timezone,
+        ext_logging,
+        ext_warnings,
+        ext_import_modules,
+        ext_orjson,
+        ext_set_secretkey,
+        ext_compress,
+        ext_code_based_extension,
+        ext_database,
+        ext_app_metrics,
+        ext_migrate,
+        ext_redis,
+        ext_storage,
+        ext_celery,
+        ext_login,
+        ext_mail,
+        ext_hosting_provider,
+        ext_sentry,
+        ext_proxy_fix,
+        ext_blueprints,
+        ext_commands,
+        ext_otel,
+        ext_request_logging,
+    ]
+    for ext in extensions:
+        short_name = ext.__name__.split(".")[-1]
+        is_enabled = ext.is_enabled() if hasattr(ext, "is_enabled") else True
+        if not is_enabled:
+            if dify_config.DEBUG:
+                logger.info("Skipped %s", short_name)
+            continue
+
+        start_time = time.perf_counter()
+        ext.init_app(app)
+        end_time = time.perf_counter()
+        if dify_config.DEBUG:
+            logger.info("Loaded %s (%s ms)", short_name, round((end_time - start_time) * 1000, 2))
+
+
+def create_migrations_app():
+    app = create_flask_app_with_configs()
+    from extensions import ext_database, ext_migrate
+
+    # Initialize only required extensions
+    ext_database.init_app(app)
+    ext_migrate.init_app(app)
+
+    return app

dify/api/celery_entrypoint.py

@@ -0,0 +1,13 @@
+import psycogreen.gevent as pscycogreen_gevent  # type: ignore
+from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+# grpc gevent
+grpc_gevent.init_gevent()
+print("gRPC patched with gevent.", flush=True)  # noqa: T201
+pscycogreen_gevent.patch_psycopg()
+print("psycopg2 patched with gevent.", flush=True)  # noqa: T201
+
+
+from app import app, celery
+
+__all__ = ["app", "celery"]

dify/api/cnt_base.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euxo pipefail
+
+for pattern in "Base" "TypeBase"; do
+    printf "%s " "$pattern"
+    grep "($pattern):" -r --include='*.py' --exclude-dir=".venv" --exclude-dir="tests" . | wc -l
+done

dify/api/configs/__init__.py

@@ -0,0 +1,3 @@
+from .app_config import DifyConfig
+
+dify_config = DifyConfig()  # type: ignore

dify/api/configs/app_config.py

@@ -0,0 +1,113 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource
+
+from libs.file_utils import search_file_upwards
+
+from .deploy import DeploymentConfig
+from .enterprise import EnterpriseFeatureConfig
+from .extra import ExtraServiceConfig
+from .feature import FeatureConfig
+from .middleware import MiddlewareConfig
+from .observability import ObservabilityConfig
+from .packaging import PackagingInfo
+from .remote_settings_sources import RemoteSettingsSource, RemoteSettingsSourceConfig, RemoteSettingsSourceName
+from .remote_settings_sources.apollo import ApolloSettingsSource
+from .remote_settings_sources.nacos import NacosSettingsSource
+
+logger = logging.getLogger(__name__)
+
+
+class RemoteSettingsSourceFactory(PydanticBaseSettingsSource):
+    def __init__(self, settings_cls: type[BaseSettings]):
+        super().__init__(settings_cls)
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        raise NotImplementedError
+
+    def __call__(self) -> dict[str, Any]:
+        current_state = self.current_state
+        remote_source_name = current_state.get("REMOTE_SETTINGS_SOURCE_NAME")
+        if not remote_source_name:
+            return {}
+
+        remote_source: RemoteSettingsSource | None = None
+        match remote_source_name:
+            case RemoteSettingsSourceName.APOLLO:
+                remote_source = ApolloSettingsSource(current_state)
+            case RemoteSettingsSourceName.NACOS:
+                remote_source = NacosSettingsSource(current_state)
+            case _:
+                logger.warning("Unsupported remote source: %s", remote_source_name)
+                return {}
+
+        d: dict[str, Any] = {}
+
+        for field_name, field in self.settings_cls.model_fields.items():
+            field_value, field_key, value_is_complex = remote_source.get_field_value(field, field_name)
+            field_value = remote_source.prepare_field_value(field_name, field, field_value, value_is_complex)
+            if field_value is not None:
+                d[field_key] = field_value
+
+        return d
+
+
+class DifyConfig(
+    # Packaging info
+    PackagingInfo,
+    # Deployment configs
+    DeploymentConfig,
+    # Feature configs
+    FeatureConfig,
+    # Middleware configs
+    MiddlewareConfig,
+    # Extra service configs
+    ExtraServiceConfig,
+    # Observability configs
+    ObservabilityConfig,
+    # Remote source configs
+    RemoteSettingsSourceConfig,
+    # Enterprise feature configs
+    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    EnterpriseFeatureConfig,
+):
+    model_config = SettingsConfigDict(
+        # read from dotenv format config file
+        env_file=".env",
+        env_file_encoding="utf-8",
+        # ignore extra attributes
+        extra="ignore",
+    )
+
+    # Before adding any config,
+    # please consider to arrange it in the proper config group of existed or added
+    # for better readability and maintainability.
+    # Thanks for your concentration and consideration.
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        return (
+            init_settings,
+            env_settings,
+            RemoteSettingsSourceFactory(settings_cls),
+            dotenv_settings,
+            file_secret_settings,
+            TomlConfigSettingsSource(
+                settings_cls=settings_cls,
+                toml_file=search_file_upwards(
+                    base_dir_path=Path(__file__).parent,
+                    target_file_name="pyproject.toml",
+                    max_search_parent_depth=2,
+                ),
+            ),
+        )

dify/api/configs/deploy/__init__.py

@@ -0,0 +1,34 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class DeploymentConfig(BaseSettings):
+    """
+    Configuration settings for application deployment
+    """
+
+    APPLICATION_NAME: str = Field(
+        description="Name of the application, used for identification and logging purposes",
+        default="langgenius/dify",
+    )
+
+    DEBUG: bool = Field(
+        description="Enable debug mode for additional logging and development features",
+        default=False,
+    )
+
+    # Request logging configuration
+    ENABLE_REQUEST_LOGGING: bool = Field(
+        description="Enable request and response body logging",
+        default=False,
+    )
+
+    EDITION: str = Field(
+        description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
+        default="SELF_HOSTED",
+    )
+
+    DEPLOY_ENV: str = Field(
+        description="Deployment environment (e.g., 'PRODUCTION', 'DEVELOPMENT'), default to PRODUCTION",
+        default="PRODUCTION",
+    )

dify/api/configs/enterprise/__init__.py

@@ -0,0 +1,20 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class EnterpriseFeatureConfig(BaseSettings):
+    """
+    Configuration for enterprise-level features.
+    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    """
+
+    ENTERPRISE_ENABLED: bool = Field(
+        description="Enable or disable enterprise-level features."
+        "Before using, please contact business@dify.ai by email to inquire about licensing matters.",
+        default=False,
+    )
+
+    CAN_REPLACE_LOGO: bool = Field(
+        description="Allow customization of the enterprise logo.",
+        default=False,
+    )

dify/api/configs/extra/__init__.py

@@ -0,0 +1,10 @@
+from configs.extra.notion_config import NotionConfig
+from configs.extra.sentry_config import SentryConfig
+
+
+class ExtraServiceConfig(
+    # place the configs in alphabet order
+    NotionConfig,
+    SentryConfig,
+):
+    pass

dify/api/configs/extra/notion_config.py

@@ -0,0 +1,34 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class NotionConfig(BaseSettings):
+    """
+    Configuration settings for Notion integration
+    """
+
+    NOTION_CLIENT_ID: str | None = Field(
+        description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
+        default=None,
+    )
+
+    NOTION_CLIENT_SECRET: str | None = Field(
+        description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TYPE: str | None = Field(
+        description="Type of Notion integration."
+        " Set to 'internal' for internal integrations, or None for public integrations.",
+        default=None,
+    )
+
+    NOTION_INTERNAL_SECRET: str | None = Field(
+        description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TOKEN: str | None = Field(
+        description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
+        default=None,
+    )

dify/api/configs/extra/sentry_config.py

@@ -0,0 +1,26 @@
+from pydantic import Field, NonNegativeFloat
+from pydantic_settings import BaseSettings
+
+
+class SentryConfig(BaseSettings):
+    """
+    Configuration settings for Sentry error tracking and performance monitoring
+    """
+
+    SENTRY_DSN: str | None = Field(
+        description="Sentry Data Source Name (DSN)."
+        " This is the unique identifier of your Sentry project, used to send events to the correct project.",
+        default=None,
+    )
+
+    SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description="Sample rate for Sentry performance monitoring traces."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of traces are sent to Sentry.",
+        default=1.0,
+    )
+
+    SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description="Sample rate for Sentry profiling."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of profiles are sent to Sentry.",
+        default=1.0,
+    )

dify/api/configs/feature/hosted_service/__init__.py

@@ -0,0 +1,254 @@
+from pydantic import Field, NonNegativeInt
+from pydantic_settings import BaseSettings
+
+
+class HostedCreditConfig(BaseSettings):
+    HOSTED_MODEL_CREDIT_CONFIG: str = Field(
+        description="Model credit configuration in format 'model:credits,model:credits', e.g., 'gpt-4:20,gpt-4o:10'",
+        default="",
+    )
+
+    def get_model_credits(self, model_name: str) -> int:
+        """
+        Get credit value for a specific model name.
+        Returns 1 if model is not found in configuration (default credit).
+
+        :param model_name: The name of the model to search for
+        :return: The credit value for the model
+        """
+        if not self.HOSTED_MODEL_CREDIT_CONFIG:
+            return 1
+
+        try:
+            credit_map = dict(
+                item.strip().split(":", 1) for item in self.HOSTED_MODEL_CREDIT_CONFIG.split(",") if ":" in item
+            )
+
+            # Search for matching model pattern
+            for pattern, credit in credit_map.items():
+                if pattern.strip() == model_name:
+                    return int(credit)
+            return 1  # Default quota if no match found
+        except (ValueError, AttributeError):
+            return 1  # Return default quota if parsing fails
+
+
+class HostedOpenAiConfig(BaseSettings):
+    """
+    Configuration for hosted OpenAI service
+    """
+
+    HOSTED_OPENAI_API_KEY: str | None = Field(
+        description="API key for hosted OpenAI service",
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_BASE: str | None = Field(
+        description="Base URL for hosted OpenAI API",
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_ORGANIZATION: str | None = Field(
+        description="Organization ID for hosted OpenAI service",
+        default=None,
+    )
+
+    HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted OpenAI service",
+        default=False,
+    )
+
+    HOSTED_OPENAI_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for trial access",
+        default="gpt-3.5-turbo,"
+        "gpt-3.5-turbo-1106,"
+        "gpt-3.5-turbo-instruct,"
+        "gpt-3.5-turbo-16k,"
+        "gpt-3.5-turbo-16k-0613,"
+        "gpt-3.5-turbo-0613,"
+        "gpt-3.5-turbo-0125,"
+        "text-davinci-003",
+    )
+
+    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted OpenAI service usage",
+        default=200,
+    )
+
+    HOSTED_OPENAI_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted OpenAI service",
+        default=False,
+    )
+
+    HOSTED_OPENAI_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="gpt-4,"
+        "gpt-4-turbo-preview,"
+        "gpt-4-turbo-2024-04-09,"
+        "gpt-4-1106-preview,"
+        "gpt-4-0125-preview,"
+        "gpt-3.5-turbo,"
+        "gpt-3.5-turbo-16k,"
+        "gpt-3.5-turbo-16k-0613,"
+        "gpt-3.5-turbo-1106,"
+        "gpt-3.5-turbo-0613,"
+        "gpt-3.5-turbo-0125,"
+        "gpt-3.5-turbo-instruct,"
+        "text-davinci-003",
+    )
+
+
+class HostedAzureOpenAiConfig(BaseSettings):
+    """
+    Configuration for hosted Azure OpenAI service
+    """
+
+    HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
+        description="Enable hosted Azure OpenAI service",
+        default=False,
+    )
+
+    HOSTED_AZURE_OPENAI_API_KEY: str | None = Field(
+        description="API key for hosted Azure OpenAI service",
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_API_BASE: str | None = Field(
+        description="Base URL for hosted Azure OpenAI API",
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted Azure OpenAI service usage",
+        default=200,
+    )
+
+
+class HostedAnthropicConfig(BaseSettings):
+    """
+    Configuration for hosted Anthropic service
+    """
+
+    HOSTED_ANTHROPIC_API_BASE: str | None = Field(
+        description="Base URL for hosted Anthropic API",
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_API_KEY: str | None = Field(
+        description="API key for hosted Anthropic service",
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted Anthropic service",
+        default=False,
+    )
+
+    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
+        description="Quota limit for hosted Anthropic service usage",
+        default=600000,
+    )
+
+    HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted Anthropic service",
+        default=False,
+    )
+
+
+class HostedMinmaxConfig(BaseSettings):
+    """
+    Configuration for hosted Minmax service
+    """
+
+    HOSTED_MINIMAX_ENABLED: bool = Field(
+        description="Enable hosted Minmax service",
+        default=False,
+    )
+
+
+class HostedSparkConfig(BaseSettings):
+    """
+    Configuration for hosted Spark service
+    """
+
+    HOSTED_SPARK_ENABLED: bool = Field(
+        description="Enable hosted Spark service",
+        default=False,
+    )
+
+
+class HostedZhipuAIConfig(BaseSettings):
+    """
+    Configuration for hosted ZhipuAI service
+    """
+
+    HOSTED_ZHIPUAI_ENABLED: bool = Field(
+        description="Enable hosted ZhipuAI service",
+        default=False,
+    )
+
+
+class HostedModerationConfig(BaseSettings):
+    """
+    Configuration for hosted Moderation service
+    """
+
+    HOSTED_MODERATION_ENABLED: bool = Field(
+        description="Enable hosted Moderation service",
+        default=False,
+    )
+
+    HOSTED_MODERATION_PROVIDERS: str = Field(
+        description="Comma-separated list of moderation providers",
+        default="",
+    )
+
+
+class HostedFetchAppTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching app templates
+    """
+
+    HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching app templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote app templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
+class HostedFetchPipelineTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching pipeline templates
+    """
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching pipeline templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote pipeline templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
+class HostedServiceConfig(
+    # place the configs in alphabet order
+    HostedAnthropicConfig,
+    HostedAzureOpenAiConfig,
+    HostedFetchAppTemplateConfig,
+    HostedFetchPipelineTemplateConfig,
+    HostedMinmaxConfig,
+    HostedOpenAiConfig,
+    HostedSparkConfig,
+    HostedZhipuAIConfig,
+    # moderation
+    HostedModerationConfig,
+    # credit config
+    HostedCreditConfig,
+):
+    pass

dify/api/configs/middleware/__init__.py

@@ -0,0 +1,366 @@
+import os
+from typing import Any, Literal
+from urllib.parse import parse_qsl, quote_plus
+
+from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
+from pydantic_settings import BaseSettings
+
+from .cache.redis_config import RedisConfig
+from .storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from .storage.amazon_s3_storage_config import S3StorageConfig
+from .storage.azure_blob_storage_config import AzureBlobStorageConfig
+from .storage.baidu_obs_storage_config import BaiduOBSStorageConfig
+from .storage.clickzetta_volume_storage_config import ClickZettaVolumeStorageConfig
+from .storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from .storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
+from .storage.oci_storage_config import OCIStorageConfig
+from .storage.opendal_storage_config import OpenDALStorageConfig
+from .storage.supabase_storage_config import SupabaseStorageConfig
+from .storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from .storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
+from .vdb.alibabacloud_mysql_config import AlibabaCloudMySQLConfig
+from .vdb.analyticdb_config import AnalyticdbConfig
+from .vdb.baidu_vector_config import BaiduVectorDBConfig
+from .vdb.chroma_config import ChromaConfig
+from .vdb.clickzetta_config import ClickzettaConfig
+from .vdb.couchbase_config import CouchbaseConfig
+from .vdb.elasticsearch_config import ElasticsearchConfig
+from .vdb.huawei_cloud_config import HuaweiCloudConfig
+from .vdb.lindorm_config import LindormConfig
+from .vdb.matrixone_config import MatrixoneConfig
+from .vdb.milvus_config import MilvusConfig
+from .vdb.myscale_config import MyScaleConfig
+from .vdb.oceanbase_config import OceanBaseVectorConfig
+from .vdb.opengauss_config import OpenGaussConfig
+from .vdb.opensearch_config import OpenSearchConfig
+from .vdb.oracle_config import OracleConfig
+from .vdb.pgvector_config import PGVectorConfig
+from .vdb.pgvectors_config import PGVectoRSConfig
+from .vdb.qdrant_config import QdrantConfig
+from .vdb.relyt_config import RelytConfig
+from .vdb.tablestore_config import TableStoreConfig
+from .vdb.tencent_vector_config import TencentVectorDBConfig
+from .vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
+from .vdb.tidb_vector_config import TiDBVectorConfig
+from .vdb.upstash_config import UpstashConfig
+from .vdb.vastbase_vector_config import VastbaseVectorConfig
+from .vdb.vikingdb_config import VikingDBConfig
+from .vdb.weaviate_config import WeaviateConfig
+
+
+class StorageConfig(BaseSettings):
+    STORAGE_TYPE: Literal[
+        "opendal",
+        "s3",
+        "aliyun-oss",
+        "azure-blob",
+        "baidu-obs",
+        "clickzetta-volume",
+        "google-storage",
+        "huawei-obs",
+        "oci-storage",
+        "tencent-cos",
+        "volcengine-tos",
+        "supabase",
+        "local",
+    ] = Field(
+        description="Type of storage to use."
+        " Options: 'opendal', '(deprecated) local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', "
+        "'clickzetta-volume', 'google-storage', 'huawei-obs', 'oci-storage', 'tencent-cos', "
+        "'volcengine-tos', 'supabase'. Default is 'opendal'.",
+        default="opendal",
+    )
+
+    STORAGE_LOCAL_PATH: str = Field(
+        description="Path for local storage when STORAGE_TYPE is set to 'local'.",
+        default="storage",
+        deprecated=True,
+    )
+
+
+class VectorStoreConfig(BaseSettings):
+    VECTOR_STORE: str | None = Field(
+        description="Type of vector store to use for efficient similarity search."
+        " Set to None if not using a vector store.",
+        default=None,
+    )
+
+    VECTOR_STORE_WHITELIST_ENABLE: bool | None = Field(
+        description="Enable whitelist for vector store.",
+        default=False,
+    )
+
+    VECTOR_INDEX_NAME_PREFIX: str | None = Field(
+        description="Prefix used to create collection name in vector database",
+        default="Vector_index",
+    )
+
+
+class KeywordStoreConfig(BaseSettings):
+    KEYWORD_STORE: str = Field(
+        description="Method for keyword extraction and storage."
+        " Default is 'jieba', a Chinese text segmentation library.",
+        default="jieba",
+    )
+
+
+class DatabaseConfig(BaseSettings):
+    # Database type selector
+    DB_TYPE: Literal["postgresql", "mysql", "oceanbase"] = Field(
+        description="Database type to use. OceanBase is MySQL-compatible.",
+        default="postgresql",
+    )
+
+    DB_HOST: str = Field(
+        description="Hostname or IP address of the database server.",
+        default="localhost",
+    )
+
+    DB_PORT: PositiveInt = Field(
+        description="Port number for database connection.",
+        default=5432,
+    )
+
+    DB_USERNAME: str = Field(
+        description="Username for database authentication.",
+        default="postgres",
+    )
+
+    DB_PASSWORD: str = Field(
+        description="Password for database authentication.",
+        default="",
+    )
+
+    DB_DATABASE: str = Field(
+        description="Name of the database to connect to.",
+        default="dify",
+    )
+
+    DB_CHARSET: str = Field(
+        description="Character set for database connection.",
+        default="",
+    )
+
+    DB_EXTRAS: str = Field(
+        description="Additional database connection parameters. Example: 'keepalives_idle=60&keepalives=1'",
+        default="",
+    )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_DATABASE_URI_SCHEME(self) -> str:
+        return "postgresql" if self.DB_TYPE == "postgresql" else "mysql+pymysql"
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_DATABASE_URI(self) -> str:
+        db_extras = (
+            f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else self.DB_EXTRAS
+        ).strip("&")
+        db_extras = f"?{db_extras}" if db_extras else ""
+        return (
+            f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
+            f"{quote_plus(self.DB_USERNAME)}:{quote_plus(self.DB_PASSWORD)}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
+            f"{db_extras}"
+        )
+
+    SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
+        description="Maximum number of database connections in the pool.",
+        default=30,
+    )
+
+    SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
+        description="Maximum number of connections that can be created beyond the pool_size.",
+        default=10,
+    )
+
+    SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
+        description="Number of seconds after which a connection is automatically recycled.",
+        default=3600,
+    )
+
+    SQLALCHEMY_POOL_USE_LIFO: bool = Field(
+        description="If True, SQLAlchemy will use last-in-first-out way to retrieve connections from pool.",
+        default=False,
+    )
+
+    SQLALCHEMY_POOL_PRE_PING: bool = Field(
+        description="If True, enables connection pool pre-ping feature to check connections.",
+        default=False,
+    )
+
+    SQLALCHEMY_ECHO: bool | str = Field(
+        description="If True, SQLAlchemy will log all SQL statements.",
+        default=False,
+    )
+
+    SQLALCHEMY_POOL_TIMEOUT: NonNegativeInt = Field(
+        description="Number of seconds to wait for a connection from the pool before raising a timeout error.",
+        default=30,
+    )
+
+    RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
+        description="Number of processes for the retrieval service, default to CPU cores.",
+        default=os.cpu_count() or 1,
+    )
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
+        # Parse DB_EXTRAS for 'options'
+        db_extras_dict = dict(parse_qsl(self.DB_EXTRAS))
+        options = db_extras_dict.get("options", "")
+        connect_args = {}
+        # Use the dynamic SQLALCHEMY_DATABASE_URI_SCHEME property
+        if self.SQLALCHEMY_DATABASE_URI_SCHEME.startswith("postgresql"):
+            timezone_opt = "-c timezone=UTC"
+            if options:
+                merged_options = f"{options} {timezone_opt}"
+            else:
+                merged_options = timezone_opt
+            connect_args = {"options": merged_options}
+
+        return {
+            "pool_size": self.SQLALCHEMY_POOL_SIZE,
+            "max_overflow": self.SQLALCHEMY_MAX_OVERFLOW,
+            "pool_recycle": self.SQLALCHEMY_POOL_RECYCLE,
+            "pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
+            "connect_args": connect_args,
+            "pool_use_lifo": self.SQLALCHEMY_POOL_USE_LIFO,
+            "pool_reset_on_return": None,
+            "pool_timeout": self.SQLALCHEMY_POOL_TIMEOUT,
+        }
+
+
+class CeleryConfig(DatabaseConfig):
+    CELERY_BACKEND: str = Field(
+        description="Backend for Celery task results. Options: 'database', 'redis', 'rabbitmq'.",
+        default="redis",
+    )
+
+    CELERY_BROKER_URL: str | None = Field(
+        description="URL of the message broker for Celery tasks.",
+        default=None,
+    )
+
+    CELERY_USE_SENTINEL: bool | None = Field(
+        description="Whether to use Redis Sentinel for high availability.",
+        default=False,
+    )
+
+    CELERY_SENTINEL_MASTER_NAME: str | None = Field(
+        description="Name of the Redis Sentinel master.",
+        default=None,
+    )
+
+    CELERY_SENTINEL_PASSWORD: str | None = Field(
+        description="Password of the Redis Sentinel master.",
+        default=None,
+    )
+    CELERY_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
+        description="Timeout for Redis Sentinel socket operations in seconds.",
+        default=0.1,
+    )
+
+    @computed_field
+    def CELERY_RESULT_BACKEND(self) -> str | None:
+        if self.CELERY_BACKEND in ("database", "rabbitmq"):
+            return f"db+{self.SQLALCHEMY_DATABASE_URI}"
+        elif self.CELERY_BACKEND == "redis":
+            return self.CELERY_BROKER_URL
+        else:
+            return None
+
+    @property
+    def BROKER_USE_SSL(self) -> bool:
+        return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False
+
+
+class InternalTestConfig(BaseSettings):
+    """
+    Configuration settings for Internal Test
+    """
+
+    AWS_SECRET_ACCESS_KEY: str | None = Field(
+        description="Internal test AWS secret access key",
+        default=None,
+    )
+
+    AWS_ACCESS_KEY_ID: str | None = Field(
+        description="Internal test AWS access key ID",
+        default=None,
+    )
+
+
+class DatasetQueueMonitorConfig(BaseSettings):
+    """
+    Configuration settings for Dataset Queue Monitor
+    """
+
+    QUEUE_MONITOR_THRESHOLD: NonNegativeInt | None = Field(
+        description="Threshold for dataset queue monitor",
+        default=200,
+    )
+    QUEUE_MONITOR_ALERT_EMAILS: str | None = Field(
+        description="Emails for dataset queue monitor alert, separated by commas",
+        default=None,
+    )
+    QUEUE_MONITOR_INTERVAL: NonNegativeFloat | None = Field(
+        description="Interval for dataset queue monitor in minutes",
+        default=30,
+    )
+
+
+class MiddlewareConfig(
+    # place the configs in alphabet order
+    CeleryConfig,  # Note: CeleryConfig already inherits from DatabaseConfig
+    KeywordStoreConfig,
+    RedisConfig,
+    # configs of storage and storage providers
+    StorageConfig,
+    AliyunOSSStorageConfig,
+    AzureBlobStorageConfig,
+    BaiduOBSStorageConfig,
+    ClickZettaVolumeStorageConfig,
+    GoogleCloudStorageConfig,
+    HuaweiCloudOBSStorageConfig,
+    OCIStorageConfig,
+    OpenDALStorageConfig,
+    S3StorageConfig,
+    SupabaseStorageConfig,
+    TencentCloudCOSStorageConfig,
+    VolcengineTOSStorageConfig,
+    # configs of vdb and vdb providers
+    VectorStoreConfig,
+    AnalyticdbConfig,
+    ChromaConfig,
+    ClickzettaConfig,
+    HuaweiCloudConfig,
+    MilvusConfig,
+    AlibabaCloudMySQLConfig,
+    MyScaleConfig,
+    OpenSearchConfig,
+    OracleConfig,
+    PGVectorConfig,
+    VastbaseVectorConfig,
+    PGVectoRSConfig,
+    QdrantConfig,
+    RelytConfig,
+    TencentVectorDBConfig,
+    TiDBVectorConfig,
+    WeaviateConfig,
+    ElasticsearchConfig,
+    CouchbaseConfig,
+    InternalTestConfig,
+    VikingDBConfig,
+    UpstashConfig,
+    TidbOnQdrantConfig,
+    LindormConfig,
+    OceanBaseVectorConfig,
+    BaiduVectorDBConfig,
+    OpenGaussConfig,
+    TableStoreConfig,
+    DatasetQueueMonitorConfig,
+    MatrixoneConfig,
+):
+    pass

dify/api/configs/middleware/cache/redis_config.py

@@ -0,0 +1,113 @@
+from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RedisConfig(BaseSettings):
+    """
+    Configuration settings for Redis connection
+    """
+
+    REDIS_HOST: str = Field(
+        description="Hostname or IP address of the Redis server",
+        default="localhost",
+    )
+
+    REDIS_PORT: PositiveInt = Field(
+        description="Port number on which the Redis server is listening",
+        default=6379,
+    )
+
+    REDIS_USERNAME: str | None = Field(
+        description="Username for Redis authentication (if required)",
+        default=None,
+    )
+
+    REDIS_PASSWORD: str | None = Field(
+        description="Password for Redis authentication (if required)",
+        default=None,
+    )
+
+    REDIS_DB: NonNegativeInt = Field(
+        description="Redis database number to use (0-15)",
+        default=0,
+    )
+
+    REDIS_USE_SSL: bool = Field(
+        description="Enable SSL/TLS for the Redis connection",
+        default=False,
+    )
+
+    REDIS_SSL_CERT_REQS: str = Field(
+        description="SSL certificate requirements (CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED)",
+        default="CERT_NONE",
+    )
+
+    REDIS_SSL_CA_CERTS: str | None = Field(
+        description="Path to the CA certificate file for SSL verification",
+        default=None,
+    )
+
+    REDIS_SSL_CERTFILE: str | None = Field(
+        description="Path to the client certificate file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_SSL_KEYFILE: str | None = Field(
+        description="Path to the client private key file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_USE_SENTINEL: bool | None = Field(
+        description="Enable Redis Sentinel mode for high availability",
+        default=False,
+    )
+
+    REDIS_SENTINELS: str | None = Field(
+        description="Comma-separated list of Redis Sentinel nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SERVICE_NAME: str | None = Field(
+        description="Name of the Redis Sentinel service to monitor",
+        default=None,
+    )
+
+    REDIS_SENTINEL_USERNAME: str | None = Field(
+        description="Username for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_PASSWORD: str | None = Field(
+        description="Password for Redis Sentinel authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
+        description="Socket timeout in seconds for Redis Sentinel connections",
+        default=0.1,
+    )
+
+    REDIS_USE_CLUSTERS: bool = Field(
+        description="Enable Redis Clusters mode for high availability",
+        default=False,
+    )
+
+    REDIS_CLUSTERS: str | None = Field(
+        description="Comma-separated list of Redis Clusters nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_CLUSTERS_PASSWORD: str | None = Field(
+        description="Password for Redis Clusters authentication (if required)",
+        default=None,
+    )
+
+    REDIS_SERIALIZATION_PROTOCOL: int = Field(
+        description="Redis serialization protocol (RESP) version",
+        default=3,
+    )
+
+    REDIS_ENABLE_CLIENT_SIDE_CACHE: bool = Field(
+        description="Enable client side cache in redis",
+        default=False,
+    )

dify/api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -0,0 +1,43 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AliyunOSSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Aliyun Object Storage Service (OSS)
+    """
+
+    ALIYUN_OSS_BUCKET_NAME: str | None = Field(
+        description="Name of the Aliyun OSS bucket to store and retrieve objects",
+        default=None,
+    )
+
+    ALIYUN_OSS_ACCESS_KEY: str | None = Field(
+        description="Access key ID for authenticating with Aliyun OSS",
+        default=None,
+    )
+
+    ALIYUN_OSS_SECRET_KEY: str | None = Field(
+        description="Secret access key for authenticating with Aliyun OSS",
+        default=None,
+    )
+
+    ALIYUN_OSS_ENDPOINT: str | None = Field(
+        description="URL of the Aliyun OSS endpoint for your chosen region",
+        default=None,
+    )
+
+    ALIYUN_OSS_REGION: str | None = Field(
+        description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
+        default=None,
+    )
+
+    ALIYUN_OSS_AUTH_VERSION: str | None = Field(
+        description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
+        default=None,
+    )
+
+    ALIYUN_OSS_PATH: str | None = Field(
+        description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
+        default=None,
+    )

dify/api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -0,0 +1,45 @@
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class S3StorageConfig(BaseSettings):
+    """
+    Configuration settings for S3-compatible object storage
+    """
+
+    S3_ENDPOINT: str | None = Field(
+        description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
+        default=None,
+    )
+
+    S3_REGION: str | None = Field(
+        description="Region where the S3 bucket is located (e.g., 'us-east-1')",
+        default=None,
+    )
+
+    S3_BUCKET_NAME: str | None = Field(
+        description="Name of the S3 bucket to store and retrieve objects",
+        default=None,
+    )
+
+    S3_ACCESS_KEY: str | None = Field(
+        description="Access key ID for authenticating with the S3 service",
+        default=None,
+    )
+
+    S3_SECRET_KEY: str | None = Field(
+        description="Secret access key for authenticating with the S3 service",
+        default=None,
+    )
+
+    S3_ADDRESS_STYLE: Literal["auto", "virtual", "path"] = Field(
+        description="S3 addressing style: 'auto', 'path', or 'virtual'",
+        default="auto",
+    )
+
+    S3_USE_AWS_MANAGED_IAM: bool = Field(
+        description="Use AWS managed IAM roles for authentication instead of access/secret keys",
+        default=False,
+    )

dify/api/configs/middleware/storage/azure_blob_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AzureBlobStorageConfig(BaseSettings):
+    """
+    Configuration settings for Azure Blob Storage
+    """
+
+    AZURE_BLOB_ACCOUNT_NAME: str | None = Field(
+        description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_KEY: str | None = Field(
+        description="Access key for authenticating with the Azure Storage account",
+        default=None,
+    )
+
+    AZURE_BLOB_CONTAINER_NAME: str | None = Field(
+        description="Name of the Azure Blob container to store and retrieve objects",
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_URL: str | None = Field(
+        description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
+        default=None,
+    )

dify/api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class BaiduOBSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Baidu Object Storage Service (OBS)
+    """
+
+    BAIDU_OBS_BUCKET_NAME: str | None = Field(
+        description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
+        default=None,
+    )
+
+    BAIDU_OBS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Baidu OBS",
+        default=None,
+    )
+
+    BAIDU_OBS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Baidu OBS",
+        default=None,
+    )
+
+    BAIDU_OBS_ENDPOINT: str | None = Field(
+        description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
+        default=None,
+    )

dify/api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -0,0 +1,63 @@
+"""ClickZetta Volume Storage Configuration"""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ClickZettaVolumeStorageConfig(BaseSettings):
+    """Configuration for ClickZetta Volume storage."""
+
+    CLICKZETTA_VOLUME_USERNAME: str | None = Field(
+        description="Username for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_PASSWORD: str | None = Field(
+        description="Password for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_INSTANCE: str | None = Field(
+        description="ClickZetta instance identifier",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_SERVICE: str = Field(
+        description="ClickZetta service endpoint",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_VOLUME_WORKSPACE: str = Field(
+        description="ClickZetta workspace name",
+        default="quick_start",
+    )
+
+    CLICKZETTA_VOLUME_VCLUSTER: str = Field(
+        description="ClickZetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_VOLUME_SCHEMA: str = Field(
+        description="ClickZetta schema name",
+        default="dify",
+    )
+
+    CLICKZETTA_VOLUME_TYPE: str = Field(
+        description="ClickZetta volume type (table|user|external)",
+        default="user",
+    )
+
+    CLICKZETTA_VOLUME_NAME: str | None = Field(
+        description="ClickZetta volume name for external volumes",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_TABLE_PREFIX: str = Field(
+        description="Prefix for ClickZetta volume table names",
+        default="dataset_",
+    )
+
+    CLICKZETTA_VOLUME_DIFY_PREFIX: str = Field(
+        description="Directory prefix for User Volume to organize Dify files",
+        default="dify_km",
+    )

dify/api/configs/middleware/storage/google_cloud_storage_config.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class GoogleCloudStorageConfig(BaseSettings):
+    """
+    Configuration settings for Google Cloud Storage
+    """
+
+    GOOGLE_STORAGE_BUCKET_NAME: str | None = Field(
+        description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
+        default=None,
+    )
+
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: str | None = Field(
+        description="Base64-encoded JSON key file for Google Cloud service account authentication",
+        default=None,
+    )

dify/api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -0,0 +1,28 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class HuaweiCloudOBSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Huawei Cloud Object Storage Service (OBS)
+    """
+
+    HUAWEI_OBS_BUCKET_NAME: str | None = Field(
+        description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
+        default=None,
+    )
+
+    HUAWEI_OBS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Huawei Cloud OBS",
+        default=None,
+    )
+
+    HUAWEI_OBS_SERVER: str | None = Field(
+        description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
+        default=None,
+    )

dify/api/configs/middleware/storage/oci_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OCIStorageConfig(BaseSettings):
+    """
+    Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
+    """
+
+    OCI_ENDPOINT: str | None = Field(
+        description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
+        default=None,
+    )
+
+    OCI_REGION: str | None = Field(
+        description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
+        default=None,
+    )
+
+    OCI_BUCKET_NAME: str | None = Field(
+        description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
+        default=None,
+    )
+
+    OCI_ACCESS_KEY: str | None = Field(
+        description="Access key (also known as API key) for authenticating with OCI Object Storage",
+        default=None,
+    )
+
+    OCI_SECRET_KEY: str | None = Field(
+        description="Secret key associated with the access key for authenticating with OCI Object Storage",
+        default=None,
+    )

dify/api/configs/middleware/storage/opendal_storage_config.py

@@ -0,0 +1,9 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OpenDALStorageConfig(BaseSettings):
+    OPENDAL_SCHEME: str = Field(
+        default="fs",
+        description="OpenDAL scheme.",
+    )

dify/api/configs/middleware/storage/supabase_storage_config.py

@@ -0,0 +1,23 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class SupabaseStorageConfig(BaseSettings):
+    """
+    Configuration settings for Supabase Object Storage Service
+    """
+
+    SUPABASE_BUCKET_NAME: str | None = Field(
+        description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
+        default=None,
+    )
+
+    SUPABASE_API_KEY: str | None = Field(
+        description="API KEY for authenticating with Supabase",
+        default=None,
+    )
+
+    SUPABASE_URL: str | None = Field(
+        description="URL of the Supabase",
+        default=None,
+    )

dify/api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class TencentCloudCOSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Tencent Cloud Object Storage (COS)
+    """
+
+    TENCENT_COS_BUCKET_NAME: str | None = Field(
+        description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
+        default=None,
+    )
+
+    TENCENT_COS_REGION: str | None = Field(
+        description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_ID: str | None = Field(
+        description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_KEY: str | None = Field(
+        description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
+        default=None,
+    )
+
+    TENCENT_COS_SCHEME: str | None = Field(
+        description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
+        default=None,
+    )

dify/api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class VolcengineTOSStorageConfig(BaseSettings):
+    """
+    Configuration settings for Volcengine Tinder Object Storage (TOS)
+    """
+
+    VOLCENGINE_TOS_BUCKET_NAME: str | None = Field(
+        description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ACCESS_KEY: str | None = Field(
+        description="Access Key ID for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_SECRET_KEY: str | None = Field(
+        description="Secret Access Key for authenticating with Volcengine TOS",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_ENDPOINT: str | None = Field(
+        description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
+        default=None,
+    )
+
+    VOLCENGINE_TOS_REGION: str | None = Field(
+        description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
+        default=None,
+    )

dify/api/configs/middleware/vdb/alibabacloud_mysql_config.py

@@ -0,0 +1,54 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AlibabaCloudMySQLConfig(BaseSettings):
+    """
+    Configuration settings for AlibabaCloud MySQL vector database
+    """
+
+    ALIBABACLOUD_MYSQL_HOST: str = Field(
+        description="Hostname or IP address of the AlibabaCloud MySQL server (e.g., 'localhost' or 'mysql.aliyun.com')",
+        default="localhost",
+    )
+
+    ALIBABACLOUD_MYSQL_PORT: PositiveInt = Field(
+        description="Port number on which the AlibabaCloud MySQL server is listening (default is 3306)",
+        default=3306,
+    )
+
+    ALIBABACLOUD_MYSQL_USER: str = Field(
+        description="Username for authenticating with AlibabaCloud MySQL (default is 'root')",
+        default="root",
+    )
+
+    ALIBABACLOUD_MYSQL_PASSWORD: str = Field(
+        description="Password for authenticating with AlibabaCloud MySQL (default is an empty string)",
+        default="",
+    )
+
+    ALIBABACLOUD_MYSQL_DATABASE: str = Field(
+        description="Name of the AlibabaCloud MySQL database to connect to (default is 'dify')",
+        default="dify",
+    )
+
+    ALIBABACLOUD_MYSQL_MAX_CONNECTION: PositiveInt = Field(
+        description="Maximum number of connections in the connection pool",
+        default=5,
+    )
+
+    ALIBABACLOUD_MYSQL_CHARSET: str = Field(
+        description="Character set for AlibabaCloud MySQL connection (default is 'utf8mb4')",
+        default="utf8mb4",
+    )
+
+    ALIBABACLOUD_MYSQL_DISTANCE_FUNCTION: str = Field(
+        description="Distance function used for vector similarity search in AlibabaCloud MySQL "
+        "(e.g., 'cosine', 'euclidean')",
+        default="cosine",
+    )
+
+    ALIBABACLOUD_MYSQL_HNSW_M: PositiveInt = Field(
+        description="Maximum number of connections per layer for HNSW vector index (default is 6, range: 3-200)",
+        default=6,
+    )

dify/api/configs/middleware/vdb/analyticdb_config.py

@@ -0,0 +1,49 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AnalyticdbConfig(BaseSettings):
+    """
+    Configuration for connecting to Alibaba Cloud AnalyticDB for PostgreSQL.
+    Refer to the following documentation for details on obtaining credentials:
+    https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
+    """
+
+    ANALYTICDB_KEY_ID: str | None = Field(
+        default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
+    )
+    ANALYTICDB_KEY_SECRET: str | None = Field(
+        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
+    )
+    ANALYTICDB_REGION_ID: str | None = Field(
+        default=None,
+        description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
+    )
+    ANALYTICDB_INSTANCE_ID: str | None = Field(
+        default=None,
+        description="The unique identifier of the AnalyticDB instance you want to connect to.",
+    )
+    ANALYTICDB_ACCOUNT: str | None = Field(
+        default=None,
+        description="The account name used to log in to the AnalyticDB instance"
+        " (usually the initial account created with the instance).",
+    )
+    ANALYTICDB_PASSWORD: str | None = Field(
+        default=None, description="The password associated with the AnalyticDB account for database authentication."
+    )
+    ANALYTICDB_NAMESPACE: str | None = Field(
+        default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
+    )
+    ANALYTICDB_NAMESPACE_PASSWORD: str | None = Field(
+        default=None,
+        description="The password for accessing the specified namespace within the AnalyticDB instance"
+        " (if namespace feature is enabled).",
+    )
+    ANALYTICDB_HOST: str | None = Field(
+        default=None, description="The host of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_PORT: PositiveInt = Field(
+        default=5432, description="The port of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_MIN_CONNECTION: PositiveInt = Field(default=1, description="Min connection of the AnalyticDB database.")
+    ANALYTICDB_MAX_CONNECTION: PositiveInt = Field(default=5, description="Max connection of the AnalyticDB database.")

dify/api/configs/middleware/vdb/baidu_vector_config.py

@@ -0,0 +1,53 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class BaiduVectorDBConfig(BaseSettings):
+    """
+    Configuration settings for Baidu Vector Database
+    """
+
+    BAIDU_VECTOR_DB_ENDPOINT: str | None = Field(
+        description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: PositiveInt = Field(
+        description="Timeout in milliseconds for Baidu Vector Database operations (default is 30000 milliseconds)",
+        default=30000,
+    )
+
+    BAIDU_VECTOR_DB_ACCOUNT: str | None = Field(
+        description="Account for authenticating with the Baidu Vector Database",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_API_KEY: str | None = Field(
+        description="API key for authenticating with the Baidu Vector Database service",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_DATABASE: str | None = Field(
+        description="Name of the specific Baidu Vector Database to connect to",
+        default=None,
+    )
+
+    BAIDU_VECTOR_DB_SHARD: PositiveInt = Field(
+        description="Number of shards for the Baidu Vector Database (default is 1)",
+        default=1,
+    )
+
+    BAIDU_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+        description="Number of replicas for the Baidu Vector Database (default is 3)",
+        default=3,
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER: str = Field(
+        description="Analyzer type for inverted index in Baidu Vector Database (default is DEFAULT_ANALYZER)",
+        default="DEFAULT_ANALYZER",
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE: str = Field(
+        description="Parser mode for inverted index in Baidu Vector Database (default is COARSE_MODE)",
+        default="COARSE_MODE",
+    )

dify/api/configs/middleware/vdb/chroma_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class ChromaConfig(BaseSettings):
+    """
+    Configuration settings for Chroma vector database
+    """
+
+    CHROMA_HOST: str | None = Field(
+        description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
+        default=None,
+    )
+
+    CHROMA_PORT: PositiveInt = Field(
+        description="Port number on which the Chroma server is listening (default is 8000)",
+        default=8000,
+    )
+
+    CHROMA_TENANT: str | None = Field(
+        description="Tenant identifier for multi-tenancy support in Chroma",
+        default=None,
+    )
+
+    CHROMA_DATABASE: str | None = Field(
+        description="Name of the Chroma database to connect to",
+        default=None,
+    )
+
+    CHROMA_AUTH_PROVIDER: str | None = Field(
+        description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
+        default=None,
+    )
+
+    CHROMA_AUTH_CREDENTIALS: str | None = Field(
+        description="Authentication credentials for Chroma (format depends on the auth provider)",
+        default=None,
+    )

dify/api/configs/middleware/vdb/clickzetta_config.py

@@ -0,0 +1,68 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ClickzettaConfig(BaseSettings):
+    """
+    Clickzetta Lakehouse vector database configuration
+    """
+
+    CLICKZETTA_USERNAME: str | None = Field(
+        description="Username for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_PASSWORD: str | None = Field(
+        description="Password for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_INSTANCE: str | None = Field(
+        description="Clickzetta Lakehouse instance ID",
+        default=None,
+    )
+
+    CLICKZETTA_SERVICE: str | None = Field(
+        description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_WORKSPACE: str | None = Field(
+        description="Clickzetta workspace name",
+        default="default",
+    )
+
+    CLICKZETTA_VCLUSTER: str | None = Field(
+        description="Clickzetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_SCHEMA: str | None = Field(
+        description="Database schema name in Clickzetta",
+        default="public",
+    )
+
+    CLICKZETTA_BATCH_SIZE: int | None = Field(
+        description="Batch size for bulk insert operations",
+        default=100,
+    )
+
+    CLICKZETTA_ENABLE_INVERTED_INDEX: bool | None = Field(
+        description="Enable inverted index for full-text search capabilities",
+        default=True,
+    )
+
+    CLICKZETTA_ANALYZER_TYPE: str | None = Field(
+        description="Analyzer type for full-text search: keyword, english, chinese, unicode",
+        default="chinese",
+    )
+
+    CLICKZETTA_ANALYZER_MODE: str | None = Field(
+        description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
+        default="smart",
+    )
+
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: str | None = Field(
+        description="Distance function for vector similarity: l2_distance or cosine_distance",
+        default="cosine_distance",
+    )

dify/api/configs/middleware/vdb/couchbase_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class CouchbaseConfig(BaseSettings):
+    """
+    Couchbase configs
+    """
+
+    COUCHBASE_CONNECTION_STRING: str | None = Field(
+        description="COUCHBASE connection string",
+        default=None,
+    )
+
+    COUCHBASE_USER: str | None = Field(
+        description="COUCHBASE user",
+        default=None,
+    )
+
+    COUCHBASE_PASSWORD: str | None = Field(
+        description="COUCHBASE password",
+        default=None,
+    )
+
+    COUCHBASE_BUCKET_NAME: str | None = Field(
+        description="COUCHBASE bucket name",
+        default=None,
+    )
+
+    COUCHBASE_SCOPE_NAME: str | None = Field(
+        description="COUCHBASE scope name",
+        default=None,
+    )

dify/api/configs/middleware/vdb/elasticsearch_config.py

@@ -0,0 +1,74 @@
+from pydantic import Field, PositiveInt, model_validator
+from pydantic_settings import BaseSettings
+
+
+class ElasticsearchConfig(BaseSettings):
+    """
+    Configuration settings for both self-managed and Elastic Cloud deployments.
+    Can load from environment variables or .env files.
+    """
+
+    ELASTICSEARCH_HOST: str | None = Field(
+        description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
+        default="127.0.0.1",
+    )
+
+    ELASTICSEARCH_PORT: PositiveInt = Field(
+        description="Port number on which the Elasticsearch server is listening (default is 9200)",
+        default=9200,
+    )
+
+    ELASTICSEARCH_USERNAME: str | None = Field(
+        description="Username for authenticating with Elasticsearch (default is 'elastic')",
+        default="elastic",
+    )
+
+    ELASTICSEARCH_PASSWORD: str | None = Field(
+        description="Password for authenticating with Elasticsearch (default is 'elastic')",
+        default="elastic",
+    )
+
+    # Elastic Cloud (optional)
+    ELASTICSEARCH_USE_CLOUD: bool | None = Field(
+        description="Set to True to use Elastic Cloud instead of self-hosted Elasticsearch", default=False
+    )
+    ELASTICSEARCH_CLOUD_URL: str | None = Field(
+        description="Full URL for Elastic Cloud deployment (e.g., 'https://example.es.region.aws.found.io:443')",
+        default=None,
+    )
+    ELASTICSEARCH_API_KEY: str | None = Field(description="API key for authenticating with Elastic Cloud", default=None)
+
+    # Common options
+    ELASTICSEARCH_CA_CERTS: str | None = Field(
+        description="Path to CA certificate file for SSL verification", default=None
+    )
+    ELASTICSEARCH_VERIFY_CERTS: bool = Field(
+        description="Whether to verify SSL certificates (default is False)", default=False
+    )
+    ELASTICSEARCH_REQUEST_TIMEOUT: int = Field(
+        description="Request timeout in milliseconds (default is 100000)", default=100000
+    )
+    ELASTICSEARCH_RETRY_ON_TIMEOUT: bool = Field(
+        description="Whether to retry requests on timeout (default is True)", default=True
+    )
+    ELASTICSEARCH_MAX_RETRIES: int = Field(
+        description="Maximum number of retry attempts (default is 10000)", default=10000
+    )
+
+    @model_validator(mode="after")
+    def validate_elasticsearch_config(self):
+        """Validate Elasticsearch configuration based on deployment type."""
+        if self.ELASTICSEARCH_USE_CLOUD:
+            if not self.ELASTICSEARCH_CLOUD_URL:
+                raise ValueError("ELASTICSEARCH_CLOUD_URL is required when using Elastic Cloud")
+            if not self.ELASTICSEARCH_API_KEY:
+                raise ValueError("ELASTICSEARCH_API_KEY is required when using Elastic Cloud")
+        else:
+            if not self.ELASTICSEARCH_HOST:
+                raise ValueError("ELASTICSEARCH_HOST is required for self-hosted Elasticsearch")
+            if not self.ELASTICSEARCH_USERNAME:
+                raise ValueError("ELASTICSEARCH_USERNAME is required for self-hosted Elasticsearch")
+            if not self.ELASTICSEARCH_PASSWORD:
+                raise ValueError("ELASTICSEARCH_PASSWORD is required for self-hosted Elasticsearch")
+
+        return self

dify/api/configs/middleware/vdb/huawei_cloud_config.py

@@ -0,0 +1,23 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class HuaweiCloudConfig(BaseSettings):
+    """
+    Configuration settings for Huawei cloud search service
+    """
+
+    HUAWEI_CLOUD_HOSTS: str | None = Field(
+        description="Hostname or IP address of the Huawei cloud search service instance",
+        default=None,
+    )
+
+    HUAWEI_CLOUD_USER: str | None = Field(
+        description="Username for authenticating with Huawei cloud search service",
+        default=None,
+    )
+
+    HUAWEI_CLOUD_PASSWORD: str | None = Field(
+        description="Password for authenticating with Huawei cloud search service",
+        default=None,
+    )

dify/api/configs/middleware/vdb/lindorm_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class LindormConfig(BaseSettings):
+    """
+    Lindorm configs
+    """
+
+    LINDORM_URL: str | None = Field(
+        description="Lindorm url",
+        default=None,
+    )
+    LINDORM_USERNAME: str | None = Field(
+        description="Lindorm user",
+        default=None,
+    )
+    LINDORM_PASSWORD: str | None = Field(
+        description="Lindorm password",
+        default=None,
+    )
+    LINDORM_INDEX_TYPE: str | None = Field(
+        description="Lindorm Vector Index Type, hnsw or flat is available in dify",
+        default="hnsw",
+    )
+    LINDORM_DISTANCE_TYPE: str | None = Field(
+        description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
+    )
+    LINDORM_USING_UGC: bool | None = Field(
+        description="Using UGC index will store indexes with the same IndexType/Dimension in a single big index.",
+        default=True,
+    )
+    LINDORM_QUERY_TIMEOUT: float | None = Field(description="The lindorm search request timeout (s)", default=2.0)

dify/api/configs/middleware/vdb/matrixone_config.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class MatrixoneConfig(BaseSettings):
+    """Matrixone vector database configuration."""
+
+    MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server")
+    MATRIXONE_PORT: int = Field(default=6001, description="Port number of the Matrixone server")
+    MATRIXONE_USER: str = Field(default="dump", description="Username for authenticating with Matrixone")
+    MATRIXONE_PASSWORD: str = Field(default="111", description="Password for authenticating with Matrixone")
+    MATRIXONE_DATABASE: str = Field(default="dify", description="Name of the Matrixone database to connect to")
+    MATRIXONE_METRIC: str = Field(
+        default="l2", description="Distance metric type for vector similarity search (cosine or l2)"
+    )

dify/api/configs/middleware/vdb/milvus_config.py

@@ -0,0 +1,44 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class MilvusConfig(BaseSettings):
+    """
+    Configuration settings for Milvus vector database
+    """
+
+    MILVUS_URI: str | None = Field(
+        description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
+        default="http://127.0.0.1:19530",
+    )
+
+    MILVUS_TOKEN: str | None = Field(
+        description="Authentication token for Milvus, if token-based authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_USER: str | None = Field(
+        description="Username for authenticating with Milvus, if username/password authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_PASSWORD: str | None = Field(
+        description="Password for authenticating with Milvus, if username/password authentication is enabled",
+        default=None,
+    )
+
+    MILVUS_DATABASE: str = Field(
+        description="Name of the Milvus database to connect to (default is 'default')",
+        default="default",
+    )
+
+    MILVUS_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features (requires Milvus >= 2.5.0). Set to false for compatibility with "
+        "older versions",
+        default=True,
+    )
+
+    MILVUS_ANALYZER_PARAMS: str | None = Field(
+        description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
+        default=None,
+    )

dify/api/configs/middleware/vdb/myscale_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class MyScaleConfig(BaseSettings):
+    """
+    Configuration settings for MyScale vector database
+    """
+
+    MYSCALE_HOST: str = Field(
+        description="Hostname or IP address of the MyScale server (e.g., 'localhost' or 'myscale.example.com')",
+        default="localhost",
+    )
+
+    MYSCALE_PORT: PositiveInt = Field(
+        description="Port number on which the MyScale server is listening (default is 8123)",
+        default=8123,
+    )
+
+    MYSCALE_USER: str = Field(
+        description="Username for authenticating with MyScale (default is 'default')",
+        default="default",
+    )
+
+    MYSCALE_PASSWORD: str = Field(
+        description="Password for authenticating with MyScale (default is an empty string)",
+        default="",
+    )
+
+    MYSCALE_DATABASE: str = Field(
+        description="Name of the MyScale database to connect to (default is 'default')",
+        default="default",
+    )
+
+    MYSCALE_FTS_PARAMS: str = Field(
+        description="Additional parameters for MyScale Full Text Search index)",
+        default="",
+    )

dify/api/configs/middleware/vdb/oceanbase_config.py

@@ -0,0 +1,51 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OceanBaseVectorConfig(BaseSettings):
+    """
+    Configuration settings for OceanBase Vector database
+    """
+
+    OCEANBASE_VECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_PORT: PositiveInt | None = Field(
+        description="Port number on which the OceanBase Vector server is listening (default is 2881)",
+        default=2881,
+    )
+
+    OCEANBASE_VECTOR_USER: str | None = Field(
+        description="Username for authenticating with the OceanBase Vector database",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the OceanBase Vector database",
+        default=None,
+    )
+
+    OCEANBASE_VECTOR_DATABASE: str | None = Field(
+        description="Name of the OceanBase Vector database to connect to",
+        default=None,
+    )
+
+    OCEANBASE_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features (requires OceanBase >= 4.3.5.1). Set to false for compatibility "
+        "with older versions",
+        default=False,
+    )
+
+    OCEANBASE_FULLTEXT_PARSER: str | None = Field(
+        description=(
+            "Fulltext parser to use for text indexing. "
+            "Built-in options: 'ngram' (N-gram tokenizer for English/numbers), "
+            "'beng' (Basic English tokenizer), 'space' (Space-based tokenizer), "
+            "'ngram2' (Improved N-gram tokenizer), 'ik' (Chinese tokenizer). "
+            "External plugins (require installation): 'japanese_ftparser' (Japanese tokenizer), "
+            "'thai_ftparser' (Thai tokenizer). Default is 'ik'"
+        ),
+        default="ik",
+    )

dify/api/configs/middleware/vdb/opengauss_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OpenGaussConfig(BaseSettings):
+    """
+    Configuration settings for OpenGauss
+    """
+
+    OPENGAUSS_HOST: str | None = Field(
+        description="Hostname or IP address of the OpenGauss server(e.g., 'localhost')",
+        default=None,
+    )
+
+    OPENGAUSS_PORT: PositiveInt = Field(
+        description="Port number on which the OpenGauss server is listening (default is 6600)",
+        default=6600,
+    )
+
+    OPENGAUSS_USER: str | None = Field(
+        description="Username for authenticating with the OpenGauss database",
+        default=None,
+    )
+
+    OPENGAUSS_PASSWORD: str | None = Field(
+        description="Password for authenticating with the OpenGauss database",
+        default=None,
+    )
+
+    OPENGAUSS_DATABASE: str | None = Field(
+        description="Name of the OpenGauss database to connect to",
+        default=None,
+    )
+
+    OPENGAUSS_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the OpenGauss database",
+        default=1,
+    )
+
+    OPENGAUSS_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the OpenGauss database",
+        default=5,
+    )
+
+    OPENGAUSS_ENABLE_PQ: bool = Field(
+        description="Enable openGauss PQ acceleration feature",
+        default=False,
+    )

dify/api/configs/middleware/vdb/opensearch_config.py

@@ -0,0 +1,64 @@
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AuthMethod(StrEnum):
+    """
+    Authentication method for OpenSearch
+    """
+
+    BASIC = "basic"
+    AWS_MANAGED_IAM = "aws_managed_iam"
+
+
+class OpenSearchConfig(BaseSettings):
+    """
+    Configuration settings for OpenSearch
+    """
+
+    OPENSEARCH_HOST: str | None = Field(
+        description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
+        default=None,
+    )
+
+    OPENSEARCH_PORT: PositiveInt = Field(
+        description="Port number on which the OpenSearch server is listening (default is 9200)",
+        default=9200,
+    )
+
+    OPENSEARCH_SECURE: bool = Field(
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
+        default=False,
+    )
+
+    OPENSEARCH_VERIFY_CERTS: bool = Field(
+        description="Whether to verify SSL certificates for HTTPS connections (recommended to set True in production)",
+        default=True,
+    )
+
+    OPENSEARCH_AUTH_METHOD: AuthMethod = Field(
+        description="Authentication method for OpenSearch connection (default is 'basic')",
+        default=AuthMethod.BASIC,
+    )
+
+    OPENSEARCH_USER: str | None = Field(
+        description="Username for authenticating with OpenSearch",
+        default=None,
+    )
+
+    OPENSEARCH_PASSWORD: str | None = Field(
+        description="Password for authenticating with OpenSearch",
+        default=None,
+    )
+
+    OPENSEARCH_AWS_REGION: str | None = Field(
+        description="AWS region for OpenSearch (e.g. 'us-west-2')",
+        default=None,
+    )
+
+    OPENSEARCH_AWS_SERVICE: Literal["es", "aoss"] | None = Field(
+        description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
+    )

dify/api/configs/middleware/vdb/oracle_config.py

@@ -0,0 +1,44 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OracleConfig(BaseSettings):
+    """
+    Configuration settings for Oracle database
+    """
+
+    ORACLE_USER: str | None = Field(
+        description="Username for authenticating with the Oracle database",
+        default=None,
+    )
+
+    ORACLE_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Oracle database",
+        default=None,
+    )
+
+    ORACLE_DSN: str | None = Field(
+        description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
+        "For autonomous database, use the service name from tnsnames.ora in the wallet",
+        default=None,
+    )
+
+    ORACLE_CONFIG_DIR: str | None = Field(
+        description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_LOCATION: str | None = Field(
+        description="Oracle wallet directory path containing the wallet files for secure connection",
+        default=None,
+    )
+
+    ORACLE_WALLET_PASSWORD: str | None = Field(
+        description="Password to decrypt the Oracle wallet, if it is encrypted",
+        default=None,
+    )
+
+    ORACLE_IS_AUTONOMOUS: bool = Field(
+        description="Flag indicating whether connecting to Oracle Autonomous Database",
+        default=False,
+    )

dify/api/configs/middleware/vdb/pgvector_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectorConfig(BaseSettings):
+    """
+    Configuration settings for PGVector (PostgreSQL with vector extension)
+    """
+
+    PGVECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    PGVECTOR_PORT: PositiveInt = Field(
+        description="Port number on which the PostgreSQL server is listening (default is 5433)",
+        default=5433,
+    )
+
+    PGVECTOR_USER: str | None = Field(
+        description="Username for authenticating with the PostgreSQL database",
+        default=None,
+    )
+
+    PGVECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the PostgreSQL database",
+        default=None,
+    )
+
+    PGVECTOR_DATABASE: str | None = Field(
+        description="Name of the PostgreSQL database to connect to",
+        default=None,
+    )
+
+    PGVECTOR_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the PostgreSQL database",
+        default=1,
+    )
+
+    PGVECTOR_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the PostgreSQL database",
+        default=5,
+    )
+
+    PGVECTOR_PG_BIGM: bool = Field(
+        description="Whether to use pg_bigm module for full text search",
+        default=False,
+    )

dify/api/configs/middleware/vdb/pgvectors_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectoRSConfig(BaseSettings):
+    """
+    Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
+    """
+
+    PGVECTO_RS_HOST: str | None = Field(
+        description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    PGVECTO_RS_PORT: PositiveInt = Field(
+        description="Port number on which the PostgreSQL server with PGVecto.RS is listening (default is 5431)",
+        default=5431,
+    )
+
+    PGVECTO_RS_USER: str | None = Field(
+        description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
+        default=None,
+    )
+
+    PGVECTO_RS_PASSWORD: str | None = Field(
+        description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
+        default=None,
+    )
+
+    PGVECTO_RS_DATABASE: str | None = Field(
+        description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
+        default=None,
+    )

dify/api/configs/middleware/vdb/qdrant_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class QdrantConfig(BaseSettings):
+    """
+    Configuration settings for Qdrant vector database
+    """
+
+    QDRANT_URL: str | None = Field(
+        description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
+        default=None,
+    )
+
+    QDRANT_API_KEY: str | None = Field(
+        description="API key for authenticating with the Qdrant server",
+        default=None,
+    )
+
+    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description="Timeout in seconds for Qdrant client operations (default is 20 seconds)",
+        default=20,
+    )
+
+    QDRANT_GRPC_ENABLED: bool = Field(
+        description="Whether to enable gRPC support for Qdrant connection (True for gRPC, False for HTTP)",
+        default=False,
+    )
+
+    QDRANT_GRPC_PORT: PositiveInt = Field(
+        description="Port number for gRPC connection to Qdrant server (default is 6334)",
+        default=6334,
+    )
+
+    QDRANT_REPLICATION_FACTOR: PositiveInt = Field(
+        description="Replication factor for Qdrant collections (default is 1)",
+        default=1,
+    )

dify/api/configs/middleware/vdb/relyt_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RelytConfig(BaseSettings):
+    """
+    Configuration settings for Relyt database
+    """
+
+    RELYT_HOST: str | None = Field(
+        description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
+        default=None,
+    )
+
+    RELYT_PORT: PositiveInt = Field(
+        description="Port number on which the Relyt server is listening (default is 9200)",
+        default=9200,
+    )
+
+    RELYT_USER: str | None = Field(
+        description="Username for authenticating with the Relyt database",
+        default=None,
+    )
+
+    RELYT_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Relyt database",
+        default=None,
+    )
+
+    RELYT_DATABASE: str | None = Field(
+        description="Name of the Relyt database to connect to (default is 'default')",
+        default="default",
+    )

dify/api/configs/middleware/vdb/tablestore_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class TableStoreConfig(BaseSettings):
+    """
+    Configuration settings for TableStore.
+    """
+
+    TABLESTORE_ENDPOINT: str | None = Field(
+        description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')",
+        default=None,
+    )
+
+    TABLESTORE_INSTANCE_NAME: str | None = Field(
+        description="Instance name to access TableStore server (eg. 'instance-name')",
+        default=None,
+    )
+
+    TABLESTORE_ACCESS_KEY_ID: str | None = Field(
+        description="AccessKey id for the instance name",
+        default=None,
+    )
+
+    TABLESTORE_ACCESS_KEY_SECRET: str | None = Field(
+        description="AccessKey secret for the instance name",
+        default=None,
+    )
+
+    TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: bool = Field(
+        description="Whether to normalize full-text search scores to [0, 1]",
+        default=False,
+    )

dify/api/configs/middleware/vdb/tencent_vector_config.py

@@ -0,0 +1,53 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TencentVectorDBConfig(BaseSettings):
+    """
+    Configuration settings for Tencent Vector Database
+    """
+
+    TENCENT_VECTOR_DB_URL: str | None = Field(
+        description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_API_KEY: str | None = Field(
+        description="API key for authenticating with the Tencent Vector Database service",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
+        description="Timeout in seconds for Tencent Vector Database operations (default is 30 seconds)",
+        default=30,
+    )
+
+    TENCENT_VECTOR_DB_USERNAME: str | None = Field(
+        description="Username for authenticating with the Tencent Vector Database (if required)",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Tencent Vector Database (if required)",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
+        description="Number of shards for the Tencent Vector Database (default is 1)",
+        default=1,
+    )
+
+    TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+        description="Number of replicas for the Tencent Vector Database (default is 2)",
+        default=2,
+    )
+
+    TENCENT_VECTOR_DB_DATABASE: str | None = Field(
+        description="Name of the specific Tencent Vector Database to connect to",
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: bool = Field(
+        description="Enable hybrid search features",
+        default=False,
+    )

dify/api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -0,0 +1,68 @@
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TidbOnQdrantConfig(BaseSettings):
+    """
+    Tidb on Qdrant configs
+    """
+
+    TIDB_ON_QDRANT_URL: str | None = Field(
+        description="Tidb on Qdrant url",
+        default=None,
+    )
+
+    TIDB_ON_QDRANT_API_KEY: str | None = Field(
+        description="Tidb on Qdrant api key",
+        default=None,
+    )
+
+    TIDB_ON_QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description="Tidb on Qdrant client timeout in seconds",
+        default=20,
+    )
+
+    TIDB_ON_QDRANT_GRPC_ENABLED: bool = Field(
+        description="whether enable grpc support for Tidb on Qdrant connection",
+        default=False,
+    )
+
+    TIDB_ON_QDRANT_GRPC_PORT: PositiveInt = Field(
+        description="Tidb on Qdrant grpc port",
+        default=6334,
+    )
+
+    TIDB_PUBLIC_KEY: str | None = Field(
+        description="Tidb account public key",
+        default=None,
+    )
+
+    TIDB_PRIVATE_KEY: str | None = Field(
+        description="Tidb account private key",
+        default=None,
+    )
+
+    TIDB_API_URL: str | None = Field(
+        description="Tidb API url",
+        default=None,
+    )
+
+    TIDB_IAM_API_URL: str | None = Field(
+        description="Tidb IAM API url",
+        default=None,
+    )
+
+    TIDB_REGION: str | None = Field(
+        description="Tidb serverless region",
+        default="regions/aws-us-east-1",
+    )
+
+    TIDB_PROJECT_ID: str | None = Field(
+        description="Tidb project id",
+        default=None,
+    )
+
+    TIDB_SPEND_LIMIT: int | None = Field(
+        description="Tidb spend limit",
+        default=100,
+    )

dify/api/configs/middleware/vdb/tidb_vector_config.py

@@ -0,0 +1,33 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TiDBVectorConfig(BaseSettings):
+    """
+    Configuration settings for TiDB Vector database
+    """
+
+    TIDB_VECTOR_HOST: str | None = Field(
+        description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
+        default=None,
+    )
+
+    TIDB_VECTOR_PORT: PositiveInt | None = Field(
+        description="Port number on which the TiDB Vector server is listening (default is 4000)",
+        default=4000,
+    )
+
+    TIDB_VECTOR_USER: str | None = Field(
+        description="Username for authenticating with the TiDB Vector database",
+        default=None,
+    )
+
+    TIDB_VECTOR_PASSWORD: str | None = Field(
+        description="Password for authenticating with the TiDB Vector database",
+        default=None,
+    )
+
+    TIDB_VECTOR_DATABASE: str | None = Field(
+        description="Name of the TiDB Vector database to connect to",
+        default=None,
+    )

dify/api/configs/middleware/vdb/upstash_config.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class UpstashConfig(BaseSettings):
+    """
+    Configuration settings for Upstash vector database
+    """
+
+    UPSTASH_VECTOR_URL: str | None = Field(
+        description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
+        default=None,
+    )
+
+    UPSTASH_VECTOR_TOKEN: str | None = Field(
+        description="Token for authenticating with the upstash server",
+        default=None,
+    )

dify/api/configs/middleware/vdb/vastbase_vector_config.py

@@ -0,0 +1,43 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class VastbaseVectorConfig(BaseSettings):
+    """
+    Configuration settings for Vector (Vastbase with vector extension)
+    """
+
+    VASTBASE_HOST: str | None = Field(
+        description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
+        default=None,
+    )
+
+    VASTBASE_PORT: PositiveInt = Field(
+        description="Port number on which the Vastbase server is listening (default is 5432)",
+        default=5432,
+    )
+
+    VASTBASE_USER: str | None = Field(
+        description="Username for authenticating with the Vastbase database",
+        default=None,
+    )
+
+    VASTBASE_PASSWORD: str | None = Field(
+        description="Password for authenticating with the Vastbase database",
+        default=None,
+    )
+
+    VASTBASE_DATABASE: str | None = Field(
+        description="Name of the Vastbase database to connect to",
+        default=None,
+    )
+
+    VASTBASE_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the Vastbase database",
+        default=1,
+    )
+
+    VASTBASE_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the Vastbase database",
+        default=5,
+    )

dify/api/configs/middleware/vdb/vikingdb_config.py

@@ -0,0 +1,48 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class VikingDBConfig(BaseSettings):
+    """
+    Configuration for connecting to Volcengine VikingDB.
+    Refer to the following documentation for details on obtaining credentials:
+    https://www.volcengine.com/docs/6291/65568
+    """
+
+    VIKINGDB_ACCESS_KEY: str | None = Field(
+        description="The Access Key provided by Volcengine VikingDB for API authentication."
+        "Refer to the following documentation for details on obtaining credentials:"
+        "https://www.volcengine.com/docs/6291/65568",
+        default=None,
+    )
+
+    VIKINGDB_SECRET_KEY: str | None = Field(
+        description="The Secret Key provided by Volcengine VikingDB for API authentication.",
+        default=None,
+    )
+
+    VIKINGDB_REGION: str = Field(
+        description="The region of the Volcengine VikingDB service.(e.g., 'cn-shanghai', 'cn-beijing').",
+        default="cn-shanghai",
+    )
+
+    VIKINGDB_HOST: str = Field(
+        description="The host of the Volcengine VikingDB service.(e.g., 'api-vikingdb.volces.com', \
+            'api-vikingdb.mlp.cn-shanghai.volces.com')",
+        default="api-vikingdb.mlp.cn-shanghai.volces.com",
+    )
+
+    VIKINGDB_SCHEME: str = Field(
+        description="The scheme of the Volcengine VikingDB service.(e.g., 'http', 'https').",
+        default="http",
+    )
+
+    VIKINGDB_CONNECTION_TIMEOUT: int = Field(
+        description="The connection timeout of the Volcengine VikingDB service.",
+        default=30,
+    )
+
+    VIKINGDB_SOCKET_TIMEOUT: int = Field(
+        description="The socket timeout of the Volcengine VikingDB service.",
+        default=30,
+    )

dify/api/configs/middleware/vdb/weaviate_config.py

@@ -0,0 +1,38 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class WeaviateConfig(BaseSettings):
+    """
+    Configuration settings for Weaviate vector database
+    """
+
+    WEAVIATE_ENDPOINT: str | None = Field(
+        description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
+        default=None,
+    )
+
+    WEAVIATE_API_KEY: str | None = Field(
+        description="API key for authenticating with the Weaviate server",
+        default=None,
+    )
+
+    WEAVIATE_GRPC_ENABLED: bool = Field(
+        description="Whether to enable gRPC for Weaviate connection (True for gRPC, False for HTTP)",
+        default=True,
+    )
+
+    WEAVIATE_GRPC_ENDPOINT: str | None = Field(
+        description="URL of the Weaviate gRPC server (e.g., 'grpc://localhost:50051' or 'grpcs://weaviate.example.com:443')",
+        default=None,
+    )
+
+    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
+        description="Number of objects to be processed in a single batch operation (default is 100)",
+        default=100,
+    )
+
+    WEAVIATE_TOKENIZATION: str | None = Field(
+        description="Tokenization for Weaviate (default is word)",
+        default="word",
+    )

dify/api/configs/observability/__init__.py

@@ -0,0 +1,9 @@
+from configs.observability.otel.otel_config import OTelConfig
+
+
+class ObservabilityConfig(OTelConfig):
+    """
+    Observability configuration settings
+    """
+
+    pass

dify/api/configs/observability/otel/otel_config.py

@@ -0,0 +1,59 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OTelConfig(BaseSettings):
+    """
+    OpenTelemetry configuration settings
+    """
+
+    ENABLE_OTEL: bool = Field(
+        description="Whether to enable OpenTelemetry",
+        default=False,
+    )
+
+    OTLP_TRACE_ENDPOINT: str = Field(
+        description="OTLP trace endpoint",
+        default="",
+    )
+
+    OTLP_METRIC_ENDPOINT: str = Field(
+        description="OTLP metric endpoint",
+        default="",
+    )
+
+    OTLP_BASE_ENDPOINT: str = Field(
+        description="OTLP base endpoint",
+        default="http://localhost:4318",
+    )
+
+    OTLP_API_KEY: str = Field(
+        description="OTLP API key",
+        default="",
+    )
+
+    OTEL_EXPORTER_TYPE: str = Field(
+        description="OTEL exporter type",
+        default="otlp",
+    )
+
+    OTEL_EXPORTER_OTLP_PROTOCOL: str = Field(
+        description="OTLP exporter protocol ('grpc' or 'http')",
+        default="http",
+    )
+
+    OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
+
+    OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(
+        default=5000, description="Batch export schedule delay in milliseconds"
+    )
+
+    OTEL_MAX_QUEUE_SIZE: int = Field(default=2048, description="Maximum queue size for the batch span processor")
+
+    OTEL_MAX_EXPORT_BATCH_SIZE: int = Field(default=512, description="Maximum export batch size")
+
+    OTEL_METRIC_EXPORT_INTERVAL: int = Field(default=60000, description="Metric export interval in milliseconds")
+
+    OTEL_BATCH_EXPORT_TIMEOUT: int = Field(default=10000, description="Batch export timeout in milliseconds")
+
+    OTEL_METRIC_EXPORT_TIMEOUT: int = Field(default=30000, description="Metric export timeout in milliseconds")

dify/api/configs/packaging/__init__.py

@@ -0,0 +1,14 @@
+from pydantic import Field
+
+from configs.packaging.pyproject import PyProjectTomlConfig
+
+
+class PackagingInfo(PyProjectTomlConfig):
+    """
+    Packaging build information
+    """
+
+    COMMIT_SHA: str = Field(
+        description="SHA-1 checksum of the git commit used to build the app",
+        default="",
+    )

dify/api/configs/packaging/pyproject.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+from pydantic_settings import BaseSettings
+
+
+class PyProjectConfig(BaseModel):
+    version: str = Field(description="Dify version", default="")
+
+
+class PyProjectTomlConfig(BaseSettings):
+    """
+    configs in api/pyproject.toml
+    """
+
+    project: PyProjectConfig = Field(
+        description="configs in the project section of pyproject.toml",
+        default=PyProjectConfig(),
+    )

dify/api/configs/remote_settings_sources/__init__.py

@@ -0,0 +1,15 @@
+from pydantic import Field
+
+from .apollo import ApolloSettingsSourceInfo
+from .base import RemoteSettingsSource
+from .enums import RemoteSettingsSourceName
+
+
+class RemoteSettingsSourceConfig(ApolloSettingsSourceInfo):
+    REMOTE_SETTINGS_SOURCE_NAME: RemoteSettingsSourceName | str = Field(
+        description="name of remote config source",
+        default="",
+    )
+
+
+__all__ = ["RemoteSettingsSource", "RemoteSettingsSourceConfig", "RemoteSettingsSourceName"]

dify/api/configs/remote_settings_sources/apollo/__init__.py

@@ -0,0 +1,55 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import Field
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings
+
+from configs.remote_settings_sources.base import RemoteSettingsSource
+
+from .client import ApolloClient
+
+
+class ApolloSettingsSourceInfo(BaseSettings):
+    """
+    Packaging build information
+    """
+
+    APOLLO_APP_ID: str | None = Field(
+        description="apollo app_id",
+        default=None,
+    )
+
+    APOLLO_CLUSTER: str | None = Field(
+        description="apollo cluster",
+        default=None,
+    )
+
+    APOLLO_CONFIG_URL: str | None = Field(
+        description="apollo config url",
+        default=None,
+    )
+
+    APOLLO_NAMESPACE: str | None = Field(
+        description="apollo namespace",
+        default=None,
+    )
+
+
+class ApolloSettingsSource(RemoteSettingsSource):
+    def __init__(self, configs: Mapping[str, Any]):
+        self.client = ApolloClient(
+            app_id=configs["APOLLO_APP_ID"],
+            cluster=configs["APOLLO_CLUSTER"],
+            config_url=configs["APOLLO_CONFIG_URL"],
+            start_hot_update=False,
+            _notification_map={configs["APOLLO_NAMESPACE"]: -1},
+        )
+        self.namespace = configs["APOLLO_NAMESPACE"]
+        self.remote_configs = self.client.get_all_dicts(self.namespace)
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        if not isinstance(self.remote_configs, dict):
+            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
+        field_value = self.remote_configs.get(field_name)
+        return field_value, field_name, False

dify/api/configs/remote_settings_sources/apollo/client.py

@@ -0,0 +1,306 @@
+import hashlib
+import json
+import logging
+import os
+import threading
+import time
+from collections.abc import Callable, Mapping
+from pathlib import Path
+from typing import Any
+
+from .python_3x import http_request, makedirs_wrapper
+from .utils import (
+    CONFIGURATIONS,
+    NAMESPACE_NAME,
+    NOTIFICATION_ID,
+    get_value_from_dict,
+    init_ip,
+    no_key_cache_key,
+    signature,
+    url_encode_wrapper,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ApolloClient:
+    def __init__(
+        self,
+        config_url: str,
+        app_id: str,
+        cluster: str = "default",
+        secret: str = "",
+        start_hot_update: bool = True,
+        change_listener: Callable[[str, str, str, Any], None] | None = None,
+        _notification_map: dict[str, int] | None = None,
+    ):
+        # Core routing parameters
+        self.config_url = config_url
+        self.cluster = cluster
+        self.app_id = app_id
+
+        # Non-core parameters
+        self.ip = init_ip()
+        self.secret = secret
+
+        # Check the parameter variables
+
+        # Private control variables
+        self._cycle_time = 5
+        self._stopping = False
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._no_key: dict[str, str] = {}
+        self._hash: dict[str, str] = {}
+        self._pull_timeout = 75
+        self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
+        self._long_poll_thread: threading.Thread | None = None
+        self._change_listener = change_listener  # "add" "delete" "update"
+        if _notification_map is None:
+            _notification_map = {"application": -1}
+        self._notification_map = _notification_map
+        self.last_release_key: str | None = None
+        # Private startup method
+        self._path_checker()
+        if start_hot_update:
+            self._start_hot_update()
+
+        # start the heartbeat thread
+        heartbeat = threading.Thread(target=self._heart_beat)
+        heartbeat.daemon = True
+        heartbeat.start()
+
+    def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None:
+        url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
+            self.config_url, self.app_id, self.cluster, namespace, "", self.ip
+        )
+        try:
+            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
+            if code == 200:
+                if not body:
+                    logger.error("get_json_from_net load configs failed, body is %s", body)
+                    return None
+                data = json.loads(body)
+                data = data["configurations"]
+                return_data = {CONFIGURATIONS: data}
+                return return_data
+            else:
+                return None
+        except Exception:
+            logger.exception("an error occurred in get_json_from_net")
+            return None
+
+    def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any:
+        try:
+            # read memory configuration
+            namespace_cache = self._cache.get(namespace)
+            val = get_value_from_dict(namespace_cache, key)
+            if val is not None:
+                return val
+
+            no_key = no_key_cache_key(namespace, key)
+            if no_key in self._no_key:
+                return default_val
+
+            # read the network configuration
+            namespace_data = self.get_json_from_net(namespace)
+            val = get_value_from_dict(namespace_data, key)
+            if val is not None:
+                if namespace_data is not None:
+                    self._update_cache_and_file(namespace_data, namespace)
+                return val
+
+            # read the file configuration
+            namespace_cache = self._get_local_cache(namespace)
+            val = get_value_from_dict(namespace_cache, key)
+            if val is not None:
+                self._update_cache_and_file(namespace_cache, namespace)
+                return val
+
+            # If all of them are not obtained, the default value is returned
+            # and the local cache is set to None
+            self._set_local_cache_none(namespace, key)
+            return default_val
+        except Exception:
+            logger.exception("get_value has error, [key is %s], [namespace is %s]", key, namespace)
+            return default_val
+
+    # Set the key of a namespace to none, and do not set default val
+    # to ensure the real-time correctness of the function call.
+    # If the user does not have the same default val twice
+    # and the default val is used here, there may be a problem.
+    def _set_local_cache_none(self, namespace: str, key: str) -> None:
+        no_key = no_key_cache_key(namespace, key)
+        self._no_key[no_key] = key
+
+    def _start_hot_update(self) -> None:
+        self._long_poll_thread = threading.Thread(target=self._listener)
+        # When the asynchronous thread is started, the daemon thread will automatically exit
+        # when the main thread is launched.
+        self._long_poll_thread.daemon = True
+        self._long_poll_thread.start()
+
+    def stop(self) -> None:
+        self._stopping = True
+        logger.info("Stopping listener...")
+
+    # Call the set callback function, and if it is abnormal, try it out
+    def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None:
+        if self._change_listener is None:
+            return
+        if old_kv is None:
+            old_kv = {}
+        if new_kv is None:
+            new_kv = {}
+        try:
+            for key in old_kv:
+                new_value = new_kv.get(key)
+                old_value = old_kv.get(key)
+                if new_value is None:
+                    # If newValue is empty, it means key, and the value is deleted.
+                    self._change_listener("delete", namespace, key, old_value)
+                    continue
+                if new_value != old_value:
+                    self._change_listener("update", namespace, key, new_value)
+                    continue
+            for key in new_kv:
+                new_value = new_kv.get(key)
+                old_value = old_kv.get(key)
+                if old_value is None:
+                    self._change_listener("add", namespace, key, new_value)
+        except BaseException as e:
+            logger.warning(str(e))
+
+    def _path_checker(self) -> None:
+        if not os.path.isdir(self._cache_file_path):
+            makedirs_wrapper(self._cache_file_path)
+
+    # update the local cache and file cache
+    def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None:
+        # update the local cache
+        self._cache[namespace] = namespace_data
+        # update the file cache
+        new_string = json.dumps(namespace_data)
+        new_hash = hashlib.md5(new_string.encode("utf-8")).hexdigest()
+        if self._hash.get(namespace) == new_hash:
+            pass
+        else:
+            file_path = Path(self._cache_file_path) / f"{self.app_id}_configuration_{namespace}.txt"
+            file_path.write_text(new_string)
+            self._hash[namespace] = new_hash
+
+    # get the configuration from the local file
+    def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]:
+        cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
+        if os.path.isfile(cache_file_path):
+            with open(cache_file_path) as f:
+                result = json.loads(f.readline())
+            return result
+        return {}
+
+    def _long_poll(self) -> None:
+        notifications: list[dict[str, Any]] = []
+        for key in self._cache:
+            namespace_data = self._cache[key]
+            notification_id = -1
+            if NOTIFICATION_ID in namespace_data:
+                notification_id = self._cache[key][NOTIFICATION_ID]
+            notifications.append({NAMESPACE_NAME: key, NOTIFICATION_ID: notification_id})
+        try:
+            # if the length is 0 it is returned directly
+            if len(notifications) == 0:
+                return
+            url = f"{self.config_url}/notifications/v2"
+            params = {
+                "appId": self.app_id,
+                "cluster": self.cluster,
+                "notifications": json.dumps(notifications, ensure_ascii=False),
+            }
+            param_str = url_encode_wrapper(params)
+            url = url + "?" + param_str
+            code, body = http_request(url, self._pull_timeout, headers=self._sign_headers(url))
+            http_code = code
+            if http_code == 304:
+                logger.debug("No change, loop...")
+                return
+            if http_code == 200:
+                if not body:
+                    logger.error("_long_poll load configs failed,body is %s", body)
+                    return
+                data = json.loads(body)
+                for entry in data:
+                    namespace = entry[NAMESPACE_NAME]
+                    n_id = entry[NOTIFICATION_ID]
+                    logger.info("%s has changes: notificationId=%d", namespace, n_id)
+                    self._get_net_and_set_local(namespace, n_id, call_change=True)
+                    return
+            else:
+                logger.warning("Sleep...")
+        except Exception as e:
+            logger.warning(str(e))
+
+    def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None:
+        namespace_data = self.get_json_from_net(namespace)
+        if not namespace_data:
+            return
+        namespace_data[NOTIFICATION_ID] = n_id
+        old_namespace = self._cache.get(namespace)
+        self._update_cache_and_file(namespace_data, namespace)
+        if self._change_listener is not None and call_change and old_namespace:
+            old_kv = old_namespace.get(CONFIGURATIONS)
+            new_kv = namespace_data.get(CONFIGURATIONS)
+            self._call_listener(namespace, old_kv, new_kv)
+
+    def _listener(self) -> None:
+        logger.info("start long_poll")
+        while not self._stopping:
+            self._long_poll()
+            time.sleep(self._cycle_time)
+        logger.info("stopped, long_poll")
+
+    # add the need for endorsement to the header
+    def _sign_headers(self, url: str) -> Mapping[str, str]:
+        headers: dict[str, str] = {}
+        if self.secret == "":
+            return headers
+        uri = url[len(self.config_url) : len(url)]
+        time_unix_now = str(int(round(time.time() * 1000)))
+        headers["Authorization"] = "Apollo " + self.app_id + ":" + signature(time_unix_now, uri, self.secret)
+        headers["Timestamp"] = time_unix_now
+        return headers
+
+    def _heart_beat(self) -> None:
+        while not self._stopping:
+            for namespace in self._notification_map:
+                self._do_heart_beat(namespace)
+            time.sleep(60 * 10)  # 10 minutes
+
+    def _do_heart_beat(self, namespace: str) -> None:
+        url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}"
+        try:
+            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
+            if code == 200:
+                if not body:
+                    logger.error("_do_heart_beat load configs failed,body is %s", body)
+                    return None
+                data = json.loads(body)
+                if self.last_release_key == data["releaseKey"]:
+                    return None
+                self.last_release_key = data["releaseKey"]
+                data = data["configurations"]
+                self._update_cache_and_file(data, namespace)
+            else:
+                return None
+        except Exception:
+            logger.exception("an error occurred in _do_heart_beat")
+            return None
+
+    def get_all_dicts(self, namespace: str) -> dict[str, Any] | None:
+        namespace_data = self._cache.get(namespace)
+        if namespace_data is None:
+            net_namespace_data = self.get_json_from_net(namespace)
+            if not net_namespace_data:
+                return namespace_data
+            namespace_data = net_namespace_data.get(CONFIGURATIONS)
+            if namespace_data:
+                self._update_cache_and_file(namespace_data, namespace)
+        return namespace_data

dify/api/configs/remote_settings_sources/apollo/python_3x.py

@@ -0,0 +1,43 @@
+import logging
+import os
+import ssl
+import urllib.request
+from collections.abc import Mapping
+from typing import Any
+from urllib import parse
+from urllib.error import HTTPError
+
+# Create an SSL context that allows for a lower level of security
+ssl_context = ssl.create_default_context()
+ssl_context.set_ciphers("HIGH:!DH:!aNULL")
+ssl_context.check_hostname = False
+ssl_context.verify_mode = ssl.CERT_NONE
+
+# Create an opener object and pass in a custom SSL context
+opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ssl_context))
+
+urllib.request.install_opener(opener)
+
+logger = logging.getLogger(__name__)
+
+
+def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]:
+    try:
+        request = urllib.request.Request(url, headers=dict(headers))
+        res = urllib.request.urlopen(request, timeout=timeout)
+        body = res.read().decode("utf-8")
+        return res.code, body
+    except HTTPError as e:
+        if e.code == 304:
+            logger.warning("http_request error,code is 304, maybe you should check secret")
+            return 304, None
+        logger.warning("http_request error,code is %d, msg is %s", e.code, e.msg)
+        raise e
+
+
+def url_encode(params: dict[str, Any]) -> str:
+    return parse.urlencode(params)
+
+
+def makedirs_wrapper(path: str) -> None:
+    os.makedirs(path, exist_ok=True)

dify/api/configs/remote_settings_sources/apollo/utils.py

@@ -0,0 +1,52 @@
+import hashlib
+import socket
+from typing import Any
+
+from .python_3x import url_encode
+
+# define constants
+CONFIGURATIONS = "configurations"
+NOTIFICATION_ID = "notificationId"
+NAMESPACE_NAME = "namespaceName"
+
+
+# add timestamps uris and keys
+def signature(timestamp: str, uri: str, secret: str) -> str:
+    import base64
+    import hmac
+
+    string_to_sign = "" + timestamp + "\n" + uri
+    hmac_code = hmac.new(secret.encode(), string_to_sign.encode(), hashlib.sha1).digest()
+    return base64.b64encode(hmac_code).decode()
+
+
+def url_encode_wrapper(params: dict[str, Any]) -> str:
+    return url_encode(params)
+
+
+def no_key_cache_key(namespace: str, key: str) -> str:
+    return f"{namespace}{len(namespace)}{key}"
+
+
+# Returns whether the obtained value is obtained, and None if it does not
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any:
+    if namespace_cache:
+        kv_data = namespace_cache.get(CONFIGURATIONS)
+        if kv_data is None:
+            return None
+        if key in kv_data:
+            return kv_data[key]
+    return None
+
+
+def init_ip() -> str:
+    ip = ""
+    s = None
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("8.8.8.8", 53))
+        ip = s.getsockname()[0]
+    finally:
+        if s:
+            s.close()
+    return ip

dify/api/configs/remote_settings_sources/base.py

@@ -0,0 +1,15 @@
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic.fields import FieldInfo
+
+
+class RemoteSettingsSource:
+    def __init__(self, configs: Mapping[str, Any]):
+        pass
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        raise NotImplementedError
+
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool):
+        return value

dify/api/configs/remote_settings_sources/enums.py

@@ -0,0 +1,6 @@
+from enum import StrEnum
+
+
+class RemoteSettingsSourceName(StrEnum):
+    APOLLO = "apollo"
+    NACOS = "nacos"

dify/api/configs/remote_settings_sources/nacos/__init__.py

@@ -0,0 +1,49 @@
+import logging
+import os
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic.fields import FieldInfo
+
+from .http_request import NacosHttpClient
+
+logger = logging.getLogger(__name__)
+
+from configs.remote_settings_sources.base import RemoteSettingsSource
+
+from .utils import parse_config
+
+
+class NacosSettingsSource(RemoteSettingsSource):
+    def __init__(self, configs: Mapping[str, Any]):
+        self.configs = configs
+        self.remote_configs: dict[str, str] = {}
+        self.async_init()
+
+    def async_init(self) -> None:
+        data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
+        group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
+        tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
+
+        params = {"dataId": data_id, "group": group, "tenant": tenant}
+        try:
+            content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
+            self.remote_configs = self._parse_config(content)
+        except Exception:
+            logger.exception("[get-access-token] exception occurred")
+            raise
+
+    def _parse_config(self, content: str) -> dict[str, str]:
+        if not content:
+            return {}
+        try:
+            return parse_config(content)
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse config: {e}")
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        field_value = self.remote_configs.get(field_name)
+        if field_value is None:
+            return None, field_name, False
+
+        return field_value, field_name, False

dify/api/configs/remote_settings_sources/nacos/http_request.py

@@ -0,0 +1,90 @@
+import base64
+import hashlib
+import hmac
+import logging
+import os
+import time
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class NacosHttpClient:
+    def __init__(self):
+        self.username = os.getenv("DIFY_ENV_NACOS_USERNAME")
+        self.password = os.getenv("DIFY_ENV_NACOS_PASSWORD")
+        self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
+        self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
+        self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
+        self.token: str | None = None
+        self.token_ttl = 18000
+        self.token_expire_time: float = 0
+
+    def http_request(
+        self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None
+    ) -> str:
+        if headers is None:
+            headers = {}
+        if params is None:
+            params = {}
+        try:
+            self._inject_auth_info(headers, params)
+            response = httpx.request(method, url="http://" + self.server + url, headers=headers, params=params)
+            response.raise_for_status()
+            return response.text
+        except httpx.RequestError as e:
+            return f"Request to Nacos failed: {e}"
+
+    def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
+        headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
+
+        if module == "login":
+            return
+
+        ts = str(int(time.time() * 1000))
+
+        if self.ak and self.sk:
+            sign_str = self.get_sign_str(params["group"], params["tenant"], ts)
+            headers["Spas-AccessKey"] = self.ak
+            headers["Spas-Signature"] = self.__do_sign(sign_str, self.sk)
+            headers["timeStamp"] = ts
+        if self.username and self.password:
+            self.get_access_token(force_refresh=False)
+            if self.token is not None:
+                params["accessToken"] = self.token
+
+    def __do_sign(self, sign_str: str, sk: str) -> str:
+        return (
+            base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
+            .decode()
+            .strip()
+        )
+
+    def get_sign_str(self, group: str, tenant: str, ts: str) -> str:
+        sign_str = ""
+        if tenant:
+            sign_str = tenant + "+"
+        if group:
+            sign_str = sign_str + group + "+"
+        sign_str += ts  # Directly concatenate ts without conditional checks, because the nacos auth header forced it.
+        return sign_str
+
+    def get_access_token(self, force_refresh: bool = False) -> str | None:
+        current_time = time.time()
+        if self.token and not force_refresh and self.token_expire_time > current_time:
+            return self.token
+
+        params = {"username": self.username, "password": self.password}
+        url = "http://" + self.server + "/nacos/v1/auth/login"
+        try:
+            resp = httpx.request("POST", url, headers=None, params=params)
+            resp.raise_for_status()
+            response_data = resp.json()
+            self.token = response_data.get("accessToken")
+            self.token_ttl = response_data.get("tokenTtl", 18000)
+            self.token_expire_time = current_time + self.token_ttl - 10
+            return self.token
+        except Exception:
+            logger.exception("[get-access-token] exception occur")
+            raise

dify/api/configs/remote_settings_sources/nacos/utils.py

@@ -0,0 +1,31 @@
+def parse_config(content: str) -> dict[str, str]:
+    config: dict[str, str] = {}
+    if not content:
+        return config
+
+    for line in content.splitlines():
+        cleaned_line = line.strip()
+        if not cleaned_line or cleaned_line.startswith(("#", "!")):
+            continue
+
+        separator_index = -1
+        for i, c in enumerate(cleaned_line):
+            if c in ("=", ":") and (i == 0 or cleaned_line[i - 1] != "\\"):
+                separator_index = i
+                break
+
+        if separator_index == -1:
+            continue
+
+        key = cleaned_line[:separator_index].strip()
+        raw_value = cleaned_line[separator_index + 1 :].strip()
+
+        try:
+            decoded_value = bytes(raw_value, "utf-8").decode("unicode_escape")
+            decoded_value = decoded_value.replace(r"\=", "=").replace(r"\:", ":")
+        except UnicodeDecodeError:
+            decoded_value = raw_value
+
+        config[key] = decoded_value
+
+    return config

dify/api/constants/__init__.py

@@ -0,0 +1,70 @@
+from configs import dify_config
+from libs.collection_utils import convert_to_lower_and_upper_set
+
+HIDDEN_VALUE = "[__HIDDEN__]"
+UNKNOWN_VALUE = "[__UNKNOWN__]"
+UUID_NIL = "00000000-0000-0000-0000-000000000000"
+
+DEFAULT_FILE_NUMBER_LIMITS = 3
+
+IMAGE_EXTENSIONS = convert_to_lower_and_upper_set({"jpg", "jpeg", "png", "webp", "gif", "svg"})
+
+VIDEO_EXTENSIONS = convert_to_lower_and_upper_set({"mp4", "mov", "mpeg", "webm"})
+
+AUDIO_EXTENSIONS = convert_to_lower_and_upper_set({"mp3", "m4a", "wav", "amr", "mpga"})
+
+_doc_extensions: set[str]
+if dify_config.ETL_TYPE == "Unstructured":
+    _doc_extensions = {
+        "txt",
+        "markdown",
+        "md",
+        "mdx",
+        "pdf",
+        "html",
+        "htm",
+        "xlsx",
+        "xls",
+        "vtt",
+        "properties",
+        "doc",
+        "docx",
+        "csv",
+        "eml",
+        "msg",
+        "pptx",
+        "xml",
+        "epub",
+    }
+    if dify_config.UNSTRUCTURED_API_URL:
+        _doc_extensions.add("ppt")
+else:
+    _doc_extensions = {
+        "txt",
+        "markdown",
+        "md",
+        "mdx",
+        "pdf",
+        "html",
+        "htm",
+        "xlsx",
+        "xls",
+        "docx",
+        "csv",
+        "vtt",
+        "properties",
+    }
+DOCUMENT_EXTENSIONS: set[str] = convert_to_lower_and_upper_set(_doc_extensions)
+
+# console
+COOKIE_NAME_ACCESS_TOKEN = "access_token"
+COOKIE_NAME_REFRESH_TOKEN = "refresh_token"
+COOKIE_NAME_CSRF_TOKEN = "csrf_token"
+
+# webapp
+COOKIE_NAME_WEBAPP_ACCESS_TOKEN = "webapp_access_token"
+COOKIE_NAME_PASSPORT = "passport"
+
+HEADER_NAME_CSRF_TOKEN = "X-CSRF-Token"
+HEADER_NAME_APP_CODE = "X-App-Code"
+HEADER_NAME_PASSPORT = "X-App-Passport"

dify/api/constants/languages.py

@@ -0,0 +1,39 @@
+language_timezone_mapping = {
+    "en-US": "America/New_York",
+    "zh-Hans": "Asia/Shanghai",
+    "zh-Hant": "Asia/Taipei",
+    "pt-BR": "America/Sao_Paulo",
+    "es-ES": "Europe/Madrid",
+    "fr-FR": "Europe/Paris",
+    "de-DE": "Europe/Berlin",
+    "ja-JP": "Asia/Tokyo",
+    "ko-KR": "Asia/Seoul",
+    "ru-RU": "Europe/Moscow",
+    "it-IT": "Europe/Rome",
+    "uk-UA": "Europe/Kyiv",
+    "vi-VN": "Asia/Ho_Chi_Minh",
+    "ro-RO": "Europe/Bucharest",
+    "pl-PL": "Europe/Warsaw",
+    "hi-IN": "Asia/Kolkata",
+    "tr-TR": "Europe/Istanbul",
+    "fa-IR": "Asia/Tehran",
+    "sl-SI": "Europe/Ljubljana",
+    "th-TH": "Asia/Bangkok",
+    "id-ID": "Asia/Jakarta",
+}
+
+languages = list(language_timezone_mapping.keys())
+
+
+def supported_language(lang):
+    if lang in languages:
+        return lang
+
+    error = f"{lang} is not a valid language."
+    raise ValueError(error)
+
+
+def get_valid_language(lang: str | None) -> str:
+    if lang and lang in languages:
+        return lang
+    return languages[0]

dify/api/constants/mimetypes.py

@@ -0,0 +1,7 @@
+# The two constants below should keep in sync.
+# Default content type for files which have no explicit content type.
+
+DEFAULT_MIME_TYPE = "application/octet-stream"
+# Default file extension for files which have no explicit content type, should
+# correspond to the `DEFAULT_MIME_TYPE` above.
+DEFAULT_EXTENSION = ".bin"

dify/api/constants/model_template.py

@@ -0,0 +1,84 @@
+import json
+from collections.abc import Mapping
+
+from models.model import AppMode
+
+default_app_templates: Mapping[AppMode, Mapping] = {
+    # workflow default mode
+    AppMode.WORKFLOW: {
+        "app": {
+            "mode": AppMode.WORKFLOW,
+            "enable_site": True,
+            "enable_api": True,
+        }
+    },
+    # completion default mode
+    AppMode.COMPLETION: {
+        "app": {
+            "mode": AppMode.COMPLETION,
+            "enable_site": True,
+            "enable_api": True,
+        },
+        "model_config": {
+            "model": {
+                "provider": "openai",
+                "name": "gpt-4o",
+                "mode": "chat",
+                "completion_params": {},
+            },
+            "user_input_form": json.dumps(
+                [
+                    {
+                        "paragraph": {
+                            "label": "Query",
+                            "variable": "query",
+                            "required": True,
+                            "default": "",
+                        },
+                    },
+                ]
+            ),
+            "pre_prompt": "{{query}}",
+        },
+    },
+    # chat default mode
+    AppMode.CHAT: {
+        "app": {
+            "mode": AppMode.CHAT,
+            "enable_site": True,
+            "enable_api": True,
+        },
+        "model_config": {
+            "model": {
+                "provider": "openai",
+                "name": "gpt-4o",
+                "mode": "chat",
+                "completion_params": {},
+            },
+        },
+    },
+    # advanced-chat default mode
+    AppMode.ADVANCED_CHAT: {
+        "app": {
+            "mode": AppMode.ADVANCED_CHAT,
+            "enable_site": True,
+            "enable_api": True,
+        },
+    },
+    # agent-chat default mode
+    AppMode.AGENT_CHAT: {
+        "app": {
+            "mode": AppMode.AGENT_CHAT,
+            "enable_site": True,
+            "enable_api": True,
+        },
+        "model_config": {
+            "model": {
+                "provider": "openai",
+                "name": "gpt-4o",
+                "mode": "chat",
+                "completion_params": {},
+            },
+        },
+    },
+}

dify/api/constants/tts_auto_play_timeout.py

@@ -0,0 +1,4 @@
+TTS_AUTO_PLAY_TIMEOUT = 5
+
+# sleep 20 ms ( 40ms => 1280 byte audio file,20ms => 640 byte audio file)
+TTS_AUTO_PLAY_YIELD_CPU_TIME = 0.02

dify/api/contexts/__init__.py

@@ -0,0 +1,52 @@
+from contextvars import ContextVar
+from threading import Lock
+from typing import TYPE_CHECKING
+
+from contexts.wrapper import RecyclableContextVar
+
+if TYPE_CHECKING:
+    from core.datasource.__base.datasource_provider import DatasourcePluginProviderController
+    from core.model_runtime.entities.model_entities import AIModelEntity
+    from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
+    from core.tools.plugin_tool.provider import PluginToolProviderController
+    from core.trigger.provider import PluginTriggerProviderController
+
+
+"""
+To avoid race-conditions caused by gunicorn thread recycling, using RecyclableContextVar to replace with
+"""
+plugin_tool_providers: RecyclableContextVar[dict[str, "PluginToolProviderController"]] = RecyclableContextVar(
+    ContextVar("plugin_tool_providers")
+)
+
+plugin_tool_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(ContextVar("plugin_tool_providers_lock"))
+
+plugin_model_providers: RecyclableContextVar[list["PluginModelProviderEntity"] | None] = RecyclableContextVar(
+    ContextVar("plugin_model_providers")
+)
+
+plugin_model_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("plugin_model_providers_lock")
+)
+
+plugin_model_schema_lock: RecyclableContextVar[Lock] = RecyclableContextVar(ContextVar("plugin_model_schema_lock"))
+
+plugin_model_schemas: RecyclableContextVar[dict[str, "AIModelEntity"]] = RecyclableContextVar(
+    ContextVar("plugin_model_schemas")
+)
+
+datasource_plugin_providers: RecyclableContextVar[dict[str, "DatasourcePluginProviderController"]] = (
+    RecyclableContextVar(ContextVar("datasource_plugin_providers"))
+)
+
+datasource_plugin_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("datasource_plugin_providers_lock")
+)
+
+plugin_trigger_providers: RecyclableContextVar[dict[str, "PluginTriggerProviderController"]] = RecyclableContextVar(
+    ContextVar("plugin_trigger_providers")
+)
+
+plugin_trigger_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("plugin_trigger_providers_lock")
+)

dify/api/contexts/wrapper.py

@@ -0,0 +1,65 @@
+from contextvars import ContextVar
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+class HiddenValue:
+    pass
+
+
+_default = HiddenValue()
+
+
+class RecyclableContextVar(Generic[T]):
+    """
+    RecyclableContextVar is a wrapper around ContextVar
+    It's safe to use in gunicorn with thread recycling, but features like `reset` are not available for now
+
+    NOTE: you need to call `increment_thread_recycles` before requests
+    """
+
+    _thread_recycles: ContextVar[int] = ContextVar("thread_recycles")
+
+    @classmethod
+    def increment_thread_recycles(cls):
+        try:
+            recycles = cls._thread_recycles.get()
+            cls._thread_recycles.set(recycles + 1)
+        except LookupError:
+            cls._thread_recycles.set(0)
+
+    def __init__(self, context_var: ContextVar[T]):
+        self._context_var = context_var
+        self._updates = ContextVar[int](context_var.name + "_updates", default=0)
+
+    def get(self, default: T | HiddenValue = _default) -> T:
+        thread_recycles = self._thread_recycles.get(0)
+        self_updates = self._updates.get()
+        if thread_recycles > self_updates:
+            self._updates.set(thread_recycles)
+
+        # check if thread is recycled and should be updated
+        if thread_recycles < self_updates:
+            return self._context_var.get()
+        else:
+            # thread_recycles >= self_updates, means current context is invalid
+            if isinstance(default, HiddenValue) or default is _default:
+                raise LookupError
+            else:
+                return default
+
+    def set(self, value: T):
+        # it leads to a situation that self.updates is less than cls.thread_recycles if `set` was never called before
+        # increase it manually
+        thread_recycles = self._thread_recycles.get(0)
+        self_updates = self._updates.get()
+        if thread_recycles > self_updates:
+            self._updates.set(thread_recycles)
+
+        if self._updates.get() == self._thread_recycles.get(0):
+            # after increment,
+            self._updates.set(self._updates.get() + 1)
+
+        # set the context
+        self._context_var.set(value)

dify/api/controllers/__init__.py

@@ -0,0 +1 @@
+

dify/api/controllers/common/errors.py

@@ -0,0 +1,43 @@
+from werkzeug.exceptions import HTTPException
+
+from libs.exception import BaseHTTPException
+
+
+class FilenameNotExistsError(HTTPException):
+    code = 400
+    description = "The specified filename does not exist."
+
+
+class RemoteFileUploadError(HTTPException):
+    code = 400
+    description = "Error uploading remote file."
+
+
+class FileTooLargeError(BaseHTTPException):
+    error_code = "file_too_large"
+    description = "File size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = "unsupported_file_type"
+    description = "File type not allowed."
+    code = 415
+
+
+class BlockedFileExtensionError(BaseHTTPException):
+    error_code = "file_extension_blocked"
+    description = "The file extension is blocked for security reasons."
+    code = 400
+
+
+class TooManyFilesError(BaseHTTPException):
+    error_code = "too_many_files"
+    description = "Only one file is allowed."
+    code = 400
+
+
+class NoFileUploadedError(BaseHTTPException):
+    error_code = "no_file_uploaded"
+    description = "Please upload your file."
+    code = 400

dify/api/controllers/common/fields.py

@@ -0,0 +1,62 @@
+from flask_restx import Api, Namespace, fields
+
+from libs.helper import AppIconUrlField
+
+parameters__system_parameters = {
+    "image_file_size_limit": fields.Integer,
+    "video_file_size_limit": fields.Integer,
+    "audio_file_size_limit": fields.Integer,
+    "file_size_limit": fields.Integer,
+    "workflow_file_upload_limit": fields.Integer,
+}
+
+
+def build_system_parameters_model(api_or_ns: Api | Namespace):
+    """Build the system parameters model for the API or Namespace."""
+    return api_or_ns.model("SystemParameters", parameters__system_parameters)
+
+
+parameters_fields = {
+    "opening_statement": fields.String,
+    "suggested_questions": fields.Raw,
+    "suggested_questions_after_answer": fields.Raw,
+    "speech_to_text": fields.Raw,
+    "text_to_speech": fields.Raw,
+    "retriever_resource": fields.Raw,
+    "annotation_reply": fields.Raw,
+    "more_like_this": fields.Raw,
+    "user_input_form": fields.Raw,
+    "sensitive_word_avoidance": fields.Raw,
+    "file_upload": fields.Raw,
+    "system_parameters": fields.Nested(parameters__system_parameters),
+}
+
+
+def build_parameters_model(api_or_ns: Api | Namespace):
+    """Build the parameters model for the API or Namespace."""
+    copied_fields = parameters_fields.copy()
+    copied_fields["system_parameters"] = fields.Nested(build_system_parameters_model(api_or_ns))
+    return api_or_ns.model("Parameters", copied_fields)
+
+
+site_fields = {
+    "title": fields.String,
+    "chat_color_theme": fields.String,
+    "chat_color_theme_inverted": fields.Boolean,
+    "icon_type": fields.String,
+    "icon": fields.String,
+    "icon_background": fields.String,
+    "icon_url": AppIconUrlField,
+    "description": fields.String,
+    "copyright": fields.String,
+    "privacy_policy": fields.String,
+    "custom_disclaimer": fields.String,
+    "default_language": fields.String,
+    "show_workflow_steps": fields.Boolean,
+    "use_icon_as_answer_icon": fields.Boolean,
+}
+
+
+def build_site_model(api_or_ns: Api | Namespace):
+    """Build the site model for the API or Namespace."""
+    return api_or_ns.model("Site", site_fields)

dify/api/controllers/common/helpers.py

@@ -0,0 +1,84 @@
+import contextlib
+import mimetypes
+import os
+import platform
+import re
+import urllib.parse
+import warnings
+from uuid import uuid4
+
+import httpx
+
+try:
+    import magic
+except ImportError:
+    if platform.system() == "Windows":
+        warnings.warn(
+            "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2
+        )
+    elif platform.system() == "Darwin":
+        warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)
+    elif platform.system() == "Linux":
+        warnings.warn(
+            "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2
+        )
+    else:
+        warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)
+    magic = None  # type: ignore[assignment]
+
+from pydantic import BaseModel
+
+
+class FileInfo(BaseModel):
+    filename: str
+    extension: str
+    mimetype: str
+    size: int
+
+
+def guess_file_info_from_response(response: httpx.Response):
+    url = str(response.url)
+    # Try to extract filename from URL
+    parsed_url = urllib.parse.urlparse(url)
+    url_path = parsed_url.path
+    filename = os.path.basename(url_path)
+
+    # If filename couldn't be extracted, use Content-Disposition header
+    if not filename:
+        content_disposition = response.headers.get("Content-Disposition")
+        if content_disposition:
+            filename_match = re.search(r'filename="?(.+)"?', content_disposition)
+            if filename_match:
+                filename = filename_match.group(1)
+
+    # If still no filename, generate a unique one
+    if not filename:
+        unique_name = str(uuid4())
+        filename = f"{unique_name}"
+
+    # Guess MIME type from filename first, then URL
+    mimetype, _ = mimetypes.guess_type(filename)
+    if mimetype is None:
+        mimetype, _ = mimetypes.guess_type(url)
+    if mimetype is None:
+        # If guessing fails, use Content-Type from response headers
+        mimetype = response.headers.get("Content-Type", "application/octet-stream")
+
+    # Use python-magic to guess MIME type if still unknown or generic
+    if mimetype == "application/octet-stream" and magic is not None:
+        with contextlib.suppress(magic.MagicException):
+            mimetype = magic.from_buffer(response.content[:1024], mime=True)
+
+    extension = os.path.splitext(filename)[1]
+
+    # Ensure filename has an extension
+    if not extension:
+        extension = mimetypes.guess_extension(mimetype) or ".bin"
+        filename = f"{filename}{extension}"
+
+    return FileInfo(
+        filename=filename,
+        extension=extension,
+        mimetype=mimetype,
+        size=int(response.headers.get("Content-Length", -1)),
+    )

dify/api/controllers/console/__init__.py

@@ -0,0 +1,211 @@
+from importlib import import_module
+
+from flask import Blueprint
+from flask_restx import Namespace
+
+from libs.external_api import ExternalApi
+
+bp = Blueprint("console", __name__, url_prefix="/console/api")
+
+api = ExternalApi(
+    bp,
+    version="1.0",
+    title="Console API",
+    description="Console management APIs for app configuration, monitoring, and administration",
+)
+
+console_ns = Namespace("console", description="Console management API operations", path="/")
+
+RESOURCE_MODULES = (
+    "controllers.console.app.app_import",
+    "controllers.console.explore.audio",
+    "controllers.console.explore.completion",
+    "controllers.console.explore.conversation",
+    "controllers.console.explore.message",
+    "controllers.console.explore.workflow",
+    "controllers.console.files",
+    "controllers.console.remote_files",
+)
+
+for module_name in RESOURCE_MODULES:
+    import_module(module_name)
+
+# Ensure resource modules are imported so route decorators are evaluated.
+# Import other controllers
+from . import (
+    admin,
+    apikey,
+    extension,
+    feature,
+    init_validate,
+    ping,
+    setup,
+    spec,
+    version,
+)
+
+# Import app controllers
+from .app import (
+    advanced_prompt_template,
+    agent,
+    annotation,
+    app,
+    audio,
+    completion,
+    conversation,
+    conversation_variables,
+    generator,
+    mcp_server,
+    message,
+    model_config,
+    ops_trace,
+    site,
+    statistic,
+    workflow,
+    workflow_app_log,
+    workflow_draft_variable,
+    workflow_run,
+    workflow_statistic,
+    workflow_trigger,
+)
+
+# Import auth controllers
+from .auth import (
+    activate,
+    data_source_bearer_auth,
+    data_source_oauth,
+    email_register,
+    forgot_password,
+    login,
+    oauth,
+    oauth_server,
+)
+
+# Import billing controllers
+from .billing import billing, compliance
+
+# Import datasets controllers
+from .datasets import (
+    data_source,
+    datasets,
+    datasets_document,
+    datasets_segments,
+    external,
+    hit_testing,
+    metadata,
+    website,
+)
+from .datasets.rag_pipeline import (
+    datasource_auth,
+    datasource_content_preview,
+    rag_pipeline,
+    rag_pipeline_datasets,
+    rag_pipeline_draft_variable,
+    rag_pipeline_import,
+    rag_pipeline_workflow,
+)
+
+# Import explore controllers
+from .explore import (
+    installed_app,
+    parameter,
+    recommended_app,
+    saved_message,
+)
+
+# Import tag controllers
+from .tag import tags
+
+# Import workspace controllers
+from .workspace import (
+    account,
+    agent_providers,
+    endpoint,
+    load_balancing_config,
+    members,
+    model_providers,
+    models,
+    plugin,
+    tool_providers,
+    trigger_providers,
+    workspace,
+)
+
+api.add_namespace(console_ns)
+
+__all__ = [
+    "account",
+    "activate",
+    "admin",
+    "advanced_prompt_template",
+    "agent",
+    "agent_providers",
+    "annotation",
+    "api",
+    "apikey",
+    "app",
+    "audio",
+    "billing",
+    "bp",
+    "completion",
+    "compliance",
+    "console_ns",
+    "conversation",
+    "conversation_variables",
+    "data_source",
+    "data_source_bearer_auth",
+    "data_source_oauth",
+    "datasets",
+    "datasets_document",
+    "datasets_segments",
+    "datasource_auth",
+    "datasource_content_preview",
+    "email_register",
+    "endpoint",
+    "extension",
+    "external",
+    "feature",
+    "forgot_password",
+    "generator",
+    "hit_testing",
+    "init_validate",
+    "installed_app",
+    "load_balancing_config",
+    "login",
+    "mcp_server",
+    "members",
+    "message",
+    "metadata",
+    "model_config",
+    "model_providers",
+    "models",
+    "oauth",
+    "oauth_server",
+    "ops_trace",
+    "parameter",
+    "ping",
+    "plugin",
+    "rag_pipeline",
+    "rag_pipeline_datasets",
+    "rag_pipeline_draft_variable",
+    "rag_pipeline_import",
+    "rag_pipeline_workflow",
+    "recommended_app",
+    "saved_message",
+    "setup",
+    "site",
+    "spec",
+    "statistic",
+    "tags",
+    "tool_providers",
+    "trigger_providers",
+    "version",
+    "website",
+    "workflow",
+    "workflow_app_log",
+    "workflow_draft_variable",
+    "workflow_run",
+    "workflow_statistic",
+    "workflow_trigger",
+    "workspace",
+]

dify/api/controllers/console/admin.py

@@ -0,0 +1,173 @@
+from collections.abc import Callable
+from functools import wraps
+from typing import ParamSpec, TypeVar
+
+from flask import request
+from flask_restx import Resource, fields, reqparse
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import NotFound, Unauthorized
+
+P = ParamSpec("P")
+R = TypeVar("R")
+from configs import dify_config
+from constants.languages import supported_language
+from controllers.console import console_ns
+from controllers.console.wraps import only_edition_cloud
+from extensions.ext_database import db
+from libs.token import extract_access_token
+from models.model import App, InstalledApp, RecommendedApp
+
+
+def admin_required(view: Callable[P, R]):
+    @wraps(view)
+    def decorated(*args: P.args, **kwargs: P.kwargs):
+        if not dify_config.ADMIN_API_KEY:
+            raise Unauthorized("API key is invalid.")
+
+        auth_token = extract_access_token(request)
+        if not auth_token:
+            raise Unauthorized("Authorization header is missing.")
+        if auth_token != dify_config.ADMIN_API_KEY:
+            raise Unauthorized("API key is invalid.")
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
+@console_ns.route("/admin/insert-explore-apps")
+class InsertExploreAppListApi(Resource):
+    @console_ns.doc("insert_explore_app")
+    @console_ns.doc(description="Insert or update an app in the explore list")
+    @console_ns.expect(
+        console_ns.model(
+            "InsertExploreAppRequest",
+            {
+                "app_id": fields.String(required=True, description="Application ID"),
+                "desc": fields.String(description="App description"),
+                "copyright": fields.String(description="Copyright information"),
+                "privacy_policy": fields.String(description="Privacy policy"),
+                "custom_disclaimer": fields.String(description="Custom disclaimer"),
+                "language": fields.String(required=True, description="Language code"),
+                "category": fields.String(required=True, description="App category"),
+                "position": fields.Integer(required=True, description="Display position"),
+            },
+        )
+    )
+    @console_ns.response(200, "App updated successfully")
+    @console_ns.response(201, "App inserted successfully")
+    @console_ns.response(404, "App not found")
+    @only_edition_cloud
+    @admin_required
+    def post(self):
+        parser = (
+            reqparse.RequestParser()
+            .add_argument("app_id", type=str, required=True, nullable=False, location="json")
+            .add_argument("desc", type=str, location="json")
+            .add_argument("copyright", type=str, location="json")
+            .add_argument("privacy_policy", type=str, location="json")
+            .add_argument("custom_disclaimer", type=str, location="json")
+            .add_argument("language", type=supported_language, required=True, nullable=False, location="json")
+            .add_argument("category", type=str, required=True, nullable=False, location="json")
+            .add_argument("position", type=int, required=True, nullable=False, location="json")
+        )
+        args = parser.parse_args()
+
+        app = db.session.execute(select(App).where(App.id == args["app_id"])).scalar_one_or_none()
+        if not app:
+            raise NotFound(f"App '{args['app_id']}' is not found")
+
+        site = app.site
+        if not site:
+            desc = args["desc"] or ""
+            copy_right = args["copyright"] or ""
+            privacy_policy = args["privacy_policy"] or ""
+            custom_disclaimer = args["custom_disclaimer"] or ""
+        else:
+            desc = site.description or args["desc"] or ""
+            copy_right = site.copyright or args["copyright"] or ""
+            privacy_policy = site.privacy_policy or args["privacy_policy"] or ""
+            custom_disclaimer = site.custom_disclaimer or args["custom_disclaimer"] or ""
+
+        with Session(db.engine) as session:
+            recommended_app = session.execute(
+                select(RecommendedApp).where(RecommendedApp.app_id == args["app_id"])
+            ).scalar_one_or_none()
+
+            if not recommended_app:
+                recommended_app = RecommendedApp(
+                    app_id=app.id,
+                    description=desc,
+                    copyright=copy_right,
+                    privacy_policy=privacy_policy,
+                    custom_disclaimer=custom_disclaimer,
+                    language=args["language"],
+                    category=args["category"],
+                    position=args["position"],
+                )
+
+                db.session.add(recommended_app)
+
+                app.is_public = True
+                db.session.commit()
+
+                return {"result": "success"}, 201
+            else:
+                recommended_app.description = desc
+                recommended_app.copyright = copy_right
+                recommended_app.privacy_policy = privacy_policy
+                recommended_app.custom_disclaimer = custom_disclaimer
+                recommended_app.language = args["language"]
+                recommended_app.category = args["category"]
+                recommended_app.position = args["position"]
+
+                app.is_public = True
+
+                db.session.commit()
+
+                return {"result": "success"}, 200
+
+
+@console_ns.route("/admin/insert-explore-apps/<uuid:app_id>")
+class InsertExploreAppApi(Resource):
+    @console_ns.doc("delete_explore_app")
+    @console_ns.doc(description="Remove an app from the explore list")
+    @console_ns.doc(params={"app_id": "Application ID to remove"})
+    @console_ns.response(204, "App removed successfully")
+    @only_edition_cloud
+    @admin_required
+    def delete(self, app_id):
+        with Session(db.engine) as session:
+            recommended_app = session.execute(
+                select(RecommendedApp).where(RecommendedApp.app_id == str(app_id))
+            ).scalar_one_or_none()
+
+        if not recommended_app:
+            return {"result": "success"}, 204
+
+        with Session(db.engine) as session:
+            app = session.execute(select(App).where(App.id == recommended_app.app_id)).scalar_one_or_none()
+
+        if app:
+            app.is_public = False
+
+        with Session(db.engine) as session:
+            installed_apps = (
+                session.execute(
+                    select(InstalledApp).where(
+                        InstalledApp.app_id == recommended_app.app_id,
+                        InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id,
+                    )
+                )
+                .scalars()
+                .all()
+            )
+
+            for installed_app in installed_apps:
+                session.delete(installed_app)
+
+        db.session.delete(recommended_app)
+        db.session.commit()
+
+        return {"result": "success"}, 204

dify/api/controllers/console/apikey.py

@@ -0,0 +1,217 @@
+import flask_restx
+from flask_restx import Resource, fields, marshal_with
+from flask_restx._http import HTTPStatus
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
+from libs.helper import TimestampField
+from libs.login import current_account_with_tenant, login_required
+from models.dataset import Dataset
+from models.model import ApiToken, App
+
+from . import console_ns
+from .wraps import account_initialization_required, edit_permission_required, setup_required
+
+api_key_fields = {
+    "id": fields.String,
+    "type": fields.String,
+    "token": fields.String,
+    "last_used_at": TimestampField,
+    "created_at": TimestampField,
+}
+
+api_key_list = {"data": fields.List(fields.Nested(api_key_fields), attribute="items")}
+
+api_key_item_model = console_ns.model("ApiKeyItem", api_key_fields)
+
+api_key_list_model = console_ns.model(
+    "ApiKeyList", {"data": fields.List(fields.Nested(api_key_item_model), attribute="items")}
+)
+
+
+def _get_resource(resource_id, tenant_id, resource_model):
+    if resource_model == App:
+        with Session(db.engine) as session:
+            resource = session.execute(
+                select(resource_model).filter_by(id=resource_id, tenant_id=tenant_id)
+            ).scalar_one_or_none()
+    else:
+        with Session(db.engine) as session:
+            resource = session.execute(
+                select(resource_model).filter_by(id=resource_id, tenant_id=tenant_id)
+            ).scalar_one_or_none()
+
+    if resource is None:
+        flask_restx.abort(HTTPStatus.NOT_FOUND, message=f"{resource_model.__name__} not found.")
+
+    return resource
+
+
+class BaseApiKeyListResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type: str | None = None
+    resource_model: type | None = None
+    resource_id_field: str | None = None
+    token_prefix: str | None = None
+    max_keys = 10
+
+    @marshal_with(api_key_list_model)
+    def get(self, resource_id):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
+        resource_id = str(resource_id)
+        _, current_tenant_id = current_account_with_tenant()
+
+        _get_resource(resource_id, current_tenant_id, self.resource_model)
+        keys = db.session.scalars(
+            select(ApiToken).where(
+                ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id
+            )
+        ).all()
+        return {"items": keys}
+
+    @marshal_with(api_key_item_model)
+    @edit_permission_required
+    def post(self, resource_id):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
+        resource_id = str(resource_id)
+        _, current_tenant_id = current_account_with_tenant()
+        _get_resource(resource_id, current_tenant_id, self.resource_model)
+        current_key_count = (
+            db.session.query(ApiToken)
+            .where(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
+            .count()
+        )
+
+        if current_key_count >= self.max_keys:
+            flask_restx.abort(
+                HTTPStatus.BAD_REQUEST,
+                message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
+                custom="max_keys_exceeded",
+            )
+
+        key = ApiToken.generate_api_key(self.token_prefix or "", 24)
+        api_token = ApiToken()
+        setattr(api_token, self.resource_id_field, resource_id)
+        api_token.tenant_id = current_tenant_id
+        api_token.token = key
+        api_token.type = self.resource_type
+        db.session.add(api_token)
+        db.session.commit()
+        return api_token, 201
+
+
+class BaseApiKeyResource(Resource):
+    method_decorators = [account_initialization_required, login_required, setup_required]
+
+    resource_type: str | None = None
+    resource_model: type | None = None
+    resource_id_field: str | None = None
+
+    def delete(self, resource_id: str, api_key_id: str):
+        assert self.resource_id_field is not None, "resource_id_field must be set"
+        current_user, current_tenant_id = current_account_with_tenant()
+        _get_resource(resource_id, current_tenant_id, self.resource_model)
+
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+
+        key = (
+            db.session.query(ApiToken)
+            .where(
+                getattr(ApiToken, self.resource_id_field) == resource_id,
+                ApiToken.type == self.resource_type,
+                ApiToken.id == api_key_id,
+            )
+            .first()
+        )
+
+        if key is None:
+            flask_restx.abort(HTTPStatus.NOT_FOUND, message="API key not found")
+
+        db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
+        db.session.commit()
+
+        return {"result": "success"}, 204
+
+
+@console_ns.route("/apps/<uuid:resource_id>/api-keys")
+class AppApiKeyListResource(BaseApiKeyListResource):
+    @console_ns.doc("get_app_api_keys")
+    @console_ns.doc(description="Get all API keys for an app")
+    @console_ns.doc(params={"resource_id": "App ID"})
+    @console_ns.response(200, "Success", api_key_list_model)
+    def get(self, resource_id):  # type: ignore
+        """Get all API keys for an app"""
+        return super().get(resource_id)
+
+    @console_ns.doc("create_app_api_key")
+    @console_ns.doc(description="Create a new API key for an app")
+    @console_ns.doc(params={"resource_id": "App ID"})
+    @console_ns.response(201, "API key created successfully", api_key_item_model)
+    @console_ns.response(400, "Maximum keys exceeded")
+    def post(self, resource_id):  # type: ignore
+        """Create a new API key for an app"""
+        return super().post(resource_id)
+
+    resource_type = "app"
+    resource_model = App
+    resource_id_field = "app_id"
+    token_prefix = "app-"
+
+
+@console_ns.route("/apps/<uuid:resource_id>/api-keys/<uuid:api_key_id>")
+class AppApiKeyResource(BaseApiKeyResource):
+    @console_ns.doc("delete_app_api_key")
+    @console_ns.doc(description="Delete an API key for an app")
+    @console_ns.doc(params={"resource_id": "App ID", "api_key_id": "API key ID"})
+    @console_ns.response(204, "API key deleted successfully")
+    def delete(self, resource_id, api_key_id):
+        """Delete an API key for an app"""
+        return super().delete(resource_id, api_key_id)
+
+    resource_type = "app"
+    resource_model = App
+    resource_id_field = "app_id"
+
+
+@console_ns.route("/datasets/<uuid:resource_id>/api-keys")
+class DatasetApiKeyListResource(BaseApiKeyListResource):
+    @console_ns.doc("get_dataset_api_keys")
+    @console_ns.doc(description="Get all API keys for a dataset")
+    @console_ns.doc(params={"resource_id": "Dataset ID"})
+    @console_ns.response(200, "Success", api_key_list_model)
+    def get(self, resource_id):  # type: ignore
+        """Get all API keys for a dataset"""
+        return super().get(resource_id)
+
+    @console_ns.doc("create_dataset_api_key")
+    @console_ns.doc(description="Create a new API key for a dataset")
+    @console_ns.doc(params={"resource_id": "Dataset ID"})
+    @console_ns.response(201, "API key created successfully", api_key_item_model)
+    @console_ns.response(400, "Maximum keys exceeded")
+    def post(self, resource_id):  # type: ignore
+        """Create a new API key for a dataset"""
+        return super().post(resource_id)
+
+    resource_type = "dataset"
+    resource_model = Dataset
+    resource_id_field = "dataset_id"
+    token_prefix = "ds-"
+
+
+@console_ns.route("/datasets/<uuid:resource_id>/api-keys/<uuid:api_key_id>")
+class DatasetApiKeyResource(BaseApiKeyResource):
+    @console_ns.doc("delete_dataset_api_key")
+    @console_ns.doc(description="Delete an API key for a dataset")
+    @console_ns.doc(params={"resource_id": "Dataset ID", "api_key_id": "API key ID"})
+    @console_ns.response(204, "API key deleted successfully")
+    def delete(self, resource_id, api_key_id):
+        """Delete an API key for a dataset"""
+        return super().delete(resource_id, api_key_id)
+
+    resource_type = "dataset"
+    resource_model = Dataset
+    resource_id_field = "dataset_id"