dify

2025-12-01 17:21:38 +08:00
parent 32fee2b8ab
commit fab8c13cb3
7511 changed files with 996300 additions and 0 deletions
--- a/dify/dev/pytest/pytest_all_tests.sh
+++ b/dify/dev/pytest/pytest_all_tests.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+# ModelRuntime
+dev/pytest/pytest_model_runtime.sh
+
+# Tools
+dev/pytest/pytest_tools.sh
+
+# Workflow
+dev/pytest/pytest_workflow.sh
+
+# Unit tests
+dev/pytest/pytest_unit_tests.sh
+
+# TestContainers tests
+dev/pytest/pytest_testcontainers.sh
--- a/dify/dev/pytest/pytest_artifacts.sh
+++ b/dify/dev/pytest/pytest_artifacts.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-120}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/artifact_tests/
--- a/dify/dev/pytest/pytest_config_tests.py
+++ b/dify/dev/pytest/pytest_config_tests.py
@@ -0,0 +1,116 @@
+from pathlib import Path
+
+import yaml  # type: ignore
+from dotenv import dotenv_values
+
+BASE_API_AND_DOCKER_CONFIG_SET_DIFF = {
+    "APP_MAX_EXECUTION_TIME",
+    "BATCH_UPLOAD_LIMIT",
+    "CELERY_BEAT_SCHEDULER_TIME",
+    "CODE_EXECUTION_API_KEY",
+    "HTTP_REQUEST_MAX_CONNECT_TIMEOUT",
+    "HTTP_REQUEST_MAX_READ_TIMEOUT",
+    "HTTP_REQUEST_MAX_WRITE_TIMEOUT",
+    "INNER_API_KEY",
+    "INNER_API_KEY_FOR_PLUGIN",
+    "KEYWORD_DATA_SOURCE_TYPE",
+    "LOGIN_LOCKOUT_DURATION",
+    "LOG_FORMAT",
+    "OCI_ACCESS_KEY",
+    "OCI_BUCKET_NAME",
+    "OCI_ENDPOINT",
+    "OCI_REGION",
+    "OCI_SECRET_KEY",
+    "PLUGIN_DAEMON_KEY",
+    "PLUGIN_DAEMON_URL",
+    "PLUGIN_REMOTE_INSTALL_HOST",
+    "PLUGIN_REMOTE_INSTALL_PORT",
+    "REDIS_DB",
+    "RESEND_API_URL",
+    "RESPECT_XFORWARD_HEADERS_ENABLED",
+    "SENTRY_DSN",
+    "SSRF_DEFAULT_CONNECT_TIME_OUT",
+    "SSRF_DEFAULT_MAX_RETRIES",
+    "SSRF_DEFAULT_READ_TIME_OUT",
+    "SSRF_DEFAULT_TIME_OUT",
+    "SSRF_DEFAULT_WRITE_TIME_OUT",
+    "UPSTASH_VECTOR_TOKEN",
+    "UPSTASH_VECTOR_URL",
+    "USING_UGC_INDEX",
+    "WEAVIATE_BATCH_SIZE",
+    "WEAVIATE_GRPC_ENABLED",
+}
+
+BASE_API_AND_DOCKER_COMPOSE_CONFIG_SET_DIFF = {
+    "BATCH_UPLOAD_LIMIT",
+    "CELERY_BEAT_SCHEDULER_TIME",
+    "HTTP_REQUEST_MAX_CONNECT_TIMEOUT",
+    "HTTP_REQUEST_MAX_READ_TIMEOUT",
+    "HTTP_REQUEST_MAX_WRITE_TIMEOUT",
+    "INNER_API_KEY",
+    "INNER_API_KEY_FOR_PLUGIN",
+    "KEYWORD_DATA_SOURCE_TYPE",
+    "LOGIN_LOCKOUT_DURATION",
+    "LOG_FORMAT",
+    "OPENDAL_FS_ROOT",
+    "OPENDAL_S3_ACCESS_KEY_ID",
+    "OPENDAL_S3_BUCKET",
+    "OPENDAL_S3_ENDPOINT",
+    "OPENDAL_S3_REGION",
+    "OPENDAL_S3_ROOT",
+    "OPENDAL_S3_SECRET_ACCESS_KEY",
+    "OPENDAL_S3_SERVER_SIDE_ENCRYPTION",
+    "PGVECTOR_MAX_CONNECTION",
+    "PGVECTOR_MIN_CONNECTION",
+    "PGVECTO_RS_DATABASE",
+    "PGVECTO_RS_HOST",
+    "PGVECTO_RS_PASSWORD",
+    "PGVECTO_RS_PORT",
+    "PGVECTO_RS_USER",
+    "PLUGIN_DAEMON_KEY",
+    "PLUGIN_DAEMON_URL",
+    "PLUGIN_REMOTE_INSTALL_HOST",
+    "PLUGIN_REMOTE_INSTALL_PORT",
+    "RESPECT_XFORWARD_HEADERS_ENABLED",
+    "SCARF_NO_ANALYTICS",
+    "SSRF_DEFAULT_CONNECT_TIME_OUT",
+    "SSRF_DEFAULT_MAX_RETRIES",
+    "SSRF_DEFAULT_READ_TIME_OUT",
+    "SSRF_DEFAULT_TIME_OUT",
+    "SSRF_DEFAULT_WRITE_TIME_OUT",
+    "STORAGE_OPENDAL_SCHEME",
+    "SUPABASE_API_KEY",
+    "SUPABASE_BUCKET_NAME",
+    "SUPABASE_URL",
+    "USING_UGC_INDEX",
+    "VIKINGDB_CONNECTION_TIMEOUT",
+    "VIKINGDB_SOCKET_TIMEOUT",
+    "WEAVIATE_BATCH_SIZE",
+    "WEAVIATE_GRPC_ENABLED",
+}
+
+API_CONFIG_SET = set(dotenv_values(Path("api") / Path(".env.example")).keys())
+DOCKER_CONFIG_SET = set(dotenv_values(Path("docker") / Path(".env.example")).keys())
+DOCKER_COMPOSE_CONFIG_SET = set()
+
+with open(Path("docker") / Path("docker-compose.yaml")) as f:
+    DOCKER_COMPOSE_CONFIG_SET = set(yaml.safe_load(f.read())["x-shared-env"].keys())
+
+
+def test_yaml_config():
+    # python set == operator is used to compare two sets
+    DIFF_API_WITH_DOCKER = API_CONFIG_SET - DOCKER_CONFIG_SET - BASE_API_AND_DOCKER_CONFIG_SET_DIFF
+    if DIFF_API_WITH_DOCKER:
+        print(f"API and Docker config sets are different with key: {DIFF_API_WITH_DOCKER}")
+        raise Exception("API and Docker config sets are different")
+    DIFF_API_WITH_DOCKER_COMPOSE = (
+        API_CONFIG_SET - DOCKER_COMPOSE_CONFIG_SET - BASE_API_AND_DOCKER_COMPOSE_CONFIG_SET_DIFF
+    )
+    if DIFF_API_WITH_DOCKER_COMPOSE:
+        print(f"API and Docker Compose config sets are different with key: {DIFF_API_WITH_DOCKER_COMPOSE}")
+        raise Exception("API and Docker Compose config sets are different")
+    print("All tests passed!")
+
+
+if __name__ == "__main__":
+    test_yaml_config()
--- a/dify/dev/pytest/pytest_model_runtime.sh
+++ b/dify/dev/pytest/pytest_model_runtime.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-180}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/integration_tests/model_runtime/anthropic \
+  api/tests/integration_tests/model_runtime/azure_openai \
+  api/tests/integration_tests/model_runtime/openai api/tests/integration_tests/model_runtime/chatglm \
+  api/tests/integration_tests/model_runtime/google api/tests/integration_tests/model_runtime/xinference \
+  api/tests/integration_tests/model_runtime/huggingface_hub/test_llm.py \
+  api/tests/integration_tests/model_runtime/upstage \
+  api/tests/integration_tests/model_runtime/fireworks \
+  api/tests/integration_tests/model_runtime/nomic \
+  api/tests/integration_tests/model_runtime/mixedbread \
+  api/tests/integration_tests/model_runtime/voyage
--- a/dify/dev/pytest/pytest_testcontainers.sh
+++ b/dify/dev/pytest/pytest_testcontainers.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-120}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/test_containers_integration_tests
--- a/dify/dev/pytest/pytest_tools.sh
+++ b/dify/dev/pytest/pytest_tools.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-120}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/integration_tests/tools
--- a/dify/dev/pytest/pytest_unit_tests.sh
+++ b/dify/dev/pytest/pytest_unit_tests.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-20}"
+
+# libs
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/unit_tests
--- a/dify/dev/pytest/pytest_vdb.sh
+++ b/dify/dev/pytest/pytest_vdb.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-180}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/integration_tests/vdb/chroma \
+  api/tests/integration_tests/vdb/milvus \
+  api/tests/integration_tests/vdb/pgvecto_rs \
+  api/tests/integration_tests/vdb/pgvector \
+  api/tests/integration_tests/vdb/qdrant \
+  api/tests/integration_tests/vdb/weaviate \
+  api/tests/integration_tests/vdb/elasticsearch \
+  api/tests/integration_tests/vdb/vikingdb \
+  api/tests/integration_tests/vdb/baidu \
+  api/tests/integration_tests/vdb/tcvectordb \
+  api/tests/integration_tests/vdb/upstash \
+  api/tests/integration_tests/vdb/couchbase \
+  api/tests/integration_tests/vdb/oceanbase \
+  api/tests/integration_tests/vdb/tidb_vector \
+  api/tests/integration_tests/vdb/huawei \
--- a/dify/dev/pytest/pytest_workflow.sh
+++ b/dify/dev/pytest/pytest_workflow.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+cd "$SCRIPT_DIR/../.."
+
+PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-120}"
+
+pytest --timeout "${PYTEST_TIMEOUT}" api/tests/integration_tests/workflow