test: add scripts for running tests on api module both locally and CI jobs (#3497)

.github/workflows/api-tests.yml

@@ -50,10 +50,10 @@ jobs:
         run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt
 
       - name: Run ModelRuntime
-        run: pytest api/tests/integration_tests/model_runtime/anthropic api/tests/integration_tests/model_runtime/azure_openai api/tests/integration_tests/model_runtime/openai api/tests/integration_tests/model_runtime/chatglm api/tests/integration_tests/model_runtime/google api/tests/integration_tests/model_runtime/xinference api/tests/integration_tests/model_runtime/huggingface_hub/test_llm.py
+        run: dev/pytest/pytest_model_runtime.sh
 
       - name: Run Tool
-        run: pytest api/tests/integration_tests/tools/test_all_provider.py
+        run: dev/pytest/pytest_tools.sh
 
       - name: Run Workflow
-        run: pytest api/tests/integration_tests/workflow
+        run: dev/pytest/pytest_workflow.sh

api/README.md

@@ -55,3 +55,16 @@
 9. If you need to debug local async processing, please start the worker service by running 
 `celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail`.
 The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
+
+
+## Testing
+
+1. Install dependencies for both the backend and the test environment
+   ```bash
+   pip install -r requirements.txt -r requirements-dev.txt
+   ``` 
+   
+2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
+   ```bash
+   dev/pytest/pytest_all_tests.sh
+   ```

api/pyproject.toml

@@ -25,3 +25,23 @@ ignore = [
     "UP007", # non-pep604-annotation
     "UP032", # f-string
 ]
+
+
+[tool.pytest_env]
+OPENAI_API_KEY = "sk-IamNotARealKeyJustForMockTestKawaiiiiiiiiii"
+AZURE_OPENAI_API_BASE = "https://difyai-openai.openai.azure.com"
+AZURE_OPENAI_API_KEY = "xxxxb1707exxxxxxxxxxaaxxxxxf94"
+ANTHROPIC_API_KEY = "sk-ant-api11-IamNotARealKeyJustForMockTestKawaiiiiiiiiii-NotBaka-ASkksz"
+CHATGLM_API_BASE = "http://a.abc.com:11451"
+XINFERENCE_SERVER_URL = "http://a.abc.com:11451"
+XINFERENCE_GENERATION_MODEL_UID = "generate"
+XINFERENCE_CHAT_MODEL_UID = "chat"
+XINFERENCE_EMBEDDINGS_MODEL_UID = "embedding"
+XINFERENCE_RERANK_MODEL_UID = "rerank"
+GOOGLE_API_KEY = "abcdefghijklmnopqrstuvwxyz"
+HUGGINGFACE_API_KEY = "hf-awuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwuwu"
+HUGGINGFACE_TEXT_GEN_ENDPOINT_URL = "a"
+HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL = "b"
+HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL = "c"
+MOCK_SWITCH = "true"
+CODE_MAX_STRING_LENGTH = "80000"

api/requirements-dev.txt

@@ -1,4 +1,5 @@
 coverage~=7.2.4
-pytest~=7.3.1
+pytest~=8.1.1
-pytest-mock~=3.11.1
 pytest-benchmark~=4.0.0
+pytest-env~=1.1.3
+pytest-mock~=3.14.0

dev/pytest/pytest_all_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+
+# ModelRuntime
+dev/pytest/pytest_model_runtime.sh
+
+# Tools
+dev/pytest/pytest_tools.sh
+
+# Workflow
+dev/pytest/pytest_workflow.sh

dev/pytest/pytest_model_runtime.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -x
+
+pytest api/tests/integration_tests/model_runtime/anthropic \
+  api/tests/integration_tests/model_runtime/azure_openai \
+  api/tests/integration_tests/model_runtime/openai api/tests/integration_tests/model_runtime/chatglm \
+  api/tests/integration_tests/model_runtime/google api/tests/integration_tests/model_runtime/xinference \
+  api/tests/integration_tests/model_runtime/huggingface_hub/test_llm.py

dev/pytest/pytest_tools.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x
+
+pytest api/tests/integration_tests/tools/test_all_provider.py

dev/pytest/pytest_workflow.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x
+
+pytest api/tests/integration_tests/workflow