From 9a5c423d593f1fb214ddeeffd876ab17226a61ab Mon Sep 17 00:00:00 2001 From: takatost Date: Sat, 22 Jun 2024 02:05:50 +0800 Subject: [PATCH] chore: remove pip support for api service (#5453) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Bowen Liang --- .devcontainer/post_start_command.sh | 2 +- .github/workflows/api-tests.yml | 70 --------------------- api/README.md | 65 +------------------ api/requirements-dev.txt | 5 -- api/requirements.txt | 96 ----------------------------- 5 files changed, 4 insertions(+), 234 deletions(-) delete mode 100644 api/requirements-dev.txt diff --git a/.devcontainer/post_start_command.sh b/.devcontainer/post_start_command.sh index 633da9649c..e3d5a6d59d 100755 --- a/.devcontainer/post_start_command.sh +++ b/.devcontainer/post_start_command.sh @@ -1,3 +1,3 @@ #!/bin/bash -cd api && pip install -r requirements.txt \ No newline at end of file +poetry install -C api \ No newline at end of file diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 76ea795de5..4b75f886fd 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -14,76 +14,6 @@ concurrency: jobs: test: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: - - "3.10" - - "3.11" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: | - ./api/requirements.txt - ./api/requirements-dev.txt - - - name: Install dependencies - run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt - - - name: Run Unit tests - run: dev/pytest/pytest_unit_tests.sh - - - name: Run ModelRuntime - run: dev/pytest/pytest_model_runtime.sh - - - name: Run Tool - run: dev/pytest/pytest_tools.sh - - - name: Set up Sandbox - uses: hoverkraft-tech/compose-action@v2.0.0 - with: - compose-file: | - docker/docker-compose.middleware.yaml - services: | - sandbox - ssrf_proxy - - - name: Run Workflow - run: dev/pytest/pytest_workflow.sh - - - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma) - uses: hoverkraft-tech/compose-action@v2.0.0 - with: - compose-file: | - docker/docker-compose.middleware.yaml - docker/docker-compose.qdrant.yaml - docker/docker-compose.milvus.yaml - docker/docker-compose.pgvecto-rs.yaml - docker/docker-compose.pgvector.yaml - docker/docker-compose.chroma.yaml - docker/docker-compose.oracle.yaml - services: | - weaviate - qdrant - etcd - minio - milvus-standalone - pgvecto-rs - pgvector - chroma - oracle - - - name: Test Vector Stores - run: dev/pytest/pytest_vdb.sh - - test-in-poetry: name: API Tests runs-on: ubuntu-latest strategy: diff --git a/api/README.md b/api/README.md index f49f2cf6d8..5f71dbe5f0 100644 --- a/api/README.md +++ b/api/README.md @@ -2,6 +2,9 @@ ## Usage +> [!IMPORTANT] +> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`. + 1. Start the docker-compose stack The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`. @@ -29,8 +32,6 @@ Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment. - > Using pip can be found [below](#usage-with-pip). - 5. Install dependencies ```bash @@ -84,63 +85,3 @@ cd ../ poetry run -C api bash dev/pytest/pytest_all_tests.sh ``` - -## Usage with pip - -> [!NOTE] -> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist. - -1. Start the docker-compose stack - - The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`. - - ```bash - cd ../docker - docker-compose -f docker-compose.middleware.yaml -p dify up -d - cd ../api - ``` - -2. Copy `.env.example` to `.env` -3. Generate a `SECRET_KEY` in the `.env` file. - - ```bash - sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env - ``` - -4. Create environment. - - If you use Anaconda, create a new environment and activate it - - ```bash - conda create --name dify python=3.10 - conda activate dify - ``` - -5. Install dependencies - - ```bash - pip install -r requirements.txt - ``` - -6. Run migrate - - Before the first launch, migrate the database to the latest version. - - ```bash - flask db upgrade - ``` - -7. Start backend: - - ```bash - flask run --host 0.0.0.0 --port=5001 --debug - ``` - -8. Setup your application by visiting or other apis... -9. If you need to debug local async processing, please start the worker service. - - ```bash - celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail - ``` - - The started celery app handles the async tasks, e.g. dataset importing and documents indexing. diff --git a/api/requirements-dev.txt b/api/requirements-dev.txt deleted file mode 100644 index 0391ac5969..0000000000 --- a/api/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -coverage~=7.2.4 -pytest~=8.1.1 -pytest-benchmark~=4.0.0 -pytest-env~=1.1.3 -pytest-mock~=3.14.0 diff --git a/api/requirements.txt b/api/requirements.txt index 831c23953a..e69de29bb2 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,96 +0,0 @@ -beautifulsoup4==4.12.2 -flask~=3.0.1 -Flask-SQLAlchemy~=3.0.5 -SQLAlchemy~=2.0.29 -Flask-Compress~=1.14 -flask-login~=0.6.3 -flask-migrate~=4.0.5 -flask-restful~=0.3.10 -flask-cors~=4.0.0 -gunicorn~=22.0.0 -gevent~=23.9.1 -openai~=1.29.0 -tiktoken~=0.7.0 -psycopg2-binary~=2.9.6 -pycryptodome==3.19.1 -python-dotenv==1.0.0 -Authlib==1.3.1 -boto3==1.34.123 -cachetools~=5.3.0 -weaviate-client~=3.21.0 -mailchimp-transactional~=1.0.50 -scikit-learn==1.2.2 -sentry-sdk[flask]~=1.39.2 -sympy==1.12 -jieba==0.42.1 -celery~=5.3.6 -redis[hiredis]~=5.0.3 -chardet~=5.1.0 -python-docx~=1.1.0 -pypdfium2~=4.17.0 -resend~=0.7.0 -pyjwt~=2.8.0 -anthropic~=0.23.1 -newspaper3k==0.2.8 -wikipedia==1.4.0 -readabilipy==0.2.0 -google-ai-generativelanguage==0.6.1 -google-api-core==2.18.0 -google-api-python-client==2.90.0 -google-auth==2.29.0 -google-auth-httplib2==0.2.0 -google-generativeai==0.5.0 -googleapis-common-protos==1.63.0 -google-cloud-storage==2.16.0 -replicate~=0.22.0 -websocket-client~=1.7.0 -dashscope[tokenizer]~=1.17.0 -huggingface_hub~=0.16.4 -transformers~=4.35.0 -tokenizers~=0.15.0 -pandas[performance,excel]~=2.2.2 -xinference-client==0.9.4 -safetensors~=0.4.3 -zhipuai==1.0.7 -werkzeug~=3.0.1 -pymilvus==2.3.1 -qdrant-client==1.7.3 -cohere~=5.2.4 -pyyaml~=6.0.1 -numpy~=1.26.4 -unstructured[docx,pptx,msg,md,ppt,epub]~=0.10.27 -bs4~=0.0.1 -markdown~=3.5.1 -httpx[socks]~=0.27.0 -matplotlib~=3.8.2 -yfinance~=0.2.40 -pydub~=0.25.1 -gmpy2~=2.1.5 -numexpr~=2.9.0 -duckduckgo-search~=6.1.5 -arxiv==2.1.0 -yarl~=1.9.4 -twilio~=9.0.4 -qrcode~=7.4.2 -azure-storage-blob==12.13.0 -azure-identity==1.16.1 -lxml==5.1.0 -pydantic~=2.7.4 -pydantic_extra_types~=2.8.1 -pydantic-settings~=2.3.3 -pgvecto-rs==0.1.4 -tcvectordb==1.3.2 -firecrawl-py==0.0.5 -oss2==2.18.5 -pgvector==0.2.5 -pymysql==1.1.1 -tidb-vector==0.0.9 -google-cloud-aiplatform==1.49.0 -vanna[postgres,mysql,clickhouse,duckdb]==0.5.5 -tencentcloud-sdk-python-hunyuan~=3.0.1158 -chromadb~=0.5.1 -novita_client~=0.5.6 -tenacity~=8.3.0 -opensearch-py==2.4.0 -cos-python-sdk-v5==1.9.30 -oracledb~=2.2.1