diff --git a/api/.env.example b/api/.env.example index 04b3a9224a..e7e704e135 100644 --- a/api/.env.example +++ b/api/.env.example @@ -137,7 +137,7 @@ WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,* # Vector database configuration -# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase, opengauss +# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase, opengauss, tablestore VECTOR_STORE=weaviate # Weaviate configuration @@ -212,6 +212,12 @@ PGVECTOR_DATABASE=postgres PGVECTOR_MIN_CONNECTION=1 PGVECTOR_MAX_CONNECTION=5 +# TableStore Vector configuration +TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com +TABLESTORE_INSTANCE_NAME=instance-name +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx + # Tidb Vector configuration TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com TIDB_VECTOR_PORT=4000 diff --git a/api/commands.py b/api/commands.py index ea82fef397..e70d6e0b49 100644 --- a/api/commands.py +++ b/api/commands.py @@ -276,6 +276,7 @@ def migrate_knowledge_vector_database(): VectorType.ORACLE, VectorType.ELASTICSEARCH, VectorType.OPENGAUSS, + VectorType.TABLESTORE, } lower_collection_vector_types = { VectorType.ANALYTICDB, diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index 3bd638bc74..15dfe0063b 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -33,6 +33,7 @@ from .vdb.pgvector_config import PGVectorConfig from .vdb.pgvectors_config import PGVectoRSConfig from .vdb.qdrant_config import QdrantConfig from .vdb.relyt_config import RelytConfig +from .vdb.tablestore_config import TableStoreConfig from .vdb.tencent_vector_config import TencentVectorDBConfig from .vdb.tidb_on_qdrant_config import TidbOnQdrantConfig from .vdb.tidb_vector_config import TiDBVectorConfig @@ -283,5 +284,6 @@ class MiddlewareConfig( OceanBaseVectorConfig, BaiduVectorDBConfig, OpenGaussConfig, + TableStoreConfig, ): pass diff --git a/api/configs/middleware/vdb/tablestore_config.py b/api/configs/middleware/vdb/tablestore_config.py new file mode 100644 index 0000000000..c4dcc0d465 --- /dev/null +++ b/api/configs/middleware/vdb/tablestore_config.py @@ -0,0 +1,30 @@ +from typing import Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class TableStoreConfig(BaseSettings): + """ + Configuration settings for TableStore. + """ + + TABLESTORE_ENDPOINT: Optional[str] = Field( + description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')", + default=None, + ) + + TABLESTORE_INSTANCE_NAME: Optional[str] = Field( + description="Instance name to access TableStore server (eg. 'instance-name')", + default=None, + ) + + TABLESTORE_ACCESS_KEY_ID: Optional[str] = Field( + description="AccessKey id for the instance name", + default=None, + ) + + TABLESTORE_ACCESS_KEY_SECRET: Optional[str] = Field( + description="AccessKey secret for the instance name", + default=None, + ) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 52dd0242e2..396dae7a57 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -664,6 +664,7 @@ class DatasetRetrievalSettingApi(Resource): | VectorType.MILVUS | VectorType.OPENGAUSS | VectorType.OCEANBASE + | VectorType.TABLESTORE ): return { "retrieval_method": [ @@ -708,6 +709,7 @@ class DatasetRetrievalSettingMockApi(Resource): | VectorType.LINDORM | VectorType.OPENGAUSS | VectorType.OCEANBASE + | VectorType.TABLESTORE ): return { "retrieval_method": [ diff --git a/api/core/rag/datasource/vdb/tablestore/__init__.py b/api/core/rag/datasource/vdb/tablestore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py b/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py new file mode 100644 index 0000000000..a124faa503 --- /dev/null +++ b/api/core/rag/datasource/vdb/tablestore/tablestore_vector.py @@ -0,0 +1,295 @@ +import json +import logging +from typing import Any, Optional + +import tablestore # type: ignore +from pydantic import BaseModel, model_validator + +from configs import dify_config +from core.rag.datasource.vdb.field import Field +from core.rag.datasource.vdb.vector_base import BaseVector +from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory +from core.rag.datasource.vdb.vector_type import VectorType +from core.rag.embedding.embedding_base import Embeddings +from core.rag.models.document import Document +from extensions.ext_redis import redis_client +from models import Dataset + + +class TableStoreConfig(BaseModel): + access_key_id: Optional[str] = None + access_key_secret: Optional[str] = None + instance_name: Optional[str] = None + endpoint: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def validate_config(cls, values: dict) -> dict: + if not values["access_key_id"]: + raise ValueError("config ACCESS_KEY_ID is required") + if not values["access_key_secret"]: + raise ValueError("config ACCESS_KEY_SECRET is required") + if not values["instance_name"]: + raise ValueError("config INSTANCE_NAME is required") + if not values["endpoint"]: + raise ValueError("config ENDPOINT is required") + return values + + +class TableStoreVector(BaseVector): + def __init__(self, collection_name: str, config: TableStoreConfig): + super().__init__(collection_name) + self._config = config + self._tablestore_client = tablestore.OTSClient( + config.endpoint, + config.access_key_id, + config.access_key_secret, + config.instance_name, + ) + self._table_name = f"{collection_name}" + self._index_name = f"{collection_name}_idx" + self._tags_field = f"{Field.METADATA_KEY.value}_tags" + + def get_type(self) -> str: + return VectorType.TABLESTORE + + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + dimension = len(embeddings[0]) + self._create_collection(dimension) + self.add_texts(documents=texts, embeddings=embeddings, **kwargs) + + def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + uuids = self._get_uuids(documents) + + for i in range(len(documents)): + self._write_row( + primary_key=uuids[i], + attributes={ + Field.CONTENT_KEY.value: documents[i].page_content, + Field.VECTOR.value: embeddings[i], + Field.METADATA_KEY.value: documents[i].metadata, + }, + ) + return uuids + + def text_exists(self, id: str) -> bool: + _, return_row, _ = self._tablestore_client.get_row( + table_name=self._table_name, primary_key=[("id", id)], columns_to_get=["id"] + ) + + return return_row is not None + + def delete_by_ids(self, ids: list[str]) -> None: + if not ids: + return + for id in ids: + self._delete_row(id=id) + + def get_ids_by_metadata_field(self, key: str, value: str): + return self._search_by_metadata(key, value) + + def delete_by_metadata_field(self, key: str, value: str) -> None: + ids = self.get_ids_by_metadata_field(key, value) + self.delete_by_ids(ids) + + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: + top_k = kwargs.get("top_k", 4) + return self._search_by_vector(query_vector, top_k) + + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: + return self._search_by_full_text(query) + + def delete(self) -> None: + self._delete_table_if_exist() + + def _create_collection(self, dimension: int): + lock_name = f"vector_indexing_lock_{self._collection_name}" + with redis_client.lock(lock_name, timeout=20): + collection_exist_cache_key = f"vector_indexing_{self._collection_name}" + if redis_client.get(collection_exist_cache_key): + logging.info(f"Collection {self._collection_name} already exists.") + return + + self._create_table_if_not_exist() + self._create_search_index_if_not_exist(dimension) + redis_client.set(collection_exist_cache_key, 1, ex=3600) + + def _create_table_if_not_exist(self) -> None: + table_list = self._tablestore_client.list_table() + if self._table_name in table_list: + logging.info("Tablestore system table[%s] already exists", self._table_name) + return None + + schema_of_primary_key = [("id", "STRING")] + table_meta = tablestore.TableMeta(self._table_name, schema_of_primary_key) + table_options = tablestore.TableOptions() + reserved_throughput = tablestore.ReservedThroughput(tablestore.CapacityUnit(0, 0)) + self._tablestore_client.create_table(table_meta, table_options, reserved_throughput) + logging.info("Tablestore create table[%s] successfully.", self._table_name) + + def _create_search_index_if_not_exist(self, dimension: int) -> None: + search_index_list = self._tablestore_client.list_search_index(table_name=self._table_name) + if self._index_name in [t[1] for t in search_index_list]: + logging.info("Tablestore system index[%s] already exists", self._index_name) + return None + + field_schemas = [ + tablestore.FieldSchema( + Field.CONTENT_KEY.value, + tablestore.FieldType.TEXT, + analyzer=tablestore.AnalyzerType.MAXWORD, + index=True, + enable_sort_and_agg=False, + store=False, + ), + tablestore.FieldSchema( + Field.VECTOR.value, + tablestore.FieldType.VECTOR, + vector_options=tablestore.VectorOptions( + data_type=tablestore.VectorDataType.VD_FLOAT_32, + dimension=dimension, + metric_type=tablestore.VectorMetricType.VM_COSINE, + ), + ), + tablestore.FieldSchema( + Field.METADATA_KEY.value, + tablestore.FieldType.KEYWORD, + index=True, + store=False, + ), + tablestore.FieldSchema( + self._tags_field, + tablestore.FieldType.KEYWORD, + index=True, + store=False, + is_array=True, + ), + ] + + index_meta = tablestore.SearchIndexMeta(field_schemas) + self._tablestore_client.create_search_index(self._table_name, self._index_name, index_meta) + logging.info("Tablestore create system index[%s] successfully.", self._index_name) + + def _delete_table_if_exist(self): + search_index_list = self._tablestore_client.list_search_index(table_name=self._table_name) + for resp_tuple in search_index_list: + self._tablestore_client.delete_search_index(resp_tuple[0], resp_tuple[1]) + logging.info("Tablestore delete index[%s] successfully.", self._index_name) + + self._tablestore_client.delete_table(self._table_name) + logging.info("Tablestore delete system table[%s] successfully.", self._index_name) + + def _delete_search_index(self) -> None: + self._tablestore_client.delete_search_index(self._table_name, self._index_name) + logging.info("Tablestore delete index[%s] successfully.", self._index_name) + + def _write_row(self, primary_key: str, attributes: dict[str, Any]) -> None: + pk = [("id", primary_key)] + + tags = [] + for key, value in attributes[Field.METADATA_KEY.value].items(): + tags.append(str(key) + "=" + str(value)) + + attribute_columns = [ + (Field.CONTENT_KEY.value, attributes[Field.CONTENT_KEY.value]), + (Field.VECTOR.value, json.dumps(attributes[Field.VECTOR.value])), + ( + Field.METADATA_KEY.value, + json.dumps(attributes[Field.METADATA_KEY.value]), + ), + (self._tags_field, json.dumps(tags)), + ] + row = tablestore.Row(pk, attribute_columns) + self._tablestore_client.put_row(self._table_name, row) + + def _delete_row(self, id: str) -> None: + primary_key = [("id", id)] + row = tablestore.Row(primary_key) + self._tablestore_client.delete_row(self._table_name, row, None) + logging.info("Tablestore delete row successfully. id:%s", id) + + def _search_by_metadata(self, key: str, value: str) -> list[str]: + query = tablestore.SearchQuery( + tablestore.TermQuery(self._tags_field, str(key) + "=" + str(value)), + limit=100, + get_total_count=False, + ) + + search_response = self._tablestore_client.search( + table_name=self._table_name, + index_name=self._index_name, + search_query=query, + columns_to_get=tablestore.ColumnsToGet(return_type=tablestore.ColumnReturnType.ALL_FROM_INDEX), + ) + + return [row[0][0][1] for row in search_response.rows] + + def _search_by_vector(self, query_vector: list[float], top_k: int) -> list[Document]: + ots_query = tablestore.KnnVectorQuery( + field_name=Field.VECTOR.value, + top_k=top_k, + float32_query_vector=query_vector, + ) + sort = tablestore.Sort(sorters=[tablestore.ScoreSort(sort_order=tablestore.SortOrder.DESC)]) + search_query = tablestore.SearchQuery(ots_query, limit=top_k, get_total_count=False, sort=sort) + + search_response = self._tablestore_client.search( + table_name=self._table_name, + index_name=self._index_name, + search_query=search_query, + columns_to_get=tablestore.ColumnsToGet(return_type=tablestore.ColumnReturnType.ALL_FROM_INDEX), + ) + logging.info( + "Tablestore search successfully. request_id:%s", + search_response.request_id, + ) + return self._to_query_result(search_response) + + def _to_query_result(self, search_response: tablestore.SearchResponse) -> list[Document]: + documents = [] + for row in search_response.rows: + documents.append( + Document( + page_content=row[1][2][1], + vector=json.loads(row[1][3][1]), + metadata=json.loads(row[1][0][1]), + ) + ) + + return documents + + def _search_by_full_text(self, query: str) -> list[Document]: + search_query = tablestore.SearchQuery( + query=tablestore.MatchQuery(text=query, field_name=Field.CONTENT_KEY.value), + sort=tablestore.Sort(sorters=[tablestore.ScoreSort(sort_order=tablestore.SortOrder.DESC)]), + limit=100, + ) + search_response = self._tablestore_client.search( + table_name=self._table_name, + index_name=self._index_name, + search_query=search_query, + columns_to_get=tablestore.ColumnsToGet(return_type=tablestore.ColumnReturnType.ALL_FROM_INDEX), + ) + + return self._to_query_result(search_response) + + +class TableStoreVectorFactory(AbstractVectorFactory): + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> TableStoreVector: + if dataset.index_struct_dict: + class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"] + collection_name = class_prefix + else: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.TABLESTORE, collection_name)) + + return TableStoreVector( + collection_name=collection_name, + config=TableStoreConfig( + endpoint=dify_config.TABLESTORE_ENDPOINT, + instance_name=dify_config.TABLESTORE_INSTANCE_NAME, + access_key_id=dify_config.TABLESTORE_ACCESS_KEY_ID, + access_key_secret=dify_config.TABLESTORE_ACCESS_KEY_SECRET, + ), + ) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 6b4d7b0426..00601c38a1 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -152,6 +152,10 @@ class Vector: from core.rag.datasource.vdb.opengauss.opengauss import OpenGaussFactory return OpenGaussFactory + case VectorType.TABLESTORE: + from core.rag.datasource.vdb.tablestore.tablestore_vector import TableStoreVectorFactory + + return TableStoreVectorFactory case _: raise ValueError(f"Vector store {vector_type} is not supported.") diff --git a/api/core/rag/datasource/vdb/vector_type.py b/api/core/rag/datasource/vdb/vector_type.py index e06988bfcb..940f12caef 100644 --- a/api/core/rag/datasource/vdb/vector_type.py +++ b/api/core/rag/datasource/vdb/vector_type.py @@ -25,3 +25,4 @@ class VectorType(StrEnum): TIDB_ON_QDRANT = "tidb_on_qdrant" OCEANBASE = "oceanbase" OPENGAUSS = "opengauss" + TABLESTORE = "tablestore" diff --git a/api/poetry.lock b/api/poetry.lock index aed61ddded..d7b6eabfee 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1770,6 +1770,123 @@ files = [ [package.extras] toml = ["tomli ; python_full_version <= \"3.11.0a6\""] +[[package]] +name = "crc32c" +version = "2.7.1" +description = "A python package implementing the crc32c algorithm in hardware and software" +optional = false +python-versions = ">=3.7" +groups = ["vdb"] +files = [ + {file = "crc32c-2.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1fd1f9c6b50d7357736676278a1b8c8986737b8a1c76d7eab4baa71d0b6af67f"}, + {file = "crc32c-2.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:805c2be1bc0e251c48439a62b0422385899c15289483692bc70e78473c1039f1"}, + {file = "crc32c-2.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f4333e62b7844dfde112dbb8489fd2970358eddc3310db21e943a9f6994df749"}, + {file = "crc32c-2.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f0fadc741e79dc705e2d9ee967473e8a061d26b04310ed739f1ee292f33674f"}, + {file = "crc32c-2.7.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91ced31055d26d59385d708bbd36689e1a1d604d4b0ceb26767eb5a83156f85d"}, + {file = "crc32c-2.7.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36ffa999b72e3c17f6a066ae9e970b40f8c65f38716e436c39a33b809bc6ed9f"}, + {file = "crc32c-2.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e80114dd7f462297e54d5da1b9ff472e5249c5a2b406aa51c371bb0edcbf76bd"}, + {file = "crc32c-2.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:676f5b46da268b5190f9fb91b3f037a00d114b411313664438525db876adc71f"}, + {file = "crc32c-2.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8d0e660c9ed269e90692993a4457a932fc22c9cc96caf79dd1f1a84da85bb312"}, + {file = "crc32c-2.7.1-cp310-cp310-win32.whl", hash = "sha256:17a2c3f8c6d85b04b5511af827b5dbbda4e672d188c0b9f20a8156e93a1aa7b6"}, + {file = "crc32c-2.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3208764c29688f91a35392073229975dd7687b6cb9f76b919dae442cabcd5126"}, + {file = "crc32c-2.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:19e03a50545a3ef400bd41667d5525f71030488629c57d819e2dd45064f16192"}, + {file = "crc32c-2.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c03286b1e5ce9bed7090084f206aacd87c5146b4b10de56fe9e86cbbbf851cf"}, + {file = "crc32c-2.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80ebbf144a1a56a532b353e81fa0f3edca4f4baa1bf92b1dde2c663a32bb6a15"}, + {file = "crc32c-2.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96b794fd11945298fdd5eb1290a812efb497c14bc42592c5c992ca077458eeba"}, + {file = "crc32c-2.7.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df7194dd3c0efb5a21f5d70595b7a8b4fd9921fbbd597d6d8e7a11eca3e2d27"}, + {file = "crc32c-2.7.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d698eec444b18e296a104d0b9bb6c596c38bdcb79d24eba49604636e9d747305"}, + {file = "crc32c-2.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e07cf10ef852d219d179333fd706d1c415626f1f05e60bd75acf0143a4d8b225"}, + {file = "crc32c-2.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d2a051f296e6e92e13efee3b41db388931cdb4a2800656cd1ed1d9fe4f13a086"}, + {file = "crc32c-2.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a1738259802978cdf428f74156175da6a5fdfb7256f647fdc0c9de1bc6cd7173"}, + {file = "crc32c-2.7.1-cp311-cp311-win32.whl", hash = "sha256:f7786d219a1a1bf27d0aa1869821d11a6f8e90415cfffc1e37791690d4a848a1"}, + {file = "crc32c-2.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:887f6844bb3ad35f0778cd10793ad217f7123a5422e40041231b8c4c7329649d"}, + {file = "crc32c-2.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f7d1c4e761fe42bf856130daf8b2658df33fe0ced3c43dadafdfeaa42b57b950"}, + {file = "crc32c-2.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:73361c79a6e4605204457f19fda18b042a94508a52e53d10a4239da5fb0f6a34"}, + {file = "crc32c-2.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:afd778fc8ac0ed2ffbfb122a9aa6a0e409a8019b894a1799cda12c01534493e0"}, + {file = "crc32c-2.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56ef661b34e9f25991fface7f9ad85e81bbc1b3fe3b916fd58c893eabe2fa0b8"}, + {file = "crc32c-2.7.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:571aa4429444b5d7f588e4377663592145d2d25eb1635abb530f1281794fc7c9"}, + {file = "crc32c-2.7.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c02a3bd67dea95cdb25844aaf44ca2e1b0c1fd70b287ad08c874a95ef4bb38db"}, + {file = "crc32c-2.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:99d17637c4867672cb8adeea007294e3c3df9d43964369516cfe2c1f47ce500a"}, + {file = "crc32c-2.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f4a400ac3c69a32e180d8753fd7ec7bccb80ade7ab0812855dce8a208e72495f"}, + {file = "crc32c-2.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:588587772e55624dd9c7a906ec9e8773ae0b6ac5e270fc0bc84ee2758eba90d5"}, + {file = "crc32c-2.7.1-cp312-cp312-win32.whl", hash = "sha256:9f14b60e5a14206e8173dd617fa0c4df35e098a305594082f930dae5488da428"}, + {file = "crc32c-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:7c810a246660a24dc818047dc5f89c7ce7b2814e1e08a8e99993f4103f7219e8"}, + {file = "crc32c-2.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:24949bffb06fc411cc18188d33357923cb935273642164d0bb37a5f375654169"}, + {file = "crc32c-2.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d5d326e7e118d4fa60187770d86b66af2fdfc63ce9eeb265f0d3e7d49bebe0b"}, + {file = "crc32c-2.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba110df60c64c8e2d77a9425b982a520ccdb7abe42f06604f4d98a45bb1fff62"}, + {file = "crc32c-2.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c277f9d16a3283e064d54854af0976b72abaa89824955579b2b3f37444f89aae"}, + {file = "crc32c-2.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881af0478a01331244e27197356929edbdeaef6a9f81b5c6bacfea18d2139289"}, + {file = "crc32c-2.7.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:724d5ff4d29ff093a983ae656be3307093706d850ea2a233bf29fcacc335d945"}, + {file = "crc32c-2.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2416c4d88696ac322632555c0f81ab35e15f154bc96055da6cf110d642dbc10"}, + {file = "crc32c-2.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:60254251b88ec9b9795215f0f9ec015a6b5eef8b2c5fba1267c672d83c78fc02"}, + {file = "crc32c-2.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:edefc0e46f3c37372183f70338e5bdee42f6789b62fcd36ec53aa933e9dfbeaf"}, + {file = "crc32c-2.7.1-cp313-cp313-win32.whl", hash = "sha256:813af8111218970fe2adb833c5e5239f091b9c9e76f03b4dd91aaba86e99b499"}, + {file = "crc32c-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:7d9ede7be8e4ec1c9e90aaf6884decbeef10e3473e6ddac032706d710cab5888"}, + {file = "crc32c-2.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db9ac92294284b22521356715784b91cc9094eee42a5282ab281b872510d1831"}, + {file = "crc32c-2.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8fcd7f2f29a30dc92af64a9ee3d38bde0c82bd20ad939999427aac94bbd87373"}, + {file = "crc32c-2.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5c056ef043393085523e149276a7ce0cb534b872e04f3e20d74d9a94a75c0ad7"}, + {file = "crc32c-2.7.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03a92551a343702629af91f78d205801219692b6909f8fa126b830e332bfb0e0"}, + {file = "crc32c-2.7.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb9424ec1a8ca54763155a703e763bcede82e6569fe94762614bb2de1412d4e1"}, + {file = "crc32c-2.7.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88732070f6175530db04e0bb36880ac45c33d49f8ac43fa0e50cfb1830049d23"}, + {file = "crc32c-2.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:57a20dfc27995f568f64775eea2bbb58ae269f1a1144561df5e4a4955f79db32"}, + {file = "crc32c-2.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f7186d098bfd2cff25eac6880b7c7ad80431b90610036131c1c7dd0eab42a332"}, + {file = "crc32c-2.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:55a77e29a265418fa34bef15bd0f2c60afae5348988aaf35ed163b4bbf93cf37"}, + {file = "crc32c-2.7.1-cp313-cp313t-win32.whl", hash = "sha256:ae38a4b6aa361595d81cab441405fbee905c72273e80a1c010fb878ae77ac769"}, + {file = "crc32c-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:eee2a43b663feb6c79a6c1c6e5eae339c2b72cfac31ee54ec0209fa736cf7ee5"}, + {file = "crc32c-2.7.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:04a56e9f4995559fa86bcf5d0ed5c48505a36e2be1c41d70cae5c080d9a00b74"}, + {file = "crc32c-2.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88c5c9c21cd9fff593bb7dfe97d3287438c8aecbcc73d227f2366860a0663521"}, + {file = "crc32c-2.7.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:595146cb94ba0055301d273113add2af5859b467db41b50367f47870c2d0a81c"}, + {file = "crc32c-2.7.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9f3792872f1320961f33aaf0198edea371aee393bcc221fab66d10ecffd77d"}, + {file = "crc32c-2.7.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:999a40d75cd1696e779f6f99c29fa52be777197d1d9e3ae69cb919a05a369c1e"}, + {file = "crc32c-2.7.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:eff485526172cee7e6d1fa9c23913f92c7d38ab05674b0b578767c7b693faf5d"}, + {file = "crc32c-2.7.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:541dac90c64ed9ce05f85a71066567e854c1b40743a01d83fa2c66419a2e97b6"}, + {file = "crc32c-2.7.1-cp37-cp37m-win32.whl", hash = "sha256:7138ec26e79100c4cf4294ef40027a1cff26a1e23b7e5eb70efe5d7ff37cbc66"}, + {file = "crc32c-2.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:35a3ed12ac2e2551a07d246b7e6512ac39db021e006205a40c1cfd32ea73fcc3"}, + {file = "crc32c-2.7.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af062f11aea283b7e9c95f3a97fb6bb96ac08a9063f71621c2140237df141ada"}, + {file = "crc32c-2.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f25ca521ecf7cccfff0ecae4d0538b5c0c7235d27bf098241f3e2bf86aed713"}, + {file = "crc32c-2.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1410bcd909be36ccbf8a52c45e4bddca77adfd4e80789ac3cd575c024086516d"}, + {file = "crc32c-2.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33fc8cb32f82685ebefd078e740925ea9da37a008ed5f43b68fc8324f8ca4a37"}, + {file = "crc32c-2.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad3dc6283ce53ad7d1dc5775003460110ab7eebf348efebe0486a531b28f8184"}, + {file = "crc32c-2.7.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:758ead20e122496764ae50db26bb90fb47fc4b6d242c8e99e87c3f1dae1f1dce"}, + {file = "crc32c-2.7.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:e436d9044bbd51936f7aeb8b322543c516bf22371a17970a370a10af1661fa54"}, + {file = "crc32c-2.7.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:47e5be99057264b603e3cd88cf091985f33c16d3c8609f1c83ed6e72ec4179b4"}, + {file = "crc32c-2.7.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:280509210e622a236f16f031856847fd0d6704df662d7209da819ccfb40c6167"}, + {file = "crc32c-2.7.1-cp38-cp38-win32.whl", hash = "sha256:4ab48e048cfa123a9f9bdc5d4d687a3461723132c749c721a6d358605e6d470d"}, + {file = "crc32c-2.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:65471d1b1b6e10a404ca8200a4271d5bc0a552c3f5dcd943c1c7835f766ea02d"}, + {file = "crc32c-2.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:39ca842586084bca24f9c4ab43e2d99191b1186b2f89b2122b470d0730254d1b"}, + {file = "crc32c-2.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a911abc33d453b3f171a3200b1e18b3fc39c204670b5b0a353cca99e4c664332"}, + {file = "crc32c-2.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:22a72e81ec08a7ece6a35ac68d1ed32dd4a8be7949b164db88d4b4a4bade5c5a"}, + {file = "crc32c-2.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54d6f8c5be6815eabd6e3e90fa0bc13045183a6aa33a30dd684eb0f062b92213"}, + {file = "crc32c-2.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c855726d71dee7ae25f81c6b54293455fc66802f34d334d22bea1f6ce8bc21c"}, + {file = "crc32c-2.7.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98d5f7fc364bb9c4c4123d149406fbee063f2e8c2cff19a12f13e50faa146237"}, + {file = "crc32c-2.7.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:51ffba582c95a281e5a3f71eacdafc96b9a1835ddae245385639458fff197034"}, + {file = "crc32c-2.7.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3950d3c340c9d70889630ef81fba8666abfd0cf0aa19fd9c3a55634e0b383b0f"}, + {file = "crc32c-2.7.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:522fba1770aad8f7eb189f21fca591a51d96dcc749859088f462281324aec30b"}, + {file = "crc32c-2.7.1-cp39-cp39-win32.whl", hash = "sha256:812723e222b6a9fe0562554d72f4f072c3a95720c60ee500984e7d0e568caac3"}, + {file = "crc32c-2.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:6793fcfe9d4130230d196abbe4021c01ffe8e85c92633bf3c8559f9836c227f5"}, + {file = "crc32c-2.7.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2e83fedebcdeb80c19e76b7a0e5103528bb062521c40702bf34516a429e81df3"}, + {file = "crc32c-2.7.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30004a7383538ef93bda9b22f7b3805bc0aa5625ab2675690e1b676b19417d4b"}, + {file = "crc32c-2.7.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a01b0983aa87f517c12418f9898ecf2083bf86f4ea04122e053357c3edb0d73f"}, + {file = "crc32c-2.7.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb2b963c42128b38872e9ed63f04a73ce1ff89a1dfad7ea38add6fe6296497b8"}, + {file = "crc32c-2.7.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cdd5e576fee5d255c1e68a4dae4420f21e57e6f05900b38d5ae47c713fc3330d"}, + {file = "crc32c-2.7.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:79f0ff50863aeb441fbfa87e9db6542ddfe3e941189dece832b0af2e454dbab0"}, + {file = "crc32c-2.7.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cd27a1e400d77e9872fa1303e8f9d30bd050df35ee4858354ce0b59f8227d32"}, + {file = "crc32c-2.7.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:274739b3e1591bd4b7ec98764f2f79c6fbcc0f7d7676d5f17369832fe14ee4f0"}, + {file = "crc32c-2.7.1-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:050f52045b4a033a245e0ee4357e1a793de5af6496c82250ef13d8cb90a21e20"}, + {file = "crc32c-2.7.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ceb4ca126f75694bda020a307221563d3c522719c0acedcc81ffb985b4867c94"}, + {file = "crc32c-2.7.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eabefe7a6fb5dfc6318fb35f4d98893baef17ebda9b311498e870526d32168e7"}, + {file = "crc32c-2.7.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:217edd9ba8c5f0c3ad60c82a11fa78f01162fa106fd7f5d17175dac6bf1eedf9"}, + {file = "crc32c-2.7.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15d640d9d4aa213aec6c837f602081a17d1522f8cd78b52334b62ee27b083410"}, + {file = "crc32c-2.7.1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:519878822bf9bdead63c25a5e4bdc26d2eae9da6056f92b9b5f3023c08f1d016"}, + {file = "crc32c-2.7.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2bf69cfa4c3ea9f060fe06db00b7e34f771c83f73dd2c3568c2c9019479e34c2"}, + {file = "crc32c-2.7.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e89d51c90f6730b67b12c97d49099ba18d0fdce18541fab94d2be95d1c939adb"}, + {file = "crc32c-2.7.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:488a0feba1bb005d0dd2f702c1da4849d083e88d82cd27b83ac2d2d93af80755"}, + {file = "crc32c-2.7.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:919262b7a12ef63f222ec19c0e092f39268802652e11669315257ae6249ec79f"}, + {file = "crc32c-2.7.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4181240f6080c38eec9dd1539cd23a304a12100d3f4ffe43234f32064fae5ef0"}, + {file = "crc32c-2.7.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fedde1e53507d0ede1980e8109442edd108c04ab100abcd5145c274820dacd4f"}, + {file = "crc32c-2.7.1.tar.gz", hash = "sha256:f91b144a21eef834d64178e01982bb9179c354b3e9e5f4c803b0e5096384968c"}, +] + [[package]] name = "crcmod" version = "1.7" @@ -2022,6 +2139,19 @@ files = [ [package.extras] dev = ["coverage", "pytest (>=7.4.4)"] +[[package]] +name = "enum34" +version = "1.1.10" +description = "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4" +optional = false +python-versions = "*" +groups = ["vdb"] +files = [ + {file = "enum34-1.1.10-py2-none-any.whl", hash = "sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53"}, + {file = "enum34-1.1.10-py3-none-any.whl", hash = "sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328"}, + {file = "enum34-1.1.10.tar.gz", hash = "sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248"}, +] + [[package]] name = "esdk-obs-python" version = "3.24.6.1" @@ -8302,6 +8432,28 @@ mpmath = ">=1.1.0,<1.4" [package.extras] dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] +[[package]] +name = "tablestore" +version = "6.1.0" +description = "Aliyun TableStore(OTS) SDK" +optional = false +python-versions = "*" +groups = ["vdb"] +files = [ + {file = "tablestore-6.1.0.tar.gz", hash = "sha256:bfe6a3e0fe88a230729723c357f4a46b8869a06a4b936db20692ed587a721c1c"}, +] + +[package.dependencies] +certifi = ">=2016.2.28" +crc32c = ">=2.7.1" +enum34 = ">=1.1.6" +flatbuffers = ">=22.9.24" +future = ">=0.16.0" +numpy = ">=1.11.0" +protobuf = ">=3.20.0,<=5.27.4" +six = ">=1.11.0" +urllib3 = ">=1.14" + [[package]] name = "tabulate" version = "0.9.0" @@ -9996,4 +10148,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "5ee6591e61f05986ab0bb50e5a540ef4eee4b086e8da8e9f01ac1176730ad051" +content-hash = "a094082d2fd4d8ea480ac800e54029bdb604de70a7b4348778bdcddb39b06c7e" diff --git a/api/pyproject.toml b/api/pyproject.toml index 4b5ec05e9a..0cea0293a6 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -130,6 +130,7 @@ pymilvus = "~2.5.0" pymochow = "1.3.1" pyobvector = "~0.1.6" qdrant-client = "1.7.3" +tablestore = "6.1.0" tcvectordb = "~1.6.4" tidb-vector = "0.0.9" upstash-vector = "0.6.0" diff --git a/api/tests/integration_tests/vdb/tablestore/__init__.py b/api/tests/integration_tests/vdb/tablestore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/tests/integration_tests/vdb/tablestore/test_tablestore.py b/api/tests/integration_tests/vdb/tablestore/test_tablestore.py new file mode 100644 index 0000000000..da890d0b7c --- /dev/null +++ b/api/tests/integration_tests/vdb/tablestore/test_tablestore.py @@ -0,0 +1,34 @@ +import os + +from core.rag.datasource.vdb.tablestore.tablestore_vector import ( + TableStoreConfig, + TableStoreVector, +) +from tests.integration_tests.vdb.test_vector_store import ( + AbstractVectorTest, + setup_mock_redis, +) + + +class TableStoreVectorTest(AbstractVectorTest): + def __init__(self): + super().__init__() + self.vector = TableStoreVector( + collection_name=self.collection_name, + config=TableStoreConfig( + endpoint=os.getenv("TABLESTORE_ENDPOINT"), + instance_name=os.getenv("TABLESTORE_INSTANCE_NAME"), + access_key_id=os.getenv("TABLESTORE_ACCESS_KEY_ID"), + access_key_secret=os.getenv("TABLESTORE_ACCESS_KEY_SECRET"), + ), + ) + + def get_ids_by_metadata_field(self): + ids = self.vector.get_ids_by_metadata_field(key="doc_id", value=self.example_doc_id) + assert ids is not None + assert len(ids) == 1 + assert ids[0] == self.example_doc_id + + +def test_tablestore_vector(setup_mock_redis): + TableStoreVectorTest().run_all_tests() diff --git a/docker/.env.example b/docker/.env.example index d366b18160..0da77f613f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -383,7 +383,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`, `opengauss`, `tablestore`. VECTOR_STORE=weaviate # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. @@ -570,6 +570,13 @@ OPENGAUSS_ENABLE_PQ=false UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io UPSTASH_VECTOR_TOKEN=dify +# TableStore Vector configuration +# (only used when VECTOR_STORE is tablestore) +TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com +TABLESTORE_INSTANCE_NAME=instance-name +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx + # ------------------------------ # Knowledge Configuration # ------------------------------ diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index bc92cb0922..edd2b8352f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -263,6 +263,10 @@ x-shared-env: &shared-api-worker-env OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} + TABLESTORE_ENDPOINT: ${TABLESTORE_ENDPOINT:-https://instance-name.cn-hangzhou.ots.aliyuncs.com} + TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name} + TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx} + TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx} UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} ETL_TYPE: ${ETL_TYPE:-dify}