diff --git a/api/Dockerfile b/api/Dockerfile index 4f0e7f65e3..53c33a7659 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -23,7 +23,6 @@ RUN apt-get update \ COPY pyproject.toml poetry.lock ./ RUN poetry install --sync --no-cache --no-root - # production stage FROM base AS production diff --git a/api/core/tools/provider/builtin/json_process/_assets/icon.svg b/api/core/tools/provider/builtin/json_process/_assets/icon.svg new file mode 100644 index 0000000000..b123983836 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/_assets/icon.svg @@ -0,0 +1,358 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/json_process.py b/api/core/tools/provider/builtin/json_process/json_process.py new file mode 100644 index 0000000000..f6eed3c628 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/json_process.py @@ -0,0 +1,17 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.json_process.tools.parse import JSONParseTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class JsonExtractProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + JSONParseTool().invoke(user_id='', + tool_parameters={ + 'content': '{"name": "John", "age": 30, "city": "New York"}', + 'json_filter': '$.name' + }) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/json_process.yaml b/api/core/tools/provider/builtin/json_process/json_process.yaml new file mode 100644 index 0000000000..c7896bbea7 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/json_process.yaml @@ -0,0 +1,14 @@ +identity: + author: Mingwei Zhang + name: json_process + label: + en_US: JSON Process + zh_Hans: JSON 处理 + pt_BR: JSON Process + description: + en_US: Tools for processing JSON content using jsonpath_ng + zh_Hans: 利用 jsonpath_ng 处理 JSON 内容的工具 + pt_BR: Tools for processing JSON content using jsonpath_ng + icon: icon.svg + tags: + - utilities diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.py b/api/core/tools/provider/builtin/json_process/tools/delete.py new file mode 100644 index 0000000000..b09e494881 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/delete.py @@ -0,0 +1,59 @@ +import json +from typing import Any, Union + +from jsonpath_ng import parse + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class JSONDeleteTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + Invoke the JSON delete tool + """ + # Get content + content = tool_parameters.get('content', '') + if not content: + return self.create_text_message('Invalid parameter content') + + # Get query + query = tool_parameters.get('query', '') + if not query: + return self.create_text_message('Invalid parameter query') + + try: + result = self._delete(content, query) + return self.create_text_message(str(result)) + except Exception as e: + return self.create_text_message(f'Failed to delete JSON content: {str(e)}') + + def _delete(self, origin_json: str, query: str) -> str: + try: + input_data = json.loads(origin_json) + expr = parse('$.' + query.lstrip('$.')) # Ensure query path starts with $ + + matches = expr.find(input_data) + + if not matches: + return json.dumps(input_data, ensure_ascii=True) # No changes if no matches found + + for match in matches: + if isinstance(match.context.value, dict): + # Delete key from dictionary + del match.context.value[match.path.fields[-1]] + elif isinstance(match.context.value, list): + # Remove item from list + match.context.value.remove(match.value) + else: + # For other cases, we might want to set to None or remove the parent key + parent = match.context.parent + if parent: + del parent.value[match.path.fields[-1]] + + return json.dumps(input_data, ensure_ascii=True) + except Exception as e: + raise Exception(f"Delete operation failed: {str(e)}") \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.yaml b/api/core/tools/provider/builtin/json_process/tools/delete.yaml new file mode 100644 index 0000000000..4cfa90b861 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/delete.yaml @@ -0,0 +1,40 @@ +identity: + name: json_delete + author: Mingwei Zhang + label: + en_US: JSON Delete + zh_Hans: JSON 删除 + pt_BR: JSON Delete +description: + human: + en_US: A tool for deleting JSON content + zh_Hans: 一个删除 JSON 内容的工具 + pt_BR: A tool for deleting JSON content + llm: A tool for deleting JSON content +parameters: + - name: content + type: string + required: true + label: + en_US: JSON content + zh_Hans: JSON 内容 + pt_BR: JSON content + human_description: + en_US: JSON content to be processed + zh_Hans: 待处理的 JSON 内容 + pt_BR: JSON content to be processed + llm_description: JSON content to be processed + form: llm + - name: query + type: string + required: true + label: + en_US: Query + zh_Hans: 查询 + pt_BR: Query + human_description: + en_US: JSONPath query to locate the element to delete + zh_Hans: 用于定位要删除元素的 JSONPath 查询 + pt_BR: JSONPath query to locate the element to delete + llm_description: JSONPath query to locate the element to delete + form: llm diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.py b/api/core/tools/provider/builtin/json_process/tools/insert.py new file mode 100644 index 0000000000..aa5986e2b4 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/insert.py @@ -0,0 +1,97 @@ +import json +from typing import Any, Union + +from jsonpath_ng import parse + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class JSONParseTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + # get content + content = tool_parameters.get('content', '') + if not content: + return self.create_text_message('Invalid parameter content') + + # get query + query = tool_parameters.get('query', '') + if not query: + return self.create_text_message('Invalid parameter query') + + # get new value + new_value = tool_parameters.get('new_value', '') + if not new_value: + return self.create_text_message('Invalid parameter new_value') + + # get insert position + index = tool_parameters.get('index') + + # get create path + create_path = tool_parameters.get('create_path', False) + + try: + result = self._insert(content, query, new_value, index, create_path) + return self.create_text_message(str(result)) + except Exception: + return self.create_text_message('Failed to insert JSON content') + + + def _insert(self, origin_json, query, new_value, index=None, create_path=False): + try: + input_data = json.loads(origin_json) + expr = parse(query) + try: + new_value = json.loads(new_value) + except json.JSONDecodeError: + new_value = new_value + + matches = expr.find(input_data) + + if not matches and create_path: + # create new path + path_parts = query.strip('$').strip('.').split('.') + current = input_data + for i, part in enumerate(path_parts): + if '[' in part and ']' in part: + # process array index + array_name, index = part.split('[') + index = int(index.rstrip(']')) + if array_name not in current: + current[array_name] = [] + while len(current[array_name]) <= index: + current[array_name].append({}) + current = current[array_name][index] + else: + if i == len(path_parts) - 1: + current[part] = new_value + elif part not in current: + current[part] = {} + current = current[part] + else: + for match in matches: + if isinstance(match.value, dict): + # insert new value into dict + if isinstance(new_value, dict): + match.value.update(new_value) + else: + raise ValueError("Cannot insert non-dict value into dict") + elif isinstance(match.value, list): + # insert new value into list + if index is None: + match.value.append(new_value) + else: + match.value.insert(int(index), new_value) + else: + # replace old value with new value + match.full_path.update(input_data, new_value) + + return json.dumps(input_data, ensure_ascii=True) + except Exception as e: + return str(e) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.yaml b/api/core/tools/provider/builtin/json_process/tools/insert.yaml new file mode 100644 index 0000000000..66a6ff9929 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/insert.yaml @@ -0,0 +1,77 @@ +identity: + name: json_insert + author: Mingwei Zhang + label: + en_US: JSON Insert + zh_Hans: JSON 插入 + pt_BR: JSON Insert +description: + human: + en_US: A tool for inserting JSON content + zh_Hans: 一个插入 JSON 内容的工具 + pt_BR: A tool for inserting JSON content + llm: A tool for inserting JSON content +parameters: + - name: content + type: string + required: true + label: + en_US: JSON content + zh_Hans: JSON 内容 + pt_BR: JSON content + human_description: + en_US: JSON content + zh_Hans: JSON 内容 + pt_BR: JSON content + llm_description: JSON content to be processed + form: llm + - name: query + type: string + required: true + label: + en_US: Query + zh_Hans: 查询 + pt_BR: Query + human_description: + en_US: Object to insert + zh_Hans: 待插入的对象 + pt_BR: Object to insert + llm_description: JSONPath query to locate the element to insert + form: llm + - name: new_value + type: string + required: true + label: + en_US: New Value + zh_Hans: 新值 + pt_BR: New Value + human_description: + en_US: New Value + zh_Hans: 新值 + pt_BR: New Value + llm_description: New Value to insert + form: llm + - name: create_path + type: select + required: true + default: "False" + label: + en_US: Whether to create a path + zh_Hans: 是否创建路径 + pt_BR: Whether to create a path + human_description: + en_US: Whether to create a path when the path does not exist + zh_Hans: 查询路径不存在时是否创建路径 + pt_BR: Whether to create a path when the path does not exist + options: + - value: "True" + label: + en_US: "Yes" + zh_Hans: 是 + pt_BR: "Yes" + - value: "False" + label: + en_US: "No" + zh_Hans: 否 + pt_BR: "No" + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.py b/api/core/tools/provider/builtin/json_process/tools/parse.py new file mode 100644 index 0000000000..b246afc07e --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/parse.py @@ -0,0 +1,51 @@ +import json +from typing import Any, Union + +from jsonpath_ng import parse + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class JSONParseTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + # get content + content = tool_parameters.get('content', '') + if not content: + return self.create_text_message('Invalid parameter content') + + # get json filter + json_filter = tool_parameters.get('json_filter', '') + if not json_filter: + return self.create_text_message('Invalid parameter json_filter') + + try: + result = self._extract(content, json_filter) + return self.create_text_message(str(result)) + except Exception: + return self.create_text_message('Failed to extract JSON content') + + # Extract data from JSON content + def _extract(self, content: str, json_filter: str) -> str: + try: + input_data = json.loads(content) + expr = parse(json_filter) + result = [match.value for match in expr.find(input_data)] + + if len(result) == 1: + result = result[0] + + if isinstance(result, dict | list): + return json.dumps(result, ensure_ascii=True) + elif isinstance(result, str | int | float | bool) or result is None: + return str(result) + else: + return repr(result) + except Exception as e: + return str(e) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.yaml b/api/core/tools/provider/builtin/json_process/tools/parse.yaml new file mode 100644 index 0000000000..b619dcde94 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/parse.yaml @@ -0,0 +1,40 @@ +identity: + name: parse + author: Mingwei Zhang + label: + en_US: JSON Parse + zh_Hans: JSON 解析 + pt_BR: JSON Parse +description: + human: + en_US: A tool for extracting JSON objects + zh_Hans: 一个解析JSON对象的工具 + pt_BR: A tool for extracting JSON objects + llm: A tool for extracting JSON objects +parameters: + - name: content + type: string + required: true + label: + en_US: JSON data + zh_Hans: JSON数据 + pt_BR: JSON data + human_description: + en_US: JSON data + zh_Hans: JSON数据 + pt_BR: JSON数据 + llm_description: JSON data to be processed + form: llm + - name: json_filter + type: string + required: true + label: + en_US: JSON filter + zh_Hans: JSON解析对象 + pt_BR: JSON filter + human_description: + en_US: JSON fields to be parsed + zh_Hans: 需要解析的 JSON 字段 + pt_BR: JSON fields to be parsed + llm_description: JSON fields to be parsed + form: llm diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.py b/api/core/tools/provider/builtin/json_process/tools/replace.py new file mode 100644 index 0000000000..9f127b9d06 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/replace.py @@ -0,0 +1,106 @@ +import json +from typing import Any, Union + +from jsonpath_ng import parse + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class JSONReplaceTool(BuiltinTool): + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + # get content + content = tool_parameters.get('content', '') + if not content: + return self.create_text_message('Invalid parameter content') + + # get query + query = tool_parameters.get('query', '') + if not query: + return self.create_text_message('Invalid parameter query') + + # get replace value + replace_value = tool_parameters.get('replace_value', '') + if not replace_value: + return self.create_text_message('Invalid parameter replace_value') + + # get replace model + replace_model = tool_parameters.get('replace_model', '') + if not replace_model: + return self.create_text_message('Invalid parameter replace_model') + + try: + if replace_model == 'pattern': + # get replace pattern + replace_pattern = tool_parameters.get('replace_pattern', '') + if not replace_pattern: + return self.create_text_message('Invalid parameter replace_pattern') + result = self._replace_pattern(content, query, replace_pattern, replace_value) + elif replace_model == 'key': + result = self._replace_key(content, query, replace_value) + elif replace_model == 'value': + result = self._replace_value(content, query, replace_value) + return self.create_text_message(str(result)) + except Exception: + return self.create_text_message('Failed to replace JSON content') + + # Replace pattern + def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str) -> str: + try: + input_data = json.loads(content) + expr = parse(query) + + matches = expr.find(input_data) + + for match in matches: + new_value = match.value.replace(replace_pattern, replace_value) + match.full_path.update(input_data, new_value) + + return json.dumps(input_data, ensure_ascii=True) + except Exception as e: + return str(e) + + # Replace key + def _replace_key(self, content: str, query: str, replace_value: str) -> str: + try: + input_data = json.loads(content) + expr = parse(query) + + matches = expr.find(input_data) + + for match in matches: + parent = match.context.value + if isinstance(parent, dict): + old_key = match.path.fields[0] + if old_key in parent: + value = parent.pop(old_key) + parent[replace_value] = value + elif isinstance(parent, list): + for item in parent: + if isinstance(item, dict) and old_key in item: + value = item.pop(old_key) + item[replace_value] = value + return json.dumps(input_data, ensure_ascii=True) + except Exception as e: + return str(e) + + # Replace value + def _replace_value(self, content: str, query: str, replace_value: str) -> str: + try: + input_data = json.loads(content) + expr = parse(query) + + matches = expr.find(input_data) + + for match in matches: + match.full_path.update(input_data, replace_value) + + return json.dumps(input_data, ensure_ascii=True) + except Exception as e: + return str(e) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.yaml b/api/core/tools/provider/builtin/json_process/tools/replace.yaml new file mode 100644 index 0000000000..556be5e8b2 --- /dev/null +++ b/api/core/tools/provider/builtin/json_process/tools/replace.yaml @@ -0,0 +1,95 @@ +identity: + name: json_replace + author: Mingwei Zhang + label: + en_US: JSON Replace + zh_Hans: JSON 替换 + pt_BR: JSON Replace +description: + human: + en_US: A tool for replacing JSON content + zh_Hans: 一个替换 JSON 内容的工具 + pt_BR: A tool for replacing JSON content + llm: A tool for replacing JSON content +parameters: + - name: content + type: string + required: true + label: + en_US: JSON content + zh_Hans: JSON 内容 + pt_BR: JSON content + human_description: + en_US: JSON content + zh_Hans: JSON 内容 + pt_BR: JSON content + llm_description: JSON content to be processed + form: llm + - name: query + type: string + required: true + label: + en_US: Query + zh_Hans: 查询 + pt_BR: Query + human_description: + en_US: Query + zh_Hans: 查询 + pt_BR: Query + llm_description: JSONPath query to locate the element to replace + form: llm + - name: replace_pattern + type: string + required: false + label: + en_US: String to be replaced + zh_Hans: 待替换字符串 + pt_BR: String to be replaced + human_description: + en_US: String to be replaced + zh_Hans: 待替换字符串 + pt_BR: String to be replaced + llm_description: String to be replaced + form: llm + - name: replace_value + type: string + required: true + label: + en_US: Replace Value + zh_Hans: 替换值 + pt_BR: Replace Value + human_description: + en_US: New Value + zh_Hans: New Value + pt_BR: New Value + llm_description: New Value to replace + form: llm + - name: replace_model + type: select + required: true + default: pattern + label: + en_US: Replace Model + zh_Hans: 替换模式 + pt_BR: Replace Model + human_description: + en_US: Replace Model + zh_Hans: 替换模式 + pt_BR: Replace Model + options: + - value: key + label: + en_US: replace key + zh_Hans: 键替换 + pt_BR: replace key + - value: value + label: + en_US: replace value + zh_Hans: 值替换 + pt_BR: replace value + - value: pattern + label: + en_US: replace string + zh_Hans: 字符串替换 + pt_BR: replace string + form: form diff --git a/api/poetry.lock b/api/poetry.lock index 2fdc13bb96..ce2adec183 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -3702,6 +3702,20 @@ files = [ {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] +[[package]] +name = "jsonpath-ng" +version = "1.6.1" +description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." +optional = false +python-versions = "*" +files = [ + {file = "jsonpath-ng-1.6.1.tar.gz", hash = "sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa"}, + {file = "jsonpath_ng-1.6.1-py3-none-any.whl", hash = "sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be"}, +] + +[package.dependencies] +ply = "*" + [[package]] name = "kaleido" version = "0.2.1" @@ -9081,4 +9095,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "5f6f7d8114ece4f7e865fdf9a1fc38a86238d6bc80333b878d50c95a7885b9f5" +content-hash = "d40bed69caecf3a2bcd5ec054288d7cb36a9a231fff210d4f1a42745dd3bf604" diff --git a/api/pyproject.toml b/api/pyproject.toml index ed85e79d1a..89539f93d8 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -187,6 +187,7 @@ arxiv = "2.1.0" matplotlib = "~3.8.2" newspaper3k = "0.2.8" duckduckgo-search = "~6.1.5" +jsonpath-ng = "1.6.1" numexpr = "~2.9.0" opensearch-py = "2.4.0" qrcode = "~7.4.2" @@ -246,4 +247,4 @@ optional = true [tool.poetry.group.lint.dependencies] ruff = "~0.4.8" -dotenv-linter = "~0.5.0" +dotenv-linter = "~0.5.0" \ No newline at end of file