From b415c4d933ad67a3c45d9a06c9baa1349aa486b1 Mon Sep 17 00:00:00 2001 From: Xinhong Zhang Date: Tue, 7 Jan 2025 14:28:31 +0800 Subject: [PATCH] fix(knext): fix knext project env (#211) * fix create project * fix create project * fix create project * fix create project --- knext/command/sub_command/project.py | 34 +++++++++++++++----- knext/common/env.py | 46 +++++++++++++++++++--------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/knext/command/sub_command/project.py b/knext/command/sub_command/project.py index c3d4810e..b0c844c0 100644 --- a/knext/command/sub_command/project.py +++ b/knext/command/sub_command/project.py @@ -33,6 +33,8 @@ from kag.common.vectorize_model.vectorize_model_config_checker import ( from shutil import copy2 yaml = YAML() +yaml.default_flow_style = False +yaml.indent(mapping=2, sequence=4, offset=2) def _render_template(namespace: str, tmpl: str, **kwargs): @@ -95,10 +97,10 @@ def _recover_project(prj_path: str): client = ProjectClient() project = client.get(namespace=namespace) or client.create( - name=project_name, desc=desc, namespace=namespace + name=project_name, desc=desc, namespace=namespace, config=json.dumps(env.config) ) - env.config["project"]["id"] = project.id + env._config["project"]["id"] = project.id env.dump() click.secho( @@ -117,7 +119,7 @@ def _recover_project(prj_path: str): @click.option( "--delete_cfg", help="whether delete your defined .yaml file.", - default=True, + default=False, hidden=True, ) def create_project( @@ -147,6 +149,19 @@ def create_project( tmpl = "default" project_id = None + + llm_config_checker = LLMConfigChecker() + vectorize_model_config_checker = VectorizeModelConfigChecker() + llm_config = config.get("chat_llm", {}) + vectorize_model_config = config.get("vectorizer", {}) + try: + llm_config_checker.check(json.dumps(llm_config)) + dim = vectorize_model_config_checker.check(json.dumps(vectorize_model_config)) + config["vectorizer"]["vector_dimensions"] = dim + except Exception as e: + click.secho(f"Error: {e}", fg="bright_red") + sys.exit() + if host_addr: client = ProjectClient(host_addr=host_addr) project = client.create(name=name, namespace=namespace, config=json.dumps(config)) @@ -168,6 +183,10 @@ def create_project( delete_cfg=delete_cfg, ) + current_dir = os.getcwd() + os.chdir(project_dir) + update_project(project_dir) + os.chdir(current_dir) if delete_cfg and os.path.exists(config_path): os.remove(config_path) @@ -193,14 +212,13 @@ def restore_project(host_addr, proj_path): if not project_wanted: if host_addr: client = ProjectClient(host_addr=host_addr) - project = client.create(name=env.name, namespace=env.namespace) + project = client.create(name=env.name, namespace=env.namespace, config=json.dumps(env.config)) project_id = project.id else: project_id = project_wanted.id # write project id and host addr to kag_config.yaml - - env.config["project"]["id"] = project_id - env.config["project"]["host_addr"] = host_addr + env._config["project"]["id"] = project_id + env._config["project"]["host_addr"] = host_addr env.dump() if proj_path: _recover_project(proj_path) @@ -220,7 +238,7 @@ def update_project(proj_path): try: llm_config_checker.check(json.dumps(llm_config)) dim = vectorize_model_config_checker.check(json.dumps(vectorize_model_config)) - env.config["vectorizer"]["vector_dimensions"] = dim + env._config["vectorizer"]["vector_dimensions"] = dim except Exception as e: click.secho(f"Error: {e}", fg="bright_red") sys.exit() diff --git a/knext/common/env.py b/knext/common/env.py index f71e02e7..7fefd3da 100644 --- a/knext/common/env.py +++ b/knext/common/env.py @@ -19,7 +19,8 @@ from pathlib import Path from typing import Union, Optional yaml = YAML() - +yaml.default_flow_style = False +yaml.indent(mapping=2, sequence=4, offset=2) logger = logging.getLogger(__name__) DEFAULT_HOST_ADDR = "http://127.0.0.1:8887" @@ -42,11 +43,20 @@ class Environment: @property def config(self): + + closest_config = self._closest_config() + if not hasattr(self, '_config_path') or self._config_path != closest_config: + self._config_path = closest_config + if self._config is None: self._config = self.get_config() - if self._config != self.get_config(): - with open(self.config_path, "w") as f: + + current_config = self.get_config() + if self._config != current_config: + self._config = current_config + with open(self._config_path, "w") as f: yaml.dump(self._config, f) + return self._config @property @@ -56,7 +66,9 @@ class Environment: @property def config_path(self): - return self._closest_config() + if not hasattr(self, '_config_path') or self._config_path is None: + self._config_path = self._closest_config() + return self._config_path @property def project_config(self): @@ -64,13 +76,16 @@ class Environment: @property def id(self): - if os.getenv("KAG_PROJECT_ID"): - return os.getenv("KAG_PROJECT_ID") + id = self.project_config.get("id", None) if id is None: - raise Exception( - "project id not restore in spgserver, please restore project first" - ) + logger.warning("can not find id in project config") + if os.getenv("KAG_PROJECT_ID",None): + return os.getenv("KAG_PROJECT_ID") + else: + raise Exception( + "project id not restore in spgserver, please restore project first" + ) return id @property @@ -124,20 +139,23 @@ class Environment: prev_path: Optional[Union[str, os.PathLike]] = None, ) -> str: """ - Return the path to the closest .knext.cfg file by traversing the current + Return the path to the closest kag_config.yaml file by traversing the current directory and its parents """ if prev_path is not None and str(path) == str(prev_path): return "" path = Path(path).resolve() - cfg_file = path / "kag_config.yaml" - if cfg_file.exists(): + cfg_files = list(path.glob("*.yaml")) + cfg_file = next((f for f in cfg_files if f.name == "kag_config.yaml"), cfg_files[0] if cfg_files else None) + if cfg_file and cfg_file.exists(): return str(cfg_file) + if path.parent == path: + raise FileNotFoundError("No kag_config.yaml file found in current directory or any parent directories") return self._closest_config(path.parent, path) def dump(self, path=None, **kwargs): - with open(path or self.config_path, "w") as f: - yaml.dump(self.config, f, **kwargs) + with open(path or self._config_path, "w") as f: + yaml.dump(self._config, f) env = Environment()