129 lines
4.3 KiB
Python
129 lines
4.3 KiB
Python
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from pydantic import Field, ValidationInfo, field_validator
|
|
from pydantic_settings import BaseSettings
|
|
|
|
|
|
def _calculate_info_test_path(base_path: Path, benchmark_start_time: datetime) -> Path:
|
|
"""
|
|
Calculates the path to the directory where the test report will be saved.
|
|
"""
|
|
# Ensure the reports path exists
|
|
base_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Get current UTC date-time stamp
|
|
date_stamp = benchmark_start_time.strftime("%Y%m%dT%H%M%S")
|
|
|
|
# Default run name
|
|
run_name = "full_run"
|
|
|
|
# Map command-line arguments to their respective labels
|
|
arg_labels = {
|
|
"--test": None,
|
|
"--category": None,
|
|
"--maintain": "maintain",
|
|
"--improve": "improve",
|
|
"--explore": "explore",
|
|
}
|
|
|
|
# Identify the relevant command-line argument
|
|
for arg, label in arg_labels.items():
|
|
if arg in sys.argv:
|
|
test_arg = sys.argv[sys.argv.index(arg) + 1] if label is None else None
|
|
run_name = arg.strip("--")
|
|
if test_arg:
|
|
run_name = f"{run_name}_{test_arg}"
|
|
break
|
|
|
|
# Create the full new directory path with ISO standard UTC date-time stamp
|
|
report_path = base_path / f"{date_stamp}_{run_name}"
|
|
|
|
# Ensure the new directory is created
|
|
# FIXME: this is not a desirable side-effect of loading the config
|
|
report_path.mkdir(exist_ok=True)
|
|
|
|
return report_path
|
|
|
|
|
|
class AgentBenchmarkConfig(BaseSettings, extra="allow"):
|
|
"""
|
|
Configuration model and loader for the AGBenchmark.
|
|
|
|
Projects that want to use AGBenchmark should contain an agbenchmark_config folder
|
|
with a config.json file that - at minimum - specifies the `host` at which the
|
|
subject application exposes an Agent Protocol compliant API.
|
|
"""
|
|
|
|
agbenchmark_config_dir: Path = Field(exclude=True)
|
|
"""Path to the agbenchmark_config folder of the subject agent application."""
|
|
|
|
categories: list[str] | None = None
|
|
"""Categories to benchmark the agent for. If omitted, all categories are assumed."""
|
|
|
|
host: str
|
|
"""Host (scheme://address:port) of the subject agent application."""
|
|
|
|
reports_folder: Path = Field(None)
|
|
"""
|
|
Path to the folder where new reports should be stored.
|
|
Defaults to {agbenchmark_config_dir}/reports.
|
|
"""
|
|
|
|
@classmethod
|
|
def load(cls, config_dir: Optional[Path] = None) -> "AgentBenchmarkConfig":
|
|
config_dir = config_dir or cls.find_config_folder()
|
|
with (config_dir / "config.json").open("r") as f:
|
|
return cls(
|
|
agbenchmark_config_dir=config_dir,
|
|
**json.load(f),
|
|
)
|
|
|
|
@staticmethod
|
|
def find_config_folder(for_dir: Path = Path.cwd()) -> Path:
|
|
"""
|
|
Find the closest ancestor folder containing an agbenchmark_config folder,
|
|
and returns the path of that agbenchmark_config folder.
|
|
"""
|
|
current_directory = for_dir
|
|
while current_directory != Path("/"):
|
|
if (path := current_directory / "agbenchmark_config").exists():
|
|
if (path / "config.json").is_file():
|
|
return path
|
|
current_directory = current_directory.parent
|
|
raise FileNotFoundError(
|
|
"No 'agbenchmark_config' directory found in the path hierarchy."
|
|
)
|
|
|
|
@property
|
|
def config_file(self) -> Path:
|
|
return self.agbenchmark_config_dir / "config.json"
|
|
|
|
@field_validator("reports_folder", mode="before")
|
|
def set_reports_folder(cls, value: Path, info: ValidationInfo):
|
|
if not value:
|
|
return info.data["agbenchmark_config_dir"] / "reports"
|
|
return value
|
|
|
|
def get_report_dir(self, benchmark_start_time: datetime) -> Path:
|
|
return _calculate_info_test_path(self.reports_folder, benchmark_start_time)
|
|
|
|
@property
|
|
def regression_tests_file(self) -> Path:
|
|
return self.reports_folder / "regression_tests.json"
|
|
|
|
@property
|
|
def success_rate_file(self) -> Path:
|
|
return self.reports_folder / "success_rate.json"
|
|
|
|
@property
|
|
def challenges_already_beaten_file(self) -> Path:
|
|
return self.agbenchmark_config_dir / "challenges_already_beaten.json"
|
|
|
|
@property
|
|
def temp_folder(self) -> Path:
|
|
return self.agbenchmark_config_dir / "temp_folder"
|