mirror of https://github.com/microsoft/autogen.git
Move docker code exec to autogen-ext (#3733)
* move docker code exec to autogen-ext * fix test * rename docker subpackage * add missing renamed package --------- Co-authored-by: Leonardo Pinheiro <lpinheiro@microsoft.com>
This commit is contained in:
parent
e1e9d19cb4
commit
c765a34cbf
|
@ -315,8 +315,8 @@
|
||||||
"source": [
|
"source": [
|
||||||
"from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent\n",
|
"from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent\n",
|
||||||
"from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination\n",
|
"from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination\n",
|
||||||
"from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
|
|
||||||
"from autogen_core.components.models import OpenAIChatCompletionClient\n",
|
"from autogen_core.components.models import OpenAIChatCompletionClient\n",
|
||||||
|
"from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
|
||||||
"\n",
|
"\n",
|
||||||
"async with DockerCommandLineCodeExecutor(work_dir=\"coding\") as code_executor: # type: ignore[syntax]\n",
|
"async with DockerCommandLineCodeExecutor(work_dir=\"coding\") as code_executor: # type: ignore[syntax]\n",
|
||||||
" code_executor_agent = CodeExecutorAgent(\"code_executor\", code_executor=code_executor)\n",
|
" code_executor_agent = CodeExecutorAgent(\"code_executor\", code_executor=code_executor)\n",
|
||||||
|
|
|
@ -8,7 +8,7 @@ from autogen_agentchat import EVENT_LOGGER_NAME
|
||||||
from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent
|
from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent
|
||||||
from autogen_agentchat.logging import ConsoleLogHandler
|
from autogen_agentchat.logging import ConsoleLogHandler
|
||||||
from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination
|
from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination
|
||||||
from autogen_core.components.code_executor import DockerCommandLineCodeExecutor
|
from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
|
||||||
from autogen_core.components.models import OpenAIChatCompletionClient
|
from autogen_core.components.models import OpenAIChatCompletionClient
|
||||||
|
|
||||||
logger = logging.getLogger(EVENT_LOGGER_NAME)
|
logger = logging.getLogger(EVENT_LOGGER_NAME)
|
||||||
|
|
|
@ -23,7 +23,6 @@
|
||||||
"from autogen_core.base import AgentId, AgentType, MessageContext\n",
|
"from autogen_core.base import AgentId, AgentType, MessageContext\n",
|
||||||
"from autogen_core.base.intervention import DefaultInterventionHandler, DropMessage\n",
|
"from autogen_core.base.intervention import DefaultInterventionHandler, DropMessage\n",
|
||||||
"from autogen_core.components import FunctionCall, RoutedAgent, message_handler\n",
|
"from autogen_core.components import FunctionCall, RoutedAgent, message_handler\n",
|
||||||
"from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
|
|
||||||
"from autogen_core.components.models import (\n",
|
"from autogen_core.components.models import (\n",
|
||||||
" ChatCompletionClient,\n",
|
" ChatCompletionClient,\n",
|
||||||
" LLMMessage,\n",
|
" LLMMessage,\n",
|
||||||
|
@ -32,7 +31,8 @@
|
||||||
" UserMessage,\n",
|
" UserMessage,\n",
|
||||||
")\n",
|
")\n",
|
||||||
"from autogen_core.components.tool_agent import ToolAgent, ToolException, tool_agent_caller_loop\n",
|
"from autogen_core.components.tool_agent import ToolAgent, ToolException, tool_agent_caller_loop\n",
|
||||||
"from autogen_core.components.tools import PythonCodeExecutionTool, ToolSchema"
|
"from autogen_core.components.tools import PythonCodeExecutionTool, ToolSchema\n",
|
||||||
|
"from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -157,7 +157,7 @@
|
||||||
"source": [
|
"source": [
|
||||||
"In this example, we will use a tool for Python code execution.\n",
|
"In this example, we will use a tool for Python code execution.\n",
|
||||||
"First, we create a Docker-based command-line code executor\n",
|
"First, we create a Docker-based command-line code executor\n",
|
||||||
"using {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`,\n",
|
"using {py:class}`~autogen_core.components.code_executor.docker_executorCommandLineCodeExecutor`,\n",
|
||||||
"and then use it to instantiate a built-in Python code execution tool\n",
|
"and then use it to instantiate a built-in Python code execution tool\n",
|
||||||
"{py:class}`~autogen_core.components.tools.PythonCodeExecutionTool`\n",
|
"{py:class}`~autogen_core.components.tools.PythonCodeExecutionTool`\n",
|
||||||
"that runs code in a Docker container."
|
"that runs code in a Docker container."
|
||||||
|
|
|
@ -10,12 +10,12 @@
|
||||||
"Generally speaking, it will save each code block to a file and the execute that file.\n",
|
"Generally speaking, it will save each code block to a file and the execute that file.\n",
|
||||||
"This means that each code block is executed in a new process. There are two forms of this executor:\n",
|
"This means that each code block is executed in a new process. There are two forms of this executor:\n",
|
||||||
"\n",
|
"\n",
|
||||||
"- Docker ({py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n",
|
"- Docker ({py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n",
|
||||||
"- Local ({py:class}`~autogen_core.components.code_executor.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n",
|
"- Local ({py:class}`~autogen_core.components.code_executor.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n",
|
||||||
"\n",
|
"\n",
|
||||||
"## Docker\n",
|
"## Docker\n",
|
||||||
"\n",
|
"\n",
|
||||||
"The {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor` will create a Docker container and run all commands within that container. \n",
|
"The {py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor` will create a Docker container and run all commands within that container. \n",
|
||||||
"The default image that is used is `python:3-slim`, this can be customized by passing the `image` parameter to the constructor. \n",
|
"The default image that is used is `python:3-slim`, this can be customized by passing the `image` parameter to the constructor. \n",
|
||||||
"If the image is not found locally then the class will try to pull it. \n",
|
"If the image is not found locally then the class will try to pull it. \n",
|
||||||
"Therefore, having built the image locally is enough. The only thing required for this image to be compatible with the executor is to have `sh` and `python` installed. \n",
|
"Therefore, having built the image locally is enough. The only thing required for this image to be compatible with the executor is to have `sh` and `python` installed. \n",
|
||||||
|
@ -50,7 +50,8 @@
|
||||||
"from pathlib import Path\n",
|
"from pathlib import Path\n",
|
||||||
"\n",
|
"\n",
|
||||||
"from autogen_core.base import CancellationToken\n",
|
"from autogen_core.base import CancellationToken\n",
|
||||||
"from autogen_core.components.code_executor import CodeBlock, DockerCommandLineCodeExecutor\n",
|
"from autogen_core.components.code_executor import CodeBlock\n",
|
||||||
|
"from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
|
||||||
"\n",
|
"\n",
|
||||||
"work_dir = Path(\"coding\")\n",
|
"work_dir = Path(\"coding\")\n",
|
||||||
"work_dir.mkdir(exist_ok=True)\n",
|
"work_dir.mkdir(exist_ok=True)\n",
|
||||||
|
|
|
@ -44,8 +44,8 @@
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"from autogen_core.base import CancellationToken\n",
|
"from autogen_core.base import CancellationToken\n",
|
||||||
"from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
|
|
||||||
"from autogen_core.components.tools import PythonCodeExecutionTool\n",
|
"from autogen_core.components.tools import PythonCodeExecutionTool\n",
|
||||||
|
"from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
|
||||||
"\n",
|
"\n",
|
||||||
"# Create the tool.\n",
|
"# Create the tool.\n",
|
||||||
"code_executor = DockerCommandLineCodeExecutor()\n",
|
"code_executor = DockerCommandLineCodeExecutor()\n",
|
||||||
|
@ -63,7 +63,7 @@
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {},
|
"metadata": {},
|
||||||
"source": [
|
"source": [
|
||||||
"The {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`\n",
|
"The {py:class}`~autogen_core.components.code_executor.docker_executorCommandLineCodeExecutor`\n",
|
||||||
"class is a built-in code executor that runs Python code snippets in a subprocess\n",
|
"class is a built-in code executor that runs Python code snippets in a subprocess\n",
|
||||||
"in the local command line environment.\n",
|
"in the local command line environment.\n",
|
||||||
"The {py:class}`~autogen_core.components.tools.PythonCodeExecutionTool` class wraps the code executor\n",
|
"The {py:class}`~autogen_core.components.tools.PythonCodeExecutionTool` class wraps the code executor\n",
|
||||||
|
|
|
@ -312,8 +312,8 @@
|
||||||
"import tempfile\n",
|
"import tempfile\n",
|
||||||
"\n",
|
"\n",
|
||||||
"from autogen_core.application import SingleThreadedAgentRuntime\n",
|
"from autogen_core.application import SingleThreadedAgentRuntime\n",
|
||||||
"from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
|
|
||||||
"from autogen_core.components.models import OpenAIChatCompletionClient\n",
|
"from autogen_core.components.models import OpenAIChatCompletionClient\n",
|
||||||
|
"from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
|
||||||
"\n",
|
"\n",
|
||||||
"work_dir = tempfile.mkdtemp()\n",
|
"work_dir = tempfile.mkdtemp()\n",
|
||||||
"\n",
|
"\n",
|
||||||
|
|
|
@ -23,7 +23,6 @@ dependencies = [
|
||||||
"grpcio~=1.62.0",
|
"grpcio~=1.62.0",
|
||||||
"protobuf~=4.25.1",
|
"protobuf~=4.25.1",
|
||||||
"tiktoken",
|
"tiktoken",
|
||||||
"docker~=7.0",
|
|
||||||
"opentelemetry-api~=1.27.0",
|
"opentelemetry-api~=1.27.0",
|
||||||
"asyncio_atexit"
|
"asyncio_atexit"
|
||||||
]
|
]
|
||||||
|
|
|
@ -20,7 +20,6 @@ from typing import Dict, List
|
||||||
from autogen_core.application import SingleThreadedAgentRuntime
|
from autogen_core.application import SingleThreadedAgentRuntime
|
||||||
from autogen_core.base import MessageContext
|
from autogen_core.base import MessageContext
|
||||||
from autogen_core.components import DefaultSubscription, DefaultTopicId, FunctionCall, RoutedAgent, message_handler
|
from autogen_core.components import DefaultSubscription, DefaultTopicId, FunctionCall, RoutedAgent, message_handler
|
||||||
from autogen_core.components.code_executor import DockerCommandLineCodeExecutor
|
|
||||||
from autogen_core.components.models import (
|
from autogen_core.components.models import (
|
||||||
AssistantMessage,
|
AssistantMessage,
|
||||||
ChatCompletionClient,
|
ChatCompletionClient,
|
||||||
|
@ -31,6 +30,7 @@ from autogen_core.components.models import (
|
||||||
UserMessage,
|
UserMessage,
|
||||||
)
|
)
|
||||||
from autogen_core.components.tools import PythonCodeExecutionTool, Tool
|
from autogen_core.components.tools import PythonCodeExecutionTool, Tool
|
||||||
|
from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
|
||||||
from common.utils import get_chat_completion_client_from_envs
|
from common.utils import get_chat_completion_client_from_envs
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -10,9 +10,8 @@ from ._func_with_reqs import (
|
||||||
with_requirements,
|
with_requirements,
|
||||||
)
|
)
|
||||||
from ._impl.command_line_code_result import CommandLineCodeResult
|
from ._impl.command_line_code_result import CommandLineCodeResult
|
||||||
from ._impl.docker_command_line_code_executor import DockerCommandLineCodeExecutor
|
|
||||||
from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor
|
from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor
|
||||||
from ._impl.utils import get_required_packages, lang_to_cmd
|
from ._impl.utils import get_file_name_from_content, get_required_packages, lang_to_cmd, silence_pip
|
||||||
from ._utils import extract_markdown_code_blocks
|
from ._utils import extract_markdown_code_blocks
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
@ -31,7 +30,8 @@ __all__ = [
|
||||||
"extract_markdown_code_blocks",
|
"extract_markdown_code_blocks",
|
||||||
"get_required_packages",
|
"get_required_packages",
|
||||||
"build_python_functions_file",
|
"build_python_functions_file",
|
||||||
"DockerCommandLineCodeExecutor",
|
|
||||||
"get_required_packages",
|
"get_required_packages",
|
||||||
"lang_to_cmd",
|
"lang_to_cmd",
|
||||||
|
"get_file_name_from_content",
|
||||||
|
"silence_pip",
|
||||||
]
|
]
|
||||||
|
|
|
@ -2,7 +2,6 @@
|
||||||
# Credit to original authors
|
# Credit to original authors
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
@ -12,48 +11,22 @@ import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from aiofiles import open
|
from aiofiles import open
|
||||||
from autogen_core.base import CancellationToken
|
from autogen_core.base import CancellationToken
|
||||||
from autogen_core.components.code_executor import CodeBlock, DockerCommandLineCodeExecutor, LocalCommandLineCodeExecutor
|
from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor
|
||||||
|
|
||||||
|
|
||||||
def docker_tests_enabled() -> bool:
|
|
||||||
if os.environ.get("SKIP_DOCKER", "unset").lower() == "true":
|
|
||||||
return False
|
|
||||||
|
|
||||||
try:
|
|
||||||
import docker
|
|
||||||
from docker.errors import DockerException
|
|
||||||
except ImportError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
try:
|
|
||||||
client = docker.from_env()
|
|
||||||
client.ping() # type: ignore
|
|
||||||
return True
|
|
||||||
except DockerException:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
@pytest_asyncio.fixture(scope="function") # type: ignore
|
@pytest_asyncio.fixture(scope="function") # type: ignore
|
||||||
async def executor_and_temp_dir(
|
async def executor_and_temp_dir(
|
||||||
request: pytest.FixtureRequest,
|
request: pytest.FixtureRequest,
|
||||||
) -> AsyncGenerator[tuple[LocalCommandLineCodeExecutor | DockerCommandLineCodeExecutor, str], None]:
|
) -> AsyncGenerator[tuple[LocalCommandLineCodeExecutor, str], None]:
|
||||||
if request.param == "local":
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
yield LocalCommandLineCodeExecutor(work_dir=temp_dir), temp_dir
|
||||||
yield LocalCommandLineCodeExecutor(work_dir=temp_dir), temp_dir
|
|
||||||
elif request.param == "docker":
|
|
||||||
if not docker_tests_enabled():
|
|
||||||
pytest.skip("Docker tests are disabled")
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
|
||||||
async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as executor:
|
|
||||||
yield executor, temp_dir
|
|
||||||
|
|
||||||
|
|
||||||
ExecutorFixture: TypeAlias = tuple[LocalCommandLineCodeExecutor | DockerCommandLineCodeExecutor, str]
|
ExecutorFixture: TypeAlias = tuple[LocalCommandLineCodeExecutor, str]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
|
@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
|
||||||
async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
|
async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
executor, _temp_dir = executor_and_temp_dir
|
executor, _temp_dir = executor_and_temp_dir
|
||||||
cancellation_token = CancellationToken()
|
cancellation_token = CancellationToken()
|
||||||
|
@ -101,7 +74,7 @@ async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
|
@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
|
||||||
async def test_commandline_code_executor_timeout(executor_and_temp_dir: ExecutorFixture) -> None:
|
async def test_commandline_code_executor_timeout(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
executor, temp_dir = executor_and_temp_dir
|
executor, temp_dir = executor_and_temp_dir
|
||||||
cancellation_token = CancellationToken()
|
cancellation_token = CancellationToken()
|
||||||
|
@ -111,7 +84,6 @@ async def test_commandline_code_executor_timeout(executor_and_temp_dir: Executor
|
||||||
assert code_result.exit_code and "Timeout" in code_result.output
|
assert code_result.exit_code and "Timeout" in code_result.output
|
||||||
|
|
||||||
|
|
||||||
# TODO: add docker when cancellation is supported
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_commandline_code_executor_cancellation() -> None:
|
async def test_commandline_code_executor_cancellation() -> None:
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
@ -136,7 +108,7 @@ async def test_local_commandline_code_executor_restart() -> None:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
|
@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
|
||||||
async def test_invalid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
async def test_invalid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
executor, _temp_dir = executor_and_temp_dir
|
executor, _temp_dir = executor_and_temp_dir
|
||||||
cancellation_token = CancellationToken()
|
cancellation_token = CancellationToken()
|
||||||
|
@ -151,7 +123,7 @@ print("hello world")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
|
@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
|
||||||
async def test_valid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
async def test_valid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
executor, temp_dir_str = executor_and_temp_dir
|
executor, temp_dir_str = executor_and_temp_dir
|
||||||
|
|
||||||
|
@ -171,24 +143,3 @@ print("hello world")
|
||||||
assert "test.py" in result.code_file
|
assert "test.py" in result.code_file
|
||||||
assert (temp_dir / Path("test.py")).resolve() == Path(result.code_file).resolve()
|
assert (temp_dir / Path("test.py")).resolve() == Path(result.code_file).resolve()
|
||||||
assert (temp_dir / Path("test.py")).exists()
|
assert (temp_dir / Path("test.py")).exists()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_docker_commandline_code_executor_start_stop() -> None:
|
|
||||||
if not docker_tests_enabled():
|
|
||||||
pytest.skip("Docker tests are disabled")
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
|
||||||
executor = DockerCommandLineCodeExecutor(work_dir=temp_dir)
|
|
||||||
await executor.start()
|
|
||||||
await executor.stop()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_docker_commandline_code_executor_start_stop_context_manager() -> None:
|
|
||||||
if not docker_tests_enabled():
|
|
||||||
pytest.skip("Docker tests are disabled")
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir:
|
|
||||||
async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as _exec:
|
|
||||||
pass
|
|
||||||
|
|
|
@ -22,6 +22,7 @@ dependencies = [
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
langchain-tools = ["langchain >= 0.3.1"]
|
langchain-tools = ["langchain >= 0.3.1"]
|
||||||
azure-code-executor = ["azure-core"]
|
azure-code-executor = ["azure-core"]
|
||||||
|
docker-code-executor = ["docker~=7.0"]
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["src/autogen_ext"]
|
packages = ["src/autogen_ext"]
|
||||||
|
@ -47,3 +48,8 @@ include = "../../shared_tasks.toml"
|
||||||
|
|
||||||
[tool.poe.tasks]
|
[tool.poe.tasks]
|
||||||
test = "pytest -n auto"
|
test = "pytest -n auto"
|
||||||
|
|
||||||
|
[tool.mypy]
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
module = "docker.*"
|
||||||
|
ignore_missing_imports = true
|
|
@ -0,0 +1,3 @@
|
||||||
|
from ._impl import DockerCommandLineCodeExecutor
|
||||||
|
|
||||||
|
__all__ = ["DockerCommandLineCodeExecutor"]
|
|
@ -18,16 +18,19 @@ import asyncio_atexit
|
||||||
import docker
|
import docker
|
||||||
import docker.models
|
import docker.models
|
||||||
import docker.models.containers
|
import docker.models.containers
|
||||||
from docker.errors import ImageNotFound, NotFound
|
from autogen_core.base import CancellationToken
|
||||||
|
from autogen_core.components.code_executor import (
|
||||||
from ....base._cancellation_token import CancellationToken
|
CodeBlock,
|
||||||
from ....components.code_executor._base import CodeBlock, CodeExecutor
|
CodeExecutor,
|
||||||
from ....components.code_executor._func_with_reqs import FunctionWithRequirements, FunctionWithRequirementsStr
|
CommandLineCodeResult,
|
||||||
from ....components.code_executor._impl.command_line_code_result import CommandLineCodeResult
|
FunctionWithRequirements,
|
||||||
from .._func_with_reqs import (
|
FunctionWithRequirementsStr,
|
||||||
build_python_functions_file,
|
build_python_functions_file,
|
||||||
|
get_file_name_from_content,
|
||||||
|
lang_to_cmd,
|
||||||
|
silence_pip,
|
||||||
)
|
)
|
||||||
from .utils import get_file_name_from_content, lang_to_cmd, silence_pip
|
from docker.errors import ImageNotFound, NotFound
|
||||||
|
|
||||||
if sys.version_info >= (3, 11):
|
if sys.version_info >= (3, 11):
|
||||||
from typing import Self
|
from typing import Self
|
|
@ -0,0 +1,166 @@
|
||||||
|
# mypy: disable-error-code="no-any-unimported"
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import AsyncGenerator, TypeAlias
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
from aiofiles import open
|
||||||
|
from autogen_core.base import CancellationToken
|
||||||
|
from autogen_core.components.code_executor import CodeBlock
|
||||||
|
from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
|
||||||
|
|
||||||
|
|
||||||
|
def docker_tests_enabled() -> bool:
|
||||||
|
if os.environ.get("SKIP_DOCKER", "unset").lower() == "true":
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
import docker
|
||||||
|
from docker.errors import DockerException
|
||||||
|
except ImportError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
client = docker.from_env()
|
||||||
|
client.ping() # type: ignore
|
||||||
|
return True
|
||||||
|
except DockerException:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture(scope="function") # type: ignore
|
||||||
|
async def executor_and_temp_dir(
|
||||||
|
request: pytest.FixtureRequest,
|
||||||
|
) -> AsyncGenerator[tuple[DockerCommandLineCodeExecutor, str], None]:
|
||||||
|
if not docker_tests_enabled():
|
||||||
|
pytest.skip("Docker tests are disabled")
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as executor:
|
||||||
|
yield executor, temp_dir
|
||||||
|
|
||||||
|
|
||||||
|
ExecutorFixture: TypeAlias = tuple[DockerCommandLineCodeExecutor, str]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("executor_and_temp_dir", ["docker"], indirect=True)
|
||||||
|
async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
|
executor, _temp_dir = executor_and_temp_dir
|
||||||
|
cancellation_token = CancellationToken()
|
||||||
|
|
||||||
|
# Test single code block.
|
||||||
|
code_blocks = [CodeBlock(code="import sys; print('hello world!')", language="python")]
|
||||||
|
code_result = await executor.execute_code_blocks(code_blocks, cancellation_token)
|
||||||
|
assert code_result.exit_code == 0 and "hello world!" in code_result.output and code_result.code_file is not None
|
||||||
|
|
||||||
|
# Test multiple code blocks.
|
||||||
|
code_blocks = [
|
||||||
|
CodeBlock(code="import sys; print('hello world!')", language="python"),
|
||||||
|
CodeBlock(code="a = 100 + 100; print(a)", language="python"),
|
||||||
|
]
|
||||||
|
code_result = await executor.execute_code_blocks(code_blocks, cancellation_token)
|
||||||
|
assert (
|
||||||
|
code_result.exit_code == 0
|
||||||
|
and "hello world!" in code_result.output
|
||||||
|
and "200" in code_result.output
|
||||||
|
and code_result.code_file is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
# Test bash script.
|
||||||
|
if sys.platform not in ["win32"]:
|
||||||
|
code_blocks = [CodeBlock(code="echo 'hello world!'", language="bash")]
|
||||||
|
code_result = await executor.execute_code_blocks(code_blocks, cancellation_token)
|
||||||
|
assert code_result.exit_code == 0 and "hello world!" in code_result.output and code_result.code_file is not None
|
||||||
|
|
||||||
|
# Test running code.
|
||||||
|
file_lines = ["import sys", "print('hello world!')", "a = 100 + 100", "print(a)"]
|
||||||
|
code_blocks = [CodeBlock(code="\n".join(file_lines), language="python")]
|
||||||
|
code_result = await executor.execute_code_blocks(code_blocks, cancellation_token)
|
||||||
|
assert (
|
||||||
|
code_result.exit_code == 0
|
||||||
|
and "hello world!" in code_result.output
|
||||||
|
and "200" in code_result.output
|
||||||
|
and code_result.code_file is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check saved code file.
|
||||||
|
async with open(code_result.code_file) as f:
|
||||||
|
code_lines = await f.readlines()
|
||||||
|
for file_line, code_line in zip(file_lines, code_lines, strict=False):
|
||||||
|
assert file_line.strip() == code_line.strip()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("executor_and_temp_dir", ["docker"], indirect=True)
|
||||||
|
async def test_commandline_code_executor_timeout(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
|
_executor, temp_dir = executor_and_temp_dir
|
||||||
|
cancellation_token = CancellationToken()
|
||||||
|
code_blocks = [CodeBlock(code="import time; time.sleep(10); print('hello world!')", language="python")]
|
||||||
|
|
||||||
|
async with DockerCommandLineCodeExecutor(timeout=1, work_dir=temp_dir) as executor:
|
||||||
|
code_result = await executor.execute_code_blocks(code_blocks, cancellation_token)
|
||||||
|
|
||||||
|
assert code_result.exit_code and "Timeout" in code_result.output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("executor_and_temp_dir", ["docker"], indirect=True)
|
||||||
|
async def test_invalid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
|
executor, _temp_dir = executor_and_temp_dir
|
||||||
|
cancellation_token = CancellationToken()
|
||||||
|
code = """# filename: /tmp/test.py
|
||||||
|
|
||||||
|
print("hello world")
|
||||||
|
"""
|
||||||
|
result = await executor.execute_code_blocks(
|
||||||
|
[CodeBlock(code=code, language="python")], cancellation_token=cancellation_token
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1 and "Filename is not in the workspace" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("executor_and_temp_dir", ["docker"], indirect=True)
|
||||||
|
async def test_valid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
|
||||||
|
executor, temp_dir_str = executor_and_temp_dir
|
||||||
|
|
||||||
|
cancellation_token = CancellationToken()
|
||||||
|
temp_dir = Path(temp_dir_str)
|
||||||
|
|
||||||
|
code = """# filename: test.py
|
||||||
|
|
||||||
|
print("hello world")
|
||||||
|
"""
|
||||||
|
result = await executor.execute_code_blocks(
|
||||||
|
[CodeBlock(code=code, language="python")], cancellation_token=cancellation_token
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "hello world" in result.output
|
||||||
|
assert result.code_file is not None
|
||||||
|
assert "test.py" in result.code_file
|
||||||
|
assert (temp_dir / Path("test.py")).resolve() == Path(result.code_file).resolve()
|
||||||
|
assert (temp_dir / Path("test.py")).exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_docker_commandline_code_executor_start_stop() -> None:
|
||||||
|
if not docker_tests_enabled():
|
||||||
|
pytest.skip("Docker tests are disabled")
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
executor = DockerCommandLineCodeExecutor(work_dir=temp_dir)
|
||||||
|
await executor.start()
|
||||||
|
await executor.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_docker_commandline_code_executor_start_stop_context_manager() -> None:
|
||||||
|
if not docker_tests_enabled():
|
||||||
|
pytest.skip("Docker tests are disabled")
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as temp_dir:
|
||||||
|
async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as _exec:
|
||||||
|
pass
|
|
@ -10,7 +10,8 @@ import logging
|
||||||
from autogen_core.application import SingleThreadedAgentRuntime
|
from autogen_core.application import SingleThreadedAgentRuntime
|
||||||
from autogen_core.application.logging import EVENT_LOGGER_NAME
|
from autogen_core.application.logging import EVENT_LOGGER_NAME
|
||||||
from autogen_core.base import AgentId, AgentProxy
|
from autogen_core.base import AgentId, AgentProxy
|
||||||
from autogen_core.components.code_executor import CodeBlock, DockerCommandLineCodeExecutor
|
from autogen_core.components.code_executor import CodeBlock
|
||||||
|
from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
|
||||||
from autogen_magentic_one.agents.coder import Coder, Executor
|
from autogen_magentic_one.agents.coder import Coder, Executor
|
||||||
from autogen_magentic_one.agents.orchestrator import RoundRobinOrchestrator
|
from autogen_magentic_one.agents.orchestrator import RoundRobinOrchestrator
|
||||||
from autogen_magentic_one.agents.user_proxy import UserProxy
|
from autogen_magentic_one.agents.user_proxy import UserProxy
|
||||||
|
|
|
@ -359,7 +359,6 @@ source = { editable = "packages/autogen-core" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "aiohttp" },
|
{ name = "aiohttp" },
|
||||||
{ name = "asyncio-atexit" },
|
{ name = "asyncio-atexit" },
|
||||||
{ name = "docker" },
|
|
||||||
{ name = "grpcio" },
|
{ name = "grpcio" },
|
||||||
{ name = "openai" },
|
{ name = "openai" },
|
||||||
{ name = "opentelemetry-api" },
|
{ name = "opentelemetry-api" },
|
||||||
|
@ -415,7 +414,6 @@ dev = [
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "aiohttp" },
|
{ name = "aiohttp" },
|
||||||
{ name = "asyncio-atexit" },
|
{ name = "asyncio-atexit" },
|
||||||
{ name = "docker", specifier = "~=7.0" },
|
|
||||||
{ name = "grpcio", specifier = "~=1.62.0" },
|
{ name = "grpcio", specifier = "~=1.62.0" },
|
||||||
{ name = "openai", specifier = ">=1.3" },
|
{ name = "openai", specifier = ">=1.3" },
|
||||||
{ name = "opentelemetry-api", specifier = "~=1.27.0" },
|
{ name = "opentelemetry-api", specifier = "~=1.27.0" },
|
||||||
|
@ -479,6 +477,9 @@ dependencies = [
|
||||||
azure-code-executor = [
|
azure-code-executor = [
|
||||||
{ name = "azure-core" },
|
{ name = "azure-core" },
|
||||||
]
|
]
|
||||||
|
docker-code-executor = [
|
||||||
|
{ name = "docker" },
|
||||||
|
]
|
||||||
langchain-tools = [
|
langchain-tools = [
|
||||||
{ name = "langchain" },
|
{ name = "langchain" },
|
||||||
]
|
]
|
||||||
|
@ -487,6 +488,7 @@ langchain-tools = [
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "autogen-core", editable = "packages/autogen-core" },
|
{ name = "autogen-core", editable = "packages/autogen-core" },
|
||||||
{ name = "azure-core", marker = "extra == 'azure-code-executor'" },
|
{ name = "azure-core", marker = "extra == 'azure-code-executor'" },
|
||||||
|
{ name = "docker", marker = "extra == 'docker-code-executor'", specifier = "~=7.0" },
|
||||||
{ name = "langchain", marker = "extra == 'langchain-tools'", specifier = ">=0.3.1" },
|
{ name = "langchain", marker = "extra == 'langchain-tools'", specifier = ">=0.3.1" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue