add protobuf serialization test (#4224)

* add protobuf serialization test

* proto file regeneration

---------

Co-authored-by: Ryan Sweet <rysweet@microsoft.com>
Co-authored-by: Mohammad Mazraeh <Mazraeh.Mohammad@Gmail.com>

.github/workflows/checks.yml

@@ -190,6 +190,7 @@ jobs:
         run: |
           source ${{ github.workspace }}/python/.venv/bin/activate
           poe gen-proto
+          poe gen-test-proto
         working-directory: ./python
       - name: Check if there are uncommited changes
         id: changes

python/packages/autogen-core/pyproject.toml

@@ -81,7 +81,7 @@ dev-dependencies = [
 
 [tool.ruff]
 extend = "../../pyproject.toml"
-exclude = ["build", "dist", "src/autogen_core/application/protos"]
+exclude = ["build", "dist", "src/autogen_core/application/protos", "tests/protos"]
 include = ["src/**", "samples/*.py", "docs/**/*.ipynb", "tests/**"]
 
 [tool.ruff.lint.per-file-ignores]
@@ -91,7 +91,7 @@ include = ["src/**", "samples/*.py", "docs/**/*.ipynb", "tests/**"]
 [tool.pyright]
 extends = "../../pyproject.toml"
 include = ["src", "tests", "samples"]
-exclude = ["src/autogen_core/application/protos"]
+exclude = ["src/autogen_core/application/protos", "tests/protos"]
 reportDeprecated = false
 
 [tool.pytest.ini_options]
@@ -111,7 +111,7 @@ include = "../../shared_tasks.toml"
 test = "pytest -n auto"
 mypy.default_item_type = "cmd"
 mypy.sequence = [
-    "mypy --config-file ../../pyproject.toml --exclude src/autogen_core/application/protos src tests",
+    "mypy --config-file ../../pyproject.toml --exclude src/autogen_core/application/protos --exclude tests/protos src tests",
     "nbqa mypy docs/src --config-file ../../pyproject.toml",
 ]
 

python/packages/autogen-core/tests/protos/serialization_test.proto

@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+package agents;
+
+message ProtoMessage {
+      string message = 1;
+}
+message NestingProtoMessage {
+      string message = 1;
+      ProtoMessage nested = 2;
+}

python/packages/autogen-core/tests/protos/serialization_test_pb2.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: serialization_test.proto
+# Protobuf Python Version: 4.25.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18serialization_test.proto\x12\x06\x61gents\"\x1f\n\x0cProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\"L\n\x13NestingProtoMessage\x12\x0f\n\x07message\x18\x01 \x01(\t\x12$\n\x06nested\x18\x02 \x01(\x0b\x32\x14.agents.ProtoMessageb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'serialization_test_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  _globals['_PROTOMESSAGE']._serialized_start=36
+  _globals['_PROTOMESSAGE']._serialized_end=67
+  _globals['_NESTINGPROTOMESSAGE']._serialized_start=69
+  _globals['_NESTINGPROTOMESSAGE']._serialized_end=145
+# @@protoc_insertion_point(module_scope)

python/packages/autogen-core/tests/protos/serialization_test_pb2.pyi

@@ -0,0 +1,46 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+
+import builtins
+import google.protobuf.descriptor
+import google.protobuf.message
+import typing
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing.final
+class ProtoMessage(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    MESSAGE_FIELD_NUMBER: builtins.int
+    message: builtins.str
+    def __init__(
+        self,
+        *,
+        message: builtins.str = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["message", b"message"]) -> None: ...
+
+global___ProtoMessage = ProtoMessage
+
+@typing.final
+class NestingProtoMessage(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    MESSAGE_FIELD_NUMBER: builtins.int
+    NESTED_FIELD_NUMBER: builtins.int
+    message: builtins.str
+    @property
+    def nested(self) -> global___ProtoMessage: ...
+    def __init__(
+        self,
+        *,
+        message: builtins.str = ...,
+        nested: global___ProtoMessage | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["nested", b"nested"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["message", b"message", "nested", b"nested"]) -> None: ...
+
+global___NestingProtoMessage = NestingProtoMessage

python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

python/packages/autogen-core/tests/protos/serialization_test_pb2_grpc.pyi

@@ -0,0 +1,17 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+
+import abc
+import collections.abc
+import grpc
+import grpc.aio
+import typing
+
+_T = typing.TypeVar("_T")
+
+class _MaybeAsyncIterator(collections.abc.AsyncIterator[_T], collections.abc.Iterator[_T], metaclass=abc.ABCMeta): ...
+
+class _ServicerContext(grpc.ServicerContext, grpc.aio.ServicerContext):  # type: ignore[misc, type-arg]
+    ...

python/packages/autogen-core/tests/test_serialization.py

@@ -11,6 +11,7 @@ from autogen_core.base import (
 from autogen_core.base._serialization import DataclassJsonMessageSerializer, PydanticJsonMessageSerializer
 from autogen_core.components import Image
 from PIL import Image as PILImage
+from protos.serialization_test_pb2 import NestingProtoMessage, ProtoMessage
 from pydantic import BaseModel
 
 
@@ -83,6 +84,36 @@ def test_nesting_dataclass_dataclass() -> None:
         serde.add_serializer(try_get_known_serializers_for_type(NestingDataclassMessage))
 
 
+def test_proto() -> None:
+    serde = SerializationRegistry()
+    serde.add_serializer(try_get_known_serializers_for_type(ProtoMessage))
+
+    message = ProtoMessage(message="hello")
+    name = serde.type_name(message)
+    # TODO: should be PROTO_DATA_CONTENT_TYPE
+    data = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
+    assert name == "ProtoMessage"
+    # TODO: assert data == stuff
+    deserialized = serde.deserialize(data, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
+    assert deserialized == message
+
+
+def test_nested_proto() -> None:
+    serde = SerializationRegistry()
+    serde.add_serializer(try_get_known_serializers_for_type(NestingProtoMessage))
+
+    message = NestingProtoMessage(message="hello", nested=ProtoMessage(message="world"))
+    name = serde.type_name(message)
+
+    # TODO: should be PROTO_DATA_CONTENT_TYPE
+    data = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
+
+    # TODO: assert data == stuff
+
+    deserialized = serde.deserialize(data, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
+    assert deserialized == message
+
+
 @dataclass
 class DataclassNestedUnionSyntaxOldMessage:
     message: Union[str, int]

python/pyproject.toml

@@ -79,3 +79,5 @@ test = "python run_task_in_pkgs_if_exist.py test"
 check = ["fmt", "lint", "pyright", "mypy", "test"]
 
 gen-proto = "python -m grpc_tools.protoc --python_out=./packages/autogen-core/src/autogen_core/application/protos --grpc_python_out=./packages/autogen-core/src/autogen_core/application/protos --mypy_out=./packages/autogen-core/src/autogen_core/application/protos --mypy_grpc_out=./packages/autogen-core/src/autogen_core/application/protos --proto_path ../protos/ agent_worker.proto --proto_path ../protos/ cloudevent.proto"
+
+gen-test-proto = "python -m grpc_tools.protoc --python_out=./packages/autogen-core/tests/protos --grpc_python_out=./packages/autogen-core/tests/protos --mypy_out=./packages/autogen-core/tests/protos --mypy_grpc_out=./packages/autogen-core/tests/protos --proto_path ./packages/autogen-core/tests/protos serialization_test.proto"