feat: add GeneralSerde for unified JSON serialization/deserialization

- Introduce GeneralSerde to handle both Pydantic BaseModel instances and regular JSON objects.
- Use model_dump_json for Pydantic models, falling back to json.dumps for non-Pydantic objects.
- Maintain consistency with existing serde implementations.

Author: ouatu-ro

python/restate/handler.py

@@ -17,10 +17,10 @@
 
 from dataclasses import dataclass
 from inspect import Signature
-from typing import Any, Callable, Awaitable, Generic, Literal, Optional, TypeVar
+from typing import Any, Awaitable, Callable, Generic, Literal, Optional, TypeVar
 
 from restate.exceptions import TerminalError
-from restate.serde import JsonSerde, Serde, PydanticJsonSerde
+from restate.serde import GeneralSerde, PydanticBaseModel, PydanticJsonSerde, Serde
 
 I = TypeVar('I')
 O = TypeVar('O')
@@ -29,22 +29,6 @@
 # we will use this symbol to store the handler in the function
 RESTATE_UNIQUE_HANDLER_SYMBOL = str(object())
 
-
-def try_import_pydantic_base_model():
-    """
-    Try to import PydanticBaseModel from Pydantic.
-    """
-    try:
-        from pydantic import BaseModel # type: ignore # pylint: disable=import-outside-toplevel
-        return BaseModel
-    except ImportError:
-        class Dummy: # pylint: disable=too-few-public-methods
-            """a dummy class to use when Pydantic is not available"""
-
-        return Dummy
-
-PydanticBaseModel = try_import_pydantic_base_model()
-
 @dataclass
 class ServiceTag:
     """
@@ -103,15 +87,15 @@ def extract_io_type_hints(handler_io: HandlerIO[I, O], signature: Signature):
 
     if is_pydantic(annotation):
         handler_io.input_type.is_pydantic = True
-        if isinstance(handler_io.input_serde, JsonSerde): # type: ignore
+        if isinstance(handler_io.input_serde, GeneralSerde): # type: ignore
             handler_io.input_serde = PydanticJsonSerde(annotation)
 
     annotation = signature.return_annotation
     handler_io.output_type = TypeHint(annotation=annotation, is_pydantic=False)
 
     if is_pydantic(annotation):
         handler_io.output_type.is_pydantic=True
-        if isinstance(handler_io.output_serde, JsonSerde): # type: ignore
+        if isinstance(handler_io.output_serde, GeneralSerde): # type: ignore
             handler_io.output_serde = PydanticJsonSerde(annotation)
 
 @dataclass
@@ -138,6 +122,11 @@ def make_handler(service_tag: ServiceTag,
                  signature: Signature) -> Handler[I, O]:
     """
     Factory function to create a handler.
+
+    Note:
+        This function mutates the `handler_io` parameter by updating its type hints
+        and serdes based on the function signature. Callers should be aware that the
+        passed `handler_io` instance will be modified.
     """
     # try to deduce the handler name
     handler_name = name
@@ -150,7 +139,7 @@ def make_handler(service_tag: ServiceTag,
         raise ValueError("Handler must have at least one parameter")
 
     arity = len(signature.parameters)
-    extract_io_type_hints(handler_io, signature)
+    extract_io_type_hints(handler_io, signature) # mutates handler_io
 
     handler = Handler[I, O](service_tag,
                             handler_io,

python/restate/object.py

@@ -17,7 +17,7 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
+from restate.serde import Serde, GeneralSerde
 from .handler import HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
@@ -54,8 +54,8 @@ def handler(self,
                 kind: typing.Optional[typing.Literal["exclusive", "shared"]] = "exclusive",
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O]()) -> typing.Callable: # type: ignore
+                input_serde: Serde[I] = GeneralSerde[I](), # type: ignore
+                output_serde: Serde[O] = GeneralSerde[O]()) -> typing.Callable: # type: ignore
         """
         Decorator for defining a handler function.
 

python/restate/serde.py

@@ -13,6 +13,21 @@
 import json
 import typing
 
+def try_import_pydantic_base_model():
+    """
+    Try to import PydanticBaseModel from Pydantic.
+    """
+    try:
+        from pydantic import BaseModel # type: ignore # pylint: disable=import-outside-toplevel
+        return BaseModel
+    except ImportError:
+        class Dummy: # pylint: disable=too-few-public-methods
+            """a dummy class to use when Pydantic is not available"""
+
+        return Dummy
+
+PydanticBaseModel = try_import_pydantic_base_model()
+
 T = typing.TypeVar('T')
 I = typing.TypeVar('I')
 O = typing.TypeVar('O')
@@ -107,6 +122,51 @@ def serialize(self, obj: typing.Optional[I]) -> bytes:
 
         return bytes(json.dumps(obj), "utf-8")
 
+class GeneralSerde(Serde[I]):
+    """
+    A general serializer/deserializer that first checks if the object is a Pydantic BaseModel.
+    If so, it uses the model's native JSON dumping method.
+    Otherwise, it defaults to using the standard JSON library.
+    """
+
+    def deserialize(self, buf: bytes) -> typing.Optional[I]:
+        """
+        Deserializes a byte array into a Python object.
+
+        Args:
+            buf (bytes): The byte array to deserialize.
+
+        Returns:
+            Optional[I]: The resulting Python object, or None if the input is empty.
+        """
+        print("Deserializing using GeneralSerde")
+        if not buf:
+            return None
+        print(f"json.loads(buf): {json.loads(buf)}")
+        return json.loads(buf)
+
+    def serialize(self, obj: typing.Optional[I]) -> bytes:
+        """
+        Serializes a Python object into a byte array.
+        If the object is a Pydantic BaseModel, uses its model_dump_json method.
+
+        Args:
+            obj (Optional[I]): The Python object to serialize.
+
+        Returns:
+            bytes: The serialized byte array.
+        """
+        
+        if obj is None:
+            return bytes()
+
+        if isinstance(obj, PydanticBaseModel):
+            # Use the Pydantic-specific serialization
+            return obj.model_dump_json().encode("utf-8")  # type: ignore[attr-defined]
+
+        # Fallback to standard JSON serialization
+        return json.dumps(obj).encode("utf-8")
+
 
 class PydanticJsonSerde(Serde[I]):
     """

python/restate/service.py

@@ -18,7 +18,7 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
+from restate.serde import Serde, GeneralSerde
 from .handler import Handler, HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
@@ -54,8 +54,8 @@ def handler(self,
                 name: typing.Optional[str] = None,
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O]()) -> typing.Callable: # type: ignore
+                input_serde: Serde[I] = GeneralSerde[I](), # type: ignore
+                output_serde: Serde[O] = GeneralSerde[O]()) -> typing.Callable: # type: ignore
         """
         Decorator for defining a handler function.
 

python/restate/workflow.py

@@ -18,7 +18,7 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
+from restate.serde import Serde, GeneralSerde
 from .handler import HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
@@ -57,8 +57,8 @@ def main(self,
             name: typing.Optional[str] = None,
             accept: str = "application/json",
             content_type: str = "application/json",
-            input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-            output_serde: Serde[O] = JsonSerde[O]()) -> typing.Callable: # type: ignore
+            input_serde: Serde[I] = GeneralSerde[I](), # type: ignore
+            output_serde: Serde[O] = GeneralSerde[O]()) -> typing.Callable: # type: ignore
         """Mark this handler as a workflow entry point"""
         return self._add_handler(name,
                             kind="workflow",
@@ -71,8 +71,8 @@ def handler(self,
                 name: typing.Optional[str] = None,
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O]()) -> typing.Callable: # type: ignore
+                input_serde: Serde[I] = GeneralSerde[I](), # type: ignore
+                output_serde: Serde[O] = GeneralSerde[O]()) -> typing.Callable: # type: ignore
         """
         Decorator for defining a handler function.
         """
@@ -83,8 +83,8 @@ def _add_handler(self,
                 kind: typing.Literal["workflow", "shared", "exclusive"] = "shared",
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O]()) -> typing.Callable: # type: ignore
+                input_serde: Serde[I] = GeneralSerde[I](), # type: ignore
+                output_serde: Serde[O] = GeneralSerde[O]()) -> typing.Callable: # type: ignore
         """
         Decorator for defining a handler function.