Refactor data serialization (#2211)

* Refactor data serialization

Remove classmethod sanitize_for_serialization, and fix
model_to_dict to use the same logic so that it handles list of list
properly for example.

* pre-commit fixes

* pre-commit fixes

---------

Co-authored-by: ci.datadog-api-spec <[email protected]>
Co-authored-by: amaskara-dd <[email protected]>

.generator/src/generator/templates/api_client.j2

@@ -6,8 +6,6 @@ import mimetypes
 import warnings
 import multiprocessing
 from multiprocessing.pool import ThreadPool
-from datetime import date, datetime
-from uuid import UUID
 import io
 import os
 import re
@@ -21,14 +19,12 @@ from {{ package }} import rest
 from {{ package }}.configuration import Configuration
 from {{ package }}.exceptions import ApiTypeError, ApiValueError
 from {{ package }}.model_utils import (
-    ModelNormal,
-    ModelSimple,
-    ModelComposed,
     check_allowed_values,
     check_validations,
     deserialize_file,
     file_type,
-    model_to_dict,
+    data_to_dict,
+    get_file_data_and_close_file,
     validate_and_convert_types,
     get_attribute_from_path,
     set_attribute_from_path,
@@ -154,40 +150,6 @@ class ApiClient:
                 new_params.append((k, v))
         return new_params
 
-    @classmethod
-    def sanitize_for_serialization(cls, obj):
-        """Prepares data for transmission before it is sent with the rest client.
-        If obj is None, return None.
-        If obj is str, int, long, float, bool, return directly.
-        If obj is datetime.datetime, datetime.date convert to string in iso8601 format.
-        If obj is list, sanitize each element in the list.
-        If obj is dict, return the dict.
-        If obj is OpenAPI model, return the properties dict.
-        If obj is io.IOBase, return the bytes.
-
-        :param obj: The data to serialize.
-        :return: The serialized form of data.
-        """
-        if isinstance(obj, (ModelNormal, ModelComposed)):
-            return {key: cls.sanitize_for_serialization(val) for key, val in model_to_dict(obj).items()}
-        elif isinstance(obj, io.IOBase):
-            return cls.get_file_data_and_close_file(obj)
-        elif isinstance(obj, (str, int, float, bool)) or obj is None:
-            return obj
-        elif isinstance(obj, (datetime, date)):
-            if getattr(obj, "tzinfo", None) is not None:
-                return obj.isoformat()
-            return "{}Z".format(obj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3])
-        elif isinstance(obj, UUID ):
-            return str(obj)
-        elif isinstance(obj, ModelSimple):
-            return cls.sanitize_for_serialization(obj.value)
-        elif isinstance(obj, (list, tuple)):
-            return [cls.sanitize_for_serialization(item) for item in obj]
-        if isinstance(obj, dict):
-            return {key: cls.sanitize_for_serialization(val) for key, val in obj.items()}
-        raise ApiValueError("Unable to prepare type {} for serialization".format(obj.__class__.__name__))
-
     def deserialize(self, response_data: str, response_type: Any, check_type: Optional[bool]):
         """Deserializes response into an object.
 
@@ -286,12 +248,12 @@ class ApiClient:
         header_params = header_params or {}
         header_params.update(self.default_headers)
         if header_params:
-            header_params = self.sanitize_for_serialization(header_params)
+            header_params = data_to_dict(header_params)
             header_params = dict(self.parameters_to_tuples(header_params, collection_formats))
 
         # path parameters
         if path_params:
-            path_params = self.sanitize_for_serialization(path_params)
+            path_params = data_to_dict(path_params)
             for k, v in self.parameters_to_tuples(path_params, collection_formats):
                 # specified safe chars, encode everything
                 resource_path = resource_path.replace(
@@ -302,21 +264,21 @@ class ApiClient:
 
         # query parameters
         if query_params:
-            query_params = self.sanitize_for_serialization(query_params)
+            query_params = data_to_dict(query_params)
             query_params = self.parameters_to_tuples(query_params, collection_formats)
 
         # post parameters
         if post_params or files:
             post_params = post_params or []
-            post_params = self.sanitize_for_serialization(post_params)
+            post_params = data_to_dict(post_params)
             post_params = self.parameters_to_tuples(post_params, collection_formats)
             post_params.extend(self.files_parameters(files))
             if header_params["Content-Type"].startswith("multipart"):
                 post_params = self.parameters_to_multipart(post_params)
 
         # body
         if body:
-            body = self.sanitize_for_serialization(body)
+            body = data_to_dict(body)
 
         # request url
         if host is None:
@@ -439,12 +401,6 @@ class ApiClient:
                 new_params.append((k, v))
         return new_params
 
-    @staticmethod
-    def get_file_data_and_close_file(file_instance: io.IOBase) -> bytes:
-        file_data = file_instance.read()
-        file_instance.close()
-        return file_data
-
     def files_parameters(self, files: Optional[Dict[str, List[io.FileIO]]] = None):
         """Builds form parameters.
 
@@ -469,7 +425,7 @@ class ApiClient:
                         "Cannot read a closed file. The passed in file_type " "for %s must be open." % param_name
                     )
                 filename = os.path.basename(str(file_instance.name))
-                filedata = self.get_file_data_and_close_file(file_instance)
+                filedata = get_file_data_and_close_file(file_instance)
                 mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
                 params.append(tuple([param_name, tuple([filename, filedata, mimetype])]))
 

.generator/src/generator/templates/model_utils.j2

@@ -1362,6 +1362,53 @@ def validate_and_convert_types(
     return input_value
 
 
+def get_file_data_and_close_file(file_instance: io.IOBase) -> bytes:
+    file_data = file_instance.read()
+    file_instance.close()
+    return file_data
+
+
+def data_to_dict(instance, serialize=True):
+    """Prepares data for transmission before it is sent with the rest client.
+
+    If obj is None, return None.
+    If obj is str, int, long, float, bool, return directly.
+    If obj is datetime.datetime, datetime.date convert to string in iso8601 format.
+    If obj is list, sanitize each element in the list.
+    If obj is dict, return the dict.
+    If obj is OpenAPI model, return the properties dict.
+    If obj is io.IOBase, return the bytes.
+
+    :param obj: The data to serialize.
+    :param serialize: If True, return data safe for wire. Forwarded to model_to_dict.
+    :type serialize: bool
+    :return: The serialized form of data.
+    """
+    if isinstance(instance, (ModelNormal, ModelComposed)):
+        return {key: data_to_dict(val) for key, val in model_to_dict(instance, serialize).items()}
+    elif isinstance(instance, io.IOBase):
+        return get_file_data_and_close_file(instance)
+    elif isinstance(instance, (str, int, float, bool)) or instance is None:
+        return instance
+    elif isinstance(instance, (datetime, date)):
+        if not serialize:
+            return instance
+        if getattr(instance, "tzinfo", None) is not None:
+            return instance.isoformat()
+        return "{}Z".format(instance.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3])
+    elif isinstance(instance, UUID):
+        if not serialize:
+            return instance
+        return str(instance)
+    elif isinstance(instance, ModelSimple):
+        return data_to_dict(instance.value)
+    elif isinstance(instance, (list, tuple)):
+        return [data_to_dict(item) for item in instance]
+    if isinstance(instance, dict):
+        return {key: data_to_dict(val) for key, val in instance.items()}
+    raise ApiValueError("Unable to handle type {}".format(instance.__class__.__name__))
+
+
 def model_to_dict(model_instance, serialize=True):
     """Returns the model properties as a dict.
 
@@ -1392,37 +1439,7 @@ def model_to_dict(model_instance, serialize=True):
                     seen_json_attribute_names.add(attr)
                 except KeyError:
                     used_fallback_python_attribute_names.add(attr)
-            if isinstance(value, list):
-                if not value:
-                    # empty list or None
-                    result[attr] = value
-                else:
-                    res = []
-                    for v in value:
-                        if isinstance(v, PRIMITIVE_TYPES) or v is None:
-                            res.append(v)
-                        elif isinstance(v, ModelSimple):
-                            res.append(v.value)
-                        elif isinstance(v, OpenApiModel):
-                            res.append(model_to_dict(v, serialize=serialize))
-                        else:
-                            res.append(v)
-                    result[attr] = res
-            elif isinstance(value, dict):
-                result[attr] = dict(
-                    map(
-                        lambda item: (item[0], model_to_dict(item[1], serialize=serialize))
-                        if hasattr(item[1], "_data_store")
-                        else item,
-                        value.items(),
-                    )
-                )
-            elif isinstance(value, ModelSimple):
-                result[attr] = value.value
-            elif hasattr(value, "_data_store"):
-                result[attr] = model_to_dict(value, serialize=serialize)
-            else:
-                result[attr] = value
+            result[attr] = data_to_dict(value, serialize)
     if serialize:
         for python_key in used_fallback_python_attribute_names:
             json_key = py_to_json_map.get(python_key)

docs/datadog_api_client.rst

@@ -1,48 +1,55 @@
-datadog\_api\_client
-====================
+datadog\_api\_client package
+============================
+
+Subpackages
+-----------
 
 .. toctree::
    :maxdepth: 4
 
    datadog_api_client.v1
    datadog_api_client.v2
 
+Submodules
+----------
 
-api\_client
------------
+datadog\_api\_client.api\_client module
+---------------------------------------
 
 .. automodule:: datadog_api_client.api_client
    :members:
    :show-inheritance:
 
-configuration
--------------
+datadog\_api\_client.configuration module
+-----------------------------------------
 
 .. automodule:: datadog_api_client.configuration
    :members:
    :show-inheritance:
 
-exceptions
-----------
+datadog\_api\_client.exceptions module
+--------------------------------------
 
 .. automodule:: datadog_api_client.exceptions
    :members:
    :show-inheritance:
 
-model\_utils
-------------
+datadog\_api\_client.model\_utils module
+----------------------------------------
 
 .. automodule:: datadog_api_client.model_utils
    :members:
    :show-inheritance:
 
-rest
-----
+datadog\_api\_client.rest module
+--------------------------------
 
 .. automodule:: datadog_api_client.rest
    :members:
    :show-inheritance:
 
+Module contents
+---------------
 
 .. automodule:: datadog_api_client
    :members: