How to pass context to child serializer?

[Crocmagnon]

I'm facing a RecursionError when querying a subset of fields that should not make recursion at all.

Here are some simplified models and serializers:

# disclaimer: I did not directly test this code, it's just an extract of mine.
# If you don't manage to reproduce the issue with this snippet, please let me know

class Client(models.Model):
    name = models.CharField(max_length=250)

class Project(models.Model):
    name = models.CharField(max_length=250)
    client = models.ForeignKey(Client, on_delete=models.PROTECT, related_name='projects')

class ProjectSerializer(FlexFieldsModelSerializer):
    expandable_fields = {
        'client_details': ('api.ClientSerializer', {'source': 'client', 'read_only': True}),
    }
    class Meta:
        model = Project
        fields = [
            'id',
            'name',
        ]

class ClientSerializer(FlexFieldsModelSerializer):
    expandable_fields = {
        'projects_details': (ProjectSerializer, {'source': 'projects', 'many': True, 'read_only': True}),
    }
    class Meta:
        model = Client
        fields = [
            'id',
            'name',
        ]

I queried my endpoint like this:

/api/client/5906?expand=projects_details&fields=id,name,projects_details.id

The expected result would be:

{
  "id": 5906,
  "name": "client name",
  "projects_details": [
    {
      "id": 2056
    },
    {
      "id": 3323
    }
  ]
}

Instead, I'm getting a RecursionError (see below). Did I miss something ? I understand that since I'm requesting to expand the projects and the projects themselves have a reference to the clients, but given the fields input, I feel like this should not fall in recursion.

RecursionError at /api/client/5906
maximum recursion depth exceeded

Request Method: GET
Request URL: http://localhost:81/api/client/5906?expand=projects_details&fields=id,name,projects_details.id
Django Version: 2.1.9
Python Executable: C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\Scripts\python.exe
Python Version: 3.7.3
Python Path: ****
Server time: Mon, 24 Jun 2019 17:47:09 +0200
Installed Applications:
['django.contrib.admindocs',
 'django.contrib.admin',
 'django.contrib.auth',
 'django.contrib.contenttypes',
 'django.contrib.sessions',
 'django.contrib.messages',
 'django.contrib.staticfiles',
 'rest_framework',
 'django_filters',
 'corsheaders',
 'api',
 'custom_auth']
Installed Middleware:
['corsheaders.middleware.CorsMiddleware',
 'django.middleware.security.SecurityMiddleware',
 'django.contrib.sessions.middleware.SessionMiddleware',
 'django.middleware.common.CommonMiddleware',
 'django.middleware.csrf.CsrfViewMiddleware',
 'django.contrib.auth.middleware.AuthenticationMiddleware',
 'django.contrib.auth.middleware.RemoteUserMiddleware',
 'django.contrib.messages.middleware.MessageMiddleware',
 'django.middleware.clickjacking.XFrameOptionsMiddleware']


Traceback:

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\exception.py" in inner
  34.             response = get_response(request)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\base.py" in _get_response
  126.                 response = self.process_exception_by_middleware(e, request)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\base.py" in _get_response
  124.                 response = wrapped_callback(request, *callback_args, **callback_kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\views\decorators\csrf.py" in wrapped_view
  54.         return view_func(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\views\generic\base.py" in view
  68.             return self.dispatch(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in dispatch
  495.             response = self.handle_exception(exc)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in handle_exception
  455.             self.raise_uncaught_exception(exc)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in dispatch
  492.             response = handler(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\generics.py" in get
  284.         return self.retrieve(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\mixins.py" in retrieve
  57.         serializer = self.get_serializer(instance)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\generics.py" in get_serializer
  112.         return serializer_class(*args, **kwargs)

File ".\api\serializers.py" in __init__
  230.         super().__init__(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_flex_fields\serializers.py" in __init__
  47.                 name, next_expand_fields, next_sparse_fields, next_omit_fields

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_flex_fields\serializers.py" in _make_expanded_field_serializer
  58.         serializer_settings = copy.deepcopy(field_options[1])

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  180.                     y = _reconstruct(x, memo, *rv)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _reconstruct
  281.         if hasattr(y, '__setstate__'):

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\request.py" in __getattr__
  412.             return getattr(self._request, attr)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\request.py" in __getattr__
  412.             return getattr(self._request, attr)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\request.py" in __getattr__
  412.             return getattr(self._request, attr)

[...]

Exception Type: RecursionError at /api/client/5906
Exception Value: maximum recursion depth exceeded
Request information:
USER: ****

GET:
expand = 'projects_details'
fields = 'id,name,projects_details.id'

[Crocmagnon]

Very strange... I managed to have the expected result in my first query this morning, then everything falls apart in the RecursionError I filed.

EDIT :
So, more info about my dev (and prod) environment : I'm running Django behind IIS on Windows (😢)

I can confirm that after a reboot I'm getting the expected result once and then it goes back to recursion error. In fact, if I kill the Python process and make another request, IIS creates another one and gives me the expected result, then subsequent requests end up in RecursionError.

I never saw anything like that. Do you have any idea ? Maybe it has something to do with the process being kept alive ?

[Crocmagnon]

Here are more insights:

I tried creating a small test project to reproduce the issue using the example code I provided in the original post but I was unable to. Everything worked flawlessly using manage.py runserver

So I decided to give a try and spin up my dev environment with runserver too.
Now I'm getting a TypeError [...] cannot serialize _io.BufferedReader

TypeError at /api/clients/
cannot serialize '_io.BufferedReader' object

Request Method: GET
Request URL: http://localhost:8001/api/clients/?expand=projects_details&fields=id,name,projects_details.id
Django Version: 2.1.9
Python Executable: C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\Scripts\python.exe
Python Version: 3.7.3
Python Path: ['C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\lib\\site-packages\\git\\ext\\gitdb', 'C:\\CODE_Shared\\kb\\bizdev_kb_api', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\Scripts\\python37.zip', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\DLLs', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\lib', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\Scripts', 'C:\\Program Files\\Python37\\Lib', 'C:\\Program Files\\Python37\\DLLs', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\lib\\site-packages', 'C:\\Users\\augendre\\.virtualenvs\\bizdev_kb_api-TgTXDkAC\\lib\\site-packages\\gitdb\\ext\\smmap']
Server time: Tue, 25 Jun 2019 11:03:14 +0200
Installed Applications:
['django.contrib.admindocs',
 'django.contrib.admin',
 'django.contrib.auth',
 'django.contrib.contenttypes',
 'django.contrib.sessions',
 'django.contrib.messages',
 'django.contrib.staticfiles',
 'rest_framework',
 'django_filters',
 'corsheaders',
 'api',
 'custom_auth']
Installed Middleware:
['corsheaders.middleware.CorsMiddleware',
 'django.middleware.security.SecurityMiddleware',
 'django.contrib.sessions.middleware.SessionMiddleware',
 'django.middleware.common.CommonMiddleware',
 'django.middleware.csrf.CsrfViewMiddleware',
 'django.contrib.auth.middleware.AuthenticationMiddleware',
 'django.contrib.auth.middleware.RemoteUserMiddleware',
 'django.contrib.messages.middleware.MessageMiddleware',
 'django.middleware.clickjacking.XFrameOptionsMiddleware']


Traceback:

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\exception.py" in inner
  34.             response = get_response(request)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\base.py" in _get_response
  126.                 response = self.process_exception_by_middleware(e, request)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\core\handlers\base.py" in _get_response
  124.                 response = wrapped_callback(request, *callback_args, **callback_kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\views\decorators\csrf.py" in wrapped_view
  54.         return view_func(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\django\views\generic\base.py" in view
  68.             return self.dispatch(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in dispatch
  495.             response = self.handle_exception(exc)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in handle_exception
  455.             self.raise_uncaught_exception(exc)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\views.py" in dispatch
  492.             response = handler(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\generics.py" in get
  201.         return self.list(request, *args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\mixins.py" in list
  44.             serializer = self.get_serializer(page, many=True)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\generics.py" in get_serializer
  112.         return serializer_class(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\serializers.py" in __new__
  124.             return cls.many_init(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_framework\serializers.py" in many_init
  145.         child_serializer = cls(*args, **kwargs)

File "C:\CODE_Shared\kb\bizdev_kb_api\api\serializers.py" in __init__
  230.         super().__init__(*args, **kwargs)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_flex_fields\serializers.py" in __init__
  47.                 name, next_expand_fields, next_sparse_fields, next_omit_fields

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\site-packages\rest_flex_fields\serializers.py" in _make_expanded_field_serializer
  58.         serializer_settings = copy.deepcopy(field_options[1])

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  180.                     y = _reconstruct(x, memo, *rv)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _reconstruct
  280.             state = deepcopy(state, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  180.                     y = _reconstruct(x, memo, *rv)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _reconstruct
  280.             state = deepcopy(state, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  180.                     y = _reconstruct(x, memo, *rv)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _reconstruct
  280.             state = deepcopy(state, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  150.         y = copier(x, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in _deepcopy_dict
  240.         y[deepcopy(key, memo)] = deepcopy(value, memo)

File "C:\Users\augendre\.virtualenvs\bizdev_kb_api-TgTXDkAC\lib\copy.py" in deepcopy
  169.                         rv = reductor(4)

Exception Type: TypeError at /api/clients/
Exception Value: cannot serialize '_io.BufferedReader' object

[Crocmagnon]

Ok, I managed to track it down to a custom __init__ method I have on the ClientSerializer.

Here's a project that reproduces this issue: https://github.com/Crocmagnon/demo-recursion-error-drf-ff

The culprit seems to be there: https://github.com/Crocmagnon/demo-recursion-error-drf-ff/blob/master/api/serializers.py#L22-L27

EDIT:
I need to filter some fields on the ProjectSerializer based on user permissions. Do you have any good idea of how I could do that ? I updated the example project to show how I currently do this, which requires the context to be passed to the serializer: https://github.com/Crocmagnon/demo-recursion-error-drf-ff/blob/master/api/serializers.py#L10-L17