ValueError when using gradio components in Chatbot messages and streaming response

[lippings]

Describe the bug

I have a chatbot implemented using gradio.Chatbot, where I am streaming the chatbot's response. I also want the chatbot to be able to include gradio components in it's responses, as well as text. I can get the component to show up fine, but the next yield from the generator function after the component is shown causes a ValueError.

Have you searched existing issues? 🔎

• I have searched and found no existing issues

Reproduction

import gradio as gr

def add_msg(history, human_input):
    msg = gr.ChatMessage(
        role='user',
        content=human_input
    )
    history.append(msg)

    return history, ''

def fn(history):
    history.append(gr.ChatMessage(
        role='assistant',
        content='Here\'s'
    ))
    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content='an image:'
    ))
    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content=gr.Image(
            'https://www.startpage.com/av/proxy-image?piurl=https%3A%2F%2Ftse3.mm.bing.net%2Fth%3Fid%3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK%26pid%3DApi&sp=1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad'
        )
    ))

    yield history

    history.append(gr.ChatMessage(
        role='assistant',
        content='Wow, what an image!'
    ))
    yield history

with gr.Blocks(fill_height=True) as demo:
    chatbot = gr.Chatbot(type='messages', scale=1)
    human_input = gr.Textbox()

    human_input.submit(
        add_msg, [chatbot, human_input], [chatbot, human_input]
    ).then(
        fn, chatbot, chatbot
    )


demo.launch()

The bot should respond with text, an image, and more text after that.

Screenshot

There is an error before the last piece of text, immediately after the image.

Logs

Running on local URL:  http://127.0.0.1:7860

To create a public link, set `share=True` in `launch()`.
Traceback (most recent call last):
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/queueing.py", line 575, in process_events
    response = await route_utils.call_process_api(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/route_utils.py", line 322, in call_process_api
    output = await app.get_blocks().process_api(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/blocks.py", line 1945, in process_api
    data = await self.postprocess_data(block_fn, result["prediction"], state)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/blocks.py", line 1768, in postprocess_data
    prediction_value = block.postprocess(prediction_value)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 496, in postprocess
    processed_messages = [
                         ^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 499, in <listcomp>
    for msg in self._postprocess_message_messages(cast(MessageDict, message))
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 441, in _postprocess_message_messages
    message.content = self._postprocess_content(message.content)  # type: ignore
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/sam/tmp/repro/gradio/.venv/lib/python3.11/site-packages/gradio/components/chatbot.py", line 421, in _postprocess_content
    raise ValueError(f"Invalid message for Chatbot component: {chat_message}")
ValueError: Invalid message for Chatbot component: component='image' value={'path': '/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'url': '/file=/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'size': None, 'orig_name': None, 'mime_type': None, 'is_stream': False, 'meta': {'_type': 'gradio.FileData'}} constructor_args={} props={'streamable': False, 'value': {'path': '/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'url': '/file=/tmp/gradio/2e96eca335c0769fe93a7287d9342bca0c0708742adeaa66786bb928838d41a6/proxy-imagepiurlhttps3A2F2Ftse3.mm.bing.net2Fth3Fid3DOIP.2GCSsS1HKwRa6qWotlHXHQHaEK26pid3DApisp1726513283T319971ef4351fe251279b3b562767806a0ddbbde1d1c96ced3168dd7e811bbad', 'size': None, 'orig_name': None, 'mime_type': None, 'is_stream': False, 'meta': {'_type': 'gradio.FileData'}}, 'format': 'webp', 'height': None, 'width': None, 'image_mode': 'RGB', 'sources': ['upload', 'webcam', 'clipboard'], 'type': 'numpy', 'label': None, 'show_label': True, 'show_download_button': True, 'container': True, 'scale': None, 'min_width': 160, 'interactive': None, 'visible': True, 'streaming': False, 'elem_id': None, 'elem_classes': [], 'key': None, 'mirror_webcam': True, 'show_share_button': False, 'placeholder': None, 'show_fullscreen_button': True, 'proxy_url': None, 'name': 'image', '_selectable': False}

System Info

Gradio Environment Information:
------------------------------
Operating System: Linux
gradio version: 4.44.0
gradio_client version: 1.3.0

------------------------------------------------
gradio dependencies in your environment:

aiofiles: 23.2.1
anyio: 4.4.0
fastapi: 0.114.2
ffmpy: 0.4.0
gradio-client==1.3.0 is not installed.
httpx: 0.27.2
huggingface-hub: 0.24.7
importlib-resources: 6.4.5
jinja2: 3.1.4
markupsafe: 2.1.5
matplotlib: 3.9.2
numpy: 2.1.1
orjson: 3.10.7
packaging: 24.1
pandas: 2.2.2
pillow: 10.4.0
pydantic: 2.9.1
pydub: 0.25.1
python-multipart: 0.0.9
pyyaml: 6.0.2
ruff: 0.6.5
semantic-version: 2.10.0
tomlkit==0.12.0 is not installed.
typer: 0.12.5
typing-extensions: 4.12.2
urllib3: 2.2.3
uvicorn: 0.30.6
authlib; extra == 'oauth' is not installed.
itsdangerous; extra == 'oauth' is not installed.


gradio_client dependencies in your environment:

fsspec: 2024.9.0
httpx: 0.27.2
huggingface-hub: 0.24.7
packaging: 24.1
typing-extensions: 4.12.2
websockets: 12.0

Severity

I can work around it

[lippings]

I can work around this by adding an if-clause for ComponentMessage in Chatbot._postprocess_content:

def _postprocess_content(
        self,
        chat_message: str
        | tuple
        | list
        | FileDataDict
        | FileData
        | GradioComponent
        | None,
    ) -> str | FileMessage | ComponentMessage | None:
        if chat_message is None:
            return None
        elif isinstance(chat_message, FileMessage):
            return chat_message
        elif isinstance(chat_message, FileData):
            return FileMessage(file=chat_message)
        elif isinstance(chat_message, ComponentMessage):  # <------------ This is the added clause
            return chat_message
        elif isinstance(chat_message, GradioComponent):
            component = import_component_and_data(type(chat_message).__name__)
            if component:
                component = chat_message.__class__(**chat_message.constructor_args)
                chat_message.constructor_args.pop("value", None)
                config = component.get_config()
                return ComponentMessage(
                    component=type(chat_message).__name__.lower(),
                    value=config.get("value", None),
                    constructor_args=chat_message.constructor_args,
                    props=config,
                )
        elif isinstance(chat_message, dict) and "path" in chat_message:
            filepath = chat_message["path"]
            return self._create_file_message(chat_message, filepath)
        elif isinstance(chat_message, (tuple, list)):
            filepath = str(chat_message[0])
            return self._create_file_message(chat_message, filepath)
        elif isinstance(chat_message, str):
            chat_message = inspect.cleandoc(chat_message)
            return chat_message
        else:
            raise ValueError(f"Invalid message for Chatbot component: {chat_message}")

But I'm not sure this is the "proper" way to fix this, since I'm not too familiar with Gradio's internals.

[chenmohan123]

我也遇到了这个问题。目前暂时可以用下面的方式解决：

# 图片1
history.append(ChatMessage(role="assistant", content={"path": "./images/123.jpg", "alt_text": "测试图片"}))
time.sleep(0.1)
yield history

# 图片2
history.append(ChatMessage(role="assistant", content={"path": "./images/456.jpg", "alt_text": "测试图片"}))
time.sleep(0.1)
yield history