Store the image moderation and text moderation logs

fastchat/serve/gradio_block_arena_anony.py

@@ -32,10 +32,10 @@
     get_ip,
     get_model_description_md,
 )
+from fastchat.serve.moderation.moderator import AzureAndOpenAIContentModerator
 from fastchat.serve.remote_logger import get_remote_logger
 from fastchat.utils import (
     build_logger,
-    moderation_filter,
 )
 
 logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
@@ -201,6 +201,9 @@ def get_battle_pair(
     if len(models) == 1:
         return models[0], models[0]
 
+    if len(models) == 0:
+        raise ValueError("There are no models provided. Cannot get battle pair.")
+
     model_weights = []
     for model in models:
         weight = get_sample_weight(
@@ -289,7 +292,11 @@ def add_text(
     all_conv_text = (
         all_conv_text_left[-1000:] + all_conv_text_right[-1000:] + "\nuser: " + text
     )
-    flagged = moderation_filter(all_conv_text, model_list, do_moderation=True)
+
+    content_moderator = AzureAndOpenAIContentModerator()
+    flagged = content_moderator.text_moderation_filter(
+        all_conv_text, model_list, do_moderation=True
+    )
     if flagged:
         logger.info(f"violate moderation (anony). ip: {ip}. text: {text}")
         # overwrite the original text

fastchat/serve/gradio_block_arena_named.py

@@ -28,10 +28,10 @@
     get_ip,
     get_model_description_md,
 )
+from fastchat.serve.moderation.moderator import AzureAndOpenAIContentModerator
 from fastchat.serve.remote_logger import get_remote_logger
 from fastchat.utils import (
     build_logger,
-    moderation_filter,
 )
 
 logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log")
@@ -174,19 +174,29 @@ def add_text(
                 no_change_btn,
             ]
             * 6
+            + [True]
         )
 
     model_list = [states[i].model_name for i in range(num_sides)]
-    all_conv_text_left = states[0].conv.get_prompt()
-    all_conv_text_right = states[1].conv.get_prompt()
-    all_conv_text = (
-        all_conv_text_left[-1000:] + all_conv_text_right[-1000:] + "\nuser: " + text
-    )
-    flagged = moderation_filter(all_conv_text, model_list)
-    if flagged:
-        logger.info(f"violate moderation (named). ip: {ip}. text: {text}")
-        # overwrite the original text
-        text = MODERATION_MSG
+    content_moderator = AzureAndOpenAIContentModerator()
+    text_flagged = content_moderator.text_moderation_filter(text, model_list)
+
+    if text_flagged:
+        logger.info(f"violate moderation. ip: {ip}. text: {text}")
+        content_moderator.write_to_json(get_ip(request))
+        for i in range(num_sides):
+            states[i].skip_next = True
+        gr.Warning(MODERATION_MSG)
+        return (
+            states
+            + [x.to_gradio_chatbot() for x in states]
+            + [""]
+            + [
+                no_change_btn,
+            ]
+            * 6
+            + [True]
+        )
 
     conv = states[0].conv
     if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
@@ -201,6 +211,7 @@ def add_text(
                 no_change_btn,
             ]
             * 6
+            + [True]
         )
 
     text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
@@ -217,6 +228,7 @@ def add_text(
             disable_btn,
         ]
         * 6
+        + [False]
     )
 
 
@@ -295,7 +307,11 @@ def bot_response_multi(
             break
 
 
-def flash_buttons():
+def flash_buttons(dont_show_vote_buttons: bool = False):
+    if dont_show_vote_buttons:
+        yield [no_change_btn] * 4 + [enable_btn] * 2
+        return
+
     btn_updates = [
         [disable_btn] * 4 + [enable_btn] * 2,
         [enable_btn] * 6,
@@ -321,6 +337,7 @@ def build_side_by_side_ui_named(models):
     states = [gr.State() for _ in range(num_sides)]
     model_selectors = [None] * num_sides
     chatbots = [None] * num_sides
+    dont_show_vote_buttons = gr.State(False)
 
     notice = gr.Markdown(notice_markdown, elem_id="notice_markdown")
 
@@ -476,24 +493,24 @@ def build_side_by_side_ui_named(models):
     textbox.submit(
         add_text,
         states + model_selectors + [textbox],
-        states + chatbots + [textbox] + btn_list,
+        states + chatbots + [textbox] + btn_list + [dont_show_vote_buttons],
     ).then(
         bot_response_multi,
         states + [temperature, top_p, max_output_tokens],
         states + chatbots + btn_list,
     ).then(
-        flash_buttons, [], btn_list
+        flash_buttons, [dont_show_vote_buttons], btn_list
     )
     send_btn.click(
         add_text,
         states + model_selectors + [textbox],
-        states + chatbots + [textbox] + btn_list,
+        states + chatbots + [textbox] + btn_list + [dont_show_vote_buttons],
     ).then(
         bot_response_multi,
         states + [temperature, top_p, max_output_tokens],
         states + chatbots + btn_list,
     ).then(
-        flash_buttons, [], btn_list
+        flash_buttons, [dont_show_vote_buttons], btn_list
     )
 
     return states + model_selectors

fastchat/serve/gradio_block_arena_vision.py

@@ -36,11 +36,10 @@
     get_conv_log_filename,
     get_remote_logger,
 )
+from fastchat.serve.moderation.moderator import AzureAndOpenAIContentModerator
 from fastchat.serve.vision.image import ImageFormat, Image
 from fastchat.utils import (
     build_logger,
-    moderation_filter,
-    image_moderation_filter,
 )
 
 logger = build_logger("gradio_web_server", "gradio_web_server.log")
@@ -51,8 +50,16 @@
 invisible_btn = gr.Button(interactive=False, visible=False)
 visible_image_column = gr.Image(visible=True)
 invisible_image_column = gr.Image(visible=False)
-enable_multimodal = gr.MultimodalTextbox(
-    interactive=True, visible=True, placeholder="Enter your prompt or add image here"
+enable_multimodal_keep_input = gr.MultimodalTextbox(
+    interactive=True,
+    visible=True,
+    placeholder="Enter your prompt or add image here",
+)
+enable_multimodal_clear_input = gr.MultimodalTextbox(
+    interactive=True,
+    visible=True,
+    placeholder="Enter your prompt or add image here",
+    value={"text": "", "files": []},
 )
 invisible_text = gr.Textbox(visible=False, value="", interactive=False)
 visible_text = gr.Textbox(
@@ -144,22 +151,22 @@ def clear_history(request: gr.Request):
     ip = get_ip(request)
     logger.info(f"clear_history. ip: {ip}")
     state = None
-    return (state, [], None) + (disable_btn,) * 5
+    return (state, [], enable_multimodal_clear_input) + (disable_btn,) * 5
 
 
 def clear_history_example(request: gr.Request):
     ip = get_ip(request)
     logger.info(f"clear_history_example. ip: {ip}")
     state = None
-    return (state, [], enable_multimodal) + (disable_btn,) * 5
+    return (state, [], enable_multimodal_keep_input) + (disable_btn,) * 5
 
 
 # TODO(Chris): At some point, we would like this to be a live-reporting feature.
 def report_csam_image(state, image):
     pass
 
 
-def _prepare_text_with_image(state, text, images, csam_flag):
+def _prepare_text_with_image(state, text, images):
     if len(images) > 0:
         if len(state.conv.get_images()) > 0:
             # reset convo with new image
@@ -184,31 +191,6 @@ def convert_images_to_conversation_format(images):
     return conv_images
 
 
-def moderate_input(state, text, all_conv_text, model_list, images, ip):
-    text_flagged = moderation_filter(all_conv_text, model_list)
-    # flagged = moderation_filter(text, [state.model_name])
-    nsfw_flagged, csam_flagged = False, False
-    if len(images) > 0:
-        nsfw_flagged, csam_flagged = image_moderation_filter(images[0])
-
-    image_flagged = nsfw_flagged or csam_flagged
-    if text_flagged or image_flagged:
-        logger.info(f"violate moderation. ip: {ip}. text: {all_conv_text}")
-        if text_flagged and not image_flagged:
-            # overwrite the original text
-            text = TEXT_MODERATION_MSG
-        elif not text_flagged and image_flagged:
-            text = IMAGE_MODERATION_MSG
-        elif text_flagged and image_flagged:
-            text = MODERATION_MSG
-
-    if csam_flagged:
-        state.has_csam_image = True
-        report_csam_image(state, images[0])
-
-    return text, image_flagged, csam_flagged
-
-
 def add_text(state, model_selector, chat_input, request: gr.Request):
     text, images = chat_input["text"], chat_input["files"]
     ip = get_ip(request)
@@ -226,16 +208,25 @@ def add_text(state, model_selector, chat_input, request: gr.Request):
 
     images = convert_images_to_conversation_format(images)
 
-    text, image_flagged, csam_flag = moderate_input(
-        state, text, all_conv_text, [state.model_name], images, ip
-    )
+    content_moderator = AzureAndOpenAIContentModerator()
+    text_flagged = content_moderator.text_moderation_filter(text, [state.model_name])
+    if len(images) > 0:
+        nsfw_flag, csam_flag = content_moderator.image_moderation_filter(images[0])
+        image_flagged = nsfw_flag or csam_flag
+        if csam_flag:
+            state.has_csam_image = True
+    else:
+        image_flagged = False
 
-    if image_flagged:
+    if text_flagged or image_flagged:
+        logger.info(f"violate moderation. ip: {ip}. text: {text}")
+        content_moderator.write_to_json(get_ip(request))
+
+    if image_flagged or text_flagged:
         logger.info(f"image flagged. ip: {ip}. text: {text}")
         state.skip_next = True
-        return (state, state.to_gradio_chatbot(), {"text": IMAGE_MODERATION_MSG}) + (
-            no_change_btn,
-        ) * 5
+        gr.Warning(MODERATION_MSG)
+        return (state, state.to_gradio_chatbot(), None) + (no_change_btn,) * 5
 
     if (len(state.conv.messages) - state.conv.offset) // 2 >= CONVERSATION_TURN_LIMIT:
         logger.info(f"conversation turn limit. ip: {ip}. text: {text}")
@@ -245,7 +236,7 @@ def add_text(state, model_selector, chat_input, request: gr.Request):
         ) * 5
 
     text = text[:INPUT_CHAR_LEN_LIMIT]  # Hard cut-off
-    text = _prepare_text_with_image(state, text, images, csam_flag=csam_flag)
+    text = _prepare_text_with_image(state, text, images)
     state.conv.append_message(state.conv.roles[0], text)
     state.conv.append_message(state.conv.roles[1], None)
     return (state, state.to_gradio_chatbot(), None) + (disable_btn,) * 5