Remove clip library dependency

Author: swimmiing

Eval.py

@@ -1,6 +1,5 @@
 import torch
 import os
-import clip
 import cv2
 
 import numpy as np
@@ -67,7 +66,7 @@ def eval_vggss_agg(
         labels, name = data['labels'], data['ids']
 
         # Inference
-        placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(model.device)
+        placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
         placeholder_tokens = placeholder_tokens.repeat((test_dataloader.batch_size, 1))
         audio_driven_embedding = model.encode_audio(audios.to(model.device), placeholder_tokens, text_pos_at_prompt,
                                                     prompt_length)
@@ -170,7 +169,7 @@ def eval_avsbench_agg(
         images, audios, gts, labels, name = data['images'], data['audios'], data['gts'], data['labels'], data['ids']
 
         # Inference
-        placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(model.device)
+        placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
         placeholder_tokens = placeholder_tokens.repeat((test_dataloader.batch_size, 1))
         audio_driven_embedding = model.encode_audio(audios.to(model.device), placeholder_tokens, text_pos_at_prompt,
                                                     prompt_length)
@@ -268,7 +267,7 @@ def eval_flickr_agg(
         labels, name = data['labels'], data['ids']
 
         # Inference
-        placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(model.device)
+        placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
         placeholder_tokens = placeholder_tokens.repeat((test_dataloader.batch_size, 1))
         audio_driven_embedding = model.encode_audio(audios.to(model.device), placeholder_tokens, text_pos_at_prompt,
                                                     prompt_length)
@@ -364,7 +363,7 @@ def eval_exvggss_agg(
         labels, name = data['labels'], data['ids']
 
         # Inference
-        placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(model.device)
+        placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
         placeholder_tokens = placeholder_tokens.repeat((test_dataloader.batch_size, 1))
         audio_driven_embedding = model.encode_audio(audios.to(model.device), placeholder_tokens, text_pos_at_prompt,
                                                     prompt_length)
@@ -446,7 +445,7 @@ def eval_exflickr_agg(
         labels, name = data['labels'], data['ids']
 
         # Inference
-        placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(model.device)
+        placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
         placeholder_tokens = placeholder_tokens.repeat((test_dataloader.batch_size, 1))
         audio_driven_embedding = model.encode_audio(audios.to(model.device), placeholder_tokens, text_pos_at_prompt,
                                                     prompt_length)

README.md

@@ -36,6 +36,7 @@ $ pip install tensorboard
 $ pip transformers==4.25.1
 $ pip install opencv-python
 $ pip install tqdm
+$ pip install scikit-learn
 
 ```
 

Test_PTModels.sh

@@ -8,4 +8,5 @@ python Test_PTModels.py \
 --vggss_path {put dataset directory} \
 --flickr_path {put dataset directory} \
 --avs_path {put dataset directory} \
+--save_path {put dataset directory} \
 --epochs None

Train_ACL.py

@@ -5,7 +5,6 @@
 import time
 import datetime
 import yaml
-import clip
 import shutil
 import argparse
 
@@ -205,7 +204,7 @@ def main(model_name, exp_name, train_config_name, data_path_dict, save_path):
 
             with autocast_fn():
                 # Train step
-                placeholder_tokens = clip.tokenize(prompt_template.replace('{}', '')).to(module.device)
+                placeholder_tokens = model.get_placeholder_token(prompt_template.replace('{}', ''))
                 placeholder_tokens = placeholder_tokens.repeat((train_dataloader.batch_size, 1))
                 audio_driven_embedding = module.encode_audio(audios.to(module.device), placeholder_tokens,
                                                              text_pos_at_prompt, prompt_length).half()

modules/models.py

@@ -9,6 +9,7 @@
 from modules.AudioToken.embedder import FGAEmbedder
 from modules.CLIPSeg.clipseg_for_audio import CLIPSeg
 from modules.mask_utils import ImageMasker, FeatureMasker
+from transformers import AutoTokenizer
 
 
 class ACL(nn.Module):
@@ -37,6 +38,9 @@ def __init__(self, conf_file: str, device: str):
         cfg = BEATsConfig(checkpoint['cfg'])
         self.audio_backbone = BEATs(cfg)
 
+        # Text Tokenizer for placeholder prompt
+        self.tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined")
+
         # Init audio projection layer
         self.audio_proj = FGAEmbedder(input_size=self.args.audio_proj.input_size * 3,
                                       output_size=self.args.audio_proj.output_size)
@@ -63,6 +67,20 @@ def __init__(self, conf_file: str, device: str):
         self.masker_i.to(self.device)
         self.masker_f.to(self.device)
 
+    def get_placeholder_token(self, prompt_text: str):
+        """
+        Get placeholder token from prompt text
+
+        Args:
+            prompt_text (str): prompt text without '{}'
+
+        Returns:
+            CLIPTokenizerFast result with prompt text
+        """
+        placeholder_token = self.tokenizer(prompt_text, return_tensors="pt").data['input_ids']
+        placeholder_token = F.pad(placeholder_token, (0, 77 - placeholder_token.shape[-1])).to(self.device)
+        return placeholder_token
+
     def train(self, bool: bool = True):
         """
         Set the module in training mode.