Can't pass trust_remote_code to hallucination_score

[mausch]

Describe the bug

When calling Scorer.hallucination_score I get:

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../.venv/lib/python3.9/site-packages/deepeval/scorer/scorer.py:255: in hallucination_score
    scorer = HallucinationModel(model_name=model)
../../.venv/lib/python3.9/site-packages/deepeval/singleton.py:12: in __call__
    instance = super().__call__(*args, **kwargs)
../../.venv/lib/python3.9/site-packages/deepeval/models/hallucination_model.py:27: in __init__
    self.model = CrossEncoder(model_name)
../../.venv/lib/python3.9/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:74: in __init__
    self.config = AutoConfig.from_pretrained(
../../.venv/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py:1037: in from_pretrained
    trust_remote_code = resolve_trust_remote_code(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

trust_remote_code = False, model_name = 'vectara/hallucination_evaluation_model'
has_local_code = False, has_remote_code = True

    def resolve_trust_remote_code(trust_remote_code, model_name, has_local_code, has_remote_code):
        if trust_remote_code is None:
            if has_local_code:
                trust_remote_code = False
            elif has_remote_code and TIME_OUT_REMOTE_CODE > 0:
                try:
                    signal.signal(signal.SIGALRM, _raise_timeout_error)
                    signal.alarm(TIME_OUT_REMOTE_CODE)
                    while trust_remote_code is None:
                        answer = input(
                            f"The repository for {model_name} contains custom code which must be executed to correctly"
                            f"load the model. You can inspect the repository content at https://hf.co/{model_name}.\n"
                            f"You can avoid this prompt in future by passing the argument `trust_remote_code=True`.\n\n"
                            f"Do you wish to run the custom code? [y/N] "
                        )
                        if answer.lower() in ["yes", "y", "1"]:
                            trust_remote_code = True
                        elif answer.lower() in ["no", "n", "0", ""]:
                            trust_remote_code = False
                    signal.alarm(0)
                except Exception:
                    # OS which does not support signal.SIGALRM
                    raise ValueError(
                        f"The repository for {model_name} contains custom code which must be executed to correctly"
                        f"load the model. You can inspect the repository content at https://hf.co/{model_name}.\n"
                        f"Please pass the argument `trust_remote_code=True` to allow custom code to be run."
                    )
            elif has_remote_code:
                # For the CI which puts the timeout at 0
                _raise_timeout_error(None, None)
    
        if has_remote_code and not has_local_code and not trust_remote_code:
>           raise ValueError(
                f"Loading {model_name} requires you to execute the configuration file in that"
                " repo on your local machine. Make sure you have read the code there to avoid malicious use, then"
                " set the option `trust_remote_code=True` to remove this error."
            )
E           ValueError: Loading vectara/hallucination_evaluation_model requires you to execute the configuration file in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.

There doesn't seem to be any way to pass that trust_remote_code parameter.

[penguine-ip]

@mausch I believe this was resolved in discord (?)

[mausch]

hi @penguine-ip , I didn't post this in Discord.