-
Notifications
You must be signed in to change notification settings - Fork 9
/
huggingface_api.py
138 lines (118 loc) · 4.07 KB
/
huggingface_api.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""
Use FastChat with Hugging Face generation APIs.
Usage:
python3 -m fastchat.serve.huggingface_api --model lmsys/vicuna-7b-v1.3
python3 -m fastchat.serve.huggingface_api --model lmsys/fastchat-t5-3b-v1.0
"""
import argparse
import json
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
from fastchat.model import load_model, get_conversation_template, add_model_args
@torch.inference_mode()
def main(args):
print(args)
# assert 1 == 0
model, tokenizer = load_model(
args.model_path,
args.device,
args.num_gpus,
args.max_gpu_memory,
args.load_8bit,
args.cpu_offloading,
revision=args.revision,
debug=args.debug,
)
msg = args.message
conv = get_conversation_template(args.model_path)
conv.append_message(conv.roles[0], msg)
conv.append_message(conv.roles[1], None)
prompt = conv.get_prompt()
print("prompt ", prompt)
input_ids = tokenizer([prompt]).input_ids
output_ids = model.generate(
torch.as_tensor(input_ids).cuda(),
do_sample=True,
temperature=args.temperature,
repetition_penalty=args.repetition_penalty,
max_new_tokens=args.max_new_tokens,
)
if model.config.is_encoder_decoder:
output_ids = output_ids[0]
else:
output_ids = output_ids[0][len(input_ids[0]) :]
outputs = tokenizer.decode(
output_ids, skip_special_tokens=True, spaces_between_special_tokens=False
)
print(f"{conv.roles[0]}: {msg}")
print(f"{conv.roles[1]}: {outputs}")
torch.inference_mode()
def load_vicuna(args):
model, tokenizer = load_model(
args.model_path,
args.device,
args.num_gpus,
args.max_gpu_memory,
args.load_8bit,
args.cpu_offloading,
revision=args.revision,
debug=args.debug,
)
return model, tokenizer
@torch.inference_mode()
def vicuna(args, msg, model, tokenizer, verbose=False):
# model, tokenizer = load_model(
# args.model_path,
# args.device,
# args.num_gpus,
# args.max_gpu_memory,
# args.load_8bit,
# args.cpu_offloading,
# revision=args.revision,
# debug=args.debug,
# )
conv = get_conversation_template(args.model_path)
conv.append_message(conv.roles[0], msg)
conv.append_message(conv.roles[1], None)
prompt = conv.get_prompt()
if verbose:
print("prompt ", prompt)
input_ids = tokenizer([prompt]).input_ids
output_ids = model.generate(
torch.as_tensor(input_ids).cuda(),
do_sample=True,
temperature=args.temperature,
repetition_penalty=args.repetition_penalty,
max_new_tokens=args.max_new_tokens,
)
if model.config.is_encoder_decoder:
output_ids = output_ids[0]
else:
output_ids = output_ids[0][len(input_ids[0]) :]
outputs = tokenizer.decode(
output_ids, skip_special_tokens=True, spaces_between_special_tokens=False
)
return outputs
from omegaconf import OmegaConf
from utils import nest_dict, read_unknowns
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=str, default='configs/base.yaml')
parser.add_argument('overrides', nargs='*', help="Any key=value arguments to override config values "
"(use dots for.nested=overrides)")
flags, unknown = parser.parse_known_args()
overrides = OmegaConf.from_cli(flags.overrides)
cfg = OmegaConf.load(flags.config)
base = OmegaConf.load('configs/base.yaml')
dataset_base = OmegaConf.load(cfg.base_config)
args = OmegaConf.merge(base, dataset_base, cfg, overrides)
if len(unknown) > 0:
print(unknown)
config = nest_dict(read_unknowns(unknown))
to_merge = OmegaConf.create(config)
args = OmegaConf.merge(args, to_merge)
args.yaml = flags.config
# Reset default repetition penalty for T5 models.
if "t5" in args.model_path and args.repetition_penalty == 1.0:
args.repetition_penalty = 1.2
main(args)