Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support local model with inference-engine mlx #475

Open
wants to merge 35 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
89665df
add inference mlx run local model in single node
OKHand-Zy Nov 8, 2024
b2bcc12
Merge exo f1eec9f commit version
OKHand-Zy Nov 15, 2024
0b87eb9
filter merge erro
OKHand-Zy Nov 15, 2024
e2f0723
filter f1eec9f model change
OKHand-Zy Nov 15, 2024
2a2e3b2
futuer:(i-e:mlx)suppoert local model and HF model terminal complet
OKHand-Zy Nov 18, 2024
d8bbb2b
futur:support cli and chatapi local model complet
OKHand-Zy Nov 19, 2024
9ab3513
filter run_model_cli
OKHand-Zy Nov 20, 2024
a9c345a
add init_exo_function (helpers.py)
OKHand-Zy Nov 20, 2024
1ae2648
Merge branch 'main' 1fa42f3 into support-local-model
OKHand-Zy Nov 20, 2024
cdba915
filter read me
OKHand-Zy Nov 20, 2024
0b06fe1
filter cli local model error
OKHand-Zy Nov 20, 2024
438eae4
filter some mark
OKHand-Zy Nov 20, 2024
32574b9
Merge exo 93d38e2 commits
OKHand-Zy Nov 22, 2024
535cb44
filter name miss
OKHand-Zy Nov 22, 2024
1065242
Merge exo commit 7013041 fix 'CLI reply error'
OKHand-Zy Nov 22, 2024
1e5429d
remove:add lcoal model arg
OKHand-Zy Nov 28, 2024
880682a
filter: other dir no config error
OKHand-Zy Dec 6, 2024
c4e18ae
merge: new image
OKHand-Zy Dec 6, 2024
daeb1eb
Merge exo 17411df branch
OKHand-Zy Dec 6, 2024
dd49f08
Merge exo d4cc2cf commit
OKHand-Zy Dec 12, 2024
cb270ed
filter: chat-api modelpool for Local model
OKHand-Zy Dec 12, 2024
316e58b
add: Local Model Api
OKHand-Zy Dec 12, 2024
961c0dc
filter: file&code to other palce
OKHand-Zy Dec 12, 2024
85d41f0
future: enhance lh_helpers but download_file function not complete
OKHand-Zy Dec 16, 2024
aca7eba
filter: add args setting sotre model ip and port
OKHand-Zy Dec 17, 2024
ef60ff3
future:add other node update local model to model_crads
OKHand-Zy Dec 18, 2024
e96f5ae
Merge exo cfedcec commit
OKHand-Zy Dec 19, 2024
22b4484
filter: _resolve_tokenizer find error
OKHand-Zy Dec 19, 2024
8c3a600
future: local mode auto download prototype
OKHand-Zy Dec 19, 2024
f0e1877
filter:download file path
OKHand-Zy Dec 20, 2024
5eef335
filter: stored_model dir path and http class name
OKHand-Zy Dec 20, 2024
c557fe8
feat:chatAPI can run Local model on multi node
OKHand-Zy Dec 24, 2024
8e05ad7
Merge branch 'auto dowload-local-model' into support-local-model
OKHand-Zy Dec 24, 2024
322b36c
docs: Update the README.md to add instructions for using local models…
OKHand-Zy Dec 24, 2024
677260e
Merge exo main branch fdc3b5a commit
OKHand-Zy Dec 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
filter: _resolve_tokenizer find error
  • Loading branch information
OKHand-Zy committed Dec 19, 2024
commit 22b4484c1993753c1511043e78f85d5f76a08229
31 changes: 20 additions & 11 deletions exo/main.py
Original file line number Diff line number Diff line change
@@ -207,18 +207,18 @@ def throttled_broadcast(shard: Shard, event: RepoProgressEvent):
shard_downloader.on_progress.register("broadcast").on_next(throttled_broadcast)

async def run_model_cli(node: Node, inference_engine: InferenceEngine, model_name_or_path: str, prompt: str):
if model_cards.get(model_name_or_path):
inference_class = inference_engine.__class__.__name__
shard = build_base_shard(model_name_or_path, inference_class)
if not shard:
print(f"Error: exo Unsupported model '{model_name_or_path}' for inference engine {inference_engine.__class__.__name__}")
return
tokenizer = await resolve_tokenizer(get_repo(shard.model_id, inference_class))
elif os.path.isdir(model_name_or_path):
model_path = model_name_or_path.rstrip('/')
model_config_path = Path(model_path)/'config.json'
model_name = str(model_path.split('/')[-1])

if os.path.isdir(model_name_or_path) or model_name_or_path.startswith("Local/"):
if os.path.isdir(model_name_or_path):
model_path = model_name_or_path.rstrip('/')
model_name = str(model_path.split('/')[-1])
else:
model_name = model_name_or_path.split("/")[-1]
exo_path = Path(Path.home()/".cache"/"exo")
model_path = f'{exo_path}/{model_name}'

model_config_path = Path(model_path)/'config.json'
print(model_config_path)
if os.path.isfile(model_config_path):
with open(model_config_path, 'r') as file:
config = json.load(file)
@@ -229,6 +229,15 @@ async def run_model_cli(node: Node, inference_engine: InferenceEngine, model_nam
else:
print(f"Error: Not Find Local model '{model_name}' for inference engine {inference_engine.__class__.__name__}")
return

elif model_cards.get(model_name_or_path):
inference_class = inference_engine.__class__.__name__
shard = build_base_shard(model_name_or_path, inference_class)
if not shard:
print(f"Error: exo Unsupported model '{model_name_or_path}' for inference engine {inference_engine.__class__.__name__}")
return
tokenizer = await resolve_tokenizer(get_repo(shard.model_id, inference_class))

else:
print(f"Error: Unsupported model '{model_name_or_path}' for inference engine {inference_engine.__class__.__name__}")
return