deepgeek/MiniCPM-o
2
0
Fork 0
mirror of https://github.com/OpenBMB/MiniCPM-V.git synced 2026-02-05 18:29:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #153 from whn09/main

Browse Source 
combine dtype and device to save CPU memory
This commit is contained in:
Hongji Zhu
2024-05-31 11:23:13 +08:00
committed by GitHub
parent d44bc28f86 5b67e5c822
commit fe7184f8c9
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
web_demo_2.5.py
View File

@@ -31,8 +31,7 @@ if 'int4' in model_path:
exit() exit()
model = AutoModel.from_pretrained(model_path, trust_remote_code=True) model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
else: else:
model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(dtype=torch.float16) model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.float16, device_map=device)
model = model.to(device=device)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model.eval() model.eval()