deepgeek/MiniCPM-o
2
0
Fork 0
mirror of https://github.com/OpenBMB/MiniCPM-V.git synced 2026-02-05 10:19:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

combine dtype and device to save CPU memory

Browse Source
This commit is contained in:
EC2 Default User
2024-05-27 05:35:21 +00:00
parent 2b572c9221
commit 5b67e5c822
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
web_demo_2.5.py
View File

@@ -31,8 +31,7 @@ if 'int4' in model_path:
exit() exit()
model = AutoModel.from_pretrained(model_path, trust_remote_code=True) model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
else: else:
model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(dtype=torch.float16) model = AutoModel.from_pretrained(model_path, trust_remote_code=True, torch_dtype=torch.float16, device_map=device)
model = model.to(device=device)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model.eval() model.eval()