deepgeek/MiniCPM-o
2
0
Fork 0
mirror of https://github.com/OpenBMB/MiniCPM-V.git synced 2026-02-04 17:59:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix web_demo_2.5 for int4

Browse Source
This commit is contained in:
Hongji Zhu
2024-05-20 19:05:17 +08:00
parent 00e8d4ac33
commit 7f9030254a
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
web_demo_2.5.py
View File

@@ -25,10 +25,15 @@ assert device in ['cuda', 'mps']
# Load model
model_path = 'openbmb/MiniCPM-Llama3-V-2_5'
model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(dtype=torch.float16)
if 'int4' in model_path:
if device == 'mps':
print('Error: running int4 model with bitsandbytes on Mac is not supported right now.')
exit()
model = AutoModel.from_pretrained(model_path, trust_remote_code=True)
else:
model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(dtype=torch.float16)
model = model.to(device=device)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = model.to(device=device)
model.eval()