deepgeek/MiniCPM-o
2
0
Fork 0
mirror of https://github.com/OpenBMB/MiniCPM-V.git synced 2026-02-05 18:29:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Modify eval_mm for MiniCPM-o 2.6

Browse Source
This commit is contained in:
Poppy Xu
2025-01-21 15:34:54 +08:00
parent ec68cefc17
commit d8f382e157
82 changed files with 14279 additions and 843 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
eval_mm/vlmevalkit/vlmeval/api/__init__.py
View File

@@ -1,5 +1,5 @@
from .gpt import OpenAIWrapper, GPT4V
__all__ = [
'OpenAIWrapper', 'GPT4V'
'OpenAIWrapper', 'GPT4V',
]

44
eval_mm/vlmevalkit/vlmeval/api/base.py
View File

@@ -3,7 +3,7 @@ import random as rd
from abc import abstractmethod
import os.path as osp
import copy as cp
from ..smp import get_logger, parse_file, concat_images_vlmeval
from ..smp import get_logger, parse_file, concat_images_vlmeval, LMUDataRoot, md5, decode_base64_to_image_file
class BaseAPI:
@@ -143,7 +143,9 @@ class BaseAPI:
while len(inputs):
try:
return self.generate_inner(inputs, **kwargs)
except:
except Exception as e:
if self.verbose:
self.logger.info(f'{type(e)}: {e}')
inputs = inputs[1:]
while len(inputs) and inputs[0]['role'] != 'user':
inputs = inputs[1:]
@@ -179,19 +181,38 @@ class BaseAPI:
if not isinstance(log, str):
try:
log = log.text
except:
self.logger.warning(f'Failed to parse {log} as an http response. ')
except Exception as e:
self.logger.warning(f'Failed to parse {log} as an http response: {str(e)}. ')
self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
except Exception as err:
if self.verbose:
self.logger.error(f'An error occured during try {i}:')
self.logger.error(err)
self.logger.error(f'An error occured during try {i}: ')
self.logger.error(f'{type(err)}: {err}')
# delay before each retry
T = rd.random() * self.wait * 2
time.sleep(T)
return self.fail_msg if answer in ['', None] else answer
def preprocess_message_with_role(self, message):
system_prompt = ''
new_message = []
for data in message:
assert isinstance(data, dict)
role = data.pop('role', 'user')
if role == 'system':
system_prompt += data['value'] + '\n'
else:
new_message.append(data)
if system_prompt != '':
if self.system_prompt is None:
self.system_prompt = system_prompt
else:
self.system_prompt += '\n' + system_prompt
return new_message
def generate(self, message, **kwargs1):
"""The main function to generate the answer. Will call `generate_inner` with the preprocessed input messages.
@@ -201,6 +222,9 @@ class BaseAPI:
Returns:
str: The generated answer of the Failed Message if failed to obtain answer.
"""
if self.check_content(message) == 'listdict':
message = self.preprocess_message_with_role(message)
assert self.check_content(message) in ['str', 'dict', 'liststr', 'listdict'], f'Invalid input type: {message}'
message = self.preproc_content(message)
assert message is not None and self.check_content(message) == 'listdict'
@@ -227,13 +251,13 @@ class BaseAPI:
if not isinstance(log, str):
try:
log = log.text
except:
self.logger.warning(f'Failed to parse {log} as an http response. ')
except Exception as e:
self.logger.warning(f'Failed to parse {log} as an http response: {str(e)}. ')
self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
except Exception as err:
if self.verbose:
self.logger.error(f'An error occured during try {i}:')
self.logger.error(err)
self.logger.error(f'An error occured during try {i}: ')
self.logger.error(f'{type(err)}: {err}')
# delay before each retry
T = rd.random() * self.wait * 2
time.sleep(T)

51
eval_mm/vlmevalkit/vlmeval/api/gpt.py
View File

@@ -38,7 +38,7 @@ class OpenAIWrapper(BaseAPI):
retry: int = 5,
wait: int = 5,
key: str = None,
verbose: bool = True,
verbose: bool = False,
system_prompt: str = None,
temperature: float = 0,
timeout: int = 60,
@@ -56,7 +56,7 @@ class OpenAIWrapper(BaseAPI):
self.temperature = temperature
self.use_azure = use_azure
if 'step-1v' in model:
if 'step' in model:
env_key = os.environ.get('STEPAI_API_KEY', '')
if key is None:
key = env_key
@@ -64,6 +64,14 @@ class OpenAIWrapper(BaseAPI):
env_key = os.environ.get('YI_API_KEY', '')
if key is None:
key = env_key
elif 'internvl2-pro' in model:
env_key = os.environ.get('InternVL2_PRO_KEY', '')
if key is None:
key = env_key
elif 'abab' in model:
env_key = os.environ.get('MiniMax_API_KEY', '')
if key is None:
key = env_key
else:
if use_azure:
env_key = os.environ.get('AZURE_OPENAI_API_KEY', None)
@@ -124,7 +132,7 @@ class OpenAIWrapper(BaseAPI):
self.api_base = api_base
else:
self.logger.error('Unknown API Base. ')
sys.exit(-1)
raise NotImplementedError
self.logger.info(f'Using API Base: {self.api_base}; API Key: {self.key}')
@@ -169,19 +177,22 @@ class OpenAIWrapper(BaseAPI):
temperature = kwargs.pop('temperature', self.temperature)
max_tokens = kwargs.pop('max_tokens', self.max_tokens)
context_window = GPT_context_window(self.model)
max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
if 0 < max_tokens <= 100:
self.logger.warning(
'Less than 100 tokens left, '
'may exceed the context window with some additional meta symbols. '
)
if max_tokens <= 0:
return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
# context_window = GPT_context_window(self.model)
# new_max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
# if 0 < new_max_tokens <= 100 and new_max_tokens < max_tokens:
# self.logger.warning(
# 'Less than 100 tokens left, '
# 'may exceed the context window with some additional meta symbols. '
# )
# if new_max_tokens <= 0:
# return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
# max_tokens = new_max_tokens
# Will send request if use Azure, dk how to use openai client for it
if self.use_azure:
headers = {'Content-Type': 'application/json', 'api-key': self.key}
elif 'internvl2-pro' in self.model:
headers = {'Content-Type': 'application/json', 'Authorization': self.key}
else:
headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}
payload = dict(
@@ -200,8 +211,11 @@ class OpenAIWrapper(BaseAPI):
try:
resp_struct = json.loads(response.text)
answer = resp_struct['choices'][0]['message']['content'].strip()
except:
pass
except Exception as err:
if self.verbose:
self.logger.error(f'{type(err)}: {err}')
self.logger.error(response.text if hasattr(response, 'text') else response)
return ret_code, answer, response
def get_image_token_len(self, img_path, detail='low'):
@@ -228,8 +242,13 @@ class OpenAIWrapper(BaseAPI):
import tiktoken
try:
enc = tiktoken.encoding_for_model(self.model)
except:
enc = tiktoken.encoding_for_model('gpt-4')
except Exception as err:
if 'gpt' in self.model.lower():
if self.verbose:
self.logger.warning(f'{type(err)}: {err}')
enc = tiktoken.encoding_for_model('gpt-4')
else:
return 0
assert isinstance(inputs, list)
tot = 0
for item in inputs: