deepgeek/MiniCPM-o
2
0
Fork 0
mirror of https://github.com/OpenBMB/MiniCPM-V.git synced 2026-02-05 18:29:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Modify eval_mm for MiniCPM-o 2.6

Browse Source
This commit is contained in:
Poppy Xu
2025-01-21 15:34:54 +08:00
parent ec68cefc17
commit d8f382e157
82 changed files with 14279 additions and 843 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
eval_mm/vlmevalkit/vlmeval/api/gpt.py
View File

@@ -38,7 +38,7 @@ class OpenAIWrapper(BaseAPI):
retry: int = 5,
wait: int = 5,
key: str = None,
verbose: bool = True,
verbose: bool = False,
system_prompt: str = None,
temperature: float = 0,
timeout: int = 60,
@@ -56,7 +56,7 @@ class OpenAIWrapper(BaseAPI):
self.temperature = temperature
self.use_azure = use_azure
if 'step-1v' in model:
if 'step' in model:
env_key = os.environ.get('STEPAI_API_KEY', '')
if key is None:
key = env_key
@@ -64,6 +64,14 @@ class OpenAIWrapper(BaseAPI):
env_key = os.environ.get('YI_API_KEY', '')
if key is None:
key = env_key
elif 'internvl2-pro' in model:
env_key = os.environ.get('InternVL2_PRO_KEY', '')
if key is None:
key = env_key
elif 'abab' in model:
env_key = os.environ.get('MiniMax_API_KEY', '')
if key is None:
key = env_key
else:
if use_azure:
env_key = os.environ.get('AZURE_OPENAI_API_KEY', None)
@@ -124,7 +132,7 @@ class OpenAIWrapper(BaseAPI):
self.api_base = api_base
else:
self.logger.error('Unknown API Base. ')
sys.exit(-1)
raise NotImplementedError
self.logger.info(f'Using API Base: {self.api_base}; API Key: {self.key}')
@@ -169,19 +177,22 @@ class OpenAIWrapper(BaseAPI):
temperature = kwargs.pop('temperature', self.temperature)
max_tokens = kwargs.pop('max_tokens', self.max_tokens)
context_window = GPT_context_window(self.model)
max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
if 0 < max_tokens <= 100:
self.logger.warning(
'Less than 100 tokens left, '
'may exceed the context window with some additional meta symbols. '
)
if max_tokens <= 0:
return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
# context_window = GPT_context_window(self.model)
# new_max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
# if 0 < new_max_tokens <= 100 and new_max_tokens < max_tokens:
# self.logger.warning(
# 'Less than 100 tokens left, '
# 'may exceed the context window with some additional meta symbols. '
# )
# if new_max_tokens <= 0:
# return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
# max_tokens = new_max_tokens
# Will send request if use Azure, dk how to use openai client for it
if self.use_azure:
headers = {'Content-Type': 'application/json', 'api-key': self.key}
elif 'internvl2-pro' in self.model:
headers = {'Content-Type': 'application/json', 'Authorization': self.key}
else:
headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}
payload = dict(
@@ -200,8 +211,11 @@ class OpenAIWrapper(BaseAPI):
try:
resp_struct = json.loads(response.text)
answer = resp_struct['choices'][0]['message']['content'].strip()
except:
pass
except Exception as err:
if self.verbose:
self.logger.error(f'{type(err)}: {err}')
self.logger.error(response.text if hasattr(response, 'text') else response)
return ret_code, answer, response
def get_image_token_len(self, img_path, detail='low'):
@@ -228,8 +242,13 @@ class OpenAIWrapper(BaseAPI):
import tiktoken
try:
enc = tiktoken.encoding_for_model(self.model)
except:
enc = tiktoken.encoding_for_model('gpt-4')
except Exception as err:
if 'gpt' in self.model.lower():
if self.verbose:
self.logger.warning(f'{type(err)}: {err}')
enc = tiktoken.encoding_for_model('gpt-4')
else:
return 0
assert isinstance(inputs, list)
tot = 0
for item in inputs: