mirror of
https://github.com/OpenBMB/MiniCPM-V.git
synced 2026-02-05 18:29:18 +08:00
Add eval_mm dir
This commit is contained in:
trainfanlhy
6
eval_mm/vlmevalkit/vlmeval/api/__init__.py
Normal file
6
eval_mm/vlmevalkit/vlmeval/api/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from .gpt import OpenAIWrapper, GPT4V
|
||||
from .gpt_int import OpenAIWrapperInternal, GPT4V_Internal
|
||||
|
||||
__all__ = [
|
||||
'OpenAIWrapper', 'OpenAIWrapperInternal', 'GPT4V', 'GPT4V_Internal'
|
||||
]
|
||||
195
eval_mm/vlmevalkit/vlmeval/api/base.py
Normal file
195
eval_mm/vlmevalkit/vlmeval/api/base.py
Normal file
@@ -0,0 +1,195 @@
|
||||
import time
|
||||
import random as rd
|
||||
from abc import abstractmethod
|
||||
import os.path as osp
|
||||
import copy as cp
|
||||
from ..smp import get_logger, parse_file
|
||||
|
||||
|
||||
class BaseAPI:
|
||||
|
||||
allowed_types = ['text', 'image']
|
||||
INTERLEAVE = True
|
||||
INSTALL_REQ = False
|
||||
|
||||
def __init__(self,
|
||||
retry=10,
|
||||
wait=3,
|
||||
system_prompt=None,
|
||||
verbose=True,
|
||||
fail_msg='Failed to obtain answer via API.',
|
||||
**kwargs):
|
||||
"""Base Class for all APIs.
|
||||
|
||||
Args:
|
||||
retry (int, optional): The retry times for `generate_inner`. Defaults to 10.
|
||||
wait (int, optional): The wait time after each failed retry of `generate_inner`. Defaults to 3.
|
||||
system_prompt (str, optional): Defaults to None.
|
||||
verbose (bool, optional): Defaults to True.
|
||||
fail_msg (str, optional): The message to return when failed to obtain answer.
|
||||
Defaults to 'Failed to obtain answer via API.'.
|
||||
**kwargs: Other kwargs for `generate_inner`.
|
||||
"""
|
||||
|
||||
self.wait = wait
|
||||
self.retry = retry
|
||||
self.system_prompt = system_prompt
|
||||
self.verbose = verbose
|
||||
self.fail_msg = fail_msg
|
||||
self.logger = get_logger('ChatAPI')
|
||||
|
||||
if len(kwargs):
|
||||
self.logger.info(f'BaseAPI received the following kwargs: {kwargs}')
|
||||
self.logger.info('Will try to use them as kwargs for `generate`. ')
|
||||
self.default_kwargs = kwargs
|
||||
|
||||
@abstractmethod
|
||||
def generate_inner(self, inputs, **kwargs):
|
||||
"""The inner function to generate the answer.
|
||||
|
||||
Returns:
|
||||
tuple(int, str, str): ret_code, response, log
|
||||
"""
|
||||
self.logger.warning('For APIBase, generate_inner is an abstract method. ')
|
||||
assert 0, 'generate_inner not defined'
|
||||
ret_code, answer, log = None, None, None
|
||||
# if ret_code is 0, means succeed
|
||||
return ret_code, answer, log
|
||||
|
||||
def working(self):
|
||||
"""If the API model is working, return True, else return False.
|
||||
|
||||
Returns:
|
||||
bool: If the API model is working, return True, else return False.
|
||||
"""
|
||||
retry = 3
|
||||
while retry > 0:
|
||||
ret = self.generate('hello')
|
||||
if ret is not None and ret != '' and self.fail_msg not in ret:
|
||||
return True
|
||||
retry -= 1
|
||||
return False
|
||||
|
||||
def check_content(self, msgs):
|
||||
"""Check the content type of the input. Four types are allowed: str, dict, liststr, listdict.
|
||||
|
||||
Args:
|
||||
msgs: Raw input messages.
|
||||
|
||||
Returns:
|
||||
str: The message type.
|
||||
"""
|
||||
if isinstance(msgs, str):
|
||||
return 'str'
|
||||
if isinstance(msgs, dict):
|
||||
return 'dict'
|
||||
if isinstance(msgs, list):
|
||||
types = [self.check_content(m) for m in msgs]
|
||||
if all(t == 'str' for t in types):
|
||||
return 'liststr'
|
||||
if all(t == 'dict' for t in types):
|
||||
return 'listdict'
|
||||
return 'unknown'
|
||||
|
||||
def preproc_content(self, inputs):
|
||||
"""Convert the raw input messages to a list of dicts.
|
||||
|
||||
Args:
|
||||
inputs: raw input messages.
|
||||
|
||||
Returns:
|
||||
list(dict): The preprocessed input messages. Will return None if failed to preprocess the input.
|
||||
"""
|
||||
if self.check_content(inputs) == 'str':
|
||||
return [dict(type='text', value=inputs)]
|
||||
elif self.check_content(inputs) == 'dict':
|
||||
assert 'type' in inputs and 'value' in inputs
|
||||
return [inputs]
|
||||
elif self.check_content(inputs) == 'liststr':
|
||||
res = []
|
||||
for s in inputs:
|
||||
mime, pth = parse_file(s)
|
||||
if mime is None or mime == 'unknown':
|
||||
res.append(dict(type='text', value=s))
|
||||
else:
|
||||
res.append(dict(type=mime.split('/')[0], value=pth))
|
||||
return res
|
||||
elif self.check_content(inputs) == 'listdict':
|
||||
for item in inputs:
|
||||
assert 'type' in item and 'value' in item
|
||||
mime, s = parse_file(item['value'])
|
||||
if mime is None:
|
||||
assert item['type'] == 'text', item['value']
|
||||
else:
|
||||
assert mime.split('/')[0] == item['type']
|
||||
item['value'] = s
|
||||
return inputs
|
||||
else:
|
||||
return None
|
||||
|
||||
def generate(self, message, **kwargs1):
|
||||
"""The main function to generate the answer. Will call `generate_inner` with the preprocessed input messages.
|
||||
|
||||
Args:
|
||||
message: raw input messages.
|
||||
|
||||
Returns:
|
||||
str: The generated answer of the Failed Message if failed to obtain answer.
|
||||
"""
|
||||
assert self.check_content(message) in ['str', 'dict', 'liststr', 'listdict'], f'Invalid input type: {message}'
|
||||
message = self.preproc_content(message)
|
||||
assert message is not None and self.check_content(message) == 'listdict'
|
||||
for item in message:
|
||||
assert item['type'] in self.allowed_types, f'Invalid input type: {item["type"]}'
|
||||
|
||||
# merge kwargs
|
||||
kwargs = cp.deepcopy(self.default_kwargs)
|
||||
kwargs.update(kwargs1)
|
||||
|
||||
answer = None
|
||||
# a very small random delay [0s - 0.5s]
|
||||
T = rd.random() * 0.5
|
||||
time.sleep(T)
|
||||
|
||||
for i in range(self.retry):
|
||||
try:
|
||||
ret_code, answer, log = self.generate_inner(message, **kwargs)
|
||||
if ret_code == 0 and self.fail_msg not in answer and answer != '':
|
||||
if self.verbose:
|
||||
print(answer)
|
||||
return answer
|
||||
elif self.verbose:
|
||||
if not isinstance(log, str):
|
||||
try:
|
||||
log = log.text
|
||||
except:
|
||||
self.logger.warning(f'Failed to parse {log} as an http response. ')
|
||||
self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
|
||||
except Exception as err:
|
||||
if self.verbose:
|
||||
self.logger.error(f'An error occured during try {i}:')
|
||||
self.logger.error(err)
|
||||
# delay before each retry
|
||||
T = rd.random() * self.wait * 2
|
||||
time.sleep(T)
|
||||
|
||||
return self.fail_msg if answer in ['', None] else answer
|
||||
|
||||
def message_to_promptimg(self, message):
|
||||
assert not self.INTERLEAVE
|
||||
model_name = self.__class__.__name__
|
||||
import warnings
|
||||
warnings.warn(
|
||||
f'Model {model_name} does not support interleaved input. '
|
||||
'Will use the first image and aggregated texts as prompt. ')
|
||||
num_images = len([x for x in message if x['type'] == 'image'])
|
||||
if num_images == 0:
|
||||
prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
|
||||
image = None
|
||||
elif num_images == 1:
|
||||
prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
|
||||
image = [x['value'] for x in message if x['type'] == 'image'][0]
|
||||
else:
|
||||
prompt = '\n'.join([x['value'] if x['type'] == 'text' else '<image>' for x in message])
|
||||
image = [x['value'] for x in message if x['type'] == 'image'][0]
|
||||
return prompt, image
|
||||
178
eval_mm/vlmevalkit/vlmeval/api/gpt.py
Normal file
178
eval_mm/vlmevalkit/vlmeval/api/gpt.py
Normal file
@@ -0,0 +1,178 @@
|
||||
from ..smp import *
|
||||
import os
|
||||
import sys
|
||||
from .base import BaseAPI
|
||||
|
||||
APIBASES = {
|
||||
'OFFICIAL': 'https://api.openai.com/v1/chat/completions',
|
||||
}
|
||||
|
||||
|
||||
def GPT_context_window(model):
|
||||
length_map = {
|
||||
'gpt-4-1106-preview': 128000,
|
||||
'gpt-4-vision-preview': 128000,
|
||||
'gpt-4': 8192,
|
||||
'gpt-4-32k': 32768,
|
||||
'gpt-4-0613': 8192,
|
||||
'gpt-4-32k-0613': 32768,
|
||||
'gpt-3.5-turbo-1106': 16385,
|
||||
'gpt-3.5-turbo': 4096,
|
||||
'gpt-3.5-turbo-16k': 16385,
|
||||
'gpt-3.5-turbo-instruct': 4096,
|
||||
'gpt-3.5-turbo-0613': 4096,
|
||||
'gpt-3.5-turbo-16k-0613': 16385,
|
||||
}
|
||||
if model in length_map:
|
||||
return length_map[model]
|
||||
else:
|
||||
return 128000
|
||||
|
||||
|
||||
class OpenAIWrapper(BaseAPI):
|
||||
|
||||
is_api: bool = True
|
||||
|
||||
def __init__(self,
|
||||
model: str = 'gpt-3.5-turbo-0613',
|
||||
retry: int = 5,
|
||||
wait: int = 5,
|
||||
key: str = None,
|
||||
verbose: bool = True,
|
||||
system_prompt: str = None,
|
||||
temperature: float = 0,
|
||||
timeout: int = 60,
|
||||
api_base: str = None,
|
||||
max_tokens: int = 1024,
|
||||
img_size: int = 512,
|
||||
img_detail: str = 'low',
|
||||
**kwargs):
|
||||
|
||||
self.model = model
|
||||
self.cur_idx = 0
|
||||
self.fail_msg = 'Failed to obtain answer via API. '
|
||||
self.max_tokens = max_tokens
|
||||
self.temperature = temperature
|
||||
|
||||
if 'step-1v' in model:
|
||||
env_key = os.environ.get('STEPAI_API_KEY', '')
|
||||
if key is None:
|
||||
key = env_key
|
||||
else:
|
||||
env_key = os.environ.get('OPENAI_API_KEY', '')
|
||||
if key is None:
|
||||
key = env_key
|
||||
assert isinstance(key, str) and key.startswith('sk-'), (
|
||||
f'Illegal openai_key {key}. '
|
||||
'Please set the environment variable OPENAI_API_KEY to your openai key. '
|
||||
)
|
||||
self.key = key
|
||||
assert img_size > 0 or img_size == -1
|
||||
self.img_size = img_size
|
||||
assert img_detail in ['high', 'low']
|
||||
self.img_detail = img_detail
|
||||
self.timeout = timeout
|
||||
|
||||
super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
|
||||
|
||||
if api_base is None:
|
||||
if 'OPENAI_API_BASE' in os.environ and os.environ['OPENAI_API_BASE'] != '':
|
||||
self.logger.error('Environment variable OPENAI_API_BASE is set. Will use it as api_base. ')
|
||||
api_base = os.environ['OPENAI_API_BASE']
|
||||
else:
|
||||
api_base = 'OFFICIAL'
|
||||
|
||||
assert api_base is not None
|
||||
|
||||
if api_base in APIBASES:
|
||||
self.api_base = APIBASES[api_base]
|
||||
elif api_base.startswith('http'):
|
||||
self.api_base = api_base
|
||||
else:
|
||||
self.logger.error('Unknown API Base. ')
|
||||
sys.exit(-1)
|
||||
self.logger.info(f'Using API Base: {self.api_base}; API Key: {self.key}')
|
||||
|
||||
# inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
|
||||
# content can be a string or a list of image & text
|
||||
def prepare_inputs(self, inputs):
|
||||
input_msgs = []
|
||||
if self.system_prompt is not None:
|
||||
input_msgs.append(dict(role='system', content=self.system_prompt))
|
||||
has_images = np.sum([x['type'] == 'image' for x in inputs])
|
||||
if has_images:
|
||||
content_list = []
|
||||
for msg in inputs:
|
||||
if msg['type'] == 'text':
|
||||
content_list.append(dict(type='text', text=msg['value']))
|
||||
elif msg['type'] == 'image':
|
||||
from PIL import Image
|
||||
img = Image.open(msg['value'])
|
||||
b64 = encode_image_to_base64(img, target_size=self.img_size)
|
||||
img_struct = dict(url=f'data:image/jpeg;base64,{b64}', detail=self.img_detail)
|
||||
content_list.append(dict(type='image_url', image_url=img_struct))
|
||||
input_msgs.append(dict(role='user', content=content_list))
|
||||
else:
|
||||
assert all([x['type'] == 'text' for x in inputs])
|
||||
text = '\n'.join([x['value'] for x in inputs])
|
||||
input_msgs.append(dict(role='user', content=text))
|
||||
return input_msgs
|
||||
|
||||
def generate_inner(self, inputs, **kwargs) -> str:
|
||||
input_msgs = self.prepare_inputs(inputs)
|
||||
temperature = kwargs.pop('temperature', self.temperature)
|
||||
max_tokens = kwargs.pop('max_tokens', self.max_tokens)
|
||||
|
||||
context_window = GPT_context_window(self.model)
|
||||
max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
|
||||
if 0 < max_tokens <= 100:
|
||||
self.logger.warning(
|
||||
'Less than 100 tokens left, '
|
||||
'may exceed the context window with some additional meta symbols. '
|
||||
)
|
||||
if max_tokens <= 0:
|
||||
return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
|
||||
|
||||
headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}
|
||||
payload = dict(
|
||||
model=self.model,
|
||||
messages=input_msgs,
|
||||
max_tokens=max_tokens,
|
||||
n=1,
|
||||
temperature=temperature,
|
||||
**kwargs)
|
||||
response = requests.post(self.api_base, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
|
||||
ret_code = response.status_code
|
||||
ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
|
||||
answer = self.fail_msg
|
||||
try:
|
||||
resp_struct = json.loads(response.text)
|
||||
answer = resp_struct['choices'][0]['message']['content'].strip()
|
||||
except:
|
||||
pass
|
||||
return ret_code, answer, response
|
||||
|
||||
def get_token_len(self, inputs) -> int:
|
||||
import tiktoken
|
||||
try:
|
||||
enc = tiktoken.encoding_for_model(self.model)
|
||||
except:
|
||||
enc = tiktoken.encoding_for_model('gpt-4')
|
||||
assert isinstance(inputs, list)
|
||||
tot = 0
|
||||
for item in inputs:
|
||||
if item['type'] == 'text':
|
||||
tot += len(enc.encode(item['value']))
|
||||
elif item['type'] == 'image':
|
||||
tot += 85
|
||||
if self.img_detail == 'high':
|
||||
img = Image.open(item['value'])
|
||||
npatch = np.ceil(img.size[0] / 512) * np.ceil(img.size[1] / 512)
|
||||
tot += npatch * 170
|
||||
return tot
|
||||
|
||||
|
||||
class GPT4V(OpenAIWrapper):
|
||||
|
||||
def generate(self, message, dataset=None):
|
||||
return super(GPT4V, self).generate(message)
|
||||
90
eval_mm/vlmevalkit/vlmeval/api/gpt_int.py
Normal file
90
eval_mm/vlmevalkit/vlmeval/api/gpt_int.py
Normal file
@@ -0,0 +1,90 @@
|
||||
import json
|
||||
import warnings
|
||||
import requests
|
||||
from ..smp import *
|
||||
from .gpt import GPT_context_window, OpenAIWrapper
|
||||
|
||||
url = 'http://ecs.sv.us.alles-apin.openxlab.org.cn/v1/openai/v2/text/chat'
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
|
||||
class OpenAIWrapperInternal(OpenAIWrapper):
|
||||
|
||||
is_api: bool = True
|
||||
|
||||
def __init__(self,
|
||||
model: str = 'gpt-3.5-turbo-0613',
|
||||
retry: int = 5,
|
||||
wait: int = 3,
|
||||
verbose: bool = True,
|
||||
system_prompt: str = None,
|
||||
temperature: float = 0,
|
||||
timeout: int = 60,
|
||||
max_tokens: int = 1024,
|
||||
img_size: int = 512,
|
||||
img_detail: str = 'low',
|
||||
**kwargs):
|
||||
|
||||
self.model = model
|
||||
if 'KEYS' in os.environ and osp.exists(os.environ['KEYS']):
|
||||
keys = load(os.environ['KEYS'])
|
||||
headers['alles-apin-token'] = keys.get('alles-apin-token', '')
|
||||
elif 'ALLES' in os.environ:
|
||||
headers['alles-apin-token'] = os.environ['ALLES']
|
||||
self.headers = headers
|
||||
self.temperature = temperature
|
||||
self.timeout = timeout
|
||||
self.max_tokens = max_tokens
|
||||
|
||||
assert img_size > 0 or img_size == -1
|
||||
self.img_size = img_size
|
||||
assert img_detail in ['high', 'low']
|
||||
self.img_detail = img_detail
|
||||
|
||||
super(OpenAIWrapper, self).__init__(
|
||||
wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
|
||||
|
||||
def generate_inner(self, inputs, **kwargs) -> str:
|
||||
input_msgs = self.prepare_inputs(inputs)
|
||||
|
||||
temperature = kwargs.pop('temperature', self.temperature)
|
||||
max_tokens = kwargs.pop('max_tokens', self.max_tokens)
|
||||
|
||||
# Held out 100 tokens as buffer
|
||||
context_window = GPT_context_window(self.model)
|
||||
max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
|
||||
if 0 < max_tokens <= 100:
|
||||
print('Less than 100 tokens left, may exceed the context window with some additional meta symbols. ')
|
||||
if max_tokens <= 0:
|
||||
return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
|
||||
|
||||
payload = dict(
|
||||
model=self.model,
|
||||
messages=input_msgs,
|
||||
max_tokens=max_tokens,
|
||||
n=1,
|
||||
stop=None,
|
||||
timeout=self.timeout,
|
||||
temperature=temperature,
|
||||
**kwargs)
|
||||
|
||||
response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
|
||||
ret_code = response.status_code
|
||||
ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
|
||||
|
||||
answer = self.fail_msg
|
||||
try:
|
||||
resp_struct = json.loads(response.text)
|
||||
assert resp_struct['msg'] == 'ok' and resp_struct['msgCode'] == '10000', resp_struct
|
||||
answer = resp_struct['data']['choices'][0]['message']['content'].strip()
|
||||
except:
|
||||
pass
|
||||
return ret_code, answer, response
|
||||
|
||||
|
||||
class GPT4V_Internal(OpenAIWrapperInternal):
|
||||
|
||||
def generate(self, message, dataset=None):
|
||||
return super(GPT4V_Internal, self).generate(message)
|
||||
Reference in New Issue
Block a user