Modify eval_mm for MiniCPM-V 2.6

eval_mm/vlmevalkit/vlmeval/api/__init__.py

@@ -1,6 +1,5 @@
 from .gpt import OpenAIWrapper, GPT4V
-from .gpt_int import OpenAIWrapperInternal, GPT4V_Internal
 
 __all__ = [
-    'OpenAIWrapper', 'OpenAIWrapperInternal', 'GPT4V', 'GPT4V_Internal'
+    'OpenAIWrapper', 'GPT4V'
 ]

eval_mm/vlmevalkit/vlmeval/api/base.py

@@ -3,7 +3,7 @@ import random as rd
 from abc import abstractmethod
 import os.path as osp
 import copy as cp
-from ..smp import get_logger, parse_file
+from ..smp import get_logger, parse_file, concat_images_vlmeval
 
 
 class BaseAPI:
@@ -62,12 +62,22 @@ class BaseAPI:
         Returns:
             bool: If the API model is working, return True, else return False.
         """
-        retry = 3
+        self.old_timeout = None
+        if hasattr(self, 'timeout'):
+            self.old_timeout = self.timeout
+            self.timeout = 120
+
+        retry = 5
         while retry > 0:
             ret = self.generate('hello')
             if ret is not None and ret != '' and self.fail_msg not in ret:
+                if self.old_timeout is not None:
+                    self.timeout = self.old_timeout
                 return True
             retry -= 1
+
+        if self.old_timeout is not None:
+            self.timeout = self.old_timeout
         return False
 
     def check_content(self, msgs):
@@ -127,6 +137,61 @@ class BaseAPI:
         else:
             return None
 
+    # May exceed the context windows size, so try with different turn numbers.
+    def chat_inner(self, inputs, **kwargs):
+        _ = kwargs.pop('dataset', None)
+        while len(inputs):
+            try:
+                return self.generate_inner(inputs, **kwargs)
+            except:
+                inputs = inputs[1:]
+                while len(inputs) and inputs[0]['role'] != 'user':
+                    inputs = inputs[1:]
+                continue
+        return -1, self.fail_msg + ': ' + 'Failed with all possible conversation turns.', None
+
+    def chat(self, messages, **kwargs1):
+        """The main function for multi-turn chatting. Will call `chat_inner` with the preprocessed input messages."""
+        assert hasattr(self, 'chat_inner'), 'The API model should has the `chat_inner` method. '
+        for msg in messages:
+            assert isinstance(msg, dict) and 'role' in msg and 'content' in msg, msg
+            assert self.check_content(msg['content']) in ['str', 'dict', 'liststr', 'listdict'], msg
+            msg['content'] = self.preproc_content(msg['content'])
+        # merge kwargs
+        kwargs = cp.deepcopy(self.default_kwargs)
+        kwargs.update(kwargs1)
+
+        answer = None
+        # a very small random delay [0s - 0.5s]
+        T = rd.random() * 0.5
+        time.sleep(T)
+
+        assert messages[-1]['role'] == 'user'
+
+        for i in range(self.retry):
+            try:
+                ret_code, answer, log = self.chat_inner(messages, **kwargs)
+                if ret_code == 0 and self.fail_msg not in answer and answer != '':
+                    if self.verbose:
+                        print(answer)
+                    return answer
+                elif self.verbose:
+                    if not isinstance(log, str):
+                        try:
+                            log = log.text
+                        except:
+                            self.logger.warning(f'Failed to parse {log} as an http response. ')
+                    self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
+            except Exception as err:
+                if self.verbose:
+                    self.logger.error(f'An error occured during try {i}:')
+                    self.logger.error(err)
+            # delay before each retry
+            T = rd.random() * self.wait * 2
+            time.sleep(T)
+
+        return self.fail_msg if answer in ['', None] else answer
+
     def generate(self, message, **kwargs1):
         """The main function to generate the answer. Will call `generate_inner` with the preprocessed input messages.
 
@@ -175,7 +240,7 @@ class BaseAPI:
 
         return self.fail_msg if answer in ['', None] else answer
 
-    def message_to_promptimg(self, message):
+    def message_to_promptimg(self, message, dataset=None):
         assert not self.INTERLEAVE
         model_name = self.__class__.__name__
         import warnings
@@ -191,5 +256,10 @@ class BaseAPI:
             image = [x['value'] for x in message if x['type'] == 'image'][0]
         else:
             prompt = '\n'.join([x['value'] if x['type'] == 'text' else '<image>' for x in message])
-            image = [x['value'] for x in message if x['type'] == 'image'][0]
+            if dataset == 'BLINK':
+                image = concat_images_vlmeval(
+                    [x['value'] for x in message if x['type'] == 'image'],
+                    target_size=512)
+            else:
+                image = [x['value'] for x in message if x['type'] == 'image'][0]
         return prompt, image

eval_mm/vlmevalkit/vlmeval/api/gpt.py

@@ -10,18 +10,18 @@ APIBASES = {
 
 def GPT_context_window(model):
     length_map = {
-        'gpt-4-1106-preview': 128000,
-        'gpt-4-vision-preview': 128000,
         'gpt-4': 8192,
-        'gpt-4-32k': 32768,
         'gpt-4-0613': 8192,
-        'gpt-4-32k-0613': 32768,
+        'gpt-4-turbo-preview': 128000,
+        'gpt-4-1106-preview': 128000,
+        'gpt-4-0125-preview': 128000,
+        'gpt-4-vision-preview': 128000,
+        'gpt-4-turbo': 128000,
+        'gpt-4-turbo-2024-04-09': 128000,
+        'gpt-3.5-turbo': 16385,
+        'gpt-3.5-turbo-0125': 16385,
         'gpt-3.5-turbo-1106': 16385,
-        'gpt-3.5-turbo': 4096,
-        'gpt-3.5-turbo-16k': 16385,
         'gpt-3.5-turbo-instruct': 4096,
-        'gpt-3.5-turbo-0613': 4096,
-        'gpt-3.5-turbo-16k-0613': 16385,
     }
     if model in length_map:
         return length_map[model]
@@ -46,6 +46,7 @@ class OpenAIWrapper(BaseAPI):
                  max_tokens: int = 1024,
                  img_size: int = 512,
                  img_detail: str = 'low',
+                 use_azure: bool = False,
                  **kwargs):
 
         self.model = model
@@ -53,19 +54,35 @@ class OpenAIWrapper(BaseAPI):
         self.fail_msg = 'Failed to obtain answer via API. '
         self.max_tokens = max_tokens
         self.temperature = temperature
+        self.use_azure = use_azure
 
         if 'step-1v' in model:
             env_key = os.environ.get('STEPAI_API_KEY', '')
             if key is None:
                 key = env_key
-        else:
-            env_key = os.environ.get('OPENAI_API_KEY', '')
+        elif 'yi-vision' in model:
+            env_key = os.environ.get('YI_API_KEY', '')
             if key is None:
                 key = env_key
-            assert isinstance(key, str) and key.startswith('sk-'), (
-                f'Illegal openai_key {key}. '
-                'Please set the environment variable OPENAI_API_KEY to your openai key. '
-            )
+        else:
+            if use_azure:
+                env_key = os.environ.get('AZURE_OPENAI_API_KEY', None)
+                assert env_key is not None, 'Please set the environment variable AZURE_OPENAI_API_KEY. '
+
+                if key is None:
+                    key = env_key
+                assert isinstance(key, str), (
+                    'Please set the environment variable AZURE_OPENAI_API_KEY to your openai key. '
+                )
+            else:
+                env_key = os.environ.get('OPENAI_API_KEY', '')
+                if key is None:
+                    key = env_key
+                assert isinstance(key, str) and key.startswith('sk-'), (
+                    f'Illegal openai_key {key}. '
+                    'Please set the environment variable OPENAI_API_KEY to your openai key. '
+                )
+
         self.key = key
         assert img_size > 0 or img_size == -1
         self.img_size = img_size
@@ -75,30 +92,46 @@ class OpenAIWrapper(BaseAPI):
 
         super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
 
-        if api_base is None:
-            if 'OPENAI_API_BASE' in os.environ and os.environ['OPENAI_API_BASE'] != '':
-                self.logger.error('Environment variable OPENAI_API_BASE is set. Will use it as api_base. ')
-                api_base = os.environ['OPENAI_API_BASE']
-            else:
-                api_base = 'OFFICIAL'
+        if use_azure:
+            api_base_template = (
+                '{endpoint}openai/deployments/{deployment_name}/chat/completions?api-version={api_version}'
+            )
+            endpoint = os.getenv('AZURE_OPENAI_ENDPOINT', None)
+            assert endpoint is not None, 'Please set the environment variable AZURE_OPENAI_ENDPOINT. '
+            deployment_name = os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME', None)
+            assert deployment_name is not None, 'Please set the environment variable AZURE_OPENAI_DEPLOYMENT_NAME. '
+            api_version = os.getenv('OPENAI_API_VERSION', None)
+            assert api_version is not None, 'Please set the environment variable OPENAI_API_VERSION. '
 
-        assert api_base is not None
-
-        if api_base in APIBASES:
-            self.api_base = APIBASES[api_base]
-        elif api_base.startswith('http'):
-            self.api_base = api_base
+            self.api_base = api_base_template.format(
+                endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),
+                deployment_name=os.getenv('AZURE_OPENAI_DEPLOYMENT_NAME'),
+                api_version=os.getenv('OPENAI_API_VERSION')
+            )
         else:
-            self.logger.error('Unknown API Base. ')
-            sys.exit(-1)
+            if api_base is None:
+                if 'OPENAI_API_BASE' in os.environ and os.environ['OPENAI_API_BASE'] != '':
+                    self.logger.info('Environment variable OPENAI_API_BASE is set. Will use it as api_base. ')
+                    api_base = os.environ['OPENAI_API_BASE']
+                else:
+                    api_base = 'OFFICIAL'
+
+            assert api_base is not None
+
+            if api_base in APIBASES:
+                self.api_base = APIBASES[api_base]
+            elif api_base.startswith('http'):
+                self.api_base = api_base
+            else:
+                self.logger.error('Unknown API Base. ')
+                sys.exit(-1)
+
         self.logger.info(f'Using API Base: {self.api_base}; API Key: {self.key}')
 
     # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
     # content can be a string or a list of image & text
-    def prepare_inputs(self, inputs):
-        input_msgs = []
-        if self.system_prompt is not None:
-            input_msgs.append(dict(role='system', content=self.system_prompt))
+    def prepare_itlist(self, inputs):
+        assert np.all([isinstance(x, dict) for x in inputs])
         has_images = np.sum([x['type'] == 'image' for x in inputs])
         if has_images:
             content_list = []
@@ -111,11 +144,24 @@ class OpenAIWrapper(BaseAPI):
                     b64 = encode_image_to_base64(img, target_size=self.img_size)
                     img_struct = dict(url=f'data:image/jpeg;base64,{b64}', detail=self.img_detail)
                     content_list.append(dict(type='image_url', image_url=img_struct))
-            input_msgs.append(dict(role='user', content=content_list))
         else:
             assert all([x['type'] == 'text' for x in inputs])
             text = '\n'.join([x['value'] for x in inputs])
-            input_msgs.append(dict(role='user', content=text))
+            content_list = [dict(type='text', text=text)]
+        return content_list
+
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        if self.system_prompt is not None:
+            input_msgs.append(dict(role='system', content=self.system_prompt))
+        assert isinstance(inputs, list) and isinstance(inputs[0], dict)
+        assert np.all(['type' in x for x in inputs]) or np.all(['role' in x for x in inputs]), inputs
+        if 'role' in inputs[0]:
+            assert inputs[-1]['role'] == 'user', inputs[-1]
+            for item in inputs:
+                input_msgs.append(dict(role=item['role'], content=self.prepare_itlist(item['content'])))
+        else:
+            input_msgs.append(dict(role='user', content=self.prepare_itlist(inputs)))
         return input_msgs
 
     def generate_inner(self, inputs, **kwargs) -> str:
@@ -133,7 +179,11 @@ class OpenAIWrapper(BaseAPI):
         if max_tokens <= 0:
             return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
 
-        headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}
+        # Will send request if use Azure, dk how to use openai client for it
+        if self.use_azure:
+            headers = {'Content-Type': 'application/json', 'api-key': self.key}
+        else:
+            headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.key}'}
         payload = dict(
             model=self.model,
             messages=input_msgs,
@@ -141,7 +191,9 @@ class OpenAIWrapper(BaseAPI):
             n=1,
             temperature=temperature,
             **kwargs)
-        response = requests.post(self.api_base, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
+        response = requests.post(
+            self.api_base,
+            headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
         ret_code = response.status_code
         ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
         answer = self.fail_msg
@@ -152,6 +204,26 @@ class OpenAIWrapper(BaseAPI):
             pass
         return ret_code, answer, response
 
+    def get_image_token_len(self, img_path, detail='low'):
+        import math
+        if detail == 'low':
+            return 85
+
+        im = Image.open(img_path)
+        height, width = im.size
+        if width > 1024 or height > 1024:
+            if width > height:
+                height = int(height * 1024 / width)
+                width = 1024
+            else:
+                width = int(width * 1024 / height)
+                height = 1024
+
+        h = math.ceil(height / 512)
+        w = math.ceil(width / 512)
+        total = 85 + 170 * h * w
+        return total
+
     def get_token_len(self, inputs) -> int:
         import tiktoken
         try:
@@ -161,18 +233,16 @@ class OpenAIWrapper(BaseAPI):
         assert isinstance(inputs, list)
         tot = 0
         for item in inputs:
-            if item['type'] == 'text':
+            if 'role' in item:
+                tot += self.get_token_len(item['content'])
+            elif item['type'] == 'text':
                 tot += len(enc.encode(item['value']))
             elif item['type'] == 'image':
-                tot += 85
-                if self.img_detail == 'high':
-                    img = Image.open(item['value'])
-                    npatch = np.ceil(img.size[0] / 512) * np.ceil(img.size[1] / 512)
-                    tot += npatch * 170
+                tot += self.get_image_token_len(item['value'], detail=self.img_detail)
         return tot
 
 
 class GPT4V(OpenAIWrapper):
 
     def generate(self, message, dataset=None):
-        return super(GPT4V, self).generate(message)
+        return super(GPT4V, self).generate(message)

eval_mm/vlmevalkit/vlmeval/api/gpt_int.py

@@ -1,90 +0,0 @@
-import json
-import warnings
-import requests
-from ..smp import *
-from .gpt import GPT_context_window, OpenAIWrapper
-
-url = 'http://ecs.sv.us.alles-apin.openxlab.org.cn/v1/openai/v2/text/chat'
-headers = {
-    'Content-Type': 'application/json'
-}
-
-
-class OpenAIWrapperInternal(OpenAIWrapper):
-
-    is_api: bool = True
-
-    def __init__(self,
-                 model: str = 'gpt-3.5-turbo-0613',
-                 retry: int = 5,
-                 wait: int = 3,
-                 verbose: bool = True,
-                 system_prompt: str = None,
-                 temperature: float = 0,
-                 timeout: int = 60,
-                 max_tokens: int = 1024,
-                 img_size: int = 512,
-                 img_detail: str = 'low',
-                 **kwargs):
-
-        self.model = model
-        if 'KEYS' in os.environ and osp.exists(os.environ['KEYS']):
-            keys = load(os.environ['KEYS'])
-            headers['alles-apin-token'] = keys.get('alles-apin-token', '')
-        elif 'ALLES' in os.environ:
-            headers['alles-apin-token'] = os.environ['ALLES']
-        self.headers = headers
-        self.temperature = temperature
-        self.timeout = timeout
-        self.max_tokens = max_tokens
-
-        assert img_size > 0 or img_size == -1
-        self.img_size = img_size
-        assert img_detail in ['high', 'low']
-        self.img_detail = img_detail
-
-        super(OpenAIWrapper, self).__init__(
-            wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
-
-    def generate_inner(self, inputs, **kwargs) -> str:
-        input_msgs = self.prepare_inputs(inputs)
-
-        temperature = kwargs.pop('temperature', self.temperature)
-        max_tokens = kwargs.pop('max_tokens', self.max_tokens)
-
-        # Held out 100 tokens as buffer
-        context_window = GPT_context_window(self.model)
-        max_tokens = min(max_tokens, context_window - self.get_token_len(inputs))
-        if 0 < max_tokens <= 100:
-            print('Less than 100 tokens left, may exceed the context window with some additional meta symbols. ')
-        if max_tokens <= 0:
-            return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
-
-        payload = dict(
-            model=self.model,
-            messages=input_msgs,
-            max_tokens=max_tokens,
-            n=1,
-            stop=None,
-            timeout=self.timeout,
-            temperature=temperature,
-            **kwargs)
-
-        response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
-        ret_code = response.status_code
-        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
-
-        answer = self.fail_msg
-        try:
-            resp_struct = json.loads(response.text)
-            assert resp_struct['msg'] == 'ok' and resp_struct['msgCode'] == '10000', resp_struct
-            answer = resp_struct['data']['choices'][0]['message']['content'].strip()
-        except:
-            pass
-        return ret_code, answer, response
-
-
-class GPT4V_Internal(OpenAIWrapperInternal):
-
-    def generate(self, message, dataset=None):
-        return super(GPT4V_Internal, self).generate(message)