Modify eval_mm for MiniCPM-o 2.6

eval_mm/vlmevalkit/vlmeval/smp/file.py

@@ -74,6 +74,20 @@ def LMUDataRoot():
     return root
 
 
+def HFCacheRoot():
+    cache_list = ['HUGGINGFACE_HUB_CACHE', 'HF_HOME']
+    for cache_name in cache_list:
+        if cache_name in os.environ and osp.exists(os.environ[cache_name]):
+            if os.environ[cache_name].split('/')[-1] == 'hub':
+                return os.environ[cache_name]
+            else:
+                return osp.join(os.environ[cache_name], 'hub')
+    home = osp.expanduser('~')
+    root = osp.join(home, '.cache', 'huggingface', 'hub')
+    os.makedirs(root, exist_ok=True)
+    return root
+
+
 def MMBenchOfficialServer(dataset_name):
     root = LMUDataRoot()
 
@@ -190,20 +204,20 @@ def download_file(url, filename=None):
     if filename is None:
         filename = url.split('/')[-1]
 
-    # If HF_ENDPOINT is set, replace huggingface.co with it
-    if 'huggingface.co' in url and os.environ.get('HF_ENDPOINT', '') != '':
-        url = url.replace('huggingface.co', os.environ['HF_ENDPOINT'].split('://')[1])
-
     try:
         with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
             urllib.request.urlretrieve(url, filename=filename, reporthook=t.update_to)
-    except:
+    except Exception as e:
+        import logging
+        logging.warning(f'{type(e)}: {e}')
         # Handle Failed Downloads from huggingface.co
         if 'huggingface.co' in url:
             url_new = url.replace('huggingface.co', 'hf-mirror.com')
             try:
-                os.system(f'wget {url_new} -O {filename}')
-            except:
+                download_file(url_new, filename)
+                return filename
+            except Exception as e:
+                logging.warning(f'{type(e)}: {e}')
                 raise Exception(f'Failed to download {url}')
         else:
             raise Exception(f'Failed to download {url}')
@@ -286,6 +300,18 @@ def parse_file(s):
         suffix = osp.splitext(s)[1].lower()
         mime = mimetypes.types_map.get(suffix, 'unknown')
         return (mime, s)
+    elif s.startswith('data:image/'):
+        # To be compatible with OPENAI base64 format
+        content = s[11:]
+        mime = content.split(';')[0]
+        content = ';'.join(content.split(';')[1:])
+        dname = osp.join(LMUDataRoot(), 'files')
+        assert content.startswith('base64,')
+        b64 = content[7:]
+        os.makedirs(dname, exist_ok=True)
+        tgt = osp.join(dname, md5(b64) + '.png')
+        decode_base64_to_image_file(b64, tgt)
+        return parse_file(tgt)
     elif validators.url(s):
         suffix = osp.splitext(s)[1].lower()
         if suffix in mimetypes.types_map:

eval_mm/vlmevalkit/vlmeval/smp/log.py

@@ -1,4 +1,7 @@
 import logging
+logging.basicConfig(
+    format='[%(asctime)s] %(levelname)s - %(filename)s: %(funcName)s - %(lineno)d: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S')
 
 logger_initialized = {}
 
@@ -29,7 +32,7 @@ def get_logger(name, log_file=None, log_level=logging.INFO, file_mode='w'):
         handlers.append(file_handler)
 
     formatter = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        '[%(asctime)s] %(levelname)s - %(name)s - %(filename)s: %(funcName)s - %(lineno)d: %(message)s')
     for handler in handlers:
         handler.setFormatter(formatter)
         handler.setLevel(log_level)

eval_mm/vlmevalkit/vlmeval/smp/misc.py

@@ -5,13 +5,13 @@ import csv
 import multiprocessing as mp
 import os
 import os.path as osp
+from pathlib import Path
 import copy as cp
 import random as rd
 import requests
 import shutil
 import subprocess
 import warnings
-import logging
 import pandas as pd
 from collections import OrderedDict, defaultdict
 from multiprocessing import Pool, current_process
@@ -21,8 +21,14 @@ import matplotlib.pyplot as plt
 from tabulate import tabulate
 from json import JSONDecoder
 from huggingface_hub import scan_cache_dir
+from huggingface_hub.utils._cache_manager import _scan_cached_repo
 from sty import fg, bg, ef, rs
 
+
+def modelscope_flag_set():
+    return os.environ.get('VLMEVALKIT_USE_MODELSCOPE', None) in ['1', 'True']
+
+
 def process_punctuation(inText):
     import re
     outText = inText
@@ -71,26 +77,30 @@ def bincount(lst):
         bins[item] += 1
     return bins
 
-def get_cache_path(repo_id, branch=None):
-    hf_cache_info = scan_cache_dir()
-    repos = list(hf_cache_info.repos)
-    repo = None
-    for r in repos:
-        if r.repo_id == repo_id:
-            repo = r
-            break
-    if repo is None:
+def get_cache_path(repo_id, branch='main', repo_type='datasets'):
+    try:
+        if modelscope_flag_set():
+            from modelscope.hub.file_download import create_temporary_directory_and_cache
+            if repo_type == 'datasets':
+                repo_type = 'dataset'
+            _, cache = create_temporary_directory_and_cache(model_id=repo_id, repo_type=repo_type)
+            cache_path = cache.get_root_location()
+            return cache_path
+        else:
+            from .file import HFCacheRoot
+            cache_path = HFCacheRoot()
+            org, repo_name = repo_id.split('/')
+            repo_path = Path(osp.join(cache_path, f'{repo_type}--{org}--{repo_name}/'))
+            hf_cache_info = _scan_cached_repo(repo_path=repo_path)
+            revs = {r.refs: r for r in hf_cache_info.revisions}
+            if branch is not None:
+                revs = {refs: r for refs, r in revs.items() if branch in refs}
+            rev2keep = max(revs.values(), key=lambda r: r.last_modified)
+            return str(rev2keep.snapshot_path)
+    except Exception as e:
+        import logging
+        logging.warning(f'{type(e)}: {e}')
         return None
-    revs = list(repo.revisions)
-    if branch is not None:
-        revs = [r for r in revs if r.refs == frozenset({branch})]
-    rev2keep, last_modified = None, 0
-    for rev in revs:
-        if rev.last_modified > last_modified:
-            rev2keep, last_modified = rev, rev.last_modified
-    if rev2keep is None:
-        return None
-    return str(rev2keep.snapshot_path)
 
 def proxy_set(s):
     import os
@@ -126,14 +136,47 @@ try:
 except ImportError:
     pass
 
-def timestr(second=True, minute=False):
-    s = datetime.datetime.now().strftime('%Y%m%d%H%M%S')[2:]
-    if second:
+def timestr(granularity='second'):
+    s = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+    assert granularity in ['second', 'minute', 'hour', 'day']
+    if granularity == 'second':
         return s
-    elif minute:
+    elif granularity == 'minute':
         return s[:-2]
-    else:
+    elif granularity == 'hour':
         return s[:-4]
+    elif granularity == 'day':
+        return s[:-6]
+
+def _minimal_ext_cmd(cmd, cwd=None):
+    env = {}
+    for k in ['SYSTEMROOT', 'PATH', 'HOME']:
+        v = os.environ.get(k)
+        if v is not None:
+            env[k] = v
+    env['LANGUAGE'] = 'C'
+    env['LANG'] = 'C'
+    env['LC_ALL'] = 'C'
+    out = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=env, cwd=cwd).communicate()[0]
+    return out
+
+def githash(fallback='unknown', digits=8):
+    if digits is not None and not isinstance(digits, int):
+        raise TypeError('digits must be None or an integer')
+    try:
+        import vlmeval
+    except ImportError as e:
+        import logging
+        logging.error(f'ImportError: {str(e)}')
+        return fallback
+    try:
+        out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD'], cwd=vlmeval.__path__[0])
+        sha = out.strip().decode('ascii')
+        if digits is not None:
+            sha = sha[:digits]
+    except OSError:
+        sha = fallback
+    return sha
 
 def dict_merge(dct, merge_dct):
     for k, _ in merge_dct.items():
@@ -152,17 +195,21 @@ def run_command(cmd):
     return subprocess.check_output(cmd).decode()
 
 def load_env():
-    logger = logging.getLogger('LOAD_ENV')
+    import logging
+    logging.basicConfig(
+        format='[%(asctime)s] %(levelname)s - %(filename)s: %(funcName)s - %(lineno)d: %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S')
+
     try:
         import vlmeval
     except ImportError:
-        logger.error('VLMEval is not installed. Failed to import environment variables from .env file. ')
+        logging.error('VLMEval is not installed. Failed to import environment variables from .env file. ')
         return
     pth = osp.realpath(vlmeval.__path__[0])
     pth = osp.join(pth, '../.env')
     pth = osp.realpath(pth)
     if not osp.exists(pth):
-        logger.error(f'Did not detect the .env file at {pth}, failed to load. ')
+        logging.error(f'Did not detect the .env file at {pth}, failed to load. ')
         return
 
     from dotenv import dotenv_values
@@ -170,7 +217,7 @@ def load_env():
     for k, v in values.items():
         if v is not None and len(v):
             os.environ[k] = v
-    logger.info(f'API Keys successfully loaded from {pth}')
+    logging.info(f'API Keys successfully loaded from {pth}')
 
 def pip_install_robust(package):
     import sys
@@ -214,3 +261,31 @@ def extract_json_objects(text, decoder=JSONDecoder()):
             pos = match + index
         except ValueError:
             pos = match + 1
+
+
+def get_gpu_memory():
+    import subprocess
+    try:
+        command = "nvidia-smi --query-gpu=memory.free --format=csv"
+        memory_free_info = subprocess.check_output(command.split()).decode('ascii').split('\n')[:-1][1:]
+        memory_free_values = [int(x.split()[0]) for i, x in enumerate(memory_free_info)]
+        return memory_free_values
+    except Exception as e:
+        print(f'{type(e)}: {str(e)}')
+        return []
+
+
+def auto_split_flag():
+    flag = os.environ.get('AUTO_SPLIT', '0')
+    if flag == '1':
+        return True
+    _, world_size = get_rank_and_world_size()
+    try:
+        import torch
+        device_count = torch.cuda.device_count()
+        if device_count > world_size and device_count % world_size == 0:
+            return True
+        else:
+            return False
+    except:
+        return False

eval_mm/vlmevalkit/vlmeval/smp/vlm.py

@@ -79,7 +79,7 @@ def mmqa_display(question, target_size=512):
                 print(f'{k.upper()}. {question[k]}')
 
 
-def encode_image_to_base64(img, target_size=-1):
+def encode_image_to_base64(img, target_size=-1, fmt='JPEG'):
     # if target_size == -1, will not do resizing
     # else, will set the max_size ot (target_size, target_size)
     if img.mode in ('RGBA', 'P'):
@@ -87,7 +87,7 @@ def encode_image_to_base64(img, target_size=-1):
     if target_size > 0:
         img.thumbnail((target_size, target_size))
     img_buffer = io.BytesIO()
-    img.save(img_buffer, format='JPEG')
+    img.save(img_buffer, format=fmt)
     image_data = img_buffer.getvalue()
     ret = base64.b64encode(image_data).decode('utf-8')
     return ret