Ovis是由阿里国际AI团队推出一款先进的多模态大模型，它在处理和理解多种类型的数据输入方面表现出色，比如文本、图像等。
Ovis模型不仅能够执行数学推理问答、物体识别、文本提取等任务，还能够在复杂的决策场景中发挥作用。
Ovis模型具备创新架构设计，引入了可学习的视觉嵌入词表，将连续的视觉特征转换为概率化的视觉token，并通过视觉嵌入词表加权生成结构化的视觉嵌入。
此外，Ovis支持高分图像处理，能够兼容极端长宽比及高分辨率图像，显示出出色的图像理解能力。
为了保证模型的全面性和实用性，Ovis训练时采用了广泛的多方向数据集，涵盖了Caption（标题）、VQA（视觉问答）、OCR（光学字符识别）、Table（表格）以及Chart（图表）等多种多模态数据。
根据OpenCompass平台的评测数据显示，Ovis 1.6-Gemma2-9B版本在30亿参数以下的模型中综合排名第一，超过了包括MiniCPM-V-2.6在内的多个行业优秀大模型。
github项目地址：https://github.com/AIDC-AI/Ovis。
一、环境安装
1、python环境
建议安装python版本在3.10以上。
2、pip库安装
pip install torch==2.2.0+cu118 torchvision==0.17.0+cu118 torchaudio==2.2.0 --extra-index-url https://download.pytorch.org/whl/cu118
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install -e .
3、Ovis1.6-Gemma2-9B模型下载：
git lfs install
git clone https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-9B
二、功能测试
1、运行测试：
（1）python代码调用测试
from dataclasses import dataclass, field
from typing import Optional, Union, List
import loggingimport torch
from PIL import Imagefrom ovis.model.modeling_ovis import Ovis
from ovis.util.constants import IMAGE_TOKEN@dataclass
class RunnerArguments:model_path: strmax_new_tokens: int = field(default=512)do_sample: bool = field(default=False)top_p: Optional[float] = field(default=0.95)top_k: Optional[int] = field(default=50)temperature: Optional[float] = field(default=1.0)max_partition: int = field(default=9)class OvisRunner:def __init__(self, args: RunnerArguments):self.model_path = args.model_pathself.device = torch.cuda.current_device() if torch.cuda.is_available() else 'cpu'self.dtype = torch.bfloat16self.model = Ovis.from_pretrained(self.model_path, torch_dtype=self.dtype, multimodal_max_length=8192)self.model = self.model.eval().to(device=self.device)self.eos_token_id = self.model.generation_config.eos_token_idself.text_tokenizer = self.model.get_text_tokenizer()self.pad_token_id = self.text_tokenizer.pad_token_idself.visual_tokenizer = self.model.get_visual_tokenizer()self.conversation_formatter = self.model.get_conversation_formatter()self.image_placeholder = IMAGE_TOKENself.max_partition = args.max_partitionself.gen_kwargs = dict(max_new_tokens=args.max_new_tokens,do_sample=args.do_sample,top_p=args.top_p,top_k=args.top_k,temperature=args.temperature,repetition_penalty=None,eos_token_id=self.eos_token_id,pad_token_id=self.pad_token_id,use_cache=True)def preprocess(self, inputs: List[Union[Image.Image, str]]):if len(inputs) == 2 and isinstance(inputs[0], str) and isinstance(inputs[1], Image.Image):inputs = list(reversed(inputs))query = ''images = []for data in inputs:if isinstance(data, Image.Image):query += self.image_placeholder + '\n'images.append(data)elif isinstance(data, str):query += data.replace(self.image_placeholder, '')elif data is not None:raise ValueError(f'Invalid input type, expected `PIL.Image.Image` or `str`, but got {type(data)}')prompt, input_ids, pixel_values = self.model.preprocess_inputs(query, images, max_partition=self.max_partition)attention_mask = torch.ne(input_ids, self.text_tokenizer.pad_token_id)input_ids = input_ids.unsqueeze(0).to(device=self.device)attention_mask = attention_mask.unsqueeze(0).to(device=self.device)pixel_values = [pv.to(device=self.device, dtype=self.dtype) if pv is not None else None for pv in pixel_values]return prompt, input_ids, attention_mask, pixel_valuesdef run(self, inputs: List[Union[Image.Image, str]]):try:prompt, input_ids, attention_mask, pixel_values = self.preprocess(inputs)output_ids = self.model.generate(input_ids,pixel_values=pixel_values,attention_mask=attention_mask,**self.gen_kwargs)output = self.text_tokenizer.decode(output_ids[0], skip_special_tokens=True)input_token_len = input_ids.shape[1]output_token_len = output_ids.shape[1]response = {'prompt': prompt,'output': output,'prompt_tokens': input_token_len,'total_tokens': input_token_len + output_token_len}return responseexcept Exception as e:logging.error(f"An error occurred: {e}")raiseif __name__ == '__main__':runner_args = RunnerArguments(model_path='<model_path>')runner = OvisRunner(runner_args)image = Image.open('<image_path>')text = '<prompt>'response = runner.run([image, text])print(response['output'])未完......
更多详细的欢迎关注：杰哥新技术
