llavaMore-demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "718bac1e-7b0a-4fe6-b048-e09f3e1f7d4f",
   "metadata": {},
   "source": [
    "### LLaVA-MORE: \n",
    "Enhancing Visual Instruction Tuning with LLaMA 3.1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "342d107d-8eb7-4695-af67-1308fb91a044",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:38.229379Z",
     "iopub.status.busy": "2024-08-28T02:28:38.229274Z",
     "iopub.status.idle": "2024-08-28T02:28:38.231440Z",
     "shell.execute_reply": "2024-08-28T02:28:38.231133Z",
     "shell.execute_reply.started": "2024-08-28T02:28:38.229367Z"
    }
   },
   "outputs": [],
   "source": [
    "## fix warning\n",
    "## \"bash: /miniconda/envs/llava-more/lib/libtinfo.so.6: no version information available (required by bash)\"\n",
    "## conda install -c conda-forge ncurses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1fd69662-a22e-4269-9da5-2ea04750601d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:38.231898Z",
     "iopub.status.busy": "2024-08-28T02:28:38.231793Z",
     "iopub.status.idle": "2024-08-28T02:28:38.233801Z",
     "shell.execute_reply": "2024-08-28T02:28:38.233489Z",
     "shell.execute_reply.started": "2024-08-28T02:28:38.231887Z"
    }
   },
   "outputs": [],
   "source": [
    "## fix widgets\n",
    "#!conda install -c conda-forge ipywidgets -y\n",
    "#!jupyter nbextension enable --py widgetsnbextension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b93483b4-d16e-4cd2-8087-ceb47a605244",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:38:42.868468Z",
     "iopub.status.busy": "2024-08-28T02:38:42.868239Z",
     "iopub.status.idle": "2024-08-28T02:38:42.870754Z",
     "shell.execute_reply": "2024-08-28T02:38:42.870402Z",
     "shell.execute_reply.started": "2024-08-28T02:38:42.868450Z"
    }
   },
   "outputs": [],
   "source": [
    "#!pip install flash-attn --no-build-isolation\n",
    "#!pip install ipywidgets\n",
    "#!pip install transformers==4.44.2\n",
    "\n",
    "# from huggingface_hub import notebook_login\n",
    "# notebook_login()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3516cb8e-05db-46ef-be53-13ba87d9d148",
   "metadata": {},
   "source": [
    "## Evaluation method-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a6f521fd-9610-4643-9b57-07b563cc4912",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:39:00.714749Z",
     "iopub.status.busy": "2024-08-28T02:39:00.714460Z",
     "iopub.status.idle": "2024-08-28T02:39:00.717857Z",
     "shell.execute_reply": "2024-08-28T02:39:00.717224Z",
     "shell.execute_reply.started": "2024-08-28T02:39:00.714727Z"
    }
   },
   "outputs": [],
   "source": [
    "# %%bash\n",
    "# export PYTHONPATH=.\n",
    "# export HF_TOKEN=\"token\"\n",
    "# # tokenizer_model_path (local)\n",
    "# export TOKENIZER_PATH=\"model/LLaVA_MORE-llama_3_1-8B-finetuning\"\n",
    "\n",
    "# time python -u llava/eval/run_llava.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5e2a13b8-0477-4780-87e1-72066f42ba86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:38.239494Z",
     "iopub.status.busy": "2024-08-28T02:28:38.239390Z",
     "iopub.status.idle": "2024-08-28T02:28:38.350821Z",
     "shell.execute_reply": "2024-08-28T02:28:38.350095Z",
     "shell.execute_reply.started": "2024-08-28T02:28:38.239482Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.10.14\n"
     ]
    }
   ],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0801b6e0-513c-49a5-a19e-34f0a5f5bc17",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:38.351979Z",
     "iopub.status.busy": "2024-08-28T02:28:38.351719Z",
     "iopub.status.idle": "2024-08-28T02:28:39.264581Z",
     "shell.execute_reply": "2024-08-28T02:28:39.263962Z",
     "shell.execute_reply.started": "2024-08-28T02:28:38.351939Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformers =  4.44.2\n",
      "torch = 2.3.1\n"
     ]
    }
   ],
   "source": [
    "import transformers\n",
    "print(\"transformers = \",transformers.__version__)\n",
    "import torch\n",
    "print (\"torch =\", torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bcc86e7-be20-4aa0-9463-99a39fbc84f9",
   "metadata": {},
   "source": [
    "## Evaluation method-2  Inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36b16b24-d85e-4d03-8edd-3f81577c5e21",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:39.265204Z",
     "iopub.status.busy": "2024-08-28T02:28:39.265032Z",
     "iopub.status.idle": "2024-08-28T02:28:39.271471Z",
     "shell.execute_reply": "2024-08-28T02:28:39.270980Z",
     "shell.execute_reply.started": "2024-08-28T02:28:39.265185Z"
    }
   },
   "outputs": [],
   "source": [
    "#!pip install python-dotenv\n",
    "%load_ext dotenv\n",
    "%dotenv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d55aecf9-99a7-4679-9cb3-e74d861bb14b",
   "metadata": {},
   "source": [
    "aimagelab/LLaVA_MORE-llama_3_1-8B-finetuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f192ec59-74b1-4712-b784-df9e7be39669",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:28:39.272009Z",
     "iopub.status.busy": "2024-08-28T02:28:39.271868Z",
     "iopub.status.idle": "2024-08-28T02:28:39.940178Z",
     "shell.execute_reply": "2024-08-28T02:28:39.939717Z",
     "shell.execute_reply.started": "2024-08-28T02:28:39.271996Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import warnings\n",
    "import shutil\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig\n",
    "import torch\n",
    "from llava.model import *\n",
    "from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN\n",
    "\n",
    "\n",
    "def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map=\"auto\", device=\"cuda\", \n",
    "                          use_flash_attn=False, mlp_path=None, **kwargs):\n",
    "    kwargs = {\"device_map\": device_map, **kwargs}\n",
    "\n",
    "    if device != \"cuda\":\n",
    "        kwargs['device_map'] = {\"\": device}\n",
    "\n",
    "    if load_8bit:\n",
    "        kwargs['load_in_8bit'] = True\n",
    "    elif load_4bit:\n",
    "        kwargs['load_in_4bit'] = True\n",
    "        kwargs['quantization_config'] = BitsAndBytesConfig(\n",
    "            load_in_4bit=True,\n",
    "            bnb_4bit_compute_dtype=torch.float16,\n",
    "            bnb_4bit_use_double_quant=True,\n",
    "            bnb_4bit_quant_type='nf4'\n",
    "        )\n",
    "    else:\n",
    "        kwargs['torch_dtype'] = torch.float16\n",
    "\n",
    "    if use_flash_attn:\n",
    "        kwargs['attn_implementation'] = 'flash_attention_2'\n",
    "\n",
    "    if 'llava' in model_name.lower():\n",
    "        # Load LLaVA model\n",
    "        if 'lora' in model_name.lower() and model_base is None:\n",
    "            warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.')\n",
    "            config_path = os.path.join(model_path, 'config.json')\n",
    "            try:\n",
    "                with open(config_path) as f:\n",
    "                    configuration= json.load(f)\n",
    "                model_base = configuration['_name_or_path']\n",
    "            except:\n",
    "                raise ValueError('Cannot find the model name in the configuration file. Please provide the `model_base` argument.')\n",
    "            \n",
    "        if 'lora' in model_name.lower() and model_base is not None:\n",
    "            from llava.model.language_model.llava_llama import LlavaConfig\n",
    "            lora_cfg_pretrained = LlavaConfig.from_pretrained(model_path)\n",
    "            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n",
    "            print('Loading LLaVA from base model...')\n",
    "            model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)\n",
    "            token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features\n",
    "            if model.lm_head.weight.shape[0] != token_num:\n",
    "                model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))\n",
    "                model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))\n",
    "\n",
    "            print('Loading additional LLaVA weights...')\n",
    "            if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):\n",
    "                non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')\n",
    "            else:\n",
    "                # this is probably from HF Hub\n",
    "                from huggingface_hub import hf_hub_download\n",
    "                def load_from_hf(repo_id, filename, subfolder=None):\n",
    "                    cache_file = hf_hub_download(\n",
    "                        repo_id=repo_id,\n",
    "                        filename=filename,\n",
    "                        subfolder=subfolder)\n",
    "                    return torch.load(cache_file, map_location='cpu')\n",
    "                non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')\n",
    "            non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}\n",
    "            if any(k.startswith('model.model.') for k in non_lora_trainables):\n",
    "                non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}\n",
    "            model.load_state_dict(non_lora_trainables, strict=False)\n",
    "\n",
    "            from peft import PeftModel\n",
    "            print('Loading LoRA weights...')\n",
    "            model = PeftModel.from_pretrained(model, model_path)\n",
    "            print('Merging LoRA weights...')\n",
    "            model = model.merge_and_unload()\n",
    "            print('Model is loaded...')\n",
    "        elif model_base is not None:\n",
    "            # this may be mm projector only\n",
    "            print('Loading LLaVA from base model...')\n",
    "            if 'mpt' in model_name.lower():\n",
    "                if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')):\n",
    "                    shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py'))\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True)\n",
    "                cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True)\n",
    "                model = LlavaMptForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)\n",
    "            else:\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n",
    "                cfg_pretrained = AutoConfig.from_pretrained(model_path)\n",
    "                model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)\n",
    "\n",
    "            mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')\n",
    "            mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}\n",
    "            model.load_state_dict(mm_projector_weights, strict=False)\n",
    "        else:\n",
    "            if 'mpt' in model_name.lower():\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\n",
    "                model = LlavaMptForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)\n",
    "            elif 'mistral' in model_name.lower():\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "                model = LlavaMistralForCausalLM.from_pretrained(\n",
    "                    model_path,\n",
    "                    low_cpu_mem_usage=True,\n",
    "                    **kwargs\n",
    "                )\n",
    "            else:\n",
    "                # some old checkpoints may not have the siglip parameter in configuration file                \n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n",
    "                model = LlavaLlamaForCausalLM.from_pretrained(\n",
    "                    model_path,\n",
    "                    low_cpu_mem_usage=True,\n",
    "                    **kwargs\n",
    "                )\n",
    "    else:\n",
    "        # Load language model\n",
    "        if model_base is not None:\n",
    "            # PEFT model\n",
    "            from peft import PeftModel\n",
    "            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)\n",
    "            model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs)\n",
    "            print(f\"Loading LoRA weights from {model_path}\")\n",
    "            model = PeftModel.from_pretrained(model, model_path)\n",
    "            print(f\"Merging weights\")\n",
    "            model = model.merge_and_unload()\n",
    "            print('Convert to FP16...')\n",
    "            model.to(torch.float16)\n",
    "        else:\n",
    "            use_fast = False\n",
    "            if 'mpt' in model_name.lower():\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\n",
    "                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs)\n",
    "            else:\n",
    "                tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n",
    "                model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)\n",
    "\n",
    "    image_processor = None\n",
    "\n",
    "    if 'llava' in model_name.lower() or mlp_path is not None:\n",
    "        mm_use_im_start_end = getattr(model.config, \"mm_use_im_start_end\", False)\n",
    "        mm_use_im_patch_token = getattr(model.config, \"mm_use_im_patch_token\", True)\n",
    "        if mm_use_im_patch_token:\n",
    "            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)\n",
    "        if mm_use_im_start_end:\n",
    "            tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)\n",
    "        model.resize_token_embeddings(len(tokenizer))\n",
    "\n",
    "        # # RE-CONTROL THE IF STATEMENT select the correct class, considering also S2\n",
    "        # if 'siglip' in model.config.mm_vision_tower and hasattr(model.config, 's2'):\n",
    "        #     # change the args for the new class\n",
    "        #     from llava.model.multimodal_encoder.builder import SigLIPVisionTowerS2\n",
    "        #     vision_tower = SigLIPVisionTowerS2('google/siglip-so400m-patch14-384', args=model.config)\n",
    "        #     model.model.vision_tower = vision_tower.vision_tower\n",
    "\n",
    "        # elif 'siglip' in model.config.mm_vision_tower:\n",
    "        #     from llava.model.multimodal_encoder.builder import SigLIPVisionTower\n",
    "        #     vision_tower= SigLIPVisionTower('google/siglip-so400m-patch14-384', args=model.config)\n",
    "        #     vision_tower.to(\"cuda\", dtype=torch.float16)\n",
    "        #     model.model.vision_tower = vision_tower.vision_tower\n",
    "\n",
    "        # else:\n",
    "        #     if hasattr(model.config, 's2'): # and on work\n",
    "        #         from llava.model.multimodal_encoder.builder import CLIPVisionTowerS2\n",
    "        #         vision_tower= CLIPVisionTowerS2('google/siglip-so400m-patch14-384', args=model.config)\n",
    "        #         model.model.vision_tower = vision_tower.vision_tower\n",
    "        #     else:\n",
    "        #         vision_tower = model.get_vision_tower()\n",
    "        vision_tower = model.get_vision_tower()\n",
    "\n",
    "        if not vision_tower.is_loaded:\n",
    "            vision_tower.load_model(device_map=device_map)\n",
    "        if device_map != 'auto':\n",
    "            vision_tower.to(device=device_map, dtype=torch.float16)\n",
    "        image_processor = vision_tower.image_processor\n",
    "\n",
    "    if hasattr(model.config, \"max_sequence_length\"):\n",
    "        context_len = model.config.max_sequence_length\n",
    "    else:\n",
    "        context_len = 2048\n",
    "\n",
    "    if mlp_path is not None:\n",
    "        print('Loading mm projector weights...')\n",
    "        mm_projector_weights = torch.load(mlp_path)\n",
    "        new_dict= {}\n",
    "        new_keys= ['0.weight', '0.bias', '2.weight', '2.bias']\n",
    "        for el, key in enumerate(new_keys):\n",
    "            new_dict[key] = mm_projector_weights[list(mm_projector_weights.keys())[el]]\n",
    "\n",
    "        model.model.mm_projector.load_state_dict(new_dict)\n",
    "        # model.model.mm_projector.to(device=device_map, dtype=torch.float16)\n",
    "\n",
    "    return tokenizer, model, image_processor, context_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "59f3af15-89ee-4850-9edc-f9f610b6146f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:36:13.778761Z",
     "iopub.status.busy": "2024-08-28T02:36:13.778440Z",
     "iopub.status.idle": "2024-08-28T02:36:13.795583Z",
     "shell.execute_reply": "2024-08-28T02:36:13.794870Z",
     "shell.execute_reply.started": "2024-08-28T02:36:13.778734Z"
    }
   },
   "outputs": [],
   "source": [
    "##%tb\n",
    "import argparse\n",
    "import torch\n",
    "import requests\n",
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "import re\n",
    "\n",
    "from llava.constants import (\n",
    "    IMAGE_TOKEN_INDEX,\n",
    "    DEFAULT_IMAGE_TOKEN,\n",
    "    DEFAULT_IM_START_TOKEN,\n",
    "    DEFAULT_IM_END_TOKEN,\n",
    "    IMAGE_PLACEHOLDER,\n",
    ")\n",
    "from llava.conversation import conv_templates, SeparatorStyle\n",
    "##from llava.model.builder import load_pretrained_model\n",
    "from llava.utils import disable_torch_init\n",
    "from llava.mm_utils import (\n",
    "    process_images,\n",
    "    tokenizer_image_token,\n",
    "    get_model_name_from_path,\n",
    ")\n",
    "\n",
    "def image_parser(args):\n",
    "    out = args.image_file.split(args.sep)\n",
    "    return out\n",
    "\n",
    "\n",
    "def load_image(image_file):\n",
    "    if image_file.startswith(\"http\") or image_file.startswith(\"https\"):\n",
    "        response = requests.get(image_file)\n",
    "        image = Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
    "    else:\n",
    "        image = Image.open(image_file).convert(\"RGB\")\n",
    "    return image\n",
    "\n",
    "\n",
    "def load_images(image_files):\n",
    "    out = []\n",
    "    for image_file in image_files:\n",
    "        image = load_image(image_file)\n",
    "        out.append(image)\n",
    "    return out\n",
    "\n",
    "def eval_model(args):\n",
    "    # Model\n",
    "    disable_torch_init()\n",
    "\n",
    "    model_name = get_model_name_from_path(args.model_path)\n",
    "    model_name= 'llava'\n",
    "\n",
    "    tokenizer, model, image_processor, context_len = load_pretrained_model(\n",
    "        args.model_path, args.model_base, model_name, load_8bit=True\n",
    "    )\n",
    "\n",
    "    print(model.config)\n",
    "    \n",
    "    qs = args.query\n",
    "    image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN\n",
    "    if IMAGE_PLACEHOLDER in qs:\n",
    "        if model.config.mm_use_im_start_end:\n",
    "            qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs)\n",
    "        else:\n",
    "            qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs)\n",
    "    else:\n",
    "        if model.config.mm_use_im_start_end:\n",
    "            qs = image_token_se + \"\\n\" + qs\n",
    "        else:\n",
    "            qs = DEFAULT_IMAGE_TOKEN + \"\\n\" + qs\n",
    "\n",
    "    if \"llama-2\" in model_name.lower():\n",
    "        conv_mode = \"llava_llama_2\"\n",
    "    elif \"mistral\" in model_name.lower():\n",
    "        conv_mode = \"mistral_instruct\"\n",
    "    elif \"v1.6-34b\" in model_name.lower():\n",
    "        conv_mode = \"chatml_direct\"\n",
    "    elif \"v1\" in model_name.lower():\n",
    "        conv_mode = \"llava_v1\"\n",
    "    elif \"mpt\" in model_name.lower():\n",
    "        conv_mode = \"mpt\"\n",
    "    else:\n",
    "        conv_mode = \"llava_v0\"\n",
    "\n",
    "    if args.conv_mode is not None and conv_mode != args.conv_mode:\n",
    "        print(\n",
    "            \"[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}\".format(\n",
    "                conv_mode, args.conv_mode, args.conv_mode\n",
    "            )\n",
    "        )\n",
    "    else:\n",
    "        args.conv_mode = conv_mode\n",
    "    \n",
    "    conv = conv_templates[args.conv_mode].copy()\n",
    "    conv.append_message(conv.roles[0], qs)\n",
    "    conv.append_message(conv.roles[1], None)\n",
    "    prompt = conv.get_prompt()\n",
    "    conv.tokenizer = tokenizer\n",
    "\n",
    "    image_files = image_parser(args)\n",
    "    images = load_images(image_files)\n",
    "    image_sizes = [x.size for x in images]\n",
    "    images_tensor = process_images(\n",
    "        images,\n",
    "        image_processor,\n",
    "        model.config\n",
    "    ).to(model.device, dtype=torch.float16)\n",
    "\n",
    "    input_ids = (\n",
    "        tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors=\"pt\")\n",
    "        .unsqueeze(0)\n",
    "        .cuda()\n",
    "    )\n",
    "\n",
    "    with torch.inference_mode():\n",
    "        output_ids = model.generate(\n",
    "            input_ids,\n",
    "            images=images_tensor,\n",
    "            image_sizes=image_sizes,\n",
    "            do_sample=True if args.temperature > 0 else False,\n",
    "            temperature=args.temperature,\n",
    "            top_p=args.top_p,\n",
    "            num_beams=args.num_beams,\n",
    "            max_new_tokens=args.max_new_tokens,\n",
    "            use_cache=True,\n",
    "        )\n",
    "\n",
    "    outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()\n",
    "    print(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a88b334a-3fd9-4c5f-b8e0-f8b7db427010",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:36:17.449229Z",
     "iopub.status.busy": "2024-08-28T02:36:17.448948Z",
     "iopub.status.idle": "2024-08-28T02:36:39.330548Z",
     "shell.execute_reply": "2024-08-28T02:36:39.330111Z",
     "shell.execute_reply.started": "2024-08-28T02:36:17.449208Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conversation mode: llama_3_1\n",
      "model name: ./model/LLaVA_MORE-llama_3_1-8B-finetuning\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74493596f7e2408ea527f411c48b5cb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ./model/LLaVA_MORE-llama_3_1-8B-finetuning were not used when initializing LlavaLlamaForCausalLM: ['model.vision_tower.vision_tower.vision_model.embeddings.class_embedding', 'model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight', 'model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias', 'model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight', 'model.vision_tower.vision_tower.vision_model.post_layernorm.bias', 'model.vision_tower.vision_tower.vision_model.post_layernorm.weight', 'model.vision_tower.vision_tower.vision_model.pre_layrnorm.bias', 'model.vision_tower.vision_tower.vision_model.pre_layrnorm.weight']\n",
      "- This IS expected if you are initializing LlavaLlamaForCausalLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing LlavaLlamaForCausalLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LlavaConfig {\n",
      "  \"_name_or_path\": \"./model/LLaVA_MORE-llama_3_1-8B-finetuning\",\n",
      "  \"architectures\": [\n",
      "    \"LlavaLlamaForCausalLM\"\n",
      "  ],\n",
      "  \"attention_bias\": false,\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 128000,\n",
      "  \"eos_token_id\": [\n",
      "    128001,\n",
      "    128008,\n",
      "    128009\n",
      "  ],\n",
      "  \"freeze_mm_mlp_adapter\": false,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 4096,\n",
      "  \"image_aspect_ratio\": \"pad\",\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 14336,\n",
      "  \"max_position_embeddings\": 131072,\n",
      "  \"mlp_bias\": false,\n",
      "  \"mm_hidden_size\": 1024,\n",
      "  \"mm_patch_merge_type\": \"flat\",\n",
      "  \"mm_projector_lr\": null,\n",
      "  \"mm_projector_type\": \"mlp2x_gelu\",\n",
      "  \"mm_use_im_patch_token\": false,\n",
      "  \"mm_use_im_start_end\": false,\n",
      "  \"mm_vision_select_feature\": \"patch\",\n",
      "  \"mm_vision_select_layer\": -2,\n",
      "  \"mm_vision_tower\": \"openai/clip-vit-large-patch14-336\",\n",
      "  \"model_type\": \"llava_llama\",\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_hidden_layers\": 32,\n",
      "  \"num_key_value_heads\": 8,\n",
      "  \"pretraining_tp\": 1,\n",
      "  \"quantization_config\": {\n",
      "    \"_load_in_4bit\": false,\n",
      "    \"_load_in_8bit\": true,\n",
      "    \"bnb_4bit_compute_dtype\": \"float32\",\n",
      "    \"bnb_4bit_quant_storage\": \"uint8\",\n",
      "    \"bnb_4bit_quant_type\": \"fp4\",\n",
      "    \"bnb_4bit_use_double_quant\": false,\n",
      "    \"llm_int8_enable_fp32_cpu_offload\": false,\n",
      "    \"llm_int8_has_fp16_weight\": false,\n",
      "    \"llm_int8_skip_modules\": null,\n",
      "    \"llm_int8_threshold\": 6.0,\n",
      "    \"load_in_4bit\": false,\n",
      "    \"load_in_8bit\": true,\n",
      "    \"quant_method\": \"bitsandbytes\"\n",
      "  },\n",
      "  \"rms_norm_eps\": 1e-05,\n",
      "  \"rope_scaling\": {\n",
      "    \"factor\": 8.0,\n",
      "    \"high_freq_factor\": 4.0,\n",
      "    \"low_freq_factor\": 1.0,\n",
      "    \"original_max_position_embeddings\": 8192,\n",
      "    \"rope_type\": \"llama3\"\n",
      "  },\n",
      "  \"rope_theta\": 500000.0,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"tokenizer_model_max_length\": 4096,\n",
      "  \"tokenizer_padding_side\": \"right\",\n",
      "  \"torch_dtype\": \"float16\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"tune_mm_mlp_adapter\": false,\n",
      "  \"use_cache\": true,\n",
      "  \"use_mm_proj\": true,\n",
      "  \"vocab_size\": 128256\n",
      "}\n",
      "\n",
      "[WARNING] the auto inferred conversation mode is llava_v0, while `--conv-mode` is llama_3_1, using llama_3_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The image features a receipt for a purchase made at a cafe. The receipt is for a total of $8.75, with a tax of $1.00 and a tip of $2.00. The receipt is dated January 2, 2019, and the cafe is located at 111 Green Street, New York, NY. \n",
      "\n",
      "The receipt lists the items purchased, including almond milk and bottled water. The receipt also shows the name of the host, Maggie.\n"
     ]
    }
   ],
   "source": [
    "from dataclasses import dataclass\n",
    "\n",
    "@dataclass\n",
    "class Inputs():\n",
    "    model_path: str = \"./model/LLaVA_MORE-llama_3_1-8B-finetuning\"\n",
    "    model_base: int = None\n",
    "    #image_file: str = \"https://farm2.staticflickr.com/1168/4723652147_ae14813f08_z.jpg\"\n",
    "    image_file: str = \"https://makereceipt.com/images/ItemizedBarcode.jpg\"    \n",
    "    query: str = \"Describe this image.\"\n",
    "    conv_mode: str = \"llama_3_1\"\n",
    "    sep: str = \",\"\n",
    "    temperature: float = 0.2\n",
    "    top_p:str = None\n",
    "    num_beams: int = 1\n",
    "    max_new_tokens: int = 128\n",
    "    \n",
    "args = Inputs()\n",
    "print(f\"conversation mode: {args.conv_mode}\")\n",
    "print(f\"model name: {args.model_path}\")\n",
    "    \n",
    "eval_model(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "494e921c-5f51-427d-bb22-32ca03456291",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "212f1d50-3306-46f7-a4d5-82979f01dd22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:35:45.920999Z",
     "iopub.status.busy": "2024-08-28T02:35:45.920713Z",
     "iopub.status.idle": "2024-08-28T02:35:45.925050Z",
     "shell.execute_reply": "2024-08-28T02:35:45.924654Z",
     "shell.execute_reply.started": "2024-08-28T02:35:45.920983Z"
    }
   },
   "outputs": [],
   "source": [
    "# from transformers import LlavaForConditionalGeneration, LlavaConfig, CLIPVisionConfig, LlamaConfig\n",
    "\n",
    "# # Initializing a CLIP-vision config\n",
    "\n",
    "# vision_config = CLIPVisionConfig()\n",
    "\n",
    "# # Initializing a Llama config\n",
    "\n",
    "# text_config = LlamaConfig()\n",
    "\n",
    "# # Initializing a Llava llava-1.5-7b style configuration\n",
    "\n",
    "# configuration = LlavaConfig(vision_config, text_config)\n",
    "\n",
    "# # Initializing a model from the llava-1.5-7b style configuration\n",
    "\n",
    "# model = LlavaForConditionalGeneration(configuration)\n",
    "\n",
    "# # Accessing the model configuration\n",
    "\n",
    "# configuration = model.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1691f099-0835-4833-9dc7-34336d963df4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-28T02:35:52.905666Z",
     "iopub.status.busy": "2024-08-28T02:35:52.905413Z",
     "iopub.status.idle": "2024-08-28T02:35:52.908760Z",
     "shell.execute_reply": "2024-08-28T02:35:52.908282Z",
     "shell.execute_reply.started": "2024-08-28T02:35:52.905645Z"
    }
   },
   "outputs": [],
   "source": [
    "# from PIL import Image\n",
    "# import requests\n",
    "# from transformers import AutoProcessor, LlavaForConditionalGeneration\n",
    "\n",
    "# #model = LlavaForConditionalGeneration.from_pretrained(\"llava-hf/llava-1.5-7b-hf\")\n",
    "# #processor = AutoProcessor.from_pretrained(\"llava-hf/llava-1.5-7b-hf\")\n",
    "\n",
    "# #aimagelab/LLaVA_MORE-llama_3_1-8B-finetuning\n",
    "# model = LlavaForConditionalGeneration.from_pretrained(\"aimagelab/LLaVA_MORE-llama_3_1-8B-finetuning\")\n",
    "# processor = AutoProcessor.from_pretrained(\"aimagelab/LLaVA_MORE-llama_3_1-8B-finetuning\")\n",
    "\n",
    "# prompt = \"USER: <image>\\nWhat's the content of the image? ASSISTANT:\"\n",
    "# url = \"https://www.ilankelman.org/stopsigns/australia.jpg\"\n",
    "# image = Image.open(requests.get(url, stream=True).raw)\n",
    "\n",
    "# inputs = processor(text=prompt, images=image, return_tensors=\"pt\")\n",
    "\n",
    "# # Generate\n",
    "# generate_ids = model.generate(**inputs, max_new_tokens=15)\n",
    "# processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451917db-a642-4a93-b5a2-19875eaa3b8b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}