analysis_clevr_llm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import torch\n",
    "import jacinle\n",
    "import json\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "import jactorch\n",
    "\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from typing import Optional, Union, List, Dict\n",
    "from jacinle.config.environ_v2 import configs, set_configs\n",
    "from left.domain import create_domain_from_parsing\n",
    "from left.models.model import LeftModel\n",
    "import re\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from typing import Union, List, Dict\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "\n",
    "import jacinle\n",
    "from jacinle.utils.container import GView\n",
    "from jacinle.config.environ_v2 import configs, set_configs, def_configs\n",
    "from concepts.benchmark.clevr.clevr_constants import g_attribute_concepts, g_relational_concepts\n",
    "from left.models.model import LeftModel\n",
    "import json\n",
    "import os\n",
    "\n",
    "from experiments.desc_clevr_nesycoco import *\n",
    "\n",
    "with set_configs():\n",
    "    configs.model.embedding_type = \"glove\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "import jacinle.io as io\n",
    "from os import path as osp\n",
    "\n",
    "data_dir = \"data/clevr/val/\"\n",
    "\n",
    "data_parses = [\n",
    "    f\"{data_dir}/questions-ncprogram-gt.pkl\"\n",
    "    ]\n",
    "\n",
    "all_parses = dict()\n",
    "for filename in data_parses:\n",
    "    if filename.endswith('.p'):\n",
    "        content = io.load_pkl(filename)\n",
    "    else:\n",
    "        content = io.load(filename)\n",
    "    all_parses.update(content)\n",
    "\n",
    "data_questions_json = osp.join(data_dir, 'questions.json')\n",
    "data_scenes_json = osp.join(data_dir, 'scenes.json')\n",
    "data_image_root = osp.join(data_dir, 'images')\n",
    "data_vocab_json = osp.join(data_dir, 'vocab.json')\n",
    "data_output_vocab_json = osp.join(data_dir, 'output-vocab.json')\n",
    "from left.domain import create_domain_from_parsing\n",
    "\n",
    "domain = create_domain_from_parsing(all_parses)\n",
    "\n",
    "\n",
    "\n",
    "from concepts.benchmark.clevr.dataset import make_dataset\n",
    "validation_dataset = make_dataset(\n",
    "    data_scenes_json,\n",
    "    data_questions_json,\n",
    "    data_image_root,\n",
    "    vocab_json=data_vocab_json,\n",
    "    output_vocab_json=data_output_vocab_json,\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1\n",
    "num_workers = 96\n",
    "# train_dataloader = train_dataset.make_dataloader(batch_size, shuffle=False, drop_last=True, nr_workers=num_workers)\n",
    "validation_dataloader = validation_dataset.make_dataloader(batch_size, shuffle=False, drop_last=False, nr_workers=num_workers)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "from jacinle.config.g import g\n",
    "g.concept_mapping = None\n",
    "domain.print_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jactorch.io import state_dict, load_state_dict\n",
    "from left.data.referit3d.vocabulary import Vocabulary\n",
    "from jactorch.train import TrainerEnv\n",
    "from jactorch.optim import AdamW\n",
    "from jactorch.cuda.copy import async_copy_to\n",
    "\n",
    "vocab = Vocabulary()\n",
    "model = make_model(None, domain, all_parses, validation_dataset.output_vocab if hasattr(validation_dataset, 'output_vocab') else validation_dataset.unwrapped.output_vocab, custom_transfer=None)\n",
    "model.cuda()\n",
    "trainable_parameters = filter(lambda x: x.requires_grad, model.parameters())\n",
    "\n",
    "optimizer = AdamW(trainable_parameters, 0.001 , weight_decay=1e-2)\n",
    "\n",
    "trainer = TrainerEnv(model, optimizer)\n",
    "\n",
    "\n",
    "filename = \"[path to model]\"\n",
    "trainer.load_checkpoint(filename)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "metadata": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "group_meters = jacinle.GroupMeters()\n",
    "group_meters.reset()\n",
    "from jacinle.utils.tqdm import tqdm_pbar\n",
    "from jactorch.utils.meta import as_float\n",
    "wrong_indices = []\n",
    "correct_indices = []\n",
    "parse_fails = []\n",
    "all_images = set()\n",
    "\n",
    "with tqdm_pbar(total=len(validation_dataloader)) as pbar:\n",
    "    for (index, feed_dict) in enumerate(validation_dataloader):\n",
    "        \n",
    "        feed_dict = async_copy_to(feed_dict, 0)\n",
    "        output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "        \n",
    "        if output_dict[\"results\"][0][2] is None:\n",
    "            parse_fails.append(index)\n",
    "            correct = False\n",
    "        if output_dict[\"monitors\"][\"acc/qa\"] == 1:\n",
    "            correct =  True\n",
    "        else:\n",
    "            correct = False\n",
    "\n",
    "        if not correct:\n",
    "            wrong_indices.append(index)\n",
    "            if feed_dict[\"image_index\"][0] not in all_images:\n",
    "                all_images.add(feed_dict[\"image_index\"][0])\n",
    "                print(feed_dict[\"image_index\"][0])\n",
    "        else:\n",
    "            correct_indices.append(index)\n",
    "            \n",
    "        acc = len(correct_indices) / (len(correct_indices) + len(wrong_indices))\n",
    "\n",
    "        pbar.set_description(group_meters.format_simple(\n",
    "            f'wrong_img {len(all_images)} Validation Acc: {acc}',\n",
    "            {k: v for k, v in group_meters.val.items() if k.startswith('validation') and k.count('/') <= 2},\n",
    "            compressed=True\n",
    "        ), refresh=False)\n",
    "        pbar.update()\n",
    "        if index > 400:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(parse_fails), len(wrong_indices), len(correct_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from jactorch.io import state_dict, load_state_dict\n",
    "from left.data.referit3d.vocabulary import Vocabulary\n",
    "from jactorch.train import TrainerEnv\n",
    "from jactorch.optim import AdamW\n",
    "from jactorch.cuda.copy import async_copy_to\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "import numpy as np\n",
    "\n",
    "concept_values = {concept:[] for concept in model.attribute_embedding.concept2id}\n",
    "concept_counter = {concept:0 for concept in model.attribute_embedding.concept2id}\n",
    "with tqdm_pbar(total=len(validation_dataloader)) as pbar:\n",
    "    for (index, feed_dict) in enumerate(validation_dataloader):\n",
    "        feed_dict = async_copy_to(feed_dict, 0)\n",
    "        output_dict, extra_info = trainer.evaluate(feed_dict)\n",
    "        has_false_negative = False\n",
    "        for g_index,grounding in enumerate(output_dict[\"groundings\"]):\n",
    "            if len(feed_dict[\"all_objects\"]) != grounding.compute_all_similarity(\"attribute\").shape[0]:\n",
    "                continue\n",
    "            for obj_index,obj in enumerate(feed_dict[\"all_objects\"]):\n",
    "                res = grounding.compute_all_similarity(\"attribute\")[obj_index].cpu().detach().numpy()\n",
    "                for meta_concept in [\"color\",\"shape\",\"material\",\"size\"]:\n",
    "                    concept_value = res[model.attribute_embedding.concept2id[obj[meta_concept][0]+\"_Object\"]]\n",
    "                    concept_values[obj[meta_concept][0]+\"_Object\"].append(concept_value)\n",
    "                    concept_counter[obj[meta_concept][0]+\"_Object\"] += 1\n",
    "                    if concept_value < 0.5:\n",
    "                        print(obj[meta_concept][0]+\"_Object\",concept_value)\n",
    "                        has_false_negative = True\n",
    "        if False:\n",
    "            image = Image.open(osp.join(data_image_root, feed_dict[\"image_filename\"][0])).convert('RGB')\n",
    "            print(feed_dict[\"all_objects\"])\n",
    "            for concept in model.attribute_embedding.concept2id:\n",
    "                res = grounding.compute_all_similarity(\"attribute\")[:,model.attribute_embedding.concept2id[concept]].cpu().detach().numpy()\n",
    "                # normalized_res = (res - res.min()) / (res.max() - res.min())\n",
    "                fig, ax = plt.subplots()\n",
    "                ax.imshow(image)\n",
    "                for obj_index,obj in enumerate(feed_dict[\"objects_raw\"][0]):\n",
    "                    # Create a Rectangle patch\n",
    "                    rect = patches.Rectangle((obj[0], obj[1]), obj[2] - obj[0], obj[3] - obj[1], linewidth=1, edgecolor='r', facecolor=\"none\")\n",
    "                    ax.add_patch(rect)\n",
    "                    text = f\"{obj_index} {round(float(res[obj_index]),4)} \"\n",
    "                    ax.text(obj[0], obj[1], text, color='white', fontsize=10, bbox=dict(facecolor='red', alpha=0.5))\n",
    "                plt.title(concept)\n",
    "                plt.show()\n",
    "            break\n",
    "            \n",
    "        pbar.update(1)\n",
    "        if index == 10000:\n",
    "            break       "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}