prepare-dataset/combine-mosaic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "844decdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from streaming.base.format.mds.encodings import Encoding, _encodings\n",
    "from streaming import MDSWriter, LocalDataset\n",
    "from tqdm import tqdm\n",
    "from typing import List\n",
    "import torch\n",
    "import json\n",
    "\n",
    "class ListOfDict(Encoding):\n",
    "    def encode(self, obj: List[dict]) -> bytes:\n",
    "        # Convert the list of dictionaries to a JSON-encoded string\n",
    "        json_str = json.dumps(obj)\n",
    "        return json_str.encode('utf-8')\n",
    "\n",
    "    def decode(self, data: bytes) -> List[dict]:\n",
    "\n",
    "        # Decode the JSON-encoded string back to a list of dictionaries\n",
    "        json_str = data.decode('utf-8')\n",
    "        return json.loads(json_str)\n",
    "\n",
    "# Register the custom encoding for 'list_of_dict'\n",
    "_encodings['list_of_dict'] = ListOfDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ed42d064",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['prepared-llava-en.jsonl',\n",
       " 'prepared-llava-ms.jsonl',\n",
       " 'prepared-audio-en.jsonl',\n",
       " 'prepared-audio-ms.jsonl',\n",
       " 'prepared-relationship-en.jsonl',\n",
       " 'prepared-relationship-ms.jsonl',\n",
       " 'prepared-malay.jsonl']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "files = glob('prepared-llava*.jsonl')\n",
    "files.extend(glob('prepared-audio*.jsonl'))\n",
    "files.extend(glob('prepared-relationship*.jsonl'))\n",
    "files.extend(glob('prepared-malay*'))\n",
    "\n",
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fb190c08",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "148657it [00:01, 136536.32it/s]\n",
      "148657it [00:01, 131254.08it/s]\n",
      "293752it [00:03, 76067.92it/s]\n",
      "293752it [00:03, 76063.72it/s]\n",
      "184500it [00:00, 219085.72it/s]\n",
      "184500it [00:00, 220085.90it/s]\n",
      "3238it [00:00, 199775.77it/s]\n"
     ]
    }
   ],
   "source": [
    "for f in files:\n",
    "    with open(f) as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            try:\n",
    "                l = json.loads(l)\n",
    "                if len(l['filename']) > 2:\n",
    "                    print(l)\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a7e3c185",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = {\n",
    "    'conversations': 'list_of_dict',\n",
    "    'filename': 'list_of_dict'\n",
    "}\n",
    "\n",
    "hashes = 'sha1', 'xxh64'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4ba05887",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "import numpy as np\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-16k-instructions-v3')\n",
    "tokenizer.add_tokens([\"<image>\", \"</image>\", \"<audio>\", \"</audio>\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "15f0ff70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[263]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer('a', return_tensors='pt')['input_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f785a7c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9f5f7216",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!rm -rf mosaic-multimodal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "05b8e4f5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "148657it [01:47, 1377.63it/s]\n",
      "148657it [02:00, 1236.91it/s]\n",
      "293752it [08:07, 602.88it/s]\n",
      "293752it [09:05, 538.53it/s]\n",
      "184500it [01:10, 2614.99it/s]\n",
      "184500it [01:10, 2602.95it/s]\n",
      "3238it [00:01, 2650.20it/s]\n"
     ]
    }
   ],
   "source": [
    "with MDSWriter(out='mosaic-multimodal', columns=columns, compression=None, hashes=hashes) as out:\n",
    "    for f in files:\n",
    "        with open(f) as fopen:\n",
    "            for l in tqdm(fopen):\n",
    "                try:\n",
    "                    l = json.loads(l)\n",
    "                    for i in range(len(l['filename'])):\n",
    "                        l['filename'][i] = l['filename'][i].replace('/output-audio', '/filter-audio')\n",
    "                    for i in range(len(l['conversations'])):\n",
    "                        l['conversations'][i]['content'] = l['conversations'][i]['content'].replace('\\n<image>', ' <image>').replace('<image>\\n', '<image> ').replace('\\n<audio>', ' <audio>').replace('<audio>\\n', '<audio> ').strip()\n",
    "                        l['conversations'][i]['content'] = l['conversations'][i]['content'].replace('<image>', '<image> </image>').replace('<audio>', '<audio> </audio>')\n",
    "                    \n",
    "                    try:\n",
    "                        a = tokenizer.apply_chat_template(l['conversations'], tokenize = False)\n",
    "                    except Exception as e:\n",
    "                        continue\n",
    "                        \n",
    "                    a = tokenizer(a, return_tensors = 'np')['input_ids'][0]\n",
    "                    \n",
    "                    if len(a[(a == 32000) | (a == 32002)]) != len(l['filename']):\n",
    "                        print(l)\n",
    "                        continue\n",
    "                        \n",
    "                    if len(a[(a == 32001) | (a == 32003)]) != len(l['filename']):\n",
    "                        print(l)\n",
    "                        continue\n",
    "                        \n",
    "                    if 'malay' not in f and random.random() > 0.5:\n",
    "                        continue\n",
    "                        \n",
    "                    out.write(l)\n",
    "                except Exception as e:\n",
    "                    print(l, e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "442b23a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "630301"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = LocalDataset('mosaic-multimodal')\n",
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b1f414b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "629476"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = LocalDataset('mosaic-multimodal')\n",
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5ecf0d2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'conversations': [{'role': 'user',\n",
       "   'content': '<image> </image> <image> </image> What is related between picture 1 and picture 2'},\n",
       "  {'role': 'assistant',\n",
       "   'content': 'There is no direct relation between Picture 1: the NCAA tournament bracket for March 29, and Picture 2: the Murray River. The first picture is a diagram of the NCAA basketball tournament bracket, while the second picture is a photograph of the Murray River. They are unrelated.'}],\n",
       " 'filename': ['/home/ubuntu/filtered-blip-images/00398/003989575.jpg',\n",
       "  '/home/ubuntu/filtered-blip-images/00043/000438421.jpg']}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[-10000]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}