create_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:15.242418Z",
     "start_time": "2024-04-24T14:13:15.233941Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from transformers import pipeline\n",
    "import requests\n",
    "from PIL import Image\n",
    "import os\n",
    "import subprocess\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370373\n"
     ]
    }
   ],
   "source": [
    "tsv_file_path = \"data.tsv\"\n",
    "\n",
    "# Read the TSV file into a DataFrame\n",
    "df = pd.read_csv(tsv_file_path, sep='\\t', header=0)\n",
    "df = df['image_url']\n",
    "# Display the DataFrame\n",
    "print(len(df))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:23.178240Z",
     "start_time": "2024-04-24T14:13:15.240324Z"
    }
   },
   "id": "f8da91d332d03dc"
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "outputs": [
    {
     "data": {
      "text/plain": "0    https://upload.wikimedia.org/wikipedia/commons...\n1    https://upload.wikimedia.org/wikipedia/commons...\n2    https://upload.wikimedia.org/wikipedia/commons...\n3    http://upload.wikimedia.org/wikipedia/commons/...\n4    https://upload.wikimedia.org/wikipedia/commons...\nName: image_url, dtype: object"
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:23.183514Z",
     "start_time": "2024-04-24T14:13:22.381914Z"
    }
   },
   "id": "de3dc4cb2b5904a6"
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "outputs": [],
   "source": [
    "captioner = pipeline(\"image-to-text\",model=\"Salesforce/blip-image-captioning-large\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:26.982561Z",
     "start_time": "2024-04-24T14:13:22.384974Z"
    }
   },
   "id": "66974b50664d5d85"
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "outputs": [],
   "source": [
    "def caption_image_data(image):\n",
    "    # image_caption = captioner(image)[0]['generated_text']\n",
    "    # return image_caption\n",
    "    try:\n",
    "        image_caption = captioner(image)[0]['generated_text']\n",
    "    except Exception as e:\n",
    "        print(\"Error generating caption:\", e)\n",
    "        image_caption = None\n",
    "    return image_caption"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:26.983324Z",
     "start_time": "2024-04-24T14:13:26.981015Z"
    }
   },
   "id": "6e0c066f7ff082f1"
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "outputs": [],
   "source": [
    "def download_image(url, file_path):\n",
    "    headers = {\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "    \n",
    "    # Check if the request was successful (status code 200)\n",
    "    try:\n",
    "        if response.status_code == 200:\n",
    "            # Open the file in binary write mode and write the content of the response\n",
    "            with open(file_path, 'wb') as f:\n",
    "                f.write(response.content)\n",
    "            resized_image = Image.open(file_path).resize((256, 256))\n",
    "            # Save the resized image\n",
    "            resized_image.save(file_path)\n",
    "            # print(\"Image downloaded successfully!\")\n",
    "            return True\n",
    "        \n",
    "        print(\"Failed to download image. Status code:\", response.status_code)\n",
    "        return False\n",
    "    except:\n",
    "        os.remove(file_path)\n",
    "        return False"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:26.989792Z",
     "start_time": "2024-04-24T14:13:26.984790Z"
    }
   },
   "id": "c43c33a1ea5f8eca"
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "outputs": [],
   "source": [
    "def run_primitive(input_image, output_image, num_shapes=100):\n",
    "    # Construct the primitive command\n",
    "    command = [\n",
    "        '/Users/rishabhsri/go/bin/primitive',            # Command to run\n",
    "        '-i', input_image,      # Input image path\n",
    "        '-o', output_image,     # Output image path\n",
    "        '-n', str(num_shapes),   # Number of primitive shapes\n",
    "        '-s', str(256)\n",
    "    ]\n",
    "    \n",
    "    # Execute the command\n",
    "    try:\n",
    "        subprocess.run(command, check=True)\n",
    "        return True\n",
    "    except subprocess.CalledProcessError as e:\n",
    "        print(\"Error:\", e)\n",
    "    return False"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:26.989890Z",
     "start_time": "2024-04-24T14:13:26.987794Z"
    }
   },
   "id": "760ee6b76e68f2db"
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "outputs": [],
   "source": [
    "n = 15000\n",
    "\n",
    "# Select n random rows from the DataFrame\n",
    "urls = df.sample(n).values.tolist()\n",
    "\n",
    "id = 1\n",
    "ctr = 0\n",
    "data = []\n",
    "if not os.path.exists('myds/source'):\n",
    "        os.makedirs('myds/source')\n",
    "if not os.path.exists('myds/target'):\n",
    "    os.makedirs('myds/target')\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-24T14:13:27.034644Z",
     "start_time": "2024-04-24T14:13:26.991823Z"
    }
   },
   "id": "2fef30c27bf03569"
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/transformers/generation/utils.py:1141: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/PIL/Image.py:3218: DecompressionBombWarning: Image size (127915336 pixels) exceeds limit of 89478485 pixels, could be decompression bomb DOS attack.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/PIL/Image.py:3218: DecompressionBombWarning: Image size (164208000 pixels) exceeds limit of 89478485 pixels, could be decompression bomb DOS attack.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to download image. Status code: 404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/PIL/Image.py:1000: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/PIL/TiffImagePlugin.py:890: UserWarning: Corrupt EXIF data.  Expecting to read 2 bytes but only got 0. \n",
      "  warnings.warn(str(msg))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rishabhsri/Desktop/Columbia Study/Sem 2/CV 2/Final Proj/venv/lib/python3.9/site-packages/PIL/TiffImagePlugin.py:890: UserWarning: Truncated File Read\n",
      "  warnings.warn(str(msg))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "In Process ...\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n",
      "Failed to download image. Status code: 404\n"
     ]
    }
   ],
   "source": [
    "folder = 'myds/'\n",
    "for url in urls:\n",
    "    ctr += 1\n",
    "    filename = f'{id}.png'\n",
    "    source_path = f'source/{filename}'\n",
    "    target_path = f'target/{filename}'\n",
    "    image = download_image(url, folder+source_path)\n",
    "    if image:\n",
    "        try:\n",
    "            id += 1\n",
    "            im = Image.open(folder+source_path)\n",
    "            caption = caption_image_data(im) \n",
    "            if not caption or len(caption) <=2:\n",
    "                id -= 1\n",
    "                os.remove(folder+source_path)\n",
    "                continue\n",
    "            check = run_primitive(folder+source_path, folder+target_path)\n",
    "            if not check:\n",
    "                os.remove(folder+source_path)\n",
    "                id -= 1\n",
    "                continue\n",
    "        except:\n",
    "            os.remove(folder+source_path)\n",
    "            id -= 1\n",
    "            continue\n",
    "        new_row = {'source': source_path, 'target': target_path, 'prompt': caption}\n",
    "        \n",
    "        # Write the new row to the file\n",
    "        with open(folder+'prompt.json', 'a') as file:\n",
    "            json.dump(new_row, file)\n",
    "            file.write('\\n')\n",
    "            \n",
    "        \n",
    "    if id % 100 == 0:\n",
    "        print(\"In Process ...\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-25T11:18:05.330397Z",
     "start_time": "2024-04-24T14:13:27.034131Z"
    }
   },
   "id": "df8982b755869067"
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "outputs": [
    {
     "data": {
      "text/plain": "15000"
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(urls)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-25T11:42:42.429775Z",
     "start_time": "2024-04-25T11:42:42.425191Z"
    }
   },
   "id": "bb7871c09dc5003b"
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List written to file successfully: urls.txt\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    with open('urls.txt', 'w') as f:\n",
    "            # Convert the list to a string representation\n",
    "        list_str = '\\n'.join(map(str, urls))\n",
    "        # Write the string representation to the file\n",
    "        f.write(list_str)\n",
    "        print(\"List written to file successfully:\", 'urls.txt')\n",
    "except Exception as e:\n",
    "    print(\"An error occurred:\", e)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "d3afb4b1f9a1b7f9"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "195d5a5e656aba7a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}