CodePromptConditional/text-davinci-003-0shot-imaginary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ff932a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import openai\n",
    "import os\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1c1d7ee1",
   "metadata": {},
   "source": [
    "Import openai api key from environment variable - here I have it stored as `LOCAL_OPENAI_API_KEY` but you can just paste yours in instead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "baba1916",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "openai.api_key = os.environ.get('LOCAL_OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c209f874",
   "metadata": {},
   "outputs": [],
   "source": [
    "def query(prompt, **kwargs):\n",
    "  \"\"\"\n",
    "  wrapper for the API to easily parse data\n",
    "  \"\"\"\n",
    "  \n",
    "  args = {\n",
    "    \"engine\":\"text-davinci-003\", # using the original davinci\n",
    "    \"temperature\":0, # 0 temperature means it's greedy and gives the same result every time (ish)\n",
    "    \"max_tokens\":500, # 500 tokens should be enough\n",
    "    \"stop\":\"\\n\\n\", # we'll use double newlines to separate the examples\n",
    "  }\n",
    "  \n",
    "  args = {**args, **kwargs}\n",
    "  \n",
    "  r = openai.Completion.create(prompt=prompt, **args)[\"choices\"][0][\"text\"].strip()\n",
    "  return r\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "58c80c83",
   "metadata": {},
   "source": [
    "Great, the API key is loaded. Now we can start using the API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "859af59a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query(\"q: what is 1+1?\\na:\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "39ccc207",
   "metadata": {},
   "source": [
    "This downloads the WIC dataset - you may need to install wget if you don't have it installed https://ports.macports.org/port/wget/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35fd9e50",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-02-26 20:51:47--  https://pilehvar.github.io/wic/package/WiC_dataset.zip\n",
      "Resolving pilehvar.github.io (pilehvar.github.io)... 2606:50c0:8001::153, 2606:50c0:8002::153, 2606:50c0:8003::153, ...\n",
      "Connecting to pilehvar.github.io (pilehvar.github.io)|2606:50c0:8001::153|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 275984 (270K) [application/zip]\n",
      "Saving to: ‘WiC_dataset.zip’\n",
      "\n",
      "WiC_dataset.zip     100%[===================>] 269.52K  --.-KB/s    in 0.05s   \n",
      "\n",
      "2023-02-26 20:51:47 (5.48 MB/s) - ‘WiC_dataset.zip’ saved [275984/275984]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://pilehvar.github.io/wic/package/WiC_dataset.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6a789718",
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile(\"WiC_dataset.zip\",\"r\") as zip_ref:\n",
    "    zip_ref.extractall(\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "719c200f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"train/train.data.txt\", sep='\\t', header=None)\n",
    "train.columns = [\"target\", \"pos\", \"position\", \"context-1\", \"context-2\"]\n",
    "train_gold = pd.read_csv(\"train/train.gold.txt\", sep='\\t', header=None)\n",
    "train_gold.columns = [\"label\"]\n",
    "train = pd.concat([train_gold,train], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5eeb235f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>target</th>\n",
       "      <th>pos</th>\n",
       "      <th>position</th>\n",
       "      <th>context-1</th>\n",
       "      <th>context-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5386</th>\n",
       "      <td>F</td>\n",
       "      <td>admit</td>\n",
       "      <td>V</td>\n",
       "      <td>3-1</td>\n",
       "      <td>The French doors admit onto the yard .</td>\n",
       "      <td>He admitted his errors .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5387</th>\n",
       "      <td>F</td>\n",
       "      <td>exhaust</td>\n",
       "      <td>V</td>\n",
       "      <td>0-4</td>\n",
       "      <td>Exhaust one 's savings .</td>\n",
       "      <td>This kind of work exhausts me .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5389</th>\n",
       "      <td>F</td>\n",
       "      <td>kill</td>\n",
       "      <td>V</td>\n",
       "      <td>0-2</td>\n",
       "      <td>Kill the engine .</td>\n",
       "      <td>She was killed in the collision of three cars .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5390</th>\n",
       "      <td>T</td>\n",
       "      <td>admit</td>\n",
       "      <td>V</td>\n",
       "      <td>1-1</td>\n",
       "      <td>To admit a serious thought into the mind .</td>\n",
       "      <td>She admitted us here .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5394</th>\n",
       "      <td>F</td>\n",
       "      <td>write</td>\n",
       "      <td>V</td>\n",
       "      <td>6-1</td>\n",
       "      <td>How many books did Georges Simenon write ?</td>\n",
       "      <td>Please write to me every week .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     label   target pos position                                   context-1  \\\n",
       "5386     F    admit   V      3-1      The French doors admit onto the yard .   \n",
       "5387     F  exhaust   V      0-4                    Exhaust one 's savings .   \n",
       "5389     F     kill   V      0-2                           Kill the engine .   \n",
       "5390     T    admit   V      1-1  To admit a serious thought into the mind .   \n",
       "5394     F    write   V      6-1  How many books did Georges Simenon write ?   \n",
       "\n",
       "                                            context-2  \n",
       "5386                         He admitted his errors .  \n",
       "5387                  This kind of work exhausts me .  \n",
       "5389  She was killed in the collision of three cars .  \n",
       "5390                           She admitted us here .  \n",
       "5394                  Please write to me every week .  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[train.pos==\"V\"].tail(18)[:5]\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a5d5af78",
   "metadata": {},
   "source": [
    "We can go and grab the last couple verbs of the train set to use as our few shots. I added some annotation for meaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ee2badfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fewShotVerb = \"\"\" interface comparison{\n",
    "    \"Similar\": bool, // whether the two meaings of the word are similar or used to mean different things, should be true | false\n",
    "}\n",
    "\n",
    "determineWordSimilarSense(word, context1, context2) : comparison =>{\n",
    "    senseOfFirstWord = ai.computeSense(word, context1) // return the sense of the word in the first context\n",
    "    senseOfSecondWord = ai.computeSense(word, context2) // return the sense of the word in the second context\n",
    "    return ai.compare(word, senseOfFirstWord, senseOfSecondWord) // return the comparison object\n",
    "}\n",
    "\n",
    "//This returns the keys inside of double quotes (\"KEY\") so we can parse with JSON\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "000b6d2b",
   "metadata": {},
   "source": [
    "Likewise we can do it with nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "97710a99",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fewShotNoun = \"\"\" interface comparison{\n",
    "    \"Similar\": bool, // whether the two meaings of the word are similar or used to mean different things, should be true | false\n",
    "}\n",
    "\n",
    "determineWordSimilarSense(word, context1, context2) : comparison =>{\n",
    "    senseOfFirstWord = ai.computeSense(word, context1) // return the sense of the word in the first context\n",
    "    senseOfSecondWord = ai.computeSense(word, context2) // return the sense of the word in the second context\n",
    "    return ai.compare(word, senseOfFirstWord, senseOfSecondWord) // return the comparison object\n",
    "}\n",
    "\n",
    "//This returns the keys inside of double quotes (\"KEY\") so we can parse with JSON\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cc80f075",
   "metadata": {},
   "outputs": [],
   "source": [
    "def testRow(row):\n",
    "  pos = row[1][\"pos\"]\n",
    "  examples = fewShotNoun if pos == \"N\" else fewShotVerb\n",
    "\n",
    "  context = \"\"\"determineWordSimilarSense(\"{}\", \"{}\", \"{}\")\n",
    ">>>\"\"\".format(row[1][\"context-1\"], row[1][\"context-2\"], row[1][\"target\"])\n",
    "  res = query(examples + context)\n",
    "  try:\n",
    "    parsed = json.loads(res)\n",
    "  except Exception as e:\n",
    "    print('error parsing: ', e)\n",
    "    # check if there's a true in the last 100 characters\n",
    "    parsed = {\n",
    "      \"Context\": res,\n",
    "      \"Similar\": \"true\" in res.lower()[-100:]\n",
    "    }\n",
    "\n",
    "  return parsed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b73eec67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores {'V': 0, 'N': 0}\n",
      "attempted {'V': 0, 'N': 0}\n",
      "output: {'Similar': False}\n",
      "scores {'V': 1, 'N': 0}\n",
      "attempted {'V': 1, 'N': 0}\n",
      "output: {'Similar': False}\n",
      "scores {'V': 2, 'N': 0}\n",
      "attempted {'V': 2, 'N': 0}\n",
      "output: {'Similar': True}\n",
      "scores {'V': 2, 'N': 0}\n",
      "attempted {'V': 3, 'N': 0}\n",
      "output: {'Similar': False}\n",
      "scores {'V': 2, 'N': 0}\n",
      "attempted {'V': 3, 'N': 1}\n",
      "output: {'Similar': True}\n",
      "scores {'V': 2, 'N': 0}\n",
      "attempted {'V': 3, 'N': 2}\n",
      "output: {'Similar': False}\n",
      "scores {'V': 3, 'N': 0}\n",
      "attempted {'V': 4, 'N': 2}\n",
      "output: {'Similar': True}\n",
      "scores {'V': 4, 'N': 0}\n",
      "attempted {'V': 5, 'N': 2}\n",
      "output: {'Similar': False}\n",
      "scores {'V': 5, 'N': 0}\n",
      "attempted {'V': 6, 'N': 2}\n",
      "output: {'Similar': True}\n",
      "scores {'V': 6, 'N': 0}\n",
      "attempted {'V': 7, 'N': 2}\n",
      "output: {'Similar': True}\n"
     ]
    }
   ],
   "source": [
    "scores = {'V': 0, 'N': 0}\n",
    "attempted = {'V': 0, 'N': 0}\n",
    "for row in train.head(10).iterrows():\n",
    "    print('scores', scores)\n",
    "    print('attempted', attempted)\n",
    "    actual = row[1][\"label\"]\n",
    "    output = testRow(row)\n",
    "    print('output:', output)\n",
    "    pos = row[1][\"pos\"]\n",
    "    attempted[pos] += 1\n",
    "\n",
    "    if actual == \"T\":\n",
    "        if output['Similar']:\n",
    "            scores[pos] += 1\n",
    "        \n",
    "    if actual == \"F\":\n",
    "        if not output['Similar']:\n",
    "            scores[pos] += 1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f85eb05c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'V': 6, 'N': 1}, {'V': 7, 'N': 3})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores, attempted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "936860db",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev = pd.read_csv(\"dev/dev.data.txt\", sep='\\t', header=None)\n",
    "dev.columns = [\"target\", \"pos\", \"position\", \"context-1\", \"context-2\"]\n",
    "dev_gold = pd.read_csv(\"dev/dev.gold.txt\", sep='\\t', header=None)\n",
    "dev_gold.columns = [\"label\"]\n",
    "dev = pd.concat([dev_gold,dev], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7bf1cd4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "devResults = {}\n",
    "complete = 0\n",
    "correct = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1052a35e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 0.0, Complete: 1 Correct: 0 Wrong: 1\n",
      "Score: 0.0, Complete: 2 Correct: 0 Wrong: 2\n",
      "Score: 0.33, Complete: 3 Correct: 1 Wrong: 2\n",
      "Score: 0.5, Complete: 4 Correct: 2 Wrong: 2\n",
      "Score: 0.4, Complete: 5 Correct: 2 Wrong: 3\n",
      "Score: 0.33, Complete: 6 Correct: 2 Wrong: 4\n",
      "Score: 0.29, Complete: 7 Correct: 2 Wrong: 5\n",
      "Score: 0.38, Complete: 8 Correct: 3 Wrong: 5\n",
      "Score: 0.44, Complete: 9 Correct: 4 Wrong: 5\n",
      "Score: 0.4, Complete: 10 Correct: 4 Wrong: 6\n",
      "Score: 0.36, Complete: 11 Correct: 4 Wrong: 7\n",
      "Score: 0.42, Complete: 12 Correct: 5 Wrong: 7\n",
      "Score: 0.38, Complete: 13 Correct: 5 Wrong: 8\n",
      "Score: 0.36, Complete: 14 Correct: 5 Wrong: 9\n",
      "Score: 0.33, Complete: 15 Correct: 5 Wrong: 10\n",
      "Score: 0.38, Complete: 16 Correct: 6 Wrong: 10\n",
      "Score: 0.41, Complete: 17 Correct: 7 Wrong: 10\n",
      "Score: 0.44, Complete: 18 Correct: 8 Wrong: 10\n",
      "Score: 0.47, Complete: 19 Correct: 9 Wrong: 10\n",
      "Score: 0.5, Complete: 20 Correct: 10 Wrong: 10\n",
      "Score: 0.52, Complete: 21 Correct: 11 Wrong: 10\n",
      "Score: 0.55, Complete: 22 Correct: 12 Wrong: 10\n",
      "Score: 0.57, Complete: 23 Correct: 13 Wrong: 10\n",
      "Score: 0.58, Complete: 24 Correct: 14 Wrong: 10\n",
      "Score: 0.6, Complete: 25 Correct: 15 Wrong: 10\n",
      "Score: 0.58, Complete: 26 Correct: 15 Wrong: 11\n",
      "Score: 0.59, Complete: 27 Correct: 16 Wrong: 11\n",
      "Score: 0.61, Complete: 28 Correct: 17 Wrong: 11\n",
      "Score: 0.59, Complete: 29 Correct: 17 Wrong: 12\n",
      "Score: 0.6, Complete: 30 Correct: 18 Wrong: 12\n",
      "Score: 0.58, Complete: 31 Correct: 18 Wrong: 13\n",
      "Score: 0.59, Complete: 32 Correct: 19 Wrong: 13\n",
      "Score: 0.58, Complete: 33 Correct: 19 Wrong: 14\n",
      "Score: 0.56, Complete: 34 Correct: 19 Wrong: 15\n",
      "Score: 0.57, Complete: 35 Correct: 20 Wrong: 15\n",
      "Score: 0.58, Complete: 36 Correct: 21 Wrong: 15\n",
      "Score: 0.59, Complete: 37 Correct: 22 Wrong: 15\n",
      "Score: 0.61, Complete: 38 Correct: 23 Wrong: 15\n",
      "Score: 0.62, Complete: 39 Correct: 24 Wrong: 15\n",
      "Score: 0.62, Complete: 40 Correct: 25 Wrong: 15\n",
      "Score: 0.63, Complete: 41 Correct: 26 Wrong: 15\n",
      "Score: 0.62, Complete: 42 Correct: 26 Wrong: 16\n",
      "Score: 0.6, Complete: 43 Correct: 26 Wrong: 17\n",
      "Score: 0.59, Complete: 44 Correct: 26 Wrong: 18\n",
      "Score: 0.6, Complete: 45 Correct: 27 Wrong: 18\n",
      "Score: 0.61, Complete: 46 Correct: 28 Wrong: 18\n",
      "Score: 0.6, Complete: 47 Correct: 28 Wrong: 19\n",
      "Score: 0.58, Complete: 48 Correct: 28 Wrong: 20\n",
      "Score: 0.57, Complete: 49 Correct: 28 Wrong: 21\n",
      "Score: 0.58, Complete: 50 Correct: 29 Wrong: 21\n",
      "Score: 0.59, Complete: 51 Correct: 30 Wrong: 21\n",
      "Score: 0.58, Complete: 52 Correct: 30 Wrong: 22\n",
      "Score: 0.58, Complete: 53 Correct: 31 Wrong: 22\n",
      "Score: 0.57, Complete: 54 Correct: 31 Wrong: 23\n",
      "Score: 0.56, Complete: 55 Correct: 31 Wrong: 24\n",
      "Score: 0.57, Complete: 56 Correct: 32 Wrong: 24\n",
      "Score: 0.58, Complete: 57 Correct: 33 Wrong: 24\n",
      "Score: 0.59, Complete: 58 Correct: 34 Wrong: 24\n",
      "Score: 0.59, Complete: 59 Correct: 35 Wrong: 24\n",
      "Score: 0.58, Complete: 60 Correct: 35 Wrong: 25\n",
      "Score: 0.59, Complete: 61 Correct: 36 Wrong: 25\n",
      "Score: 0.6, Complete: 62 Correct: 37 Wrong: 25\n",
      "Score: 0.59, Complete: 63 Correct: 37 Wrong: 26\n",
      "Score: 0.59, Complete: 64 Correct: 38 Wrong: 26\n",
      "Score: 0.6, Complete: 65 Correct: 39 Wrong: 26\n",
      "Score: 0.61, Complete: 66 Correct: 40 Wrong: 26\n",
      "Score: 0.61, Complete: 67 Correct: 41 Wrong: 26\n",
      "Score: 0.6, Complete: 68 Correct: 41 Wrong: 27\n",
      "Score: 0.61, Complete: 69 Correct: 42 Wrong: 27\n",
      "Score: 0.61, Complete: 70 Correct: 43 Wrong: 27\n",
      "Score: 0.62, Complete: 71 Correct: 44 Wrong: 27\n",
      "Score: 0.62, Complete: 72 Correct: 45 Wrong: 27\n",
      "Score: 0.63, Complete: 73 Correct: 46 Wrong: 27\n",
      "Score: 0.64, Complete: 74 Correct: 47 Wrong: 27\n",
      "Score: 0.64, Complete: 75 Correct: 48 Wrong: 27\n",
      "Score: 0.64, Complete: 76 Correct: 49 Wrong: 27\n",
      "Score: 0.64, Complete: 77 Correct: 49 Wrong: 28\n",
      "Score: 0.64, Complete: 78 Correct: 50 Wrong: 28\n",
      "Score: 0.63, Complete: 79 Correct: 50 Wrong: 29\n",
      "Score: 0.64, Complete: 80 Correct: 51 Wrong: 29\n",
      "Score: 0.63, Complete: 81 Correct: 51 Wrong: 30\n",
      "Score: 0.62, Complete: 82 Correct: 51 Wrong: 31\n",
      "Score: 0.63, Complete: 83 Correct: 52 Wrong: 31\n",
      "Score: 0.63, Complete: 84 Correct: 53 Wrong: 31\n",
      "Score: 0.64, Complete: 85 Correct: 54 Wrong: 31\n",
      "Score: 0.63, Complete: 86 Correct: 54 Wrong: 32\n",
      "Score: 0.63, Complete: 87 Correct: 55 Wrong: 32\n",
      "Score: 0.62, Complete: 88 Correct: 55 Wrong: 33\n",
      "Score: 0.63, Complete: 89 Correct: 56 Wrong: 33\n",
      "Score: 0.62, Complete: 90 Correct: 56 Wrong: 34\n",
      "Score: 0.62, Complete: 91 Correct: 56 Wrong: 35\n",
      "Score: 0.61, Complete: 92 Correct: 56 Wrong: 36\n",
      "Score: 0.6, Complete: 93 Correct: 56 Wrong: 37\n",
      "Score: 0.61, Complete: 94 Correct: 57 Wrong: 37\n",
      "Score: 0.61, Complete: 95 Correct: 58 Wrong: 37\n",
      "Score: 0.61, Complete: 96 Correct: 59 Wrong: 37\n",
      "Score: 0.62, Complete: 97 Correct: 60 Wrong: 37\n",
      "Score: 0.61, Complete: 98 Correct: 60 Wrong: 38\n",
      "Score: 0.62, Complete: 99 Correct: 61 Wrong: 38\n",
      "Score: 0.62, Complete: 100 Correct: 62 Wrong: 38\n",
      "Score: 0.61, Complete: 101 Correct: 62 Wrong: 39\n",
      "Score: 0.62, Complete: 102 Correct: 63 Wrong: 39\n",
      "Score: 0.62, Complete: 103 Correct: 64 Wrong: 39\n",
      "Score: 0.62, Complete: 104 Correct: 64 Wrong: 40\n",
      "Score: 0.61, Complete: 105 Correct: 64 Wrong: 41\n",
      "Score: 0.6, Complete: 106 Correct: 64 Wrong: 42\n",
      "Score: 0.6, Complete: 107 Correct: 64 Wrong: 43\n",
      "Score: 0.6, Complete: 108 Correct: 65 Wrong: 43\n",
      "Score: 0.61, Complete: 109 Correct: 66 Wrong: 43\n",
      "Score: 0.61, Complete: 110 Correct: 67 Wrong: 43\n",
      "Score: 0.61, Complete: 111 Correct: 68 Wrong: 43\n",
      "Score: 0.61, Complete: 112 Correct: 68 Wrong: 44\n",
      "Score: 0.61, Complete: 113 Correct: 69 Wrong: 44\n",
      "Score: 0.61, Complete: 114 Correct: 69 Wrong: 45\n",
      "Score: 0.61, Complete: 115 Correct: 70 Wrong: 45\n",
      "Score: 0.6, Complete: 116 Correct: 70 Wrong: 46\n",
      "Score: 0.61, Complete: 117 Correct: 71 Wrong: 46\n",
      "Score: 0.6, Complete: 118 Correct: 71 Wrong: 47\n",
      "Score: 0.61, Complete: 119 Correct: 72 Wrong: 47\n",
      "Score: 0.61, Complete: 120 Correct: 73 Wrong: 47\n",
      "Score: 0.61, Complete: 121 Correct: 74 Wrong: 47\n",
      "Score: 0.61, Complete: 122 Correct: 74 Wrong: 48\n",
      "Score: 0.61, Complete: 123 Correct: 75 Wrong: 48\n",
      "Score: 0.61, Complete: 124 Correct: 76 Wrong: 48\n",
      "Score: 0.61, Complete: 125 Correct: 76 Wrong: 49\n",
      "Score: 0.61, Complete: 126 Correct: 77 Wrong: 49\n",
      "Score: 0.61, Complete: 127 Correct: 78 Wrong: 49\n",
      "Score: 0.62, Complete: 128 Correct: 79 Wrong: 49\n",
      "Score: 0.61, Complete: 129 Correct: 79 Wrong: 50\n",
      "Score: 0.62, Complete: 130 Correct: 80 Wrong: 50\n",
      "Score: 0.61, Complete: 131 Correct: 80 Wrong: 51\n",
      "Score: 0.61, Complete: 132 Correct: 81 Wrong: 51\n",
      "Score: 0.62, Complete: 133 Correct: 82 Wrong: 51\n",
      "Score: 0.61, Complete: 134 Correct: 82 Wrong: 52\n",
      "Score: 0.61, Complete: 135 Correct: 82 Wrong: 53\n",
      "Score: 0.61, Complete: 136 Correct: 83 Wrong: 53\n",
      "Score: 0.61, Complete: 137 Correct: 84 Wrong: 53\n",
      "Score: 0.62, Complete: 138 Correct: 85 Wrong: 53\n",
      "Score: 0.61, Complete: 139 Correct: 85 Wrong: 54\n",
      "Score: 0.61, Complete: 140 Correct: 86 Wrong: 54\n",
      "Score: 0.62, Complete: 141 Correct: 87 Wrong: 54\n",
      "Score: 0.62, Complete: 142 Correct: 88 Wrong: 54\n",
      "Score: 0.62, Complete: 143 Correct: 89 Wrong: 54\n",
      "Score: 0.62, Complete: 144 Correct: 90 Wrong: 54\n",
      "Score: 0.62, Complete: 145 Correct: 90 Wrong: 55\n",
      "Score: 0.62, Complete: 146 Correct: 91 Wrong: 55\n",
      "Score: 0.63, Complete: 147 Correct: 92 Wrong: 55\n",
      "Score: 0.63, Complete: 148 Correct: 93 Wrong: 55\n",
      "Score: 0.63, Complete: 149 Correct: 94 Wrong: 55\n",
      "Score: 0.63, Complete: 150 Correct: 95 Wrong: 55\n",
      "Score: 0.64, Complete: 151 Correct: 96 Wrong: 55\n",
      "Score: 0.63, Complete: 152 Correct: 96 Wrong: 56\n",
      "Score: 0.63, Complete: 153 Correct: 96 Wrong: 57\n",
      "Score: 0.63, Complete: 154 Correct: 97 Wrong: 57\n",
      "Score: 0.63, Complete: 155 Correct: 98 Wrong: 57\n",
      "Score: 0.63, Complete: 156 Correct: 99 Wrong: 57\n",
      "Score: 0.64, Complete: 157 Correct: 100 Wrong: 57\n",
      "Score: 0.63, Complete: 158 Correct: 100 Wrong: 58\n",
      "Score: 0.63, Complete: 159 Correct: 100 Wrong: 59\n",
      "Score: 0.63, Complete: 160 Correct: 101 Wrong: 59\n",
      "Score: 0.63, Complete: 161 Correct: 102 Wrong: 59\n",
      "Score: 0.64, Complete: 162 Correct: 103 Wrong: 59\n",
      "Score: 0.63, Complete: 163 Correct: 103 Wrong: 60\n",
      "Score: 0.63, Complete: 164 Correct: 104 Wrong: 60\n",
      "Score: 0.63, Complete: 165 Correct: 104 Wrong: 61\n",
      "Score: 0.63, Complete: 166 Correct: 104 Wrong: 62\n",
      "Score: 0.63, Complete: 167 Correct: 105 Wrong: 62\n",
      "Score: 0.63, Complete: 168 Correct: 106 Wrong: 62\n",
      "Score: 0.63, Complete: 169 Correct: 107 Wrong: 62\n",
      "Score: 0.63, Complete: 170 Correct: 107 Wrong: 63\n",
      "Score: 0.63, Complete: 171 Correct: 107 Wrong: 64\n",
      "Score: 0.63, Complete: 172 Correct: 108 Wrong: 64\n",
      "Score: 0.63, Complete: 173 Correct: 109 Wrong: 64\n",
      "Score: 0.63, Complete: 174 Correct: 109 Wrong: 65\n",
      "Score: 0.63, Complete: 175 Correct: 110 Wrong: 65\n",
      "Score: 0.63, Complete: 176 Correct: 111 Wrong: 65\n",
      "Score: 0.63, Complete: 177 Correct: 112 Wrong: 65\n",
      "Score: 0.63, Complete: 178 Correct: 113 Wrong: 65\n",
      "Score: 0.64, Complete: 179 Correct: 114 Wrong: 65\n",
      "Score: 0.64, Complete: 180 Correct: 115 Wrong: 65\n",
      "Score: 0.64, Complete: 181 Correct: 116 Wrong: 65\n",
      "Score: 0.64, Complete: 182 Correct: 116 Wrong: 66\n",
      "Score: 0.63, Complete: 183 Correct: 116 Wrong: 67\n",
      "Score: 0.63, Complete: 184 Correct: 116 Wrong: 68\n",
      "Score: 0.63, Complete: 185 Correct: 117 Wrong: 68\n",
      "Score: 0.63, Complete: 186 Correct: 118 Wrong: 68\n",
      "Score: 0.63, Complete: 187 Correct: 118 Wrong: 69\n",
      "Score: 0.63, Complete: 188 Correct: 119 Wrong: 69\n",
      "Score: 0.63, Complete: 189 Correct: 119 Wrong: 70\n",
      "Score: 0.63, Complete: 190 Correct: 120 Wrong: 70\n",
      "Score: 0.63, Complete: 191 Correct: 121 Wrong: 70\n",
      "Score: 0.63, Complete: 192 Correct: 121 Wrong: 71\n",
      "Score: 0.63, Complete: 193 Correct: 122 Wrong: 71\n",
      "Score: 0.63, Complete: 194 Correct: 123 Wrong: 71\n",
      "Score: 0.64, Complete: 195 Correct: 124 Wrong: 71\n",
      "Score: 0.64, Complete: 196 Correct: 125 Wrong: 71\n",
      "Score: 0.63, Complete: 197 Correct: 125 Wrong: 72\n",
      "Score: 0.64, Complete: 198 Correct: 126 Wrong: 72\n",
      "Score: 0.63, Complete: 199 Correct: 126 Wrong: 73\n",
      "Score: 0.63, Complete: 200 Correct: 126 Wrong: 74\n",
      "Score: 0.63, Complete: 201 Correct: 126 Wrong: 75\n",
      "Score: 0.63, Complete: 202 Correct: 127 Wrong: 75\n",
      "Score: 0.63, Complete: 203 Correct: 128 Wrong: 75\n",
      "Score: 0.63, Complete: 204 Correct: 129 Wrong: 75\n",
      "Score: 0.63, Complete: 205 Correct: 129 Wrong: 76\n",
      "Score: 0.63, Complete: 206 Correct: 130 Wrong: 76\n",
      "Score: 0.63, Complete: 207 Correct: 130 Wrong: 77\n",
      "Score: 0.63, Complete: 208 Correct: 131 Wrong: 77\n",
      "Score: 0.63, Complete: 209 Correct: 132 Wrong: 77\n",
      "Score: 0.63, Complete: 210 Correct: 133 Wrong: 77\n",
      "Score: 0.64, Complete: 211 Correct: 134 Wrong: 77\n",
      "Score: 0.64, Complete: 212 Correct: 135 Wrong: 77\n",
      "Score: 0.63, Complete: 213 Correct: 135 Wrong: 78\n",
      "Score: 0.64, Complete: 214 Correct: 136 Wrong: 78\n",
      "Score: 0.63, Complete: 215 Correct: 136 Wrong: 79\n",
      "Score: 0.63, Complete: 216 Correct: 136 Wrong: 80\n",
      "Score: 0.63, Complete: 217 Correct: 137 Wrong: 80\n",
      "Score: 0.63, Complete: 218 Correct: 137 Wrong: 81\n",
      "Score: 0.63, Complete: 219 Correct: 137 Wrong: 82\n",
      "Score: 0.62, Complete: 220 Correct: 137 Wrong: 83\n",
      "Score: 0.62, Complete: 221 Correct: 138 Wrong: 83\n",
      "Score: 0.62, Complete: 222 Correct: 138 Wrong: 84\n",
      "Score: 0.62, Complete: 223 Correct: 139 Wrong: 84\n",
      "Score: 0.62, Complete: 224 Correct: 139 Wrong: 85\n",
      "Score: 0.62, Complete: 225 Correct: 140 Wrong: 85\n",
      "Score: 0.62, Complete: 226 Correct: 141 Wrong: 85\n",
      "Score: 0.62, Complete: 227 Correct: 141 Wrong: 86\n",
      "Score: 0.62, Complete: 228 Correct: 142 Wrong: 86\n",
      "Score: 0.62, Complete: 229 Correct: 142 Wrong: 87\n",
      "Score: 0.62, Complete: 230 Correct: 142 Wrong: 88\n",
      "Score: 0.62, Complete: 231 Correct: 143 Wrong: 88\n",
      "Score: 0.62, Complete: 232 Correct: 143 Wrong: 89\n",
      "Score: 0.61, Complete: 233 Correct: 143 Wrong: 90\n",
      "Score: 0.61, Complete: 234 Correct: 143 Wrong: 91\n",
      "Score: 0.61, Complete: 235 Correct: 143 Wrong: 92\n",
      "Score: 0.61, Complete: 236 Correct: 144 Wrong: 92\n",
      "Score: 0.61, Complete: 237 Correct: 145 Wrong: 92\n",
      "Score: 0.61, Complete: 238 Correct: 146 Wrong: 92\n",
      "Score: 0.61, Complete: 239 Correct: 146 Wrong: 93\n",
      "Score: 0.61, Complete: 240 Correct: 146 Wrong: 94\n",
      "Score: 0.61, Complete: 241 Correct: 147 Wrong: 94\n",
      "Score: 0.61, Complete: 242 Correct: 148 Wrong: 94\n",
      "Score: 0.61, Complete: 243 Correct: 149 Wrong: 94\n",
      "Score: 0.61, Complete: 244 Correct: 149 Wrong: 95\n",
      "Score: 0.61, Complete: 245 Correct: 149 Wrong: 96\n",
      "Score: 0.61, Complete: 246 Correct: 150 Wrong: 96\n",
      "Score: 0.61, Complete: 247 Correct: 150 Wrong: 97\n",
      "Score: 0.61, Complete: 248 Correct: 151 Wrong: 97\n",
      "Score: 0.61, Complete: 249 Correct: 151 Wrong: 98\n",
      "Score: 0.6, Complete: 250 Correct: 151 Wrong: 99\n",
      "Score: 0.61, Complete: 251 Correct: 152 Wrong: 99\n",
      "Score: 0.61, Complete: 252 Correct: 153 Wrong: 99\n",
      "Score: 0.61, Complete: 253 Correct: 154 Wrong: 99\n",
      "Score: 0.61, Complete: 254 Correct: 154 Wrong: 100\n",
      "Score: 0.6, Complete: 255 Correct: 154 Wrong: 101\n",
      "Score: 0.61, Complete: 256 Correct: 155 Wrong: 101\n",
      "Score: 0.6, Complete: 257 Correct: 155 Wrong: 102\n",
      "Score: 0.6, Complete: 258 Correct: 155 Wrong: 103\n",
      "Score: 0.6, Complete: 259 Correct: 156 Wrong: 103\n",
      "Score: 0.6, Complete: 260 Correct: 157 Wrong: 103\n",
      "Score: 0.6, Complete: 261 Correct: 157 Wrong: 104\n",
      "Score: 0.6, Complete: 262 Correct: 158 Wrong: 104\n",
      "Score: 0.6, Complete: 263 Correct: 159 Wrong: 104\n",
      "Score: 0.61, Complete: 264 Correct: 160 Wrong: 104\n",
      "Score: 0.6, Complete: 265 Correct: 160 Wrong: 105\n",
      "Score: 0.61, Complete: 266 Correct: 161 Wrong: 105\n",
      "Score: 0.61, Complete: 267 Correct: 162 Wrong: 105\n",
      "Score: 0.61, Complete: 268 Correct: 163 Wrong: 105\n",
      "Score: 0.61, Complete: 269 Correct: 164 Wrong: 105\n",
      "Score: 0.61, Complete: 270 Correct: 165 Wrong: 105\n",
      "Score: 0.61, Complete: 271 Correct: 166 Wrong: 105\n",
      "Score: 0.61, Complete: 272 Correct: 167 Wrong: 105\n",
      "Score: 0.61, Complete: 273 Correct: 167 Wrong: 106\n",
      "Score: 0.61, Complete: 274 Correct: 168 Wrong: 106\n",
      "Score: 0.61, Complete: 275 Correct: 169 Wrong: 106\n",
      "Score: 0.61, Complete: 276 Correct: 169 Wrong: 107\n",
      "Score: 0.61, Complete: 277 Correct: 169 Wrong: 108\n",
      "Score: 0.61, Complete: 278 Correct: 169 Wrong: 109\n",
      "Score: 0.61, Complete: 279 Correct: 170 Wrong: 109\n",
      "Score: 0.61, Complete: 280 Correct: 171 Wrong: 109\n",
      "Score: 0.61, Complete: 281 Correct: 171 Wrong: 110\n",
      "Score: 0.61, Complete: 282 Correct: 171 Wrong: 111\n",
      "Score: 0.61, Complete: 283 Correct: 172 Wrong: 111\n",
      "Score: 0.61, Complete: 284 Correct: 173 Wrong: 111\n",
      "Score: 0.61, Complete: 285 Correct: 174 Wrong: 111\n",
      "Score: 0.61, Complete: 286 Correct: 175 Wrong: 111\n",
      "Score: 0.61, Complete: 287 Correct: 176 Wrong: 111\n",
      "Score: 0.61, Complete: 288 Correct: 176 Wrong: 112\n",
      "Score: 0.61, Complete: 289 Correct: 177 Wrong: 112\n",
      "Score: 0.61, Complete: 290 Correct: 177 Wrong: 113\n",
      "Score: 0.61, Complete: 291 Correct: 177 Wrong: 114\n",
      "Score: 0.61, Complete: 292 Correct: 177 Wrong: 115\n",
      "Score: 0.6, Complete: 293 Correct: 177 Wrong: 116\n",
      "Score: 0.6, Complete: 294 Correct: 177 Wrong: 117\n",
      "Score: 0.6, Complete: 295 Correct: 177 Wrong: 118\n",
      "Score: 0.6, Complete: 296 Correct: 178 Wrong: 118\n",
      "Score: 0.6, Complete: 297 Correct: 179 Wrong: 118\n",
      "Score: 0.6, Complete: 298 Correct: 179 Wrong: 119\n",
      "Score: 0.6, Complete: 299 Correct: 179 Wrong: 120\n",
      "Score: 0.6, Complete: 300 Correct: 180 Wrong: 120\n",
      "Score: 0.6, Complete: 301 Correct: 180 Wrong: 121\n",
      "Score: 0.6, Complete: 302 Correct: 181 Wrong: 121\n",
      "Score: 0.6, Complete: 303 Correct: 181 Wrong: 122\n",
      "Score: 0.6, Complete: 304 Correct: 182 Wrong: 122\n",
      "Score: 0.6, Complete: 305 Correct: 182 Wrong: 123\n",
      "Score: 0.6, Complete: 306 Correct: 183 Wrong: 123\n",
      "Score: 0.6, Complete: 307 Correct: 184 Wrong: 123\n",
      "Score: 0.6, Complete: 308 Correct: 184 Wrong: 124\n",
      "Score: 0.6, Complete: 309 Correct: 185 Wrong: 124\n",
      "Score: 0.6, Complete: 310 Correct: 186 Wrong: 124\n",
      "Score: 0.6, Complete: 311 Correct: 186 Wrong: 125\n",
      "Score: 0.6, Complete: 312 Correct: 187 Wrong: 125\n",
      "Score: 0.6, Complete: 313 Correct: 187 Wrong: 126\n",
      "Score: 0.6, Complete: 314 Correct: 188 Wrong: 126\n",
      "Score: 0.6, Complete: 315 Correct: 189 Wrong: 126\n",
      "Score: 0.6, Complete: 316 Correct: 190 Wrong: 126\n",
      "Score: 0.6, Complete: 317 Correct: 190 Wrong: 127\n",
      "Score: 0.6, Complete: 318 Correct: 190 Wrong: 128\n",
      "Score: 0.6, Complete: 319 Correct: 190 Wrong: 129\n",
      "Score: 0.6, Complete: 320 Correct: 191 Wrong: 129\n",
      "Score: 0.6, Complete: 321 Correct: 192 Wrong: 129\n",
      "Score: 0.6, Complete: 322 Correct: 193 Wrong: 129\n",
      "Score: 0.6, Complete: 323 Correct: 193 Wrong: 130\n",
      "Score: 0.6, Complete: 324 Correct: 194 Wrong: 130\n",
      "Score: 0.6, Complete: 325 Correct: 195 Wrong: 130\n",
      "Score: 0.6, Complete: 326 Correct: 195 Wrong: 131\n",
      "Score: 0.6, Complete: 327 Correct: 196 Wrong: 131\n",
      "Score: 0.6, Complete: 328 Correct: 197 Wrong: 131\n",
      "Score: 0.6, Complete: 329 Correct: 197 Wrong: 132\n",
      "Score: 0.6, Complete: 330 Correct: 198 Wrong: 132\n",
      "Score: 0.6, Complete: 331 Correct: 199 Wrong: 132\n",
      "Score: 0.6, Complete: 332 Correct: 199 Wrong: 133\n",
      "Score: 0.6, Complete: 333 Correct: 199 Wrong: 134\n",
      "Score: 0.6, Complete: 334 Correct: 199 Wrong: 135\n",
      "Score: 0.6, Complete: 335 Correct: 200 Wrong: 135\n",
      "Score: 0.6, Complete: 336 Correct: 200 Wrong: 136\n",
      "Score: 0.6, Complete: 337 Correct: 201 Wrong: 136\n",
      "Score: 0.59, Complete: 338 Correct: 201 Wrong: 137\n",
      "Score: 0.59, Complete: 339 Correct: 201 Wrong: 138\n",
      "Score: 0.59, Complete: 340 Correct: 201 Wrong: 139\n",
      "Score: 0.59, Complete: 341 Correct: 201 Wrong: 140\n",
      "Score: 0.59, Complete: 342 Correct: 202 Wrong: 140\n",
      "Score: 0.59, Complete: 343 Correct: 203 Wrong: 140\n",
      "Score: 0.59, Complete: 344 Correct: 203 Wrong: 141\n",
      "Score: 0.59, Complete: 345 Correct: 204 Wrong: 141\n",
      "Score: 0.59, Complete: 346 Correct: 205 Wrong: 141\n",
      "Score: 0.59, Complete: 347 Correct: 205 Wrong: 142\n",
      "Score: 0.59, Complete: 348 Correct: 206 Wrong: 142\n",
      "Score: 0.59, Complete: 349 Correct: 207 Wrong: 142\n",
      "Score: 0.59, Complete: 350 Correct: 207 Wrong: 143\n",
      "Score: 0.59, Complete: 351 Correct: 207 Wrong: 144\n",
      "Score: 0.59, Complete: 352 Correct: 207 Wrong: 145\n",
      "Score: 0.59, Complete: 353 Correct: 208 Wrong: 145\n",
      "Score: 0.59, Complete: 354 Correct: 209 Wrong: 145\n",
      "Score: 0.59, Complete: 355 Correct: 209 Wrong: 146\n",
      "Score: 0.59, Complete: 356 Correct: 210 Wrong: 146\n",
      "Score: 0.59, Complete: 357 Correct: 211 Wrong: 146\n",
      "Score: 0.59, Complete: 358 Correct: 211 Wrong: 147\n",
      "Score: 0.59, Complete: 359 Correct: 211 Wrong: 148\n",
      "Score: 0.59, Complete: 360 Correct: 211 Wrong: 149\n",
      "Score: 0.59, Complete: 361 Correct: 212 Wrong: 149\n",
      "Score: 0.59, Complete: 362 Correct: 213 Wrong: 149\n",
      "Score: 0.59, Complete: 363 Correct: 214 Wrong: 149\n",
      "Score: 0.59, Complete: 364 Correct: 214 Wrong: 150\n",
      "Score: 0.59, Complete: 365 Correct: 214 Wrong: 151\n",
      "Score: 0.59, Complete: 366 Correct: 215 Wrong: 151\n",
      "Score: 0.59, Complete: 367 Correct: 215 Wrong: 152\n",
      "Score: 0.59, Complete: 368 Correct: 216 Wrong: 152\n",
      "Score: 0.59, Complete: 369 Correct: 217 Wrong: 152\n",
      "Score: 0.59, Complete: 370 Correct: 218 Wrong: 152\n",
      "Score: 0.59, Complete: 371 Correct: 218 Wrong: 153\n",
      "Score: 0.59, Complete: 372 Correct: 219 Wrong: 153\n",
      "Score: 0.59, Complete: 373 Correct: 219 Wrong: 154\n",
      "Score: 0.59, Complete: 374 Correct: 220 Wrong: 154\n",
      "Score: 0.59, Complete: 375 Correct: 221 Wrong: 154\n",
      "Score: 0.59, Complete: 376 Correct: 222 Wrong: 154\n",
      "Score: 0.59, Complete: 377 Correct: 223 Wrong: 154\n",
      "Score: 0.59, Complete: 378 Correct: 224 Wrong: 154\n",
      "Score: 0.59, Complete: 379 Correct: 225 Wrong: 154\n",
      "Score: 0.59, Complete: 380 Correct: 226 Wrong: 154\n",
      "Score: 0.6, Complete: 381 Correct: 227 Wrong: 154\n",
      "Score: 0.59, Complete: 382 Correct: 227 Wrong: 155\n",
      "Score: 0.59, Complete: 383 Correct: 227 Wrong: 156\n",
      "Score: 0.59, Complete: 384 Correct: 228 Wrong: 156\n",
      "Score: 0.59, Complete: 385 Correct: 229 Wrong: 156\n",
      "Score: 0.59, Complete: 386 Correct: 229 Wrong: 157\n",
      "Score: 0.59, Complete: 387 Correct: 230 Wrong: 157\n",
      "Score: 0.6, Complete: 388 Correct: 231 Wrong: 157\n",
      "Score: 0.59, Complete: 389 Correct: 231 Wrong: 158\n",
      "Score: 0.59, Complete: 390 Correct: 232 Wrong: 158\n",
      "Score: 0.59, Complete: 391 Correct: 232 Wrong: 159\n",
      "Score: 0.59, Complete: 392 Correct: 233 Wrong: 159\n",
      "Score: 0.6, Complete: 393 Correct: 234 Wrong: 159\n",
      "Score: 0.6, Complete: 394 Correct: 235 Wrong: 159\n",
      "Score: 0.6, Complete: 395 Correct: 236 Wrong: 159\n",
      "Score: 0.6, Complete: 396 Correct: 237 Wrong: 159\n",
      "Score: 0.6, Complete: 397 Correct: 237 Wrong: 160\n",
      "Score: 0.6, Complete: 398 Correct: 238 Wrong: 160\n",
      "Score: 0.6, Complete: 399 Correct: 238 Wrong: 161\n",
      "Score: 0.59, Complete: 400 Correct: 238 Wrong: 162\n",
      "Score: 0.6, Complete: 401 Correct: 239 Wrong: 162\n",
      "Score: 0.59, Complete: 402 Correct: 239 Wrong: 163\n",
      "Score: 0.59, Complete: 403 Correct: 239 Wrong: 164\n",
      "Score: 0.59, Complete: 404 Correct: 239 Wrong: 165\n",
      "Score: 0.59, Complete: 405 Correct: 240 Wrong: 165\n",
      "Score: 0.59, Complete: 406 Correct: 240 Wrong: 166\n",
      "Score: 0.59, Complete: 407 Correct: 240 Wrong: 167\n",
      "Score: 0.59, Complete: 408 Correct: 240 Wrong: 168\n",
      "Score: 0.59, Complete: 409 Correct: 240 Wrong: 169\n",
      "Score: 0.59, Complete: 410 Correct: 240 Wrong: 170\n",
      "Score: 0.59, Complete: 411 Correct: 241 Wrong: 170\n",
      "Score: 0.58, Complete: 412 Correct: 241 Wrong: 171\n",
      "Score: 0.58, Complete: 413 Correct: 241 Wrong: 172\n",
      "Score: 0.58, Complete: 414 Correct: 242 Wrong: 172\n",
      "Score: 0.58, Complete: 415 Correct: 242 Wrong: 173\n",
      "Score: 0.58, Complete: 416 Correct: 243 Wrong: 173\n",
      "Score: 0.58, Complete: 417 Correct: 243 Wrong: 174\n",
      "Score: 0.58, Complete: 418 Correct: 244 Wrong: 174\n",
      "Score: 0.58, Complete: 419 Correct: 245 Wrong: 174\n",
      "Score: 0.59, Complete: 420 Correct: 246 Wrong: 174\n",
      "Score: 0.58, Complete: 421 Correct: 246 Wrong: 175\n",
      "Score: 0.59, Complete: 422 Correct: 247 Wrong: 175\n",
      "Score: 0.59, Complete: 423 Correct: 248 Wrong: 175\n",
      "Score: 0.59, Complete: 424 Correct: 249 Wrong: 175\n",
      "Score: 0.59, Complete: 425 Correct: 249 Wrong: 176\n",
      "Score: 0.59, Complete: 426 Correct: 250 Wrong: 176\n",
      "Score: 0.59, Complete: 427 Correct: 250 Wrong: 177\n",
      "Score: 0.59, Complete: 428 Correct: 251 Wrong: 177\n",
      "Score: 0.59, Complete: 429 Correct: 252 Wrong: 177\n",
      "Score: 0.59, Complete: 430 Correct: 253 Wrong: 177\n",
      "Score: 0.59, Complete: 431 Correct: 254 Wrong: 177\n",
      "Score: 0.59, Complete: 432 Correct: 254 Wrong: 178\n",
      "Score: 0.59, Complete: 433 Correct: 255 Wrong: 178\n",
      "Score: 0.59, Complete: 434 Correct: 256 Wrong: 178\n",
      "Score: 0.59, Complete: 435 Correct: 257 Wrong: 178\n",
      "Score: 0.59, Complete: 436 Correct: 258 Wrong: 178\n",
      "Score: 0.59, Complete: 437 Correct: 259 Wrong: 178\n",
      "Score: 0.59, Complete: 438 Correct: 259 Wrong: 179\n",
      "Score: 0.59, Complete: 439 Correct: 260 Wrong: 179\n",
      "Score: 0.59, Complete: 440 Correct: 261 Wrong: 179\n",
      "Score: 0.59, Complete: 441 Correct: 261 Wrong: 180\n",
      "Score: 0.59, Complete: 442 Correct: 262 Wrong: 180\n",
      "Score: 0.59, Complete: 443 Correct: 262 Wrong: 181\n",
      "Score: 0.59, Complete: 444 Correct: 263 Wrong: 181\n",
      "Score: 0.59, Complete: 445 Correct: 264 Wrong: 181\n",
      "Score: 0.59, Complete: 446 Correct: 265 Wrong: 181\n",
      "Score: 0.6, Complete: 447 Correct: 266 Wrong: 181\n",
      "Score: 0.59, Complete: 448 Correct: 266 Wrong: 182\n",
      "Score: 0.59, Complete: 449 Correct: 267 Wrong: 182\n",
      "Score: 0.59, Complete: 450 Correct: 267 Wrong: 183\n",
      "Score: 0.59, Complete: 451 Correct: 268 Wrong: 183\n",
      "Score: 0.59, Complete: 452 Correct: 268 Wrong: 184\n",
      "Score: 0.59, Complete: 453 Correct: 269 Wrong: 184\n",
      "Score: 0.59, Complete: 454 Correct: 269 Wrong: 185\n",
      "Score: 0.59, Complete: 455 Correct: 269 Wrong: 186\n",
      "Score: 0.59, Complete: 456 Correct: 269 Wrong: 187\n",
      "Score: 0.59, Complete: 457 Correct: 270 Wrong: 187\n",
      "Score: 0.59, Complete: 458 Correct: 271 Wrong: 187\n",
      "Score: 0.59, Complete: 459 Correct: 271 Wrong: 188\n",
      "Score: 0.59, Complete: 460 Correct: 272 Wrong: 188\n",
      "Score: 0.59, Complete: 461 Correct: 273 Wrong: 188\n",
      "Score: 0.59, Complete: 462 Correct: 274 Wrong: 188\n",
      "Score: 0.59, Complete: 463 Correct: 275 Wrong: 188\n",
      "Score: 0.59, Complete: 464 Correct: 275 Wrong: 189\n",
      "Score: 0.59, Complete: 465 Correct: 275 Wrong: 190\n",
      "Score: 0.59, Complete: 466 Correct: 276 Wrong: 190\n",
      "Score: 0.59, Complete: 467 Correct: 277 Wrong: 190\n",
      "Score: 0.59, Complete: 468 Correct: 278 Wrong: 190\n",
      "Score: 0.59, Complete: 469 Correct: 279 Wrong: 190\n",
      "Score: 0.59, Complete: 470 Correct: 279 Wrong: 191\n",
      "Score: 0.59, Complete: 471 Correct: 280 Wrong: 191\n",
      "Score: 0.6, Complete: 472 Correct: 281 Wrong: 191\n",
      "Score: 0.6, Complete: 473 Correct: 282 Wrong: 191\n",
      "Score: 0.6, Complete: 474 Correct: 283 Wrong: 191\n",
      "Score: 0.6, Complete: 475 Correct: 283 Wrong: 192\n",
      "Score: 0.6, Complete: 476 Correct: 284 Wrong: 192\n",
      "Score: 0.6, Complete: 477 Correct: 285 Wrong: 192\n",
      "Score: 0.6, Complete: 478 Correct: 285 Wrong: 193\n",
      "Score: 0.6, Complete: 479 Correct: 286 Wrong: 193\n",
      "Score: 0.6, Complete: 480 Correct: 287 Wrong: 193\n",
      "Score: 0.6, Complete: 481 Correct: 288 Wrong: 193\n",
      "Score: 0.6, Complete: 482 Correct: 288 Wrong: 194\n",
      "Score: 0.6, Complete: 483 Correct: 289 Wrong: 194\n",
      "Score: 0.6, Complete: 484 Correct: 290 Wrong: 194\n",
      "Score: 0.6, Complete: 485 Correct: 290 Wrong: 195\n",
      "Score: 0.6, Complete: 486 Correct: 290 Wrong: 196\n",
      "Score: 0.6, Complete: 487 Correct: 291 Wrong: 196\n",
      "Score: 0.6, Complete: 488 Correct: 292 Wrong: 196\n",
      "Score: 0.6, Complete: 489 Correct: 292 Wrong: 197\n",
      "Score: 0.6, Complete: 490 Correct: 293 Wrong: 197\n",
      "Score: 0.6, Complete: 491 Correct: 293 Wrong: 198\n",
      "Score: 0.6, Complete: 492 Correct: 294 Wrong: 198\n",
      "Score: 0.6, Complete: 493 Correct: 295 Wrong: 198\n",
      "Score: 0.6, Complete: 494 Correct: 295 Wrong: 199\n",
      "Score: 0.6, Complete: 495 Correct: 295 Wrong: 200\n",
      "Score: 0.6, Complete: 496 Correct: 296 Wrong: 200\n",
      "Score: 0.6, Complete: 497 Correct: 296 Wrong: 201\n",
      "Score: 0.59, Complete: 498 Correct: 296 Wrong: 202\n",
      "Score: 0.6, Complete: 499 Correct: 297 Wrong: 202\n",
      "Score: 0.6, Complete: 500 Correct: 298 Wrong: 202\n",
      "Score: 0.6, Complete: 501 Correct: 299 Wrong: 202\n",
      "Score: 0.6, Complete: 502 Correct: 300 Wrong: 202\n",
      "Score: 0.6, Complete: 503 Correct: 301 Wrong: 202\n",
      "Score: 0.6, Complete: 504 Correct: 301 Wrong: 203\n",
      "Score: 0.6, Complete: 505 Correct: 302 Wrong: 203\n",
      "Score: 0.6, Complete: 506 Correct: 302 Wrong: 204\n",
      "Score: 0.6, Complete: 507 Correct: 303 Wrong: 204\n",
      "Score: 0.6, Complete: 508 Correct: 304 Wrong: 204\n",
      "Score: 0.6, Complete: 509 Correct: 304 Wrong: 205\n",
      "Score: 0.6, Complete: 510 Correct: 305 Wrong: 205\n",
      "Score: 0.6, Complete: 511 Correct: 306 Wrong: 205\n",
      "Score: 0.6, Complete: 512 Correct: 307 Wrong: 205\n",
      "Score: 0.6, Complete: 513 Correct: 308 Wrong: 205\n",
      "Score: 0.6, Complete: 514 Correct: 308 Wrong: 206\n",
      "Score: 0.6, Complete: 515 Correct: 309 Wrong: 206\n",
      "Score: 0.6, Complete: 516 Correct: 309 Wrong: 207\n",
      "Score: 0.6, Complete: 517 Correct: 310 Wrong: 207\n",
      "Score: 0.6, Complete: 518 Correct: 310 Wrong: 208\n",
      "Score: 0.6, Complete: 519 Correct: 311 Wrong: 208\n",
      "Score: 0.6, Complete: 520 Correct: 311 Wrong: 209\n",
      "Score: 0.6, Complete: 521 Correct: 312 Wrong: 209\n",
      "Score: 0.6, Complete: 522 Correct: 312 Wrong: 210\n",
      "Score: 0.6, Complete: 523 Correct: 312 Wrong: 211\n",
      "Score: 0.6, Complete: 524 Correct: 312 Wrong: 212\n",
      "Score: 0.59, Complete: 525 Correct: 312 Wrong: 213\n",
      "Score: 0.6, Complete: 526 Correct: 313 Wrong: 213\n",
      "Score: 0.6, Complete: 527 Correct: 314 Wrong: 213\n",
      "Score: 0.6, Complete: 528 Correct: 315 Wrong: 213\n",
      "Score: 0.6, Complete: 529 Correct: 316 Wrong: 213\n",
      "Score: 0.6, Complete: 530 Correct: 316 Wrong: 214\n",
      "Score: 0.6, Complete: 531 Correct: 316 Wrong: 215\n",
      "Score: 0.6, Complete: 532 Correct: 317 Wrong: 215\n",
      "Score: 0.6, Complete: 533 Correct: 318 Wrong: 215\n",
      "Score: 0.6, Complete: 534 Correct: 319 Wrong: 215\n",
      "Score: 0.6, Complete: 535 Correct: 320 Wrong: 215\n",
      "Score: 0.6, Complete: 536 Correct: 320 Wrong: 216\n",
      "Score: 0.6, Complete: 537 Correct: 321 Wrong: 216\n",
      "Score: 0.6, Complete: 538 Correct: 322 Wrong: 216\n",
      "Score: 0.6, Complete: 539 Correct: 323 Wrong: 216\n",
      "Score: 0.6, Complete: 540 Correct: 324 Wrong: 216\n",
      "Score: 0.6, Complete: 541 Correct: 325 Wrong: 216\n",
      "Score: 0.6, Complete: 542 Correct: 325 Wrong: 217\n",
      "Score: 0.6, Complete: 543 Correct: 325 Wrong: 218\n",
      "Score: 0.6, Complete: 544 Correct: 326 Wrong: 218\n",
      "Score: 0.6, Complete: 545 Correct: 327 Wrong: 218\n",
      "Score: 0.6, Complete: 546 Correct: 328 Wrong: 218\n",
      "Score: 0.6, Complete: 547 Correct: 329 Wrong: 218\n",
      "Score: 0.6, Complete: 548 Correct: 330 Wrong: 218\n",
      "Score: 0.6, Complete: 549 Correct: 331 Wrong: 218\n",
      "Score: 0.6, Complete: 550 Correct: 332 Wrong: 218\n",
      "Score: 0.6, Complete: 551 Correct: 333 Wrong: 218\n",
      "Score: 0.6, Complete: 552 Correct: 333 Wrong: 219\n",
      "Score: 0.6, Complete: 553 Correct: 333 Wrong: 220\n",
      "Score: 0.6, Complete: 554 Correct: 333 Wrong: 221\n",
      "Score: 0.6, Complete: 555 Correct: 334 Wrong: 221\n",
      "Score: 0.6, Complete: 556 Correct: 334 Wrong: 222\n",
      "Score: 0.6, Complete: 557 Correct: 335 Wrong: 222\n",
      "Score: 0.6, Complete: 558 Correct: 336 Wrong: 222\n",
      "Score: 0.6, Complete: 559 Correct: 337 Wrong: 222\n",
      "Score: 0.6, Complete: 560 Correct: 338 Wrong: 222\n",
      "Score: 0.6, Complete: 561 Correct: 338 Wrong: 223\n",
      "Score: 0.6, Complete: 562 Correct: 339 Wrong: 223\n",
      "Score: 0.6, Complete: 563 Correct: 340 Wrong: 223\n",
      "Score: 0.6, Complete: 564 Correct: 340 Wrong: 224\n",
      "Score: 0.6, Complete: 565 Correct: 340 Wrong: 225\n",
      "Score: 0.6, Complete: 566 Correct: 340 Wrong: 226\n",
      "Score: 0.6, Complete: 567 Correct: 340 Wrong: 227\n",
      "Score: 0.6, Complete: 568 Correct: 341 Wrong: 227\n",
      "Score: 0.6, Complete: 569 Correct: 341 Wrong: 228\n",
      "Score: 0.6, Complete: 570 Correct: 342 Wrong: 228\n",
      "Score: 0.6, Complete: 571 Correct: 342 Wrong: 229\n",
      "Score: 0.6, Complete: 572 Correct: 342 Wrong: 230\n",
      "Score: 0.6, Complete: 573 Correct: 342 Wrong: 231\n",
      "Score: 0.6, Complete: 574 Correct: 342 Wrong: 232\n",
      "Score: 0.59, Complete: 575 Correct: 342 Wrong: 233\n",
      "Score: 0.59, Complete: 576 Correct: 342 Wrong: 234\n",
      "Score: 0.59, Complete: 577 Correct: 343 Wrong: 234\n",
      "Score: 0.6, Complete: 578 Correct: 344 Wrong: 234\n",
      "Score: 0.6, Complete: 579 Correct: 345 Wrong: 234\n",
      "Score: 0.6, Complete: 580 Correct: 346 Wrong: 234\n",
      "Score: 0.6, Complete: 581 Correct: 347 Wrong: 234\n",
      "Score: 0.6, Complete: 582 Correct: 347 Wrong: 235\n",
      "Score: 0.6, Complete: 583 Correct: 348 Wrong: 235\n",
      "Score: 0.6, Complete: 584 Correct: 349 Wrong: 235\n",
      "Score: 0.6, Complete: 585 Correct: 349 Wrong: 236\n",
      "Score: 0.6, Complete: 586 Correct: 350 Wrong: 236\n",
      "Score: 0.6, Complete: 587 Correct: 350 Wrong: 237\n",
      "Score: 0.6, Complete: 588 Correct: 351 Wrong: 237\n",
      "Score: 0.6, Complete: 589 Correct: 351 Wrong: 238\n",
      "Score: 0.6, Complete: 590 Correct: 352 Wrong: 238\n",
      "Score: 0.6, Complete: 591 Correct: 353 Wrong: 238\n",
      "Score: 0.6, Complete: 592 Correct: 354 Wrong: 238\n",
      "Score: 0.6, Complete: 593 Correct: 355 Wrong: 238\n",
      "Score: 0.6, Complete: 594 Correct: 356 Wrong: 238\n",
      "Score: 0.6, Complete: 595 Correct: 357 Wrong: 238\n",
      "Score: 0.6, Complete: 596 Correct: 358 Wrong: 238\n",
      "Score: 0.6, Complete: 597 Correct: 358 Wrong: 239\n",
      "Score: 0.6, Complete: 598 Correct: 358 Wrong: 240\n",
      "Score: 0.6, Complete: 599 Correct: 358 Wrong: 241\n",
      "Score: 0.6, Complete: 600 Correct: 359 Wrong: 241\n",
      "Score: 0.6, Complete: 601 Correct: 359 Wrong: 242\n",
      "Score: 0.6, Complete: 602 Correct: 360 Wrong: 242\n",
      "Score: 0.6, Complete: 603 Correct: 361 Wrong: 242\n",
      "Score: 0.6, Complete: 604 Correct: 362 Wrong: 242\n",
      "Score: 0.6, Complete: 605 Correct: 363 Wrong: 242\n",
      "Score: 0.6, Complete: 606 Correct: 364 Wrong: 242\n",
      "Score: 0.6, Complete: 607 Correct: 364 Wrong: 243\n",
      "Score: 0.6, Complete: 608 Correct: 365 Wrong: 243\n",
      "Score: 0.6, Complete: 609 Correct: 366 Wrong: 243\n",
      "Score: 0.6, Complete: 610 Correct: 366 Wrong: 244\n",
      "Score: 0.6, Complete: 611 Correct: 367 Wrong: 244\n",
      "Score: 0.6, Complete: 612 Correct: 368 Wrong: 244\n",
      "Score: 0.6, Complete: 613 Correct: 368 Wrong: 245\n",
      "Score: 0.6, Complete: 614 Correct: 369 Wrong: 245\n",
      "Score: 0.6, Complete: 615 Correct: 370 Wrong: 245\n",
      "Score: 0.6, Complete: 616 Correct: 370 Wrong: 246\n",
      "Score: 0.6, Complete: 617 Correct: 370 Wrong: 247\n",
      "Score: 0.6, Complete: 618 Correct: 371 Wrong: 247\n",
      "Score: 0.6, Complete: 619 Correct: 372 Wrong: 247\n",
      "Score: 0.6, Complete: 620 Correct: 373 Wrong: 247\n",
      "Score: 0.6, Complete: 621 Correct: 373 Wrong: 248\n",
      "Score: 0.6, Complete: 622 Correct: 373 Wrong: 249\n",
      "Score: 0.6, Complete: 623 Correct: 373 Wrong: 250\n",
      "Score: 0.6, Complete: 624 Correct: 373 Wrong: 251\n",
      "Score: 0.6, Complete: 625 Correct: 373 Wrong: 252\n",
      "Score: 0.6, Complete: 626 Correct: 374 Wrong: 252\n",
      "Score: 0.6, Complete: 627 Correct: 375 Wrong: 252\n",
      "Score: 0.6, Complete: 628 Correct: 376 Wrong: 252\n",
      "Score: 0.6, Complete: 629 Correct: 377 Wrong: 252\n",
      "Score: 0.6, Complete: 630 Correct: 378 Wrong: 252\n",
      "Score: 0.6, Complete: 631 Correct: 379 Wrong: 252\n",
      "Score: 0.6, Complete: 632 Correct: 379 Wrong: 253\n",
      "Score: 0.6, Complete: 633 Correct: 380 Wrong: 253\n",
      "Score: 0.6, Complete: 634 Correct: 380 Wrong: 254\n",
      "Score: 0.6, Complete: 635 Correct: 381 Wrong: 254\n",
      "Score: 0.6, Complete: 636 Correct: 382 Wrong: 254\n",
      "Score: 0.6, Complete: 637 Correct: 383 Wrong: 254\n",
      "Score: 0.6, Complete: 638 Correct: 383 Wrong: 255\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for row in dev.iterrows():\n",
    "\n",
    "    if row[0] in devResults:\n",
    "        continue\n",
    "\n",
    "    q1 = row[1][\"context-1\"]\n",
    "    q2 = row[1][\"context-2\"]\n",
    "    target = row[1][\"target\"]\n",
    "    actual = row[1][\"label\"]\n",
    "    \n",
    "    pos = row[1][\"pos\"]\n",
    "\n",
    "    output = testRow(row)\n",
    "    \n",
    "    myResults = {}\n",
    "    myResults[\"q1\"] = q1\n",
    "    myResults[\"q2\"] = q2\n",
    "\n",
    "    myResults[\"pos\"] = row[1][\"pos\"]\n",
    "\n",
    "    myResults[\"target\"] = target\n",
    "\n",
    "    myResults[\"output\"] = output\n",
    "\n",
    "    myResults[\"actual\"] = actual\n",
    "    devResults[row[0]] = myResults\n",
    "    complete +=1\n",
    "    if actual == \"T\":\n",
    "        if output['Similar']:\n",
    "            correct += 1\n",
    "    if actual == \"F\":\n",
    "        if not output['Similar']:\n",
    "            correct += 1\n",
    "\n",
    "    pct = correct/complete\n",
    "    pct = round(pct, 2)\n",
    "\n",
    "    print (\"Score: {}, Complete: {} Correct: {} Wrong: {}\".format(pct, complete, correct, complete-correct))\n",
    "    with open('text-davinci-003-code-0shot-imaginary.json', 'w') as f:\n",
    "        json.dump(devResults, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8cd76e65",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('text-davinci-003-code-0shot-imaginary.json', 'w') as f:\n",
    "    json.dump(devResults, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcbf8037",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "c735a6024e2df8b2c383589304813f5c39729bfd28a4c8f38c29181a5cd583ca"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}