find_tokens_in_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find tokens in training data\n",
    "\n",
    "This script finds the origin of tokens in training data. It is set up for the StarCoder2 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "from ipywidgets import interact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = datasets.load_dataset(\"bigcode/the-stack-march-sample-special-tokens-stripped\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "phrases = ['ittrLoremipum','lcsStatusWlan','ucMZQg','mJNUZLIEMNV','GQGantt','BjKPZFq','INCEXPRS','ELABSCOPES',' cKVisitor','tableOBJECT',' simpleIndexQueryParserTests', 'trimBalanco', 'awsAwsjson', 'Baseldytsch', 'Ostschwizertütsch',' BibleDownload', 'ProrrateoImpor','PoliticaProducto']\n",
    "finds = {p: [] for p in phrases}\n",
    "for r in ds['train']:\n",
    "    for p in phrases:\n",
    "        if p in r['content']:\n",
    "            finds[p].append(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hashes = {p: hex(abs(sum(hash(s['content']) for s in finds[p])))[2:8] for p in phrases}\n",
    "phrases = sorted(phrases, key=lambda p: (len(finds[p]),hashes[p]))\n",
    "# group phrases derived from the same docs\n",
    "for p in phrases:\n",
    "    count = sum(s['content'].count(p) for s in finds[p])\n",
    "    print(f\"Phrase {repr(p):<40} found in {len(finds[p])}/{len(ds['train'])} samples\\t  hash {hashes[p]}\\t  {count} occurrences\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@interact(i=(1,11), phrase=phrases)\n",
    "def show_samples(phrase, i=1):\n",
    "    if i > len(finds[phrase]):\n",
    "        print(\"No more samples\")\n",
    "        return\n",
    "    content = finds[phrase][i-1]['content']\n",
    "    lines = content.count('\\n') + 1\n",
    "    print(f\"Showing sample #{i}/{len(finds[phrase])} for {phrase}, which appears {content.count(phrase)} times in {lines} lines\\n-----------------------\\n{content}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "magikarp-ODZH3vJA-py3.9",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}