process_cognates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91098c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c19a1e00",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_29292/1445220695.py:1: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(\"CogNet-v2.0.tsv\", sep=\"\\t\", quoting=3, error_bad_lines=False)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"CogNet-v2.0.tsv\", sep=\"\\t\", quoting=3, error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "ff86a54a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['lang 1', 'word 1', 'lang 2', 'word 2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "4aca66ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "cognates = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2f8628ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "57d43d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "langs = set(df['lang 1'].values) or set(df['lang 2'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "0de92a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "for lang in langs:\n",
    "    cognates[lang] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "d8037339",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kmr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7967it [00:00, 33546.21it/s]\n",
      "27880it [00:00, 31290.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing shi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "28it [00:00, 25491.75it/s]\n",
      "62it [00:00, 31191.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing glg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "178613it [00:05, 31443.62it/s]\n",
      "73006it [00:02, 30845.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tyv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1727it [00:00, 30875.70it/s]\n",
      "5823it [00:00, 31952.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing yua\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "77it [00:00, 30180.49it/s]\n",
      "322it [00:00, 31637.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "248it [00:00, 31344.16it/s]\n",
      "214it [00:00, 31761.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing slv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "149430it [00:04, 31020.29it/s]\n",
      "182794it [00:05, 31017.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pag\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "337it [00:00, 30934.97it/s]\n",
      "520it [00:00, 31439.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing eus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "138200it [00:04, 29622.68it/s]\n",
      "84847it [00:02, 29692.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ccc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "594it [00:00, 21994.99it/s]\n",
      "426it [00:00, 26209.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1509it [00:00, 28363.39it/s]\n",
      "4697it [00:00, 28463.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "96992it [00:03, 28814.41it/s]\n",
      "175139it [00:05, 30561.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nog\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1204it [00:00, 28523.82it/s]\n",
      "3300it [00:00, 27310.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7it [00:00, 10591.68it/s]\n",
      "88it [00:00, 17929.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "842it [00:00, 24905.00it/s]\n",
      "953it [00:00, 25054.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pjt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "114it [00:00, 25773.54it/s]\n",
      "169it [00:00, 26362.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mon\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9005it [00:00, 26549.96it/s]\n",
      "36319it [00:01, 26784.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kab\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1187it [00:00, 26534.77it/s]\n",
      "3287it [00:00, 26690.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ary\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "221it [00:00, 26211.43it/s]\n",
      "343it [00:00, 27590.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing aln\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46it [00:00, 18956.37it/s]\n",
      "53it [00:00, 22851.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sce\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9it [00:00, 11600.72it/s]\n",
      "12it [00:00, 21779.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lkt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "163it [00:00, 25649.87it/s]\n",
      "530it [00:00, 25928.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing amh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3897it [00:00, 26528.30it/s]\n",
      "1366it [00:00, 26526.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bak\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7061it [00:00, 28028.16it/s]\n",
      "8688it [00:00, 26188.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hak\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "973it [00:00, 24957.55it/s]\n",
      "1785it [00:00, 24596.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bul\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75447it [00:03, 24188.00it/s]\n",
      "78122it [00:03, 25069.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bar\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2769it [00:00, 15966.96it/s]\n",
      "5836it [00:00, 19509.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "354267it [00:13, 27064.81it/s]\n",
      "112183it [00:05, 18799.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing uig\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1605it [00:00, 23613.43it/s]\n",
      "4011it [00:00, 21823.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ibo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "832it [00:00, 23408.92it/s]\n",
      "1592it [00:00, 22637.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1467it [00:00, 22425.75it/s]\n",
      "5260it [00:00, 22175.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36it [00:00, 16960.01it/s]\n",
      "59it [00:00, 17039.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing jam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1471it [00:00, 24222.16it/s]\n",
      "3557it [00:00, 21470.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing slk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "112888it [00:06, 17736.04it/s]\n",
      "197904it [00:10, 18718.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bjn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "163it [00:00, 28593.54it/s]\n",
      "570it [00:00, 31132.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lbe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "756it [00:00, 27007.25it/s]\n",
      "2069it [00:00, 27363.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ell\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "62837it [00:02, 27565.14it/s]\n",
      "45686it [00:01, 26691.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nob\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "69231it [00:02, 28931.00it/s]\n",
      "129583it [00:04, 28757.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing epo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "89431it [00:03, 28724.85it/s]\n",
      "168910it [00:05, 28908.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing esh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "63it [00:00, 21829.09it/s]\n",
      "31it [00:00, 20783.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tah\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1275it [00:00, 27506.25it/s]\n",
      "1331it [00:00, 27608.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hye\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "28498it [00:01, 27119.77it/s]\n",
      "26150it [00:00, 26858.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "219502it [00:08, 25060.02it/s]\n",
      "109685it [00:03, 28134.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22003it [00:00, 30281.05it/s]\n",
      "26780it [00:00, 28745.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5944it [00:00, 30277.31it/s]\n",
      "10682it [00:00, 29741.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tet\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1507it [00:00, 29766.59it/s]\n",
      "3058it [00:00, 29283.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing udm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1743it [00:00, 31466.49it/s]\n",
      "4640it [00:00, 29238.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19it [00:00, 22297.64it/s]\n",
      "66it [00:00, 25736.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ilo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2794it [00:00, 27973.17it/s]\n",
      "6111it [00:00, 28823.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing udi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "350it [00:00, 30001.56it/s]\n",
      "1144it [00:00, 29994.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pol\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "235057it [00:08, 26614.87it/s]\n",
      "109637it [00:05, 18689.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mrc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2it [00:00, 4286.46it/s]\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tur\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "49009it [00:02, 18760.61it/s]\n",
      "97849it [00:05, 18143.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lmo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4873it [00:00, 19568.44it/s]\n",
      "13065it [00:00, 18945.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing alt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2509it [00:00, 16971.29it/s]\n",
      "3542it [00:00, 16915.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kaa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1321it [00:00, 20747.48it/s]\n",
      "2753it [00:00, 20088.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ood\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:00, 8811.56it/s]\n",
      "28it [00:00, 14935.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing zul\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8618it [00:00, 17657.19it/s]\n",
      "11932it [00:00, 17418.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kis\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "114it [00:00, 17623.78it/s]\n",
      "86it [00:00, 19197.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing haw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5052it [00:00, 21209.71it/s]\n",
      "8514it [00:00, 19551.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 5090.17it/s]\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bxr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1313it [00:00, 17859.33it/s]\n",
      "3870it [00:00, 16562.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing src\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "969it [00:00, 19174.66it/s]\n",
      "2879it [00:00, 19300.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cho\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 20065.56it/s]\n",
      "220it [00:00, 20724.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tuk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8489it [00:00, 23472.94it/s]\n",
      "25326it [00:01, 22699.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing wbp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11it [00:00, 19450.82it/s]\n",
      "55it [00:00, 29777.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ven\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "368it [00:00, 29199.85it/s]\n",
      "581it [00:00, 29171.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pnb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "662it [00:00, 31641.06it/s]\n",
      "2102it [00:00, 31382.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing khm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5556it [00:00, 32396.64it/s]\n",
      "4850it [00:00, 31818.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tzm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "151it [00:00, 28242.58it/s]\n",
      "221it [00:00, 30875.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ind\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "362586it [00:13, 26978.13it/s]\n",
      "80539it [00:02, 30425.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sne\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17it [00:00, 14563.56it/s]\n",
      "64it [00:00, 29064.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dsb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "8141it [00:00, 29680.90it/s]\n",
      "15927it [00:00, 28188.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kbd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "560it [00:00, 30686.96it/s]\n",
      "2033it [00:00, 30586.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sdc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "627it [00:00, 23160.09it/s]\n",
      "1157it [00:00, 23586.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ita\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "328001it [00:10, 29823.42it/s]\n",
      "118413it [00:03, 29620.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing brx\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24324it [00:00, 30676.62it/s]\n",
      "56520it [00:01, 29326.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pap\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6684it [00:00, 31122.24it/s]\n",
      "26489it [00:00, 30780.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing heb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13340it [00:00, 30599.84it/s]\n",
      "9829it [00:00, 29779.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "92873it [00:03, 24492.19it/s]\n",
      "117036it [00:07, 16316.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing liv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2239it [00:00, 21084.81it/s]\n",
      "1589it [00:00, 20225.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing por\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "287793it [00:13, 21199.39it/s]\n",
      "287586it [00:11, 24752.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing avd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "116it [00:00, 25823.43it/s]\n",
      "27it [00:00, 22795.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ppl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "78it [00:00, 30261.37it/s]\n",
      "143it [00:00, 29977.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ain\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "197it [00:00, 29032.95it/s]\n",
      "96it [00:00, 29689.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:00, 12276.01it/s]\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing csb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6302it [00:00, 30407.81it/s]\n",
      "8756it [00:00, 30604.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fij\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1850it [00:00, 24765.77it/s]\n",
      "1503it [00:00, 24520.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cha\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2261it [00:00, 21596.34it/s]\n",
      "8860it [00:00, 22351.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing yor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3977it [00:00, 13110.73it/s]\n",
      "12264it [00:00, 26111.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rgn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "80it [00:00, 21975.53it/s]\n",
      "292it [00:00, 24250.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gla\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17585it [00:00, 26560.01it/s]\n",
      "21256it [00:00, 27499.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rmf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22it [00:00, 20356.21it/s]\n",
      "170it [00:00, 24839.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bug\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "291it [00:00, 28121.15it/s]\n",
      "100it [00:00, 28950.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing brg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "29it [00:00, 25923.87it/s]\n",
      "34it [00:00, 27319.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kri\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "91it [00:00, 28511.37it/s]\n",
      "156it [00:00, 28945.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lao\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4581it [00:00, 29365.72it/s]\n",
      "6563it [00:00, 27634.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "124341it [00:04, 29595.75it/s]\n",
      "79218it [00:02, 28227.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sjd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "476it [00:00, 31198.06it/s]\n",
      "1949it [00:00, 31290.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vro\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3130it [00:00, 30100.82it/s]\n",
      "8745it [00:00, 27350.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mwl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1220it [00:00, 30130.96it/s]\n",
      "3865it [00:00, 28892.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kjh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1995it [00:00, 30540.83it/s]\n",
      "4578it [00:00, 27450.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kik\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "707it [00:00, 27472.42it/s]\n",
      "1034it [00:00, 27841.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing myv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2641it [00:00, 26966.35it/s]\n",
      "9472it [00:00, 24070.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kas\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11628it [00:00, 26440.45it/s]\n",
      "68730it [00:02, 27973.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ssw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "303it [00:00, 31108.46it/s]\n",
      "569it [00:00, 30921.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5536it [00:00, 28939.90it/s]\n",
      "12516it [00:00, 27573.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ket\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "161it [00:00, 28584.61it/s]\n",
      "543it [00:00, 31561.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing abq\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1146it [00:00, 29534.63it/s]\n",
      "692it [00:00, 29742.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ltz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18282it [00:00, 26716.02it/s]\n",
      "32516it [00:01, 28812.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing naq\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "330it [00:00, 29525.38it/s]\n",
      "516it [00:00, 30147.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bcl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "888it [00:00, 22929.99it/s]\n",
      "856it [00:00, 20120.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tsn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7565it [00:00, 20286.39it/s]\n",
      "16975it [00:00, 18840.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tir\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "770it [00:00, 18002.81it/s]\n",
      "1818it [00:00, 18801.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nau\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "219it [00:00, 15925.80it/s]\n",
      "528it [00:00, 16158.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mar\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14344it [00:00, 17953.86it/s]\n",
      "29127it [00:01, 18454.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gag\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2108it [00:00, 19051.13it/s]\n",
      "3844it [00:00, 18710.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sro\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "921it [00:00, 22215.06it/s]\n",
      "3643it [00:00, 21411.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing wol\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1626it [00:00, 20797.44it/s]\n",
      "2603it [00:00, 19438.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing oci\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "26038it [00:01, 17450.84it/s]\n",
      "72649it [00:04, 17514.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing akz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "420it [00:00, 21598.65it/s]\n",
      "438it [00:00, 21791.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ban\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "81it [00:00, 19076.79it/s]\n",
      "158it [00:00, 19507.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fur\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15812it [00:00, 19797.27it/s]\n",
      "22063it [00:01, 17094.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing asm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "113710it [00:06, 18364.55it/s]\n",
      "34467it [00:02, 14329.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cop\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1002it [00:00, 24344.81it/s]\n",
      "1476it [00:00, 18159.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing jav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6032it [00:00, 31404.32it/s]\n",
      "11075it [00:00, 31372.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ady\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "656it [00:00, 30342.90it/s]\n",
      "1944it [00:00, 30636.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing koi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1271it [00:00, 31103.54it/s]\n",
      "3275it [00:00, 30095.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing grc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5830it [00:00, 31720.13it/s]\n",
      "8710it [00:00, 28592.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vls\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1958it [00:00, 27957.17it/s]\n",
      "5371it [00:00, 28284.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hil\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1998it [00:00, 30500.25it/s]\n",
      "4337it [00:00, 30105.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dzo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "76it [00:00, 24116.14it/s]\n",
      "151it [00:00, 27451.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ryu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "94it [00:00, 29776.04it/s]\n",
      "179it [00:00, 30724.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing yrk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "593it [00:00, 30070.88it/s]\n",
      "1612it [00:00, 29285.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42656it [00:01, 29288.95it/s]\n",
      "44906it [00:01, 27526.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tpi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3992it [00:00, 25753.62it/s]\n",
      "8882it [00:00, 25805.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kal\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2708it [00:00, 27068.60it/s]\n",
      "2180it [00:00, 26472.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing xho\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3262it [00:00, 31278.16it/s]\n",
      "15760it [00:00, 30586.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ceb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6284it [00:00, 31259.05it/s]\n",
      "15922it [00:00, 28626.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4056it [00:00, 28401.02it/s]\n",
      "10328it [00:00, 25189.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ron\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170022it [00:05, 29759.59it/s]\n",
      "216524it [00:07, 29987.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rap\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "545it [00:00, 29813.57it/s]\n",
      "1228it [00:00, 30347.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing izh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "158it [00:00, 27278.34it/s]\n",
      "276it [00:00, 29338.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing glv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11442it [00:00, 31859.05it/s]\n",
      "23560it [00:00, 31702.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing syc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "138it [00:00, 30291.71it/s]\n",
      "274it [00:00, 21569.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tly\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "191it [00:00, 27500.33it/s]\n",
      "362it [00:00, 29657.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing afr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42527it [00:01, 30150.93it/s]\n",
      "72444it [00:02, 28510.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ems\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "41it [00:00, 23605.55it/s]\n",
      "106it [00:00, 29957.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing crh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5913it [00:00, 29013.91it/s]\n",
      "10426it [00:00, 23259.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing scn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13069it [00:00, 15554.23it/s]\n",
      "39795it [00:02, 15324.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing guj\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "64158it [00:03, 18022.21it/s]\n",
      "212613it [00:10, 20787.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ace\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "191it [00:00, 19296.00it/s]\n",
      "273it [00:00, 19959.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nld\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "203235it [00:09, 22290.05it/s]\n",
      "154646it [00:04, 31484.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing jpn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "143516it [00:05, 27414.99it/s]\n",
      "35205it [00:01, 26235.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tgl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13765it [00:00, 26968.78it/s]\n",
      "33451it [00:01, 27439.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing wln\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5858it [00:00, 26749.81it/s]\n",
      "13560it [00:00, 29790.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing smn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "172it [00:00, 27029.61it/s]\n",
      "1073it [00:00, 28345.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sna\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "502it [00:00, 30288.14it/s]\n",
      "1408it [00:00, 28851.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fra\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "343229it [00:12, 27648.03it/s]\n",
      "252738it [00:09, 27602.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sah\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4231it [00:00, 26802.06it/s]\n",
      "10460it [00:00, 22499.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mkd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42379it [00:03, 14028.94it/s]\n",
      "72311it [00:03, 18166.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sqi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38133it [00:02, 15004.30it/s]\n",
      "43786it [00:02, 14893.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kir\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12063it [00:00, 22729.22it/s]\n",
      "22574it [00:01, 22036.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bre\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22036it [00:01, 21555.70it/s]\n",
      "31982it [00:01, 21441.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing blt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19it [00:00, 16197.52it/s]\n",
      "8it [00:00, 14959.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dak\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "66it [00:00, 20278.67it/s]\n",
      "145it [00:00, 21978.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing amm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9it [00:00, 14774.46it/s]\n",
      "3it [00:00, 8139.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lez\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1950it [00:00, 23264.51it/s]\n",
      "5449it [00:00, 23032.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sgs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3921it [00:00, 23701.76it/s]\n",
      "10843it [00:00, 22702.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cuk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23it [00:00, 19398.55it/s]\n",
      "29it [00:00, 20682.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing snd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "728it [00:00, 22401.79it/s]\n",
      "1956it [00:00, 21566.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1531it [00:00, 21535.15it/s]\n",
      "3130it [00:00, 21152.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ckb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1849it [00:00, 14992.75it/s]\n",
      "6604it [00:00, 14493.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ext\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1515it [00:00, 16651.30it/s]\n",
      "2611it [00:00, 15779.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing san\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "85283it [00:04, 17517.39it/s]\n",
      "34071it [00:01, 25639.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing moe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "19it [00:00, 15334.19it/s]\n",
      "24it [00:00, 24350.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gle\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "29131it [00:01, 27816.43it/s]\n",
      "41349it [00:01, 27520.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rup\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7181it [00:00, 26838.00it/s]\n",
      "17251it [00:00, 27040.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ces\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "114275it [00:03, 29102.84it/s]\n",
      "180825it [00:07, 25754.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vot\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "441it [00:00, 28460.24it/s]\n",
      "1117it [00:00, 29079.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fas\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "26684it [00:01, 26259.44it/s]\n",
      "3859it [00:00, 25608.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dng\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:00, 28896.49it/s]\n",
      "166it [00:00, 26803.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nio\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:00, 8186.67it/s]\n",
      "3it [00:00, 12658.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "250it [00:00, 23745.47it/s]\n",
      "711it [00:00, 25508.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bis\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2088it [00:00, 24628.73it/s]\n",
      "2161it [00:00, 25643.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bpy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "338it [00:00, 22040.62it/s]\n",
      "249it [00:00, 23733.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gld\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "172it [00:00, 28414.68it/s]\n",
      "288it [00:00, 26302.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lij\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4334it [00:00, 24678.64it/s]\n",
      "12612it [00:00, 20922.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sei\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "110it [00:00, 25529.74it/s]\n",
      "178it [00:00, 26193.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing abe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "199it [00:00, 26610.55it/s]\n",
      "535it [00:00, 27719.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mni\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16634it [00:00, 26967.62it/s]\n",
      "23890it [00:00, 27971.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing arg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14224it [00:00, 27287.33it/s]\n",
      "28316it [00:01, 26824.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pli\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "192it [00:00, 17682.72it/s]\n",
      "155it [00:00, 15565.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ltg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1208it [00:00, 20897.39it/s]\n",
      "2605it [00:00, 21434.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lim\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7627it [00:00, 29012.12it/s]\n",
      "14707it [00:00, 27643.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nep\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21771it [00:00, 28020.36it/s]\n",
      "137179it [00:06, 20386.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing msa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "204273it [00:09, 21495.51it/s]\n",
      "201496it [00:09, 21981.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cmn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "16257it [00:00, 21663.83it/s]\n",
      "71480it [00:03, 22475.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing alq\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18it [00:00, 12945.38it/s]\n",
      "3it [00:00, 9355.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing smo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3522it [00:00, 30425.62it/s]\n",
      "8041it [00:00, 30227.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing szl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "936it [00:00, 30304.59it/s]\n",
      "2539it [00:00, 30213.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing som\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2902it [00:00, 30572.06it/s]\n",
      "6216it [00:00, 31413.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kok\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "61267it [00:02, 29822.55it/s]\n",
      "76143it [00:02, 30089.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing esu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "52it [00:00, 26863.38it/s]\n",
      "182it [00:00, 26574.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hrv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "212454it [00:06, 30939.77it/s]\n",
      "87802it [00:02, 30005.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42772it [00:01, 26130.68it/s]\n",
      "78964it [00:02, 29363.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cjs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "858it [00:00, 30717.27it/s]\n",
      "413it [00:00, 29952.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing zho\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "61336it [00:02, 26535.72it/s]\n",
      "563it [00:00, 23494.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bqi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "190it [00:00, 21919.24it/s]\n",
      "65it [00:00, 20814.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dlg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "313it [00:00, 26815.72it/s]\n",
      "132it [00:00, 26608.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing evn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1404it [00:00, 27596.30it/s]\n",
      "3508it [00:00, 27324.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10914it [00:00, 25937.31it/s]\n",
      "20782it [00:00, 27734.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pih\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "303it [00:00, 22936.65it/s]\n",
      "720it [00:00, 25343.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing roh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12695it [00:00, 27063.35it/s]\n",
      "28103it [00:01, 26722.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fao\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24872it [00:00, 26818.52it/s]\n",
      "29058it [00:01, 28434.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing wym\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "551it [00:00, 28902.72it/s]\n",
      "1609it [00:00, 28411.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "525it [00:00, 28108.01it/s]\n",
      "1728it [00:00, 28119.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5414it [00:00, 24649.29it/s]\n",
      "16591it [00:00, 26312.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vie\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "14414it [00:00, 27060.81it/s]\n",
      "31726it [00:01, 27869.57it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hop\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "87it [00:00, 28445.93it/s]\n",
      "69it [00:00, 28577.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "45014it [00:02, 20890.33it/s]\n",
      "60474it [00:03, 16444.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing wuu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "17it [00:00, 14522.03it/s]\n",
      "105it [00:00, 16115.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing inh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "258it [00:00, 20406.39it/s]\n",
      "295it [00:00, 19654.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cym\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24427it [00:01, 19535.06it/s]\n",
      "37308it [00:02, 16971.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ori\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "96759it [00:05, 17098.13it/s]\n",
      "82573it [00:05, 16057.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lus\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "53it [00:00, 15036.40it/s]\n",
      "327it [00:00, 14958.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pnt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12it [00:00, 14786.03it/s]\n",
      "66it [00:00, 16400.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mnk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7it [00:00, 9604.23it/s]\n",
      "25it [00:00, 21115.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mrj\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "672it [00:00, 21254.60it/s]\n",
      "2127it [00:00, 21869.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing krc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1996it [00:00, 21076.21it/s]\n",
      "3801it [00:00, 20973.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing rhg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "688it [00:00, 23027.42it/s]\n",
      "2212it [00:00, 22984.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tvl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "640it [00:00, 22811.40it/s]\n",
      "1735it [00:00, 23047.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46146it [00:02, 22406.03it/s]\n",
      "83284it [00:03, 23321.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sme\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3229it [00:00, 31356.07it/s]\n",
      "10328it [00:00, 30541.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "681it [00:00, 31731.61it/s]\n",
      "1310it [00:00, 31143.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing bod\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "525it [00:00, 29147.93it/s]\n",
      "372it [00:00, 29006.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sco\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5963it [00:00, 30403.30it/s]\n",
      "13711it [00:00, 27565.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tgk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11230it [00:00, 30926.47it/s]\n",
      "20659it [00:00, 28801.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing prg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "562it [00:00, 23622.30it/s]\n",
      "1977it [00:00, 20333.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing yue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1845it [00:00, 25738.26it/s]\n",
      "4572it [00:00, 25313.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing frp\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1917it [00:00, 29736.83it/s]\n",
      "2754it [00:00, 29647.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hau\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1628it [00:00, 19704.64it/s]\n",
      "7477it [00:00, 18603.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nap\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6119it [00:00, 24288.81it/s]\n",
      "13112it [00:00, 26231.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "63644it [00:02, 26164.03it/s]\n",
      "101210it [00:03, 28124.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mdf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3131it [00:00, 31330.27it/s]\n",
      "10246it [00:00, 30346.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing oss\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6837it [00:00, 30716.59it/s]\n",
      "15329it [00:00, 30732.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gsw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3691it [00:00, 31166.18it/s]\n",
      "7808it [00:00, 29373.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing luo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "770it [00:00, 30697.13it/s]\n",
      "1126it [00:00, 30002.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nno\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "44048it [00:01, 30433.69it/s]\n",
      "52862it [00:01, 30290.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lzz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "153it [00:00, 30496.06it/s]\n",
      "409it [00:00, 31206.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ast\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46581it [00:01, 30767.53it/s]\n",
      "84060it [00:02, 28392.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65017it [00:02, 28847.96it/s]\n",
      "25406it [00:00, 27016.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing div\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1875it [00:00, 29950.19it/s]\n",
      "1131it [00:00, 28687.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mya\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2131it [00:00, 30160.90it/s]\n",
      "5236it [00:00, 29866.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing che\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3351it [00:00, 25222.50it/s]\n",
      "6392it [00:00, 23928.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gil\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "940it [00:00, 22277.86it/s]\n",
      "875it [00:00, 21917.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing zsm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46380it [00:02, 21305.89it/s]\n",
      "210398it [00:11, 18497.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "335it [00:00, 14898.34it/s]\n",
      "195it [00:00, 18132.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing arb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21188it [00:00, 21197.58it/s]\n",
      "21691it [00:01, 20517.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mvi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "154it [00:00, 9741.84it/s]\n",
      "222it [00:00, 9573.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing spa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "324697it [00:16, 19256.25it/s]\n",
      "205911it [00:07, 28477.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing eng\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "663294it [00:22, 29804.88it/s]\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing krl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "679it [00:00, 26413.28it/s]\n",
      "1594it [00:00, 26303.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "744it [00:00, 29049.02it/s]\n",
      "351it [00:00, 27719.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ben\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "59178it [00:02, 23909.76it/s]\n",
      "121309it [00:05, 23664.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mal\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "41636it [00:02, 18120.60it/s]\n",
      "50768it [00:02, 18259.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kaz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18568it [00:01, 17874.25it/s]\n",
      "33889it [00:02, 15231.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing amu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38it [00:00, 18643.53it/s]\n",
      "45it [00:00, 19577.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "299it [00:00, 19740.85it/s]\n",
      "348it [00:00, 19853.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2918it [00:00, 15524.09it/s]\n",
      "3523it [00:00, 15907.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing new\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "753it [00:00, 22217.69it/s]\n",
      "1153it [00:00, 21620.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing war\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2062it [00:00, 20170.90it/s]\n",
      "6268it [00:00, 19481.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mlt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21161it [00:01, 18278.34it/s]\n",
      "42087it [00:02, 18752.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ewe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2172it [00:00, 16324.61it/s]\n",
      "2052it [00:00, 15029.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing vep\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1294it [00:00, 17558.98it/s]\n",
      "3992it [00:00, 18390.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing gcf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "233it [00:00, 20209.96it/s]\n",
      "328it [00:00, 20253.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ava\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1291it [00:00, 21188.24it/s]\n",
      "4676it [00:00, 20608.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing swe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "99651it [00:05, 19820.60it/s]\n",
      "149562it [00:05, 29548.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sun\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2778it [00:00, 29427.99it/s]\n",
      "6735it [00:00, 27398.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing deu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "165125it [00:05, 29747.49it/s]\n",
      "264983it [00:08, 30300.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing srn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "665it [00:00, 28895.07it/s]\n",
      "2068it [00:00, 30174.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mad\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "18it [00:00, 14282.53it/s]\n",
      "47it [00:00, 23379.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lug\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "879it [00:00, 29790.58it/s]\n",
      "2571it [00:00, 29786.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hsb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6753it [00:00, 29736.75it/s]\n",
      "14584it [00:00, 28497.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mnc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "44it [00:00, 26103.16it/s]\n",
      "394it [00:00, 28375.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kum\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2711it [00:00, 21138.22it/s]\n",
      "6764it [00:00, 25268.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mri\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6337it [00:00, 27328.32it/s]\n",
      "12172it [00:00, 27768.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10183it [00:00, 29378.27it/s]\n",
      "17026it [00:00, 28793.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "173it [00:00, 23838.32it/s]\n",
      "225it [00:00, 19491.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cdo\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "105it [00:00, 24823.96it/s]\n",
      "46it [00:00, 26531.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tha\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "57147it [00:02, 26365.61it/s]\n",
      "14298it [00:00, 30210.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing stq\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2837it [00:00, 31397.82it/s]\n",
      "7109it [00:00, 30271.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6159it [00:00, 29389.96it/s]\n",
      "10349it [00:00, 25828.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mzn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "487it [00:00, 25696.64it/s]\n",
      "1219it [00:00, 26290.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tam\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30297it [00:00, 30551.75it/s]\n",
      "21337it [00:00, 24739.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nmn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "80it [00:00, 21327.42it/s]\n",
      "270it [00:00, 22068.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing twf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "596it [00:00, 22569.77it/s]\n",
      "887it [00:00, 22409.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing fry\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20763it [00:00, 22751.30it/s]\n",
      "32605it [00:01, 23354.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing urd\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20687it [00:00, 23146.15it/s]\n",
      "92961it [00:04, 22838.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing cos\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7362it [00:00, 22987.67it/s]\n",
      "16575it [00:00, 23276.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ksh\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3762it [00:00, 22691.02it/s]\n",
      "10076it [00:00, 23205.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing xal\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1467it [00:00, 23347.84it/s]\n",
      "3652it [00:00, 23269.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing isl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "43791it [00:01, 21908.35it/s]\n",
      "52846it [00:02, 22480.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1004it [00:00, 22620.76it/s]\n",
      "1497it [00:00, 22467.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lad\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1050it [00:00, 23360.27it/s]\n",
      "1484it [00:00, 22766.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing xmf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "262it [00:00, 24706.21it/s]\n",
      "1068it [00:00, 25000.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22711it [00:01, 20913.17it/s]\n",
      "27033it [00:01, 21179.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sot\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4290it [00:00, 22822.13it/s]\n",
      "10350it [00:00, 20879.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hif\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "647it [00:00, 22935.77it/s]\n",
      "940it [00:00, 22560.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pdc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "773it [00:00, 22547.83it/s]\n",
      "1462it [00:00, 22793.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ukr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "41637it [00:01, 29267.70it/s]\n",
      "111115it [00:04, 26771.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing apw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "20it [00:00, 23790.72it/s]\n",
      "24it [00:00, 27503.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing smj\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 28405.15it/s]\n",
      "116it [00:00, 28148.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing iii\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "13it [00:00, 14243.98it/s]\n",
      "26it [00:00, 22475.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mas\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "113it [00:00, 27526.79it/s]\n",
      "230it [00:00, 30579.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nya\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "87it [00:00, 25377.60it/s]\n",
      "483it [00:00, 31766.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing kin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1281it [00:00, 32004.05it/s]\n",
      "4071it [00:00, 31909.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing arz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1945it [00:00, 32014.82it/s]\n",
      "1422it [00:00, 33331.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ave\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "175it [00:00, 29792.72it/s]\n",
      "144it [00:00, 27489.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing ton\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1583it [00:00, 32416.67it/s]\n",
      "2569it [00:00, 32424.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing sdn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "446it [00:00, 31432.79it/s]\n",
      "1020it [00:00, 31706.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lld\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1443it [00:00, 31980.71it/s]\n",
      "4523it [00:00, 32181.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing aoz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "62it [00:00, 29212.18it/s]\n",
      "32it [00:00, 26913.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chu\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3891it [00:00, 32306.29it/s]\n",
      "5808it [00:00, 32165.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing aqc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "208it [00:00, 30542.47it/s]\n",
      "425it [00:00, 32092.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing lin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4204it [00:00, 31256.06it/s]\n",
      "8627it [00:00, 32197.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mhn\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "158it [00:00, 28572.05it/s]\n",
      "372it [00:00, 32052.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing hun\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "76162it [00:02, 30394.04it/s]\n",
      "139523it [00:05, 27201.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing frr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1847it [00:00, 30759.89it/s]\n",
      "4779it [00:00, 31322.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing abk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1861it [00:00, 31431.48it/s]\n",
      "2401it [00:00, 31691.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing nso\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3455it [00:00, 31642.51it/s]\n",
      "13313it [00:00, 31270.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing mhr\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "7977it [00:00, 31818.97it/s]\n",
      "30545it [00:00, 31809.21it/s]\n"
     ]
    }
   ],
   "source": [
    "for lang in tqdm(langs):\n",
    "    data1 = df[df['lang 1'] == lang]\n",
    "    data2 = df[df['lang 2'] == lang]\n",
    "    print(\"Processing\", lang)\n",
    "    for index, (lang_1, word_1, lang_2, word_2) in tqdm(data1.iterrows()):\n",
    "        if cognates[lang].get(word_1):\n",
    "            cognates[lang][word_1].append(lang_2)\n",
    "        else:\n",
    "            cognates[lang][word_1] = [lang_2]\n",
    "    for index, (lang_1, word_1, lang_2, word_2) in tqdm(data2.iterrows()):\n",
    "        if cognates[lang].get(word_2):\n",
    "            cognates[lang][word_2].append(lang_1)\n",
    "        else:\n",
    "            cognates[lang][word_2] = [lang_1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "046e3783",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(cognates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "26939440",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████| 333/333 [00:05<00:00, 59.86it/s]\n"
     ]
    }
   ],
   "source": [
    "for lang in tqdm(cognates):\n",
    "    cognates[lang] = {k: list(set(v)) for k, v in sorted(cognates[lang].items(), key=lambda item: item[0])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "469fd6c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "with lzma.open(\"cognates.json.xz\", 'wt') as f:\n",
    "    json.dump(cognates, f, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b95d932",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lzma, json\n",
    "with lzma.open(\"cognates.json.xz\") as f:\n",
    "    cognates = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "4d449385",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cognates(TL, KL):\n",
    "    return [\n",
    "        word for word, langs in cognates[TL].items() if KL in langs\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bea80c40",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pycountry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "028d64ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "codemap = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "326e465c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for language in pycountry.languages:\n",
    "    if getattr(language, 'alpha_2', None):\n",
    "        codemap[language.alpha_3] = language.alpha_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "084e7e40",
   "metadata": {},
   "outputs": [],
   "source": [
    "newcognates = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9868c8af",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████| 333/333 [00:03<00:00, 86.03it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "for lang in tqdm(cognates):\n",
    "    newcognates[codemap.get(lang, lang)] = {}\n",
    "    for key in cognates[lang]:\n",
    "        newcognates[codemap.get(lang, lang)][key] = list(codemap.get(code, code) for code in cognates[lang][key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4d926ee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with lzma.open(\"cognates.json.xz\", 'wt') as f:\n",
    "    json.dump(newcognates, f, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2945c9c8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}