channel_difference_engine_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "vulnerable-advantage",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "regulation-cabin",
   "metadata": {},
   "outputs": [],
   "source": [
    "from input_generation import rasterize_text\n",
    "import activations_tool as activations_tool\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "supreme-acting",
   "metadata": {},
   "source": [
    "# 100k word baseline calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "miniature-peeing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 norm: 23.732423939181906\n",
      "10 norm: 262.45890910462595\n",
      "20 norm: 499.9301026475303\n",
      "30 norm: 737.0380263346843\n",
      "40 norm: 972.1369632901677\n",
      "50 norm: 1207.298057602969\n",
      "60 norm: 1443.154766362121\n",
      "70 norm: 1677.4766366195258\n",
      "80 norm: 1912.7029382649575\n",
      "90 norm: 2148.1330378672965\n",
      "100 norm: 2383.29105884175\n",
      "110 norm: 2618.164384041549\n",
      "120 norm: 2852.6398052958634\n",
      "130 norm: 3086.217534714555\n",
      "140 norm: 3319.7311404210554\n",
      "150 norm: 3549.1266950601926\n",
      "160 norm: 3782.5779137550694\n",
      "170 norm: 4014.444568626924\n",
      "180 norm: 4246.401077447542\n",
      "190 norm: 4480.868475410321\n",
      "200 norm: 4710.715889889138\n",
      "210 norm: 4941.231487571465\n",
      "220 norm: 5172.325850777289\n",
      "230 norm: 5402.68947520451\n",
      "240 norm: 5635.5310040438035\n",
      "250 norm: 5863.974198282068\n",
      "260 norm: 6093.697511356275\n",
      "270 norm: 6328.499416121478\n",
      "280 norm: 6560.531504328272\n",
      "290 norm: 6791.804010850519\n",
      "300 norm: 7023.037357738446\n",
      "310 norm: 7253.003777661025\n",
      "320 norm: 7483.38944096432\n",
      "330 norm: 7714.477750409492\n",
      "340 norm: 7945.589228912543\n",
      "350 norm: 8176.447018989377\n",
      "360 norm: 8405.226537780225\n",
      "370 norm: 8637.73101360227\n",
      "380 norm: 8869.277768342647\n",
      "390 norm: 9101.177543812413\n",
      "400 norm: 9332.358255246527\n",
      "410 norm: 9561.131891342013\n",
      "420 norm: 9793.120265622108\n",
      "430 norm: 10025.492191198975\n",
      "440 norm: 10257.544209615437\n",
      "450 norm: 10488.472259927335\n",
      "460 norm: 10718.891287031332\n",
      "470 norm: 10947.735878398997\n",
      "480 norm: 11178.84355365467\n",
      "490 norm: 11409.91250831399\n",
      "500 norm: 11640.65112998363\n",
      "510 norm: 11872.61067297982\n",
      "520 norm: 12104.102248334837\n",
      "530 norm: 12333.092094710984\n",
      "540 norm: 12564.248245799045\n",
      "550 norm: 12796.267140300968\n",
      "560 norm: 13027.100328256456\n",
      "570 norm: 13257.940543554216\n",
      "580 norm: 13489.503652295958\n",
      "590 norm: 13719.544810153095\n",
      "600 norm: 13952.167575984298\n",
      "610 norm: 14182.33127563909\n",
      "620 norm: 14415.77838172088\n",
      "630 norm: 14646.745888227711\n",
      "640 norm: 14877.53626318303\n",
      "650 norm: 15106.43766078416\n",
      "660 norm: 15337.666149742969\n",
      "670 norm: 15565.467917504953\n",
      "680 norm: 15796.679619174716\n",
      "690 norm: 16026.795916618368\n",
      "700 norm: 16258.66737287747\n",
      "710 norm: 16485.66966173827\n",
      "720 norm: 16717.108448390063\n",
      "730 norm: 16946.811390272476\n",
      "740 norm: 17177.870386385952\n",
      "750 norm: 17410.04177508891\n",
      "760 norm: 17640.050102841436\n",
      "770 norm: 17869.343403636165\n",
      "780 norm: 18100.251435627317\n",
      "790 norm: 18327.470074256777\n",
      "800 norm: 18559.19673460553\n",
      "810 norm: 18788.43584769293\n",
      "820 norm: 19020.403485581395\n",
      "830 norm: 19250.68519373869\n",
      "840 norm: 19482.778099658153\n",
      "850 norm: 19713.3241262318\n",
      "860 norm: 19944.607070322338\n",
      "870 norm: 20173.76734607511\n",
      "880 norm: 20404.188694209875\n",
      "890 norm: 20631.87907044795\n",
      "900 norm: 20859.411998842825\n",
      "910 norm: 21088.040308714764\n",
      "920 norm: 21317.235104786043\n",
      "930 norm: 21547.56605026429\n",
      "940 norm: 21778.446801123584\n",
      "950 norm: 22007.39610019141\n",
      "960 norm: 22234.35778125306\n",
      "970 norm: 22464.200236243127\n",
      "980 norm: 22694.03366473195\n",
      "990 norm: 22921.784317175014\n",
      "1000 norm: 23154.136130099672\n",
      "1010 norm: 23383.78581834427\n",
      "1020 norm: 23613.345952803356\n",
      "1030 norm: 23842.282354711584\n",
      "1040 norm: 24071.08404358435\n",
      "1050 norm: 24301.699069550003\n",
      "1060 norm: 24530.277183073682\n",
      "1070 norm: 24758.36095817974\n",
      "1080 norm: 24989.759592488517\n",
      "1090 norm: 25218.7085390126\n",
      "1100 norm: 25450.599224673068\n",
      "1110 norm: 25679.265596104233\n",
      "1120 norm: 25910.870006510268\n",
      "1130 norm: 26140.39788704335\n",
      "1140 norm: 26373.260963453224\n",
      "1150 norm: 26601.120263519326\n",
      "1160 norm: 26831.329031971472\n",
      "1170 norm: 27058.621571211363\n",
      "1180 norm: 27289.408103804577\n",
      "1190 norm: 27517.847104257027\n",
      "1200 norm: 27747.093335008405\n",
      "1210 norm: 27978.44024001172\n",
      "1220 norm: 28207.93900008517\n",
      "1230 norm: 28440.154545844773\n",
      "1240 norm: 28668.10127703759\n",
      "1250 norm: 28896.426844707195\n",
      "1260 norm: 29123.586230974106\n",
      "1270 norm: 29351.49330132783\n",
      "1280 norm: 29581.86925461292\n",
      "1290 norm: 29811.48931153656\n",
      "1300 norm: 30040.839278964286\n",
      "1310 norm: 30269.586779305926\n",
      "1320 norm: 30499.73579094861\n",
      "1330 norm: 30728.070651430102\n",
      "1340 norm: 30955.907796932064\n",
      "1350 norm: 31181.696045925582\n",
      "1360 norm: 31412.530320355447\n",
      "1370 norm: 31641.59727261472\n",
      "1380 norm: 31873.16245449944\n",
      "1390 norm: 32101.85486048899\n",
      "1400 norm: 32329.26098931578\n",
      "1410 norm: 32557.91954442315\n",
      "1420 norm: 32788.4974790744\n",
      "1430 norm: 33019.8331061554\n",
      "1440 norm: 33248.92137941571\n",
      "1450 norm: 33479.67533048814\n",
      "1460 norm: 33710.54268768635\n",
      "1470 norm: 33940.437203046655\n",
      "1480 norm: 34171.75304417623\n",
      "1490 norm: 34397.26108408862\n",
      "1500 norm: 34625.430499745155\n",
      "1510 norm: 34851.87695198735\n",
      "1520 norm: 35083.88297941032\n",
      "1530 norm: 35313.49858858507\n",
      "1540 norm: 35543.995403878704\n",
      "1550 norm: 35773.78488118497\n",
      "1560 norm: 36001.60640704876\n",
      "1570 norm: 36230.5550037476\n",
      "1580 norm: 36459.13875648004\n",
      "1590 norm: 36685.253401897324\n",
      "1600 norm: 36913.47266060776\n",
      "1610 norm: 37142.79088905653\n",
      "1620 norm: 37370.378992553866\n",
      "1630 norm: 37601.35400268044\n",
      "1640 norm: 37829.65258353299\n",
      "1650 norm: 38059.44091416098\n",
      "1660 norm: 38289.43363690493\n",
      "1670 norm: 38518.23954350658\n",
      "1680 norm: 38748.09545395205\n",
      "1690 norm: 38979.90131358312\n",
      "1700 norm: 39208.160367701385\n",
      "1710 norm: 39438.09887342788\n",
      "1720 norm: 39665.78605442946\n",
      "1730 norm: 39894.61749627449\n",
      "1740 norm: 40125.899919855925\n",
      "1750 norm: 40353.58649182324\n",
      "1760 norm: 40582.7650713533\n",
      "1770 norm: 40811.07732990145\n",
      "1780 norm: 41040.1220399261\n",
      "1790 norm: 41266.75575953704\n",
      "1800 norm: 41494.971283604384\n",
      "1810 norm: 41724.91834304122\n",
      "1820 norm: 41954.206096390364\n",
      "1830 norm: 42184.273573067345\n",
      "1840 norm: 42414.90458281712\n",
      "1850 norm: 42646.788023028574\n",
      "1860 norm: 42876.744969809275\n",
      "1870 norm: 43105.01736065228\n",
      "1880 norm: 43332.630229912465\n",
      "1890 norm: 43563.153620958175\n",
      "1900 norm: 43790.93915028718\n",
      "1910 norm: 44017.9000829923\n",
      "1920 norm: 44244.09014168149\n",
      "1930 norm: 44472.05433509175\n",
      "1940 norm: 44702.02145358439\n",
      "1950 norm: 44933.80149335234\n",
      "1960 norm: 45165.828578182794\n",
      "1970 norm: 45394.0833260199\n",
      "1980 norm: 45623.94889971163\n",
      "1990 norm: 45850.91835668185\n",
      "2000 norm: 46082.128399081565\n",
      "2010 norm: 46312.307091721246\n",
      "2020 norm: 46541.30549362253\n",
      "2030 norm: 46769.91076987021\n",
      "2040 norm: 46999.44733584926\n",
      "2050 norm: 47228.59922325568\n",
      "2060 norm: 47457.6136520832\n",
      "2070 norm: 47685.68360331155\n",
      "2080 norm: 47913.89143399248\n",
      "2090 norm: 48142.20219756463\n",
      "2100 norm: 48370.0936714236\n",
      "2110 norm: 48597.907699189156\n",
      "2120 norm: 48829.837123365935\n",
      "2130 norm: 49059.10136018764\n",
      "2140 norm: 49287.70400774293\n",
      "2150 norm: 49515.060368704646\n",
      "2160 norm: 49742.08285134896\n",
      "2170 norm: 49969.89776793009\n",
      "2180 norm: 50196.79397093602\n",
      "2190 norm: 50426.2325449735\n",
      "2200 norm: 50656.78279647591\n",
      "2210 norm: 50887.69624418858\n",
      "2220 norm: 51116.82390909873\n",
      "2230 norm: 51344.067430189694\n",
      "2240 norm: 51573.588121697765\n",
      "2250 norm: 51798.133009576995\n",
      "2260 norm: 52027.24101130762\n",
      "2270 norm: 52257.4008794602\n",
      "2280 norm: 52486.989707313965\n",
      "2290 norm: 52716.058422132875\n",
      "2300 norm: 52943.60112302511\n",
      "2310 norm: 53173.29202596466\n",
      "2320 norm: 53402.15454481313\n",
      "2330 norm: 53626.82359138096\n",
      "2340 norm: 53855.992234901285\n",
      "2350 norm: 54086.556267974236\n",
      "2360 norm: 54316.56447224379\n",
      "2370 norm: 54543.72082431176\n",
      "2380 norm: 54770.93715077667\n",
      "2390 norm: 54999.44740822992\n",
      "2400 norm: 55230.469495019905\n",
      "2410 norm: 55460.47325796906\n",
      "2420 norm: 55691.9384420529\n",
      "2430 norm: 55920.620286309866\n",
      "2440 norm: 56149.64054098379\n",
      "2450 norm: 56377.86235101862\n",
      "2460 norm: 56607.439815174104\n",
      "2470 norm: 56836.86100886563\n",
      "2480 norm: 57067.5360997589\n",
      "2490 norm: 57297.84176238855\n",
      "2500 norm: 57526.79722715322\n",
      "2510 norm: 57756.234460420186\n",
      "2520 norm: 57986.08762390432\n",
      "2530 norm: 58214.24076812667\n",
      "2540 norm: 58442.08259729512\n",
      "2550 norm: 58668.659188380436\n",
      "2560 norm: 58897.81405303571\n",
      "2570 norm: 59126.62671623031\n",
      "2580 norm: 59357.13548810452\n",
      "2590 norm: 59584.55618926201\n",
      "2600 norm: 59815.45598579816\n",
      "2610 norm: 60044.202907793246\n",
      "2620 norm: 60273.124336037436\n",
      "2630 norm: 60499.19550578132\n",
      "2640 norm: 60729.21141217162\n",
      "2650 norm: 60956.8655751656\n",
      "2660 norm: 61187.18886811106\n",
      "2670 norm: 61411.93752014889\n",
      "2680 norm: 61638.9186049981\n",
      "2690 norm: 61868.95144926902\n",
      "2700 norm: 62096.473089099185\n",
      "2710 norm: 62327.266005551304\n",
      "2720 norm: 62556.00844580479\n",
      "2730 norm: 62782.8947033444\n",
      "2740 norm: 63007.346736064654\n",
      "2750 norm: 63235.16756109381\n",
      "2760 norm: 63465.36464770181\n",
      "2770 norm: 63695.820813699735\n",
      "2780 norm: 63926.026900267934\n",
      "2790 norm: 64151.15107195041\n",
      "2800 norm: 64381.25801170422\n",
      "2810 norm: 64608.50940831717\n",
      "2820 norm: 64833.12351116705\n",
      "2830 norm: 65061.5722937429\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2840 norm: 65290.62388322305\n",
      "2850 norm: 65517.65209204287\n",
      "2860 norm: 65745.59992908439\n",
      "2870 norm: 65973.61539257417\n",
      "2880 norm: 66203.23721261509\n",
      "2890 norm: 66431.79069350527\n",
      "2900 norm: 66661.57776996515\n",
      "2910 norm: 66891.48300793003\n",
      "2920 norm: 67120.41428006587\n",
      "2930 norm: 67349.85054563248\n",
      "2940 norm: 67578.4004354675\n",
      "2950 norm: 67807.03751715654\n",
      "2960 norm: 68038.51676791548\n",
      "2970 norm: 68265.9300330242\n",
      "2980 norm: 68496.11684472788\n",
      "2990 norm: 68727.48350384415\n",
      "3000 norm: 68958.07169644609\n",
      "3010 norm: 69187.77307701414\n",
      "3020 norm: 69415.66296337552\n",
      "3030 norm: 69643.27954297922\n",
      "3040 norm: 69872.93251458221\n",
      "3050 norm: 70102.34367570322\n",
      "3060 norm: 70330.5362252488\n",
      "3070 norm: 70557.56706615792\n",
      "3080 norm: 70783.82668664739\n",
      "3090 norm: 71013.08346111144\n",
      "3100 norm: 71242.05713942117\n",
      "3110 norm: 71467.49000297593\n",
      "3120 norm: 71692.84391995126\n",
      "3130 norm: 71919.53405384274\n",
      "3140 norm: 72147.80783999326\n",
      "3150 norm: 72373.4605752406\n",
      "3160 norm: 72605.07530549906\n",
      "3170 norm: 72832.19961439064\n",
      "3180 norm: 73057.72158564243\n",
      "3190 norm: 73282.99725605157\n",
      "3200 norm: 73511.44068812492\n",
      "3210 norm: 73737.93119598163\n",
      "3220 norm: 73963.85286251128\n",
      "3230 norm: 74195.67244916878\n",
      "3240 norm: 74425.73033400363\n",
      "3250 norm: 74653.438960666\n",
      "3260 norm: 74883.15809315552\n",
      "3270 norm: 75110.84480590331\n",
      "3280 norm: 75336.75957816359\n",
      "3290 norm: 75562.94890542001\n",
      "3300 norm: 75789.18286282421\n",
      "3310 norm: 76015.33008800176\n",
      "3320 norm: 76243.91873825571\n",
      "3330 norm: 76474.37866786678\n",
      "3340 norm: 76704.40717447255\n",
      "3350 norm: 76934.8726427688\n",
      "3360 norm: 77164.047489096\n",
      "3370 norm: 77389.92675243429\n",
      "3380 norm: 77617.94383287772\n",
      "3390 norm: 77848.5185435739\n",
      "3400 norm: 78076.393390061\n",
      "3410 norm: 78304.11034877492\n",
      "3420 norm: 78533.07713671136\n",
      "3430 norm: 78761.07250097611\n",
      "3440 norm: 78988.28803320955\n",
      "3450 norm: 79214.88073293507\n",
      "3460 norm: 79441.8623815747\n",
      "3470 norm: 79670.36558138924\n",
      "3480 norm: 79900.69044747582\n",
      "3490 norm: 80128.18363699091\n",
      "3500 norm: 80353.94404350339\n",
      "3510 norm: 80582.36370418798\n",
      "3520 norm: 80812.62075540038\n",
      "3530 norm: 81043.0299539924\n",
      "3540 norm: 81271.43576178716\n",
      "3550 norm: 81500.93960072246\n",
      "3560 norm: 81730.65015560575\n",
      "3570 norm: 81958.30189778727\n",
      "3580 norm: 82187.05587205564\n",
      "3590 norm: 82416.23317930363\n",
      "3600 norm: 82645.62502699117\n",
      "3610 norm: 82873.58471959257\n",
      "3620 norm: 83102.3162236879\n",
      "3630 norm: 83325.57931095627\n",
      "3640 norm: 83553.87348611548\n",
      "3650 norm: 83783.70014360237\n",
      "3660 norm: 84010.27166572298\n",
      "3670 norm: 84239.45970252031\n",
      "3680 norm: 84464.9988385998\n",
      "3690 norm: 84694.7507673486\n",
      "3700 norm: 84919.56143939693\n",
      "3710 norm: 85150.11227853385\n",
      "3720 norm: 85381.33595479242\n",
      "3730 norm: 85608.43410757194\n",
      "3740 norm: 85835.17885428644\n",
      "3750 norm: 86060.53166674227\n",
      "3760 norm: 86289.57694210987\n",
      "3770 norm: 86518.05026454115\n",
      "3780 norm: 86742.9531536928\n",
      "3790 norm: 86970.62244680547\n",
      "3800 norm: 87196.19383648965\n",
      "3810 norm: 87425.23811402715\n",
      "3820 norm: 87652.77571208421\n",
      "3830 norm: 87880.35765125147\n",
      "3840 norm: 88110.62230426546\n",
      "3850 norm: 88335.7279696152\n",
      "3860 norm: 88559.43042161688\n",
      "3870 norm: 88788.2567188909\n",
      "3880 norm: 89013.83951388778\n",
      "3890 norm: 89239.64224797577\n",
      "3900 norm: 89464.76496015531\n",
      "3910 norm: 89692.16633327075\n",
      "3920 norm: 89921.91123016867\n",
      "3930 norm: 90147.79941808971\n",
      "3940 norm: 90376.16036286204\n",
      "3950 norm: 90605.4748496506\n",
      "3960 norm: 90834.95275774636\n",
      "3970 norm: 91062.25366770968\n",
      "3980 norm: 91290.5006285134\n",
      "3990 norm: 91518.39716267199\n",
      "4000 norm: 91747.96823747412\n",
      "4010 norm: 91976.39501403116\n",
      "4020 norm: 92203.27587173258\n",
      "4030 norm: 92431.70187871397\n",
      "4040 norm: 92658.00282905268\n",
      "4050 norm: 92888.94241832048\n",
      "4060 norm: 93116.8687473571\n",
      "4070 norm: 93346.08921434332\n",
      "4080 norm: 93574.0113458747\n",
      "4090 norm: 93802.18269443105\n",
      "4100 norm: 94031.65147909011\n",
      "4110 norm: 94260.10607828823\n",
      "4120 norm: 94490.87340490283\n",
      "4130 norm: 94717.51826837119\n",
      "4140 norm: 94942.99528763628\n",
      "4150 norm: 95169.86993221164\n",
      "4160 norm: 95399.4571946945\n",
      "4170 norm: 95625.48905016296\n",
      "4180 norm: 95851.32116811532\n",
      "4190 norm: 96076.92350592902\n",
      "4200 norm: 96306.141919633\n",
      "4210 norm: 96531.89679851485\n",
      "4220 norm: 96756.99865551532\n",
      "4230 norm: 96983.84279310692\n",
      "4240 norm: 97212.07441941454\n",
      "4250 norm: 97441.88161245453\n",
      "4260 norm: 97672.42327187625\n",
      "4270 norm: 97901.3587428187\n",
      "4280 norm: 98130.39522504015\n",
      "4290 norm: 98360.94081991141\n",
      "4300 norm: 98589.29997709411\n",
      "4310 norm: 98816.95408857957\n",
      "4320 norm: 99045.12689705771\n",
      "4330 norm: 99275.62083800952\n",
      "4340 norm: 99502.51128852987\n",
      "4350 norm: 99729.60057668677\n",
      "4360 norm: 99959.42305717338\n",
      "4370 norm: 100187.09804976468\n",
      "4380 norm: 100412.81069254647\n",
      "4390 norm: 100641.8690357266\n",
      "4400 norm: 100869.07280039736\n",
      "4410 norm: 101099.76665406709\n",
      "4420 norm: 101328.03554754698\n",
      "4430 norm: 101555.6284035515\n",
      "4440 norm: 101784.35430472984\n",
      "4450 norm: 102010.78898860603\n",
      "4460 norm: 102237.88274644842\n",
      "4470 norm: 102466.08473832297\n",
      "4480 norm: 102692.09740842333\n",
      "4490 norm: 102920.18488853509\n",
      "4500 norm: 103150.38605195619\n",
      "4510 norm: 103379.67973796134\n",
      "4520 norm: 103608.41061341205\n",
      "4530 norm: 103834.06140752356\n",
      "4540 norm: 104062.90720329464\n",
      "4550 norm: 104291.87771384578\n",
      "4560 norm: 104520.43370961405\n",
      "4570 norm: 104752.19783063536\n",
      "4580 norm: 104978.27721843426\n",
      "4590 norm: 105206.26553289848\n",
      "4600 norm: 105434.16159656452\n",
      "4610 norm: 105665.5959679997\n",
      "4620 norm: 105894.58655660057\n",
      "4630 norm: 106121.51028852438\n",
      "4640 norm: 106350.5764824374\n",
      "4650 norm: 106580.09680127245\n",
      "4660 norm: 106808.13236778275\n",
      "4670 norm: 107036.77951272122\n",
      "4680 norm: 107264.916292055\n",
      "4690 norm: 107494.1346312779\n",
      "4700 norm: 107721.27386982132\n",
      "4710 norm: 107947.07982034145\n",
      "4720 norm: 108177.30987862698\n",
      "4730 norm: 108405.31770177501\n",
      "4740 norm: 108636.06205565055\n",
      "4750 norm: 108864.13228309105\n",
      "4760 norm: 109093.44664738877\n",
      "4770 norm: 109323.44248994086\n",
      "4780 norm: 109551.80195278229\n",
      "4790 norm: 109781.22370658437\n",
      "4800 norm: 110010.18358968852\n",
      "4810 norm: 110238.2581384762\n",
      "4820 norm: 110467.26500702555\n",
      "4830 norm: 110694.68301188856\n",
      "4840 norm: 110925.44397021056\n",
      "4850 norm: 111155.27256781417\n",
      "4860 norm: 111386.55219351628\n",
      "4870 norm: 111614.99586085041\n",
      "4880 norm: 111843.72923368333\n",
      "4890 norm: 112071.69429665938\n",
      "4900 norm: 112297.13861923035\n",
      "4910 norm: 112529.58080967495\n",
      "4920 norm: 112758.18872029727\n",
      "4930 norm: 112988.98720356733\n",
      "4940 norm: 113214.86600566214\n",
      "4950 norm: 113443.21326433907\n",
      "4960 norm: 113671.0664625823\n",
      "4970 norm: 113899.83358345632\n",
      "4980 norm: 114128.65331492874\n",
      "4990 norm: 114357.84253866809\n",
      "5000 norm: 114586.25196111295\n",
      "5010 norm: 114815.30747102418\n",
      "5020 norm: 115041.25049154727\n",
      "5030 norm: 115271.39339859784\n",
      "5040 norm: 115499.39858224668\n",
      "5050 norm: 115730.18338551644\n",
      "5060 norm: 115956.94093648344\n",
      "5070 norm: 116183.96310596786\n",
      "5080 norm: 116410.09245961388\n",
      "5090 norm: 116640.37241521801\n",
      "5100 norm: 116867.96508489699\n",
      "5110 norm: 117098.18221884214\n",
      "5120 norm: 117327.60791168176\n",
      "5130 norm: 117552.9780491493\n",
      "5140 norm: 117781.39873682708\n",
      "5150 norm: 118010.9930451301\n",
      "5160 norm: 118240.62271163144\n",
      "5170 norm: 118470.84937401008\n",
      "5180 norm: 118699.14443407774\n",
      "5190 norm: 118927.79216681566\n",
      "5200 norm: 119155.66494883549\n",
      "5210 norm: 119384.34691903052\n",
      "5220 norm: 119611.32041493691\n",
      "5230 norm: 119841.38326330922\n",
      "5240 norm: 120069.86158979758\n",
      "5250 norm: 120297.18317119668\n",
      "5260 norm: 120528.14806621916\n",
      "5270 norm: 120752.82643116392\n",
      "5280 norm: 120980.76258411219\n",
      "5290 norm: 121209.0564675156\n",
      "5300 norm: 121436.25931054939\n",
      "5310 norm: 121665.20097285994\n",
      "5320 norm: 121895.51905283594\n",
      "5330 norm: 122124.05149667288\n",
      "5340 norm: 122353.77158674697\n",
      "5350 norm: 122581.81845082146\n",
      "5360 norm: 122809.43076756895\n",
      "5370 norm: 123037.32772308581\n",
      "5380 norm: 123264.25018670502\n",
      "5390 norm: 123495.60435037123\n",
      "5400 norm: 123724.9581171881\n",
      "5410 norm: 123954.41382102043\n",
      "5420 norm: 124182.89851164586\n",
      "5430 norm: 124412.48172800212\n",
      "5440 norm: 124641.23756153387\n",
      "5450 norm: 124868.4945925397\n",
      "5460 norm: 125100.37676319382\n",
      "5470 norm: 125328.8328249696\n",
      "5480 norm: 125554.69410496626\n",
      "5490 norm: 125782.24183209892\n",
      "5500 norm: 126008.19814956383\n",
      "5510 norm: 126237.45011769436\n",
      "5520 norm: 126465.78608989381\n",
      "5530 norm: 126694.69271165041\n",
      "5540 norm: 126923.19021174779\n",
      "5550 norm: 127152.93989602139\n",
      "5560 norm: 127382.18017762204\n",
      "5570 norm: 127607.7711341814\n",
      "5580 norm: 127835.42202331481\n",
      "5590 norm: 128064.34074387122\n",
      "5600 norm: 128291.74214274711\n",
      "5610 norm: 128516.02100072695\n",
      "5620 norm: 128744.94732014596\n",
      "5630 norm: 128973.7719175549\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5640 norm: 129201.8468726629\n",
      "5650 norm: 129428.94526992047\n",
      "5660 norm: 129656.49789681136\n",
      "5670 norm: 129886.3831321867\n",
      "5680 norm: 130114.0789282037\n",
      "5690 norm: 130344.53582712996\n",
      "5700 norm: 130574.14233978037\n",
      "5710 norm: 130799.33168943986\n",
      "5720 norm: 131026.99191866376\n",
      "5730 norm: 131256.02356761554\n",
      "5740 norm: 131482.23106410037\n",
      "5750 norm: 131710.52551443438\n",
      "5760 norm: 131936.29021953218\n",
      "5770 norm: 132161.36425503637\n",
      "5780 norm: 132391.07959309503\n",
      "5790 norm: 132619.10432543347\n",
      "5800 norm: 132844.30701121056\n",
      "5810 norm: 133074.92431807992\n",
      "5820 norm: 133301.9635766839\n",
      "5830 norm: 133529.8672935781\n",
      "5840 norm: 133758.92331898835\n",
      "5850 norm: 133987.04949558023\n",
      "5860 norm: 134218.66369803745\n",
      "5870 norm: 134445.77904173834\n",
      "5880 norm: 134673.9110635192\n",
      "5890 norm: 134901.56133507317\n",
      "5900 norm: 135126.35659403502\n",
      "5910 norm: 135356.59704222853\n",
      "5920 norm: 135582.58799364883\n",
      "5930 norm: 135810.05514904347\n",
      "5940 norm: 136040.51646471748\n",
      "5950 norm: 136266.34789751822\n",
      "5960 norm: 136495.66999391248\n",
      "5970 norm: 136723.90057413268\n",
      "5980 norm: 136951.08353062728\n",
      "5990 norm: 137177.21917246337\n",
      "6000 norm: 137406.05080834744\n",
      "6010 norm: 137632.06845028437\n",
      "6020 norm: 137860.63801008346\n",
      "6030 norm: 138088.3603858425\n",
      "6040 norm: 138317.65866347007\n",
      "6050 norm: 138547.6308314561\n",
      "6060 norm: 138777.08818591014\n",
      "6070 norm: 139005.628321978\n",
      "6080 norm: 139232.69786387007\n",
      "6090 norm: 139461.94119898\n",
      "6100 norm: 139691.72313250316\n",
      "6110 norm: 139918.94665236608\n",
      "6120 norm: 140145.2888480135\n",
      "6130 norm: 140373.72530144462\n",
      "6140 norm: 140603.830482295\n",
      "6150 norm: 140833.7351980492\n",
      "6160 norm: 141063.47684719134\n",
      "6170 norm: 141292.0192894218\n",
      "6180 norm: 141521.73542619258\n",
      "6190 norm: 141750.43632966193\n",
      "6200 norm: 141978.87235124613\n",
      "6210 norm: 142206.5135539561\n",
      "6220 norm: 142434.65601016936\n",
      "6230 norm: 142661.40452686863\n",
      "6240 norm: 142886.31181664797\n",
      "6250 norm: 143114.60701769072\n",
      "6260 norm: 143342.14058716656\n",
      "6270 norm: 143573.10513227936\n",
      "6280 norm: 143800.64724211118\n",
      "6290 norm: 144029.7635017331\n",
      "6300 norm: 144261.4703335519\n",
      "6310 norm: 144492.44733822817\n",
      "6320 norm: 144721.69636342247\n",
      "6330 norm: 144952.03344032686\n",
      "6340 norm: 145180.38623912685\n",
      "6350 norm: 145407.19150521682\n",
      "6360 norm: 145635.61660252776\n",
      "6370 norm: 145865.13078565142\n",
      "6380 norm: 146092.011581824\n",
      "6390 norm: 146321.2887459385\n",
      "6400 norm: 146550.0809434736\n",
      "6410 norm: 146779.77414685494\n",
      "6420 norm: 147007.1723952403\n",
      "6430 norm: 147237.17077146104\n",
      "6440 norm: 147465.82693731482\n",
      "6450 norm: 147696.52401545853\n",
      "6460 norm: 147924.2713581626\n",
      "6470 norm: 148152.00337343063\n",
      "6480 norm: 148377.2660361087\n",
      "6490 norm: 148602.65416270818\n",
      "6500 norm: 148827.2488229915\n",
      "6510 norm: 149054.52086527363\n",
      "6520 norm: 149281.9612170011\n",
      "6530 norm: 149509.06709823702\n",
      "6540 norm: 149736.3879196147\n",
      "6550 norm: 149965.2775223017\n",
      "6560 norm: 150193.69738061444\n",
      "6570 norm: 150421.54358587592\n",
      "6580 norm: 150650.56218158625\n",
      "6590 norm: 150877.51151904272\n",
      "6600 norm: 151105.06806874255\n",
      "6610 norm: 151334.8337094061\n",
      "6620 norm: 151562.44602596934\n",
      "6630 norm: 151790.8497108954\n",
      "6640 norm: 152017.04869932422\n",
      "6650 norm: 152244.6844264187\n",
      "6660 norm: 152473.39688690964\n",
      "6670 norm: 152703.66835755378\n",
      "6680 norm: 152931.14784901036\n",
      "6690 norm: 153154.56782541468\n",
      "6700 norm: 153382.0872143987\n",
      "6710 norm: 153607.04036418596\n",
      "6720 norm: 153834.96155482726\n",
      "6730 norm: 154062.35809562838\n",
      "6740 norm: 154290.78073891022\n",
      "6750 norm: 154518.40578033068\n",
      "6760 norm: 154745.48006949373\n",
      "6770 norm: 154972.24318736812\n",
      "6780 norm: 155200.29342168107\n",
      "6790 norm: 155426.65663581327\n",
      "6800 norm: 155654.33188166117\n",
      "6810 norm: 155884.94427070982\n",
      "6820 norm: 156113.08374820306\n",
      "6830 norm: 156342.15391971945\n",
      "6840 norm: 156569.4803986862\n",
      "6850 norm: 156798.35932764361\n",
      "6860 norm: 157027.90205435583\n",
      "6870 norm: 157254.44254078812\n",
      "6880 norm: 157480.40877612383\n",
      "6890 norm: 157710.56926430523\n",
      "6900 norm: 157938.51032596664\n",
      "6910 norm: 158164.74774629358\n",
      "6920 norm: 158392.80595748825\n",
      "6930 norm: 158619.28662248695\n",
      "6940 norm: 158845.86826002554\n",
      "6950 norm: 159076.059399541\n",
      "6960 norm: 159305.07781956354\n",
      "6970 norm: 159532.28851234558\n",
      "6980 norm: 159760.92837753665\n",
      "6990 norm: 159987.81522134237\n",
      "7000 norm: 160215.59462048067\n",
      "7010 norm: 160443.07596963004\n",
      "7020 norm: 160671.24172276998\n",
      "7030 norm: 160900.4561802495\n",
      "7040 norm: 161130.07261168174\n",
      "7050 norm: 161356.38971995036\n",
      "7060 norm: 161585.5847712138\n",
      "7070 norm: 161814.72882911118\n",
      "7080 norm: 162044.19320693635\n",
      "7090 norm: 162272.38752947174\n",
      "7100 norm: 162501.02291965566\n",
      "7110 norm: 162726.35714997977\n",
      "7120 norm: 162954.6346339174\n",
      "7130 norm: 163184.0193105476\n",
      "7140 norm: 163412.9419842866\n",
      "7150 norm: 163639.71600699623\n",
      "7160 norm: 163866.1595704568\n",
      "7170 norm: 164095.30992458534\n",
      "7180 norm: 164324.085826697\n",
      "7190 norm: 164551.3944201184\n",
      "7200 norm: 164780.29242489437\n",
      "7210 norm: 165009.0806153309\n",
      "7220 norm: 165235.80538926172\n",
      "7230 norm: 165465.18635823153\n",
      "7240 norm: 165690.50496142515\n",
      "7250 norm: 165918.64080968947\n",
      "7260 norm: 166147.12534457436\n",
      "7270 norm: 166376.13398743572\n",
      "7280 norm: 166605.60911000086\n",
      "7290 norm: 166835.97549496827\n",
      "7300 norm: 167066.8278052371\n",
      "7310 norm: 167293.601548624\n",
      "7320 norm: 167522.46064493194\n",
      "7330 norm: 167751.06005539862\n",
      "7340 norm: 167978.55764995166\n",
      "7350 norm: 168207.54110404933\n",
      "7360 norm: 168436.33477353404\n",
      "7370 norm: 168663.9901025345\n",
      "7380 norm: 168892.6659580575\n",
      "7390 norm: 169119.940203848\n",
      "7400 norm: 169346.72603510172\n",
      "7410 norm: 169574.70718442494\n",
      "7420 norm: 169804.4143971579\n",
      "7430 norm: 170026.78952974614\n",
      "7440 norm: 170254.72572712018\n",
      "7450 norm: 170482.5917491747\n",
      "7460 norm: 170709.08769551167\n",
      "7470 norm: 170938.5660651856\n",
      "7480 norm: 171168.1875497151\n",
      "7490 norm: 171396.98311694825\n",
      "7500 norm: 171622.72910440215\n",
      "7510 norm: 171852.27036919256\n",
      "7520 norm: 172079.00533586077\n",
      "7530 norm: 172308.67648506025\n",
      "7540 norm: 172537.2149777794\n",
      "7550 norm: 172765.49307675593\n",
      "7560 norm: 172993.98036399795\n",
      "7570 norm: 173221.6442169636\n",
      "7580 norm: 173451.2615035988\n",
      "7590 norm: 173680.84875246836\n",
      "7600 norm: 173908.16865122202\n",
      "7610 norm: 174133.58278898787\n",
      "7620 norm: 174363.89343015736\n",
      "7630 norm: 174592.8328582874\n",
      "7640 norm: 174822.16379084057\n",
      "7650 norm: 175050.72976947005\n",
      "7660 norm: 175278.37290290254\n",
      "7670 norm: 175504.1676066733\n",
      "7680 norm: 175733.77712867\n",
      "7690 norm: 175962.53460501114\n",
      "7700 norm: 176191.35972986906\n",
      "7710 norm: 176420.44423471333\n",
      "7720 norm: 176651.57898034682\n",
      "7730 norm: 176881.68706581736\n",
      "7740 norm: 177109.60807797938\n",
      "7750 norm: 177340.34687299796\n",
      "7760 norm: 177564.557976262\n",
      "7770 norm: 177794.97537186724\n",
      "7780 norm: 178019.9511410213\n",
      "7790 norm: 178248.84072182648\n",
      "7800 norm: 178475.89603734654\n",
      "7810 norm: 178701.23132523088\n",
      "7820 norm: 178926.63894305908\n",
      "7830 norm: 179157.05843125304\n",
      "7840 norm: 179385.98107096727\n",
      "7850 norm: 179612.10492857892\n",
      "7860 norm: 179839.12021467817\n",
      "7870 norm: 180063.8595676905\n",
      "7880 norm: 180290.04566692607\n",
      "7890 norm: 180516.55433312838\n",
      "7900 norm: 180743.97319539005\n",
      "7910 norm: 180976.2671476848\n",
      "7920 norm: 181200.36278228217\n",
      "7930 norm: 181432.20447533412\n",
      "7940 norm: 181657.5720422647\n",
      "7950 norm: 181885.731040542\n",
      "7960 norm: 182113.5572768209\n",
      "7970 norm: 182342.9463242137\n",
      "7980 norm: 182570.25025519243\n",
      "7990 norm: 182797.8655539687\n",
      "8000 norm: 183024.78619853727\n",
      "8010 norm: 183253.1431303079\n",
      "8020 norm: 183480.85226904077\n",
      "8030 norm: 183709.53712732374\n",
      "8040 norm: 183937.40904550505\n",
      "8050 norm: 184166.58982965676\n",
      "8060 norm: 184392.60106058692\n",
      "8070 norm: 184620.14127719722\n",
      "8080 norm: 184847.63036591618\n",
      "8090 norm: 185075.01697258643\n",
      "8100 norm: 185302.30278622743\n",
      "8110 norm: 185528.19618654877\n",
      "8120 norm: 185755.3820385727\n",
      "8130 norm: 185982.11439597362\n",
      "8140 norm: 186210.2319263811\n",
      "8150 norm: 186436.43677576646\n",
      "8160 norm: 186665.87784591716\n",
      "8170 norm: 186896.0170517665\n",
      "8180 norm: 187125.3892476577\n",
      "8190 norm: 187352.13943407472\n",
      "8200 norm: 187578.50781925695\n",
      "8210 norm: 187807.39082897516\n",
      "8220 norm: 188033.8151117204\n",
      "8230 norm: 188262.97743796388\n",
      "8240 norm: 188486.2680927556\n",
      "8250 norm: 188713.5410187689\n",
      "8260 norm: 188941.1068907626\n",
      "8270 norm: 189169.6808666881\n",
      "8280 norm: 189397.35107667642\n",
      "8290 norm: 189626.93498500512\n",
      "8300 norm: 189855.0935341373\n",
      "8310 norm: 190081.86695848813\n",
      "8320 norm: 190313.453149224\n",
      "8330 norm: 190539.3483062432\n",
      "8340 norm: 190767.16579456403\n",
      "8350 norm: 190996.91073822902\n",
      "8360 norm: 191225.84072801445\n",
      "8370 norm: 191452.3065466944\n",
      "8380 norm: 191681.88582152312\n",
      "8390 norm: 191910.41962678626\n",
      "8400 norm: 192136.57324998832\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8410 norm: 192363.1097055093\n",
      "8420 norm: 192591.76087878025\n",
      "8430 norm: 192818.304909728\n",
      "8440 norm: 193046.2207694567\n",
      "8450 norm: 193273.06277932055\n",
      "8460 norm: 193501.39246258573\n",
      "8470 norm: 193727.5161034235\n",
      "8480 norm: 193956.63150767543\n",
      "8490 norm: 194186.15674467452\n",
      "8500 norm: 194414.30187976058\n",
      "8510 norm: 194641.39650494297\n",
      "8520 norm: 194869.55817782143\n",
      "8530 norm: 195100.0083316538\n",
      "8540 norm: 195329.37811681128\n",
      "8550 norm: 195558.52539521013\n",
      "8560 norm: 195786.62146799892\n",
      "8570 norm: 196013.8014694123\n",
      "8580 norm: 196243.6408740692\n",
      "8590 norm: 196474.1008154029\n",
      "8600 norm: 196700.57262643456\n",
      "8610 norm: 196927.28652397412\n",
      "8620 norm: 197154.65220366482\n",
      "8630 norm: 197381.68917692354\n",
      "8640 norm: 197609.45826091547\n",
      "8650 norm: 197834.18782028364\n",
      "8660 norm: 198066.01181385687\n",
      "8670 norm: 198291.83711149843\n",
      "8680 norm: 198520.58080048615\n",
      "8690 norm: 198751.2569694658\n",
      "8700 norm: 198981.27396649556\n",
      "8710 norm: 199209.397231695\n",
      "8720 norm: 199436.35889700716\n",
      "8730 norm: 199665.8180340047\n",
      "8740 norm: 199892.95865352673\n",
      "8750 norm: 200123.99481386278\n",
      "8760 norm: 200353.80145426624\n",
      "8770 norm: 200582.1492539796\n",
      "8780 norm: 200809.30581048483\n",
      "8790 norm: 201036.76241150184\n",
      "8800 norm: 201261.79371808193\n",
      "8810 norm: 201492.1860376876\n",
      "8820 norm: 201719.65262915465\n",
      "8830 norm: 201949.6276175257\n",
      "8840 norm: 202175.73043977795\n",
      "8850 norm: 202406.28760441247\n",
      "8860 norm: 202635.37782528947\n",
      "8870 norm: 202863.41341301275\n",
      "8880 norm: 203092.3568008836\n",
      "8890 norm: 203320.17654051434\n",
      "8900 norm: 203549.51947376152\n",
      "8910 norm: 203776.73028687364\n",
      "8920 norm: 204006.44016766342\n",
      "8930 norm: 204236.54839882534\n",
      "8940 norm: 204461.96626330444\n",
      "8950 norm: 204690.3655937614\n",
      "8960 norm: 204918.5344153778\n",
      "8970 norm: 205147.45986370387\n",
      "8980 norm: 205377.7284961218\n",
      "8990 norm: 205610.0091201652\n",
      "9000 norm: 205840.76629260049\n",
      "9010 norm: 206063.53483500288\n",
      "9020 norm: 206290.4350754268\n",
      "9030 norm: 206519.29320076207\n",
      "9040 norm: 206750.047370663\n",
      "9050 norm: 206976.91164614772\n",
      "9060 norm: 207206.461892654\n",
      "9070 norm: 207433.94535525388\n",
      "9080 norm: 207662.39448150634\n",
      "9090 norm: 207891.26567653922\n",
      "9100 norm: 208117.3228823442\n",
      "9110 norm: 208344.4229793864\n",
      "9120 norm: 208573.73346791253\n",
      "9130 norm: 208802.96353978064\n",
      "9140 norm: 209033.15144380298\n",
      "9150 norm: 209264.09444391014\n",
      "9160 norm: 209491.90317782154\n",
      "9170 norm: 209719.8635234045\n",
      "9180 norm: 209945.2598889391\n",
      "9190 norm: 210175.740664419\n",
      "9200 norm: 210401.04344631475\n",
      "9210 norm: 210628.7423409781\n",
      "9220 norm: 210856.79261661606\n",
      "9230 norm: 211081.3466752702\n",
      "9240 norm: 211311.83614797847\n",
      "9250 norm: 211541.5537256149\n",
      "9260 norm: 211767.31855091624\n",
      "9270 norm: 211995.3432794184\n",
      "9280 norm: 212222.975534713\n",
      "9290 norm: 212450.59342672458\n",
      "9300 norm: 212679.95058484148\n",
      "9310 norm: 212909.12706195598\n",
      "9320 norm: 213138.55808799333\n",
      "9330 norm: 213366.34786835138\n",
      "9340 norm: 213591.70875892037\n",
      "9350 norm: 213817.9017395596\n",
      "9360 norm: 214047.49317678926\n",
      "9370 norm: 214277.67912357103\n",
      "9380 norm: 214506.5492394429\n",
      "9390 norm: 214735.28674523983\n",
      "9400 norm: 214963.79655538744\n",
      "9410 norm: 215191.68165347353\n",
      "9420 norm: 215420.31985823496\n",
      "9430 norm: 215649.94312074577\n",
      "9440 norm: 215872.33407407082\n",
      "9450 norm: 216101.04196674118\n",
      "9460 norm: 216330.4927909304\n",
      "9470 norm: 216557.09754975667\n",
      "9480 norm: 216781.28781920555\n",
      "9490 norm: 217008.11460604385\n",
      "9500 norm: 217234.40687890584\n",
      "9510 norm: 217460.95325396772\n",
      "9520 norm: 217689.15106308358\n",
      "9530 norm: 217919.06563081968\n",
      "9540 norm: 218149.92833657147\n",
      "9550 norm: 218377.7075198963\n",
      "9560 norm: 218604.43799895293\n",
      "9570 norm: 218832.5951596154\n",
      "9580 norm: 219060.20068111536\n",
      "9590 norm: 219290.14528800503\n",
      "9600 norm: 219517.73379301987\n",
      "9610 norm: 219747.88063877812\n",
      "9620 norm: 219975.62508306484\n",
      "9630 norm: 220202.83857883775\n",
      "9640 norm: 220428.84921163603\n",
      "9650 norm: 220657.35205337382\n",
      "9660 norm: 220885.73700703977\n",
      "9670 norm: 221114.30362299562\n",
      "9680 norm: 221343.54192031114\n",
      "9690 norm: 221569.67153005328\n",
      "9700 norm: 221798.90755246027\n",
      "9710 norm: 222022.90344006068\n",
      "9720 norm: 222251.91347583133\n",
      "9730 norm: 222479.27572056628\n",
      "9740 norm: 222706.99233822993\n",
      "9750 norm: 222935.2592002124\n",
      "9760 norm: 223163.74335768036\n",
      "9770 norm: 223392.90636283372\n",
      "9780 norm: 223618.7424488981\n",
      "9790 norm: 223846.27263088108\n",
      "9800 norm: 224074.55106990325\n",
      "9810 norm: 224306.65989453014\n",
      "9820 norm: 224533.9802269524\n",
      "9830 norm: 224760.69445410906\n",
      "9840 norm: 224985.90512101576\n",
      "9850 norm: 225213.89935791207\n",
      "9860 norm: 225442.1662388593\n",
      "9870 norm: 225669.79996184705\n",
      "9880 norm: 225896.08833847105\n",
      "9890 norm: 226123.12253501161\n",
      "9900 norm: 226351.8566296004\n",
      "9910 norm: 226577.36210171523\n",
      "9920 norm: 226804.27202632488\n",
      "9930 norm: 227032.22445050706\n",
      "9940 norm: 227260.3588547802\n",
      "9950 norm: 227486.16449682208\n",
      "9960 norm: 227714.86056208558\n",
      "9970 norm: 227939.8262954758\n",
      "9980 norm: 228168.60072068332\n",
      "9990 norm: 228396.2778224015\n"
     ]
    }
   ],
   "source": [
    "layer = activations_tool.get_layer(4,2,6)\n",
    "\n",
    "baseline = np.zeros((2048))\n",
    "\n",
    "for idx, line in enumerate(open(\"datasets/google-10000-english-usa.txt\")):\n",
    "    word = line.strip()\n",
    "    \n",
    "    _, _, activations = activations_tool.highest_channel_for_image(layer, rasterize_text(word))\n",
    "    \n",
    "    baseline = baseline + activations\n",
    "    \n",
    "    if idx % 10 == 0:\n",
    "        print(idx, \"norm:\", np.linalg.norm(baseline))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "spectacular-sucking",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23.519582811745178\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0588488586678555"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import spatial\n",
    "\n",
    "layer = activations_tool.get_layer(4,2,6)\n",
    "\n",
    "_, _, eee_acts = activations_tool.highest_channel_for_image(layer, rasterize_text(\"eeeeee\"))\n",
    "\n",
    "print(np.linalg.norm(eee_acts))\n",
    "spatial.distance.cosine(baseline, eee_acts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "imperial-democracy",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"generated/baseline_activations\", baseline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "funded-metro",
   "metadata": {},
   "source": [
    "# Using the Channel Difference Engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "changing-beaver",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inspecting layer type: ReLU(inplace=True)\n",
      "Baseline activation histogram:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQT0lEQVR4nO3df6zdd13H8eeLbgwFlM3dzdIWO0wlbibA0tThlKBDVgah0zhTotjEmYZkSyDRSCcJ4h+NQyPxR5xmAqEqMmpgruGHUiuEmMjG3dyv7oft2Nhqa1tGZBCTacfbP8635HB7Tu85955777n7PB/Jzff7/Xw/33Pe99NvX+d7vuf7PTdVhSSpDS9Y6QIkScvH0Jekhhj6ktQQQ1+SGmLoS1JDzlnpAgAuvPDC2rhx40qXIUmryt133/31qpoZZ5upCP2NGzcyOzu70mVI0qqS5GvjbuPpHUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDRgr9JE8keSDJvUlmu7YLkuxPcqibnt/X/6Ykh5M8muTqpSpekjSece7I/dmq+nrf8i7gQFXdnGRXt/yeJJcC24HLgJcD/5zkx6rquYlVPcfGXZ/57vwTN79lqZ5Gkla9xZze2Qbs6eb3ANf2td9WVc9W1ePAYWDLIp5HkjQho4Z+AZ9PcneSnV3bxVV1DKCbXtS1rwOe6tv2SNcmSVpho57eubKqjia5CNif5JGz9M2AtjP+EG/34rET4BWveMWIZUiSFmOkI/2qOtpNTwC30ztdczzJWoBueqLrfgTY0Lf5euDogMe8tao2V9XmmZmxvhlUkrRA84Z+khcneenpeeBNwIPAPmBH120HcEc3vw/YnuS8JJcAm4C7Jl24JGl8o5zeuRi4Pcnp/n9XVf+Y5CvA3iTXA08C1wFU1cEke4GHgFPADUt55Y4kaXTzhn5VfRV49YD2p4GrhmyzG9i96OokSRPlHbmS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhowc+knWJPn3JJ/uli9Isj/JoW56fl/fm5IcTvJokquXonBJ0vjGOdJ/F/Bw3/Iu4EBVbQIOdMskuRTYDlwGbAVuSbJmMuVKkhZjpNBPsh54C/ChvuZtwJ5ufg9wbV/7bVX1bFU9DhwGtkykWknSoox6pP/HwG8D3+lru7iqjgF004u69nXAU339jnRt3yPJziSzSWZPnjw5bt2SpAWYN/STvBU4UVV3j/iYGdBWZzRU3VpVm6tq88zMzIgPLUlajHNG6HMl8LYk1wAvAn4gyd8Cx5OsrapjSdYCJ7r+R4ANfduvB45OsmhJ0sLMe6RfVTdV1fqq2kjvA9p/qapfBfYBO7puO4A7uvl9wPYk5yW5BNgE3DXxyiVJYxvlSH+Ym4G9Sa4HngSuA6iqg0n2Ag8Bp4Abquq5RVcqSVq0sUK/qr4IfLGbfxq4aki/3cDuRdYmSZow78iVpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNWTe0E/yoiR3JbkvycEkv9e1X5Bkf5JD3fT8vm1uSnI4yaNJrl7KX0CSNLpRjvSfBX6uql4NvAbYmuQKYBdwoKo2AQe6ZZJcCmwHLgO2ArckWbMEtUuSxjRv6FfPt7vFc7ufArYBe7r2PcC13fw24LaqeraqHgcOA1smWbQkaWFGOqefZE2Se4ETwP6quhO4uKqOAXTTi7ru64Cn+jY/0rXNfcydSWaTzJ48eXIRv4IkaVQjhX5VPVdVrwHWA1uS/MRZumfQQwx4zFuranNVbZ6ZmRmpWEnS4ox19U5V/TfwRXrn6o8nWQvQTU903Y4AG/o2Ww8cXWyhkqTFG+XqnZkkL+vmvw94I/AIsA/Y0XXbAdzRze8Dtic5L8klwCbgrgnXLUlagHNG6LMW2NNdgfMCYG9VfTrJvwF7k1wPPAlcB1BVB5PsBR4CTgE3VNVzS1O+JGkc84Z+Vd0PvHZA+9PAVUO22Q3sXnR1kqSJ8o5cSWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkPmDf0kG5J8IcnDSQ4meVfXfkGS/UkOddPz+7a5KcnhJI8muXopfwFJ0uhGOdI/BfxmVf04cAVwQ5JLgV3AgaraBBzolunWbQcuA7YCtyRZsxTFS5LGM2/oV9Wxqrqnm/8W8DCwDtgG7Om67QGu7ea3AbdV1bNV9ThwGNgy4bolSQsw1jn9JBuB1wJ3AhdX1THovTAAF3Xd1gFP9W12pGub+1g7k8wmmT158uQCSpckjWvk0E/yEuCTwLur6pmzdR3QVmc0VN1aVZuravPMzMyoZUiSFmGk0E9yLr3A/1hVfaprPp5kbbd+LXCiaz8CbOjbfD1wdDLlSpIWY5SrdwJ8GHi4qj7Yt2ofsKOb3wHc0de+Pcl5SS4BNgF3Ta5kSdJCnTNCnyuBdwAPJLm3a/sd4GZgb5LrgSeB6wCq6mCSvcBD9K78uaGqnpt04ZKk8c0b+lX1rww+Tw9w1ZBtdgO7F1GXJGkJeEeuJDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSHzhn6SjyQ5keTBvrYLkuxPcqibnt+37qYkh5M8muTqpSpckjS+UY70PwpsndO2CzhQVZuAA90ySS4FtgOXddvckmTNxKqVJC3KvKFfVV8CvjGneRuwp5vfA1zb135bVT1bVY8Dh4EtkylVkrRYCz2nf3FVHQPophd17euAp/r6HenazpBkZ5LZJLMnT55cYBmSpHFM+oPcDGirQR2r6taq2lxVm2dmZiZchiRpkIWG/vEkawG66Ymu/Qiwoa/feuDowsuTJE3SQkN/H7Cjm98B3NHXvj3JeUkuATYBdy2uREnSpJwzX4ckHwfeAFyY5Ajwu8DNwN4k1wNPAtcBVNXBJHuBh4BTwA1V9dwS1S5JGtO8oV9Vbx+y6qoh/XcDuxdTlCRpaXhHriQ1xNCXpIYY+pLUEENfkhpi6EtSQ+a9eme12bjrM9+df+Lmt6xgJZI0fTzSl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhjzv7sgdxjt1JckjfUlqSjNH+sP4DkBSS5oM/f6gH9buC4Ck5yNP70hSQ57XR/rDjuglqVXP69BfjGEvGJ72kbSaeXpHkhpi6EtSQwx9SWqIoS9JDfGD3EXwun5Jq42hPyYvA5W0mhn6E+JRv6TVwNBfAr4ASJpWhv4S8yYvSdPE0F8hw94N+C5B0lJastBPshX4E2AN8KGqunmpnqtlc99J+EIh6WyWJPSTrAH+HPh54AjwlST7quqhpXi+1W6UK4IW8g7Adw2S5kpVTf5Bk9cB76+qq7vlmwCq6vcH9d+8eXPNzs4u+Pm8jHIyxj3NNMopqnGfd9jjL6b/2baf1GcuS/EC64v289ek/m2T3F1Vm8faZolC/5eArVX1G93yO4CfrKob+/rsBHZ2i68CHl3EU14IfH0R268Ea14e1rw8rHl5zK35R6pqZpwHWKpz+hnQ9j2vLlV1K3DrRJ4smR331W6lWfPysOblYc3LYxI1L9V37xwBNvQtrweOLtFzSZJGtFSh/xVgU5JLkrwQ2A7sW6LnkiSNaElO71TVqSQ3Av9E75LNj1TVwaV4rs5EThMtM2teHta8PKx5eSy65iX5IFeSNJ38Pn1JaoihL0kNWTWhn2RrkkeTHE6ya8D6JPnTbv39SS5fiTr76tmQ5AtJHk5yMMm7BvR5Q5JvJrm3+3nfStQ6p6YnkjzQ1XPGHXNTOM6v6hu/e5M8k+Tdc/pMxTgn+UiSE0ke7Gu7IMn+JIe66flDtj3r/r/MNf9hkke6f//bk7xsyLZn3ZeWueb3J/nPvn3gmiHbTtM4f6Kv3ieS3Dtk2/HGuaqm/ofeh8GPAa8EXgjcB1w6p881wOfo3SNwBXDnCte8Fri8m38p8B8Dan4D8OmVHt85NT0BXHiW9VM1zgP2k/+id8PK1I0z8HrgcuDBvrY/AHZ187uADwz5vc66/y9zzW8CzunmPzCo5lH2pWWu+f3Ab42w/0zNOM9Z/0fA+yYxzqvlSH8LcLiqvlpV/wvcBmyb02cb8NfV82XgZUnWLnehp1XVsaq6p5v/FvAwsG6l6pmgqRrnOa4CHquqr610IYNU1ZeAb8xp3gbs6eb3ANcO2HSU/X9JDKq5qj5fVae6xS/Tuw9nagwZ51FM1TifliTALwMfn8RzrZbQXwc81bd8hDMDdJQ+KyLJRuC1wJ0DVr8uyX1JPpfksuWtbKACPp/k7u6rMuaa2nGmdz/IsP8Y0zbOp11cVcegd6AAXDSgzzSP+a/Te+c3yHz70nK7sTsl9ZEhp9GmdZx/BjheVYeGrB9rnFdL6M/7tQ4j9ll2SV4CfBJ4d1U9M2f1PfRORbwa+DPgH5a5vEGurKrLgTcDNyR5/Zz10zrOLwTeBvz9gNXTOM7jmNYxfy9wCvjYkC7z7UvL6S+AHwVeAxyjd7pkrqkcZ+DtnP0of6xxXi2hP8rXOkzdVz8kOZde4H+sqj41d31VPVNV3+7mPwucm+TCZS5zbk1Hu+kJ4HZ6b3n7Td04d94M3FNVx+eumMZx7nP89OmxbnpiQJ+pG/MkO4C3Ar9S3YnluUbYl5ZNVR2vqueq6jvAXw2pZRrH+RzgF4FPDOsz7jivltAf5Wsd9gG/1l1dcgXwzdNvm1dCdx7uw8DDVfXBIX1+uOtHki30/j2eXr4qz6jnxUleenqe3gd2D87pNlXj3Gfo0dC0jfMc+4Ad3fwO4I4Bfabqa03S+wNJ7wHeVlX/M6TPKPvSspnzudMvDKllqsa580bgkao6MmjlgsZ5OT6ZntCn29fQuwLmMeC9Xds7gXd286H3h1seAx4ANq9wvT9N763h/cC93c81c2q+EThI7yqBLwM/tcI1v7Kr5b6urqkf566m76cX4j/Y1zZ140zvRekY8H/0jiqvB34IOAAc6qYXdH1fDny2b9sz9v8VrPkwvXPfp/frv5xb87B9aQVr/ptuf72fXpCvnfZx7to/eno/7uu7qHH2axgkqSGr5fSOJGkCDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUkP8HNUaqkJ5UcTgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rasterized image:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgCAIAAACVT/22AAAF/0lEQVR4nO3aTUuUaxyA8Xv0GI1hOoyYFTVCb64SN5llBYGukopoUSCVULToO5SrPkNuclHQol0aCqELpZdFJUILK8TpTUcqLIeIwuYsBM8w5uFow5lr8Prtnj9zz30/w8UzKkYymUyQqEoKfQDp3xio0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFtoYCnZ2djfwH/f39hT6p/rGGAlUximQymUKfoWDq6uqSyeTU1FRtbW2hz6Lf8wkqNAP9jdHR0Ugkcv78+WQyefbs2ZqampKSksePH/f29kYikWvXruW8vqqqaufOnUuXT05Onj59Oh6Pb9y48dixY69fvw4hpFKpzs7O2traaDR66NChZ8+eLV04MTFx6tSpeDy+YcOGw4cPDw4O/i/3TfRXoQ/ANT093dTUlEqlVrc8lUo1NzdPT08vXPb19Y2NjY2MjBw5cmRycnJhODIy0traOjExUVlZmb1vc3PzzMzMwuXw8HBbW9vdu3dPnDix6nspXj5BlzUwMFBTUzM4ODg3N5fJZPbv37+i5f39/YlE4tGjR+l0emxsrKGh4e3bt/v27auoqBgaGpqbmxsfH29pafn8+XN3d3fOvlVVVQ8ePPj69ev4+HhHR8f8/PzFixe/ffuW1/srEpk1LJFIhBCmpqZy5s+fPw8hRKPRN2/eZM/v3bsXQrh69WrO6ysrK3fs2JGzPBaLffz4cXF4//79EML69euTyeTi8OnTpyGEM2fOZC8sKyt7+fJl9vsfPHgwhHDnzp3V32rR8gm6rKampm3btq16+YEDB+Lx+OLlnj17QggNDQ3bt29fHNbX14cQ3r17l7Pvrl27sicdHR0hhIV81xoDXdaf1BlCqK6uzr5ct27dcsMfP35kD7du3ZrzVlu2bAkhfPny5U/OU6QMdFklJbkfTmlpaVjS0/fv39PpdB73ff/+fc7kw4cPIYTsX6TWDgNdgVgsFkJ48eJF9rCvr29+fj6Puzx58mThD1KLbt++HUJobGzM4y7FwkBXYO/evdFotLe398aNG7OzszMzMz09PZcuXcrvLj9//mxvbx8aGkqn069evers7BweHq6urm5vb8/vRkXBQFegvLz8ypUrv379unz5ciwW27Rp04ULF1pbW/P75dvW1vbp06ejR49WVFTs3r375s2bpaWl3d3d5eXledylWBjoyly/fr2rqyuRSJSVldXV1XV1dd26dSu/W2zevPnhw4fHjx+PxWLRaLSlpWVgYODkyZP53aVYrOl/FqEZHR1tbGw8d+5cT09Poc9C4RNUaAYqNAMVmj+DCs0nqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFZqBCs1AhWagQjNQoRmo0AxUaAYqNAMVmoEKzUCFZqBCM1ChGajQDFRoBio0AxWagQrNQIVmoEIzUKEZqNAMVGgGKjQDFdrftQ19ZeIxK94AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=224x224 at 0x7F6A666A7250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximally activating channel after baseline subtraction: 131\n",
      "Top ten activating channels:\n",
      "131: activation 0.145172119140625\n",
      "193: activation 0.13641357421875\n",
      "856: activation 0.102783203125\n",
      "125: activation 0.09912109375\n",
      "1021: activation 0.09610748291015625\n",
      "450: activation 0.0859375\n",
      "465: activation 0.0810546875\n",
      "119: activation 0.06640625\n",
      "28: activation 0.0662078857421875\n",
      "533: activation 0.06298828125\n",
      "Histogram of activations - baseline:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPU0lEQVR4nO3dX4xcZ33G8e/TALmAVE3qTTCO1U2R+ye5aEDbFClVRUVLouTCIJXKXFBLTWWQEgkkkOrABZGQJbcqoFYqSKaJcCtKaglQrEJbgkWFkGiSTRQSOyaNIS5ZbNlLaZtcpY359WJPYHBmd2d2/uzs6+9HWs2Z97zvmd++Xj/77tkzZ1NVSJLa8nObXYAkafwMd0lqkOEuSQ0y3CWpQYa7JDXoVZtdAMC2bdtqfn5+s8uQpC3l0Ucf/WFVzfXbNxPhPj8/z+Li4maXIUlbSpL/WG2fp2UkqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBM/EOVelSML//yz/ZPn3w9k2sRJcCV+6S1CDDXZIatG64J9mZ5OtJTiY5keT9Xfs9SX6Q5PHu47aeMXcnOZXk6SS3TPITkCS90iDn3F8CPlhVjyW5Ang0yYPdvk9W1V/0dk5yPbAHuAF4A/C1JL9SVRfGWbgkaXXrrtyr6mxVPdZtvwCcBHasMWQ3cH9VvVhVzwKngJvGUawkaTBDnXNPMg+8CXioa7oryRNJ7ktyZde2A3iuZ9gSfb4ZJNmXZDHJ4vLy8vCVS5JWNXC4J3kd8AXgA1X1PPBp4I3AjcBZ4OMvd+0zvF7RUHWoqhaqamFuru8fEpEkbdBA4Z7k1awE++eq6osAVXWuqi5U1Y+Bz/DTUy9LwM6e4dcCZ8ZXsiRpPYNcLRPgXuBkVX2ip317T7d3Ase77aPAniSXJ7kO2AU8PL6SJUnrGeRqmZuB9wBPJnm8a/sw8O4kN7JyyuU08F6AqjqR5AjwFCtX2tzplTKSNF3rhntVfZP+59G/ssaYA8CBEeqSJI3Ad6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgdcM9yc4kX09yMsmJJO/v2q9K8mCSZ7rHK3vG3J3kVJKnk9wyyU9AkvRKg6zcXwI+WFW/DrwFuDPJ9cB+4FhV7QKOdc/p9u0BbgBuBT6V5LJJFC9J6m/dcK+qs1X1WLf9AnAS2AHsBg533Q4D7+i2dwP3V9WLVfUscAq4acx1S5LWMNQ59yTzwJuAh4BrquosrHwDAK7uuu0AnusZttS1XXysfUkWkywuLy9voHRJ0moGDvckrwO+AHygqp5fq2uftnpFQ9WhqlqoqoW5ublBy5AkDWCgcE/yalaC/XNV9cWu+VyS7d3+7cD5rn0J2Nkz/FrgzHjKlSQNYpCrZQLcC5ysqk/07DoK7O229wIP9LTvSXJ5kuuAXcDD4ytZkrSeVw3Q52bgPcCTSR7v2j4MHASOJLkD+D7wLoCqOpHkCPAUK1fa3FlVF8ZduCRpdeuGe1V9k/7n0QHetsqYA8CBEeqSJI3Ad6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDRrkD2RLGrP5/V/+yfbpg7dvYiVqlSt3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQeuGe5L7kpxPcryn7Z4kP0jyePdxW8++u5OcSvJ0klsmVbgkaXWDrNw/C9zap/2TVXVj9/EVgCTXA3uAG7oxn0py2biKlSQNZt1wr6pvAD8a8Hi7gfur6sWqehY4Bdw0Qn2SpA0Y5Zz7XUme6E7bXNm17QCe6+mz1LW9QpJ9SRaTLC4vL49QhiTpYhsN908DbwRuBM4CH+/a06dv9TtAVR2qqoWqWpibm9tgGZKkfjYU7lV1rqouVNWPgc/w01MvS8DOnq7XAmdGK1GSNKwNhXuS7T1P3wm8fCXNUWBPksuTXAfsAh4erURJ0rDW/TN7ST4PvBXYlmQJ+Cjw1iQ3snLK5TTwXoCqOpHkCPAU8BJwZ1VdmEjlkqRVrRvuVfXuPs33rtH/AHBglKIkSaPxHaqS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVo3XBPcl+S80mO97RdleTBJM90j1f27Ls7yakkTye5ZVKFS5JWN8jK/bPArRe17QeOVdUu4Fj3nCTXA3uAG7oxn0py2diqlSQN5FXrdaiqbySZv6h5N/DWbvsw8K/An3bt91fVi8CzSU4BNwHfGlO90pYyv//Lm12CLlEbPed+TVWdBeger+7adwDP9fRb6tokSVM07l+opk9b9e2Y7EuymGRxeXl5zGVI0qVto+F+Lsl2gO7xfNe+BOzs6XctcKbfAarqUFUtVNXC3NzcBsuQJPWz0XA/CuzttvcCD/S070lyeZLrgF3Aw6OVKEka1rq/UE3yeVZ+ebotyRLwUeAgcCTJHcD3gXcBVNWJJEeAp4CXgDur6sKEapckrWKQq2Xevcqut63S/wBwYJSiJEmj8R2qktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUHrXgopabJ6by52+uDtm1iJWuLKXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN8sZh0gzxJmIaF1fuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAb5JiZpzHrfiCRtFlfuktQgw12SGmS4S1KDRjrnnuQ08AJwAXipqhaSXAX8AzAPnAb+sKr+a7QyJUnDGMfK/Xer6saqWuie7weOVdUu4Fj3XJI0RZM4LbMbONxtHwbeMYHXkCStYdRwL+CrSR5Nsq9ru6aqzgJ0j1f3G5hkX5LFJIvLy8sjliFJ6jXqde43V9WZJFcDDyb5zqADq+oQcAhgYWGhRqxDktRjpJV7VZ3pHs8DXwJuAs4l2Q7QPZ4ftUhJ0nA2HO5JXpvkipe3gbcDx4GjwN6u217ggVGLlCQNZ5TTMtcAX0ry8nH+vqr+OckjwJEkdwDfB941epmSpGFsONyr6nvAb/Rp/0/gbaMUJUkaje9QlaQGGe6S1CBv+SuNwSRu89t7zNMHbx/78dU2V+6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQg36EqDcF3jWqrcOUuSQ0y3CWpQYa7JDXIc+7SBk3iTpDDvpbn/bUaV+6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQV4KKfXhbQa01blyl6QGuXKXtjB/wtBqXLlLUoNcueuSNsjKd5q3GRiFq3j1cuUuSQ1y5S51tsoKXRqE4S41bq3TNZ7KadfEwj3JrcBfApcBf1NVByf1WtoaZiVILoUV+jQ+x1n591R/Ewn3JJcBfw38PrAEPJLkaFU9NYnXk2DwFeqlbK15mMQ94/0GsHkmtXK/CThVVd8DSHI/sBuYSLjP2hfQrNUDq9c0rlonfZxhw9kwn65xzfcoX6eD1DDt/4+T/n+3llTV+A+a/AFwa1X9Sff8PcBvVdVdPX32Afu6p78KPL3GIbcBPxx7oZNlzdOxFWuGrVm3NU/HMDX/UlXN9dsxqZV7+rT9zHeRqjoEHBroYMliVS2Mo7Bpsebp2Io1w9as25qnY1w1T+o69yVgZ8/za4EzE3otSdJFJhXujwC7klyX5DXAHuDohF5LknSRiZyWqaqXktwF/Asrl0LeV1UnRjjkQKdvZow1T8dWrBm2Zt3WPB1jqXkiv1CVJG0u7y0jSQ0y3CWpQTMT7kmuSvJgkme6xyv79NmZ5OtJTiY5keT9w4zfjJq7fvclOZ/k+EXt9yT5QZLHu4/btkDNszzPtyZ5OsmpJPt72qc2z6vV0LM/Sf6q2/9EkjcPOnZGaz6d5MluXhenVfOAdf9akm8leTHJh4YZO6M1DzfXVTUTH8CfA/u77f3An/Xpsx14c7d9BfDvwPWDjt+Mmrt9vwO8GTh+Ufs9wIdmbZ7XqXkm55mVX9x/F/hl4DXAt3u+NqYyz2vV0NPnNuCfWHkvyFuAhwYdO2s1d/tOA9um+TU8RN1XA78JHOj995/xue5b80bmemZW7qzcnuBwt30YeMfFHarqbFU91m2/AJwEdgw6fgIGes2q+gbwoynUM4hRa57Vef7JLS+q6n+Bl295MU2D1LAb+Nta8W/ALyTZPuDYWat5M61bd1Wdr6pHgP8bduwM1jy0WQr3a6rqLKyEOCvfwVaVZB54E/DQRsaPyThe867uR937pnGKg9FrntV53gE81/N8iZ9+44fpzPN6NazVZ5CxkzBKzbDyzvOvJnk0K7cUmZZR5muW53otQ831VO/nnuRrwOv77PrIkMd5HfAF4ANV9fw4alvjtcZS8yo+DXyMlX+0jwEfB/541INOuOaJGEPNa93yYiLzPGQN6/UZZOwkjFIzwM1VdSbJ1cCDSb7T/dQ3aaPM1yzP9VqGmuuphntV/d5q+5KcS7K9qs52P/KdX6Xfq1kJ9s9V1Rd7dg00fjNqXuPY53qO9RngHzde6c8cd2I1M7vzvOotLyY1z8PUMECf1wwwdhJGqZmqevnxfJIvsXLqYRrhPsotTjbr9igjve6wcz1Lp2WOAnu77b3AAxd3SBLgXuBkVX1i2PETMNJrXnTe8p3A8dX6jtGo8zSr87zqLS+mOM+D3HbjKPBH3RUobwH+pzvVtFm37NhwzUlem+QKgCSvBd7OdL6GB617EmNHseHX3dBcT/o3xEP8JvkXgWPAM93jVV37G4CvdNu/zcqPMU8Aj3cft601frNr7p5/HjjLyi9JloA7uva/A57sPp+jwPYtUPMsz/NtrFxB9V3gIz3tU5vnfjUA7wPe122HlT9k892upoX16p/C/G6oZlau+vh293FimjUPWPfru6/d54H/7rZ/fsbnum/NG5lrbz8gSQ2apdMykqQxMdwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg/4fnUY7OWS+UdcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximally activating channel + activation without baseline subtraction: 341 16.90625\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "target_text = \"Trump\"\n",
    "baseline_text = \"eeeeee\"\n",
    "layer = activations_tool.get_layer(3,5,6)\n",
    "\n",
    "print(\"Inspecting layer type:\", layer)\n",
    "\n",
    "print(\"Baseline activation histogram:\")\n",
    "baseline_image = rasterize_text(baseline_text)\n",
    "_, _, baseline_acts = activations_tool.highest_channel_for_image(layer, baseline_image)\n",
    "plt.hist(baseline_acts, bins=100)\n",
    "plt.show()\n",
    "\n",
    "print(\"Rasterized image:\")\n",
    "image = rasterize_text(target_text)\n",
    "display(image)\n",
    "\n",
    "activation_differences = activations_tool.channel_difference_engine(layer, image, baseline_image)\n",
    "unit, max_act, all_acts = activations_tool.highest_channel_for_image(layer, image)\n",
    "\n",
    "acts = np.subtract(all_acts, baseline_acts)\n",
    "i = np.argmax(acts)\n",
    "print(\"Maximally activating channel after baseline subtraction:\", i)\n",
    "\n",
    "print(\"Top ten activating channels:\")\n",
    "top_ten = list(activation_differences.items())[:10]\n",
    "for channel, act in top_ten:\n",
    "    print(f\"{channel}: activation {act}\")\n",
    "\n",
    "print(\"Histogram of activations - baseline:\")\n",
    "plt.hist(acts, bins=100)\n",
    "plt.show()\n",
    "print('Maximally activating channel + activation without baseline subtraction:', unit, max_act)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "corrected-fault",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOAAAADgCAIAAACVT/22AAALFklEQVR4nO3ca2xT5QPH8bVb67BssAuDTWU4MpNh5ySLCiY6JcaACyKooM6ATRSrqDMmJhIvQXGR+EJREzAYEAZojDPBF+OVF4yXLAYizs1oottUKmSgDLa5Szv6f/EkJ83anl169t9v4/t59aw9l6cPX8p2eoYrGo2mAarckz0BwA6BQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQtokBxoMBl0ul8vlys/PT/FQzc3Nfr8/Nzd3586djszNsGa4YMECBw+LUZo+76DPPfdca2vr2bNna2trz5w5M9nTgTOmT6Dt7e1mEA6H//zzz8mdDJwyfQJduHChGXg8nuLi4smdDJwyfQLdtm3b1VdfnZOT88477+Tl5U32dOCMjMmegGP8fn9LS8tkzwIOmz7voJiWCBTSnA+0q6tr586dd911V0lJic/n83g8OTk5ixcvXr9+/d69e8+dO5dwr/T0dDPo7u7evXv38uXLFyxYcMkll8yaNau0tPT+++//6KOPIpGIzXlHf0l1fDO00dTU5PP5zNlLS0s7OzsTbhYKherq6qqqqubMmePxeLKysvx+/8aNG7/55hv741sv7dNPPzWPmFWqrq6+8sorMzMzs7OzFy5cuG7dug8//NB+laaeqHPC4fCrr77q8/lsTrdixYrYXR599FHzeGlpaTQaPXz48GWXXZZs37KyspaWlmRntw6Vl5c3ETMsLi5OeMyWlpbc3FyzTVFRUXt7e/w2kUhky5YtmZmZyU5aXV196tSpEV/aa6+9Fo1GDx06VFRUNL5VmnIcC/S///67/fbbhy1Wenr6sD+Vffv2xe5lLf0NN9xw6NAht9sdu++MGTOGHXDevHmhUCjhBEYMNMUZJgy0o6PD+huVk5PT3Nwcv83AwMCdd9457LyXXnqpy+WKfaSkpOTEiRP2L23Dhg0ff/xx7I5jXaUpx7FA161bZy1QQUHBm2++2dbWNjQ0FI1G//nnn8bGxmAwOH/+/O7u7ti9rKUvKioyb2zl5eX79+//+++/L1y4YPZ9//33Y99WA4FAwgmMGGiKM4wPtLOz86qrrrKC+/bbbxOeNxgMWuctLy9vaGg4d+5cNBoNh8PffffdqlWrrGdvvPFGM59kL23u3LkmR7/fX19fHwqFxrpKU44zgX7yySfW0lRUVJw+fTrhZmY1Y1lLb6xZsyYSicTv+Ndff82ZM8ds4/V6e3p64rexDzT1GQ4L9Pz585WVleapjIyMxsbGhAf86quvrPMuX768r6/PZuZpaWn19fX2G6Slpa1atSocDsdvNppVmnKcCfTaa68165KZmfn777+PfsfYpc/NzTVvLQm9/vrr1pZffvmlzaESBpr6DGMD7e/vX7ZsmXnc5XIdOHAg2e633HKL2Sw/P//ff/9NuE1PT8/cuXPNZpWVlTZzMKvU1dWV7HQjrtKU48BP8T/99NPx48fNeMOGDSUlJeM7TiAQyM7OTvbsHXfcYY1/++23MR3ZqRkaFy5cqKmp+eKLL8yX27dvr6mpSbhle3v7kSNHzHjTpk05OTkJN/P5fA8++KAZHzt2LBQK2Zw9EAjMmjUr2bOprJImBwL9/PPPrfHatWvHfZzYxY0Xe7fb2bNnx3RkR2Zo/QC3adMm6xuGF1544amnnkq2y2effWaNV69ebXPwm266yRp///33NluOfpW6urpstpwqHAi0tbXVGlvflo2D3++3eTb22tDAwMCYjuzIDM0P+zt27Hj33XfNI8FgcOvWrTa7/PDDD2aQkZGxaNEimy2vuOIKa2zdlpVQeXm5zbOxq9Tf32+z5VThwGfx1r1teXl5Nv/6jDCPjIyCgoLUJ5OQIzPMzs5uamp6+umnzZfz5s17++237XexUotEIl6vd5QnOn/+fLKnPB6P9WPQRcKBd9Cenh4zsPkOckQzZ85MfSbJODLD3t7ee++9NxwOmy9PnTq1bds2+11sUrNh81GQ/UcM05ID76BDQ0NmMOzK85hYH3VOBEdmaN0q5fP5ent709LSXn755aqqqptvvnnE87rd7qysrFGeKP7Cu2VCV0mTA4Fab37d3d2pH20iODjDF198cfXq1UuWLBkcHBwaGnrggQeOHz+e7NN/67yFhYUnTpxI8dQXJwf+iS8sLDSDM2fOaDbqyAzT09N37dr1yiuvLF682LrcGAqFHnrooWg0mnCX+fPnm0FnZ+d0u4fj/8WBQMvKyswgGo0eO3Ys9QM6zpEZXn755Y888ogZ19bWVldXm3FjY+Mbb7yRcJdrrrnGDMLh8NGjR8d33oucA4FWVVVZ44aGhtQP6LiJmOHevXutW4o2b96c8OJl7L0pBw8edOS8FxsHAl26dKl1fXjPnj2Cv1E5ETPMz88/cOCAuXofDofvu++++BtJFy1atGTJEjPevXv3zz//nPp5LzYOBOp2u5999lkz7uvru/vuu23u+Z2Uq8cTNMNbb7118+bNZtze3v7www/Hb2Ndye/r61u5cuUvv/xic8BwONzR0THKs18knLmjPhgMWh/WHT161O/379q16+TJk+aRwcHB5ubmt9566/rrr6+rq3PkjCIz3LJly9KlS824oaHB+pDJcttttz3xxBNm3NbWVllZ+cwzzzQ1NVkfhg0MDLS2th48eDAQCBQWFm7fvn28L3Gacuquk9OnT1dUVAw7uNfrHXZV7/nnn4/dazS3wVusg2zdujX+2REPleIMk91R39HRMXv2bLNNZmbmjz/+OGyDSCSyfv36+JX3+XzxHy/V1taO46WNfpWmHMd+Jyk/P//rr79+7LHHYi8mDw4O9vX1xW7m8XicOuNYTdAMi4uL33vvPTPu7+9fu3atuYxvSU9P37dv3549e6x76oze3t7BwcHYR7xer/XfT8Bw8pfmsrKyduzY0dbWVldXt2zZssLCQq/Xm5GRMXv27IqKimAweOTIkZdeesnBM4rM8J577tm4caMZ//rrr48//nj8NoFA4I8//vjggw9qamrKyspmzpzpdrtnzJhRUFBw3XXXBQKB+vr6kydPPvnkk6m+yOnFFU1ykRlQwO/FQxqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQhqBQtr/AAY5vx2PMayhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=224x224 at 0x7F55F93C8290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]],\n",
       "\n",
       "       [[1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        ...,\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.],\n",
       "        [1., 1., 1.]]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PIL import Image, ImageDraw, ImageFont\n",
    "import numpy as np\n",
    "\n",
    "filename = \"/usr/share/fonts/truetype/noto/NotoSans-Regular.ttf\"\n",
    "DEFAULT_FONT = ImageFont.truetype(filename, 40)\n",
    "\n",
    "def draw_text(text_str, font, should_show=True):\n",
    "    blank = np.ones((224, 224))\n",
    "    blank = np.uint8(blank) * 255\n",
    "    im = Image.fromarray(blank)\n",
    "    if im.mode != \"RGB\":\n",
    "        im = im.convert(\"RGB\")\n",
    "    draw = ImageDraw.Draw(im)\n",
    "    draw.text(\n",
    "        (20 + 50 * 0, 20 + 50 * 2), text_str, (0, 0, 0), font=font,\n",
    "    )\n",
    "    draw = ImageDraw.Draw(im)\n",
    "    if should_show:\n",
    "        display(im)\n",
    "    return np.asarray(im) / 255\n",
    "\n",
    "draw_text(\"chicken\", DEFAULT_FONT)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}