import_gideon_data_3-22.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n",
      "inspecting classes\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"DJANGO_ALLOW_ASYNC_UNSAFE\"] = \"true\"\n",
    "os.environ[\"DJANGO_SETTINGS_MODULE\"] = \"apis.settings.dev_remote_db\"\n",
    "import django\n",
    "django.setup()                                                                                                          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from oebl_research_backend.models import ProfessionGroup, IRSPerson, List, ListEntry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vscode/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3098: DtypeWarning: Columns (61,62,67,89) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n"
     ]
    }
   ],
   "source": [
    "df_bios = pd.read_csv(\"_D_BIOGRAFIEN__202203031104_edited.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_literatur = pd.read_csv(\"_BIOGRAFIE_LITERATUR__202203031428.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DOMAIN</th>\n",
       "      <th>USERID</th>\n",
       "      <th>PROJECT</th>\n",
       "      <th>AUFNAHMEDATUM</th>\n",
       "      <th>MODIFIZIERT_AM</th>\n",
       "      <th>TITEL</th>\n",
       "      <th>INFO</th>\n",
       "      <th>INTERN_MASTER_ID</th>\n",
       "      <th>MODUL_MASTER_ID</th>\n",
       "      <th>...</th>\n",
       "      <th>WEB_LITERATUR_SL</th>\n",
       "      <th>WEB_LITERATUR</th>\n",
       "      <th>WEB_EKATEGORIEN_ID</th>\n",
       "      <th>WEB_CORRIGENDA_ADDENDA_SL</th>\n",
       "      <th>WEB</th>\n",
       "      <th>WEB_WERKE</th>\n",
       "      <th>WEB_FREIGABEUSER_ID</th>\n",
       "      <th>WEB_TEXTANGABEN</th>\n",
       "      <th>WEB_CORRIGENDA_ADDENDA</th>\n",
       "      <th>WEB_NACHLASSANGABE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461846.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-17 15:14:14.000</td>\n",
       "      <td>2005-02-17 15:23:27.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1828627.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1837971.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1461848.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-28 14:04:13.000</td>\n",
       "      <td>2005-02-28 14:04:58.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1461648.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-24 14:27:32.000</td>\n",
       "      <td>2005-02-24 14:27:43.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1462482.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-01-20 14:12:09.000</td>\n",
       "      <td>2005-10-06 14:36:38.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1462483.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-01-20 14:12:57.000</td>\n",
       "      <td>2005-01-20 14:13:06.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154859</th>\n",
       "      <td>1461349.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-03-02 14:32:53.000</td>\n",
       "      <td>2008-09-29 10:47:40.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154860</th>\n",
       "      <td>1428326.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2000-05-08 10:01:03.000</td>\n",
       "      <td>2009-01-19 11:04:51.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154861</th>\n",
       "      <td>1451792.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2003-09-04 18:11:56.000</td>\n",
       "      <td>2012-08-10 10:10:36.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154862</th>\n",
       "      <td>1458404.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2004-07-19 15:22:44.000</td>\n",
       "      <td>2008-10-21 09:06:25.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154863</th>\n",
       "      <td>1425024.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2000-01-12 13:58:26.000</td>\n",
       "      <td>2009-02-20 10:31:14.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>154864 rows × 119 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               ID  DOMAIN  USERID  PROJECT            AUFNAHMEDATUM  \\\n",
       "0       1461846.0     1.0    19.0      2.0  2005-02-17 15:14:14.000   \n",
       "1       1461848.0     1.0    19.0      2.0  2005-02-28 14:04:13.000   \n",
       "2       1461648.0     1.0    19.0      2.0  2005-02-24 14:27:32.000   \n",
       "3       1462482.0     1.0    19.0      2.0  2005-01-20 14:12:09.000   \n",
       "4       1462483.0     1.0    19.0      2.0  2005-01-20 14:12:57.000   \n",
       "...           ...     ...     ...      ...                      ...   \n",
       "154859  1461349.0     1.0    19.0      2.0  2005-03-02 14:32:53.000   \n",
       "154860  1428326.0     1.0    13.0      2.0  2000-05-08 10:01:03.000   \n",
       "154861  1451792.0     1.0    19.0      2.0  2003-09-04 18:11:56.000   \n",
       "154862  1458404.0     1.0    19.0      2.0  2004-07-19 15:22:44.000   \n",
       "154863  1425024.0     1.0    13.0      2.0  2000-01-12 13:58:26.000   \n",
       "\n",
       "                 MODIFIZIERT_AM  TITEL  INFO  INTERN_MASTER_ID  \\\n",
       "0       2005-02-17 15:23:27.000    NaN   NaN               NaN   \n",
       "1       2005-02-28 14:04:58.000    NaN   NaN               NaN   \n",
       "2       2005-02-24 14:27:43.000    NaN   NaN               NaN   \n",
       "3       2005-10-06 14:36:38.000    NaN   NaN               NaN   \n",
       "4       2005-01-20 14:13:06.000    NaN   NaN               NaN   \n",
       "...                         ...    ...   ...               ...   \n",
       "154859  2008-09-29 10:47:40.000    NaN   NaN               NaN   \n",
       "154860  2009-01-19 11:04:51.000    NaN   NaN               NaN   \n",
       "154861  2012-08-10 10:10:36.000    NaN   NaN               NaN   \n",
       "154862  2008-10-21 09:06:25.000    NaN   NaN               NaN   \n",
       "154863  2009-02-20 10:31:14.000    NaN   NaN               NaN   \n",
       "\n",
       "        MODUL_MASTER_ID  ...  WEB_LITERATUR_SL  WEB_LITERATUR  \\\n",
       "0                   NaN  ...               NaN           -1.0   \n",
       "1                   NaN  ...               NaN            NaN   \n",
       "2                   NaN  ...               NaN            NaN   \n",
       "3                   NaN  ...               NaN            NaN   \n",
       "4                   NaN  ...               NaN            NaN   \n",
       "...                 ...  ...               ...            ...   \n",
       "154859              NaN  ...               NaN            NaN   \n",
       "154860              NaN  ...               NaN            NaN   \n",
       "154861              NaN  ...               NaN            0.0   \n",
       "154862              NaN  ...               NaN            NaN   \n",
       "154863              NaN  ...               NaN            0.0   \n",
       "\n",
       "       WEB_EKATEGORIEN_ID WEB_CORRIGENDA_ADDENDA_SL  WEB  WEB_WERKE  \\\n",
       "0               1828627.0                       NaN -1.0       -1.0   \n",
       "1                     NaN                       NaN  NaN        NaN   \n",
       "2                     NaN                       NaN  NaN        NaN   \n",
       "3                     NaN                       NaN  NaN        NaN   \n",
       "4                     NaN                       NaN  NaN        NaN   \n",
       "...                   ...                       ...  ...        ...   \n",
       "154859                NaN                       NaN  NaN        NaN   \n",
       "154860                NaN                       NaN  NaN        NaN   \n",
       "154861                NaN                       NaN  0.0        0.0   \n",
       "154862                NaN                       NaN  NaN        NaN   \n",
       "154863                NaN                       NaN  0.0        0.0   \n",
       "\n",
       "       WEB_FREIGABEUSER_ID  WEB_TEXTANGABEN  WEB_CORRIGENDA_ADDENDA  \\\n",
       "0                1837971.0             -1.0                     NaN   \n",
       "1                      NaN              NaN                     NaN   \n",
       "2                      NaN              NaN                     NaN   \n",
       "3                      NaN              NaN                     NaN   \n",
       "4                      NaN              NaN                     NaN   \n",
       "...                    ...              ...                     ...   \n",
       "154859                 NaN              NaN                     NaN   \n",
       "154860                 NaN              NaN                     NaN   \n",
       "154861                 NaN              0.0                     0.0   \n",
       "154862                 NaN              NaN                     NaN   \n",
       "154863                 NaN              0.0                     0.0   \n",
       "\n",
       "       WEB_NACHLASSANGABE  \n",
       "0                     NaN  \n",
       "1                     NaN  \n",
       "2                     NaN  \n",
       "3                     NaN  \n",
       "4                     NaN  \n",
       "...                   ...  \n",
       "154859                NaN  \n",
       "154860                NaN  \n",
       "154861                NaN  \n",
       "154862                NaN  \n",
       "154863                NaN  \n",
       "\n",
       "[154864 rows x 119 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID\n",
      "DOMAIN\n",
      "USERID\n",
      "PROJECT\n",
      "AUFNAHMEDATUM\n",
      "MODIFIZIERT_AM\n",
      "TITEL\n",
      "INFO\n",
      "INTERN_MASTER_ID\n",
      "MODUL_MASTER_ID\n",
      "STATUS_ID\n",
      "WF\n",
      "VORNAME\n",
      "NACHNAME\n",
      "WEITERE_NAMENSFORMEN\n",
      "K_GESCHLECHT_ID\n",
      "ADELSPRAEDIKAT\n",
      "IST_ADELIG\n",
      "K_RELIGIONEN_ID\n",
      "RELIGION_ANMERKUNG\n",
      "RELIGIONSWECHSEL\n",
      "K_ORDEN_ID\n",
      "K_BERUFSGRUPPEN_ID\n",
      "BERUF\n",
      "GEBURTSTAGMONAT\n",
      "GEBURTSJAHR\n",
      "GEBURTSORT_ALT\n",
      "GEBURTSORT_HEUTE\n",
      "K_LAND_ID_GEBURT_ALT\n",
      "K_LAND_ID_GEBURT_HEUTE\n",
      "GEBURT_ANMERKUNG\n",
      "GEBURTSDATEN_OK\n",
      "STERBETAGMONAT\n",
      "STERBEJAHR\n",
      "STERBEORT_ALT\n",
      "STERBEORT_HEUTE\n",
      "K_LAND_ID_TOD_ALT\n",
      "K_LAND_ID_TOD_HEUTE\n",
      "STERBE_ANMERKUNG\n",
      "STERBEDATEN_OK\n",
      "K_REIHEN_ID\n",
      "FACHREDAKTEUR\n",
      "KK\n",
      "AKTUELL\n",
      "GEPLANT_LFG\n",
      "BIO\n",
      "PUBLIZIERT\n",
      "ERRATA\n",
      "ADDENDA\n",
      "VERWEIS\n",
      "VERWANDSCHAFT\n",
      "AUSBILDUNG\n",
      "BERUF_LAUFBAHN\n",
      "GEOGRAPH_LEBENSMITTELPUNKT\n",
      "BESONDERE_LEISTUNGEN\n",
      "PERS_EIGENSCHAFTEN\n",
      "EHRUNGEN\n",
      "WUERDIGUNGEN\n",
      "WERKSANGABEN\n",
      "ARCHIVALIEN\n",
      "MITTEILUNGEN\n",
      "CORRIGENDA\n",
      "CORRIGENDA_HERKUNFT\n",
      "PRUEF_PERSONENDATEN\n",
      "LEBENSLAUF\n",
      "SUCH_VORNAME\n",
      "SUCH_NACHNAME\n",
      "ORDEN_ANMERKUNG\n",
      "PNDID\n",
      "KOE\n",
      "ALLGEMEINE_ANMERKUNG\n",
      "VERWANDTSCHAFT_GEPRUEFT\n",
      "VERWANDTSCHAFT_GEPRUEFT_VON\n",
      "VERWANDTSCHAFT_GEPRUEFT_AM\n",
      "DATA_FOLDER_ID\n",
      "GBEURTSTAG\n",
      "PRE_GEBURT_ID\n",
      "IN_GEBURT_ID\n",
      "GEBURTSTAG_2\n",
      "GBEURTSMONAT_2\n",
      "GEBURTSJAHR_2\n",
      "PRE_STERBE_ID\n",
      "IN_STREBE_ID\n",
      "STERBETAG\n",
      "STERBETAG_2\n",
      "STERBEMONAT_2\n",
      "STERBEJAHR_2\n",
      "MODIFIZIERT_VON\n",
      "ANMERKUNG_PERSON\n",
      "ANMERKUNG_RELIGION\n",
      "ANMERKUNG_BERUF\n",
      "GEBURTSTAG_TEXT\n",
      "ANMERKUNG_GEBURTSDATUM\n",
      "ANMERKUNG_GEBURTSORT\n",
      "ANMERKUNG_STERBEDATUM\n",
      "ANMERKUNG_STERBEORT\n",
      "STERBETAG_TEXT\n",
      "ANMERKUNG_VERWANDTSCHAFT\n",
      "LITERATURANGABEN\n",
      "FACHREDAKTEUR_ID\n",
      "GEPLANT_REIHE\n",
      "TEST\n",
      "DUMMY\n",
      "LITERATURANGABEN_SL\n",
      "DUMMY_NACHNAME\n",
      "DUMMY_VORNAME\n",
      "D_BIOGRAFIEN_ORIGINAL_ID\n",
      "WEB_IKATEGORIEN_ID\n",
      "WEB_FREIGABEDATUM\n",
      "WEB_LITERATUR_SL\n",
      "WEB_LITERATUR\n",
      "WEB_EKATEGORIEN_ID\n",
      "WEB_CORRIGENDA_ADDENDA_SL\n",
      "WEB\n",
      "WEB_WERKE\n",
      "WEB_FREIGABEUSER_ID\n",
      "WEB_TEXTANGABEN\n",
      "WEB_CORRIGENDA_ADDENDA\n",
      "WEB_NACHLASSANGABE\n"
     ]
    }
   ],
   "source": [
    "for x in df_bios.columns:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    154864.000000\n",
       "mean          1.084791\n",
       "std           0.278571\n",
       "min           1.000000\n",
       "25%           1.000000\n",
       "50%           1.000000\n",
       "75%           1.000000\n",
       "max           2.000000\n",
       "Name: K_GESCHLECHT_ID, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bios[\"K_GESCHLECHT_ID\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in df_bios.columns:\n",
    "    df_bios[c].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>DOMAIN</th>\n",
       "      <th>USERID</th>\n",
       "      <th>PROJECT</th>\n",
       "      <th>AUFNAHMEDATUM</th>\n",
       "      <th>MODIFIZIERT_AM</th>\n",
       "      <th>TITEL</th>\n",
       "      <th>INFO</th>\n",
       "      <th>INTERN_MASTER_ID</th>\n",
       "      <th>MODUL_MASTER_ID</th>\n",
       "      <th>...</th>\n",
       "      <th>WEB_LITERATUR_SL</th>\n",
       "      <th>WEB_LITERATUR</th>\n",
       "      <th>WEB_EKATEGORIEN_ID</th>\n",
       "      <th>WEB_CORRIGENDA_ADDENDA_SL</th>\n",
       "      <th>WEB</th>\n",
       "      <th>WEB_WERKE</th>\n",
       "      <th>WEB_FREIGABEUSER_ID</th>\n",
       "      <th>WEB_TEXTANGABEN</th>\n",
       "      <th>WEB_CORRIGENDA_ADDENDA</th>\n",
       "      <th>WEB_NACHLASSANGABE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461846.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-17 15:14:14.000</td>\n",
       "      <td>2005-02-17 15:23:27.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1828627.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1837971.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1461848.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-28 14:04:13.000</td>\n",
       "      <td>2005-02-28 14:04:58.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1461648.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-02-24 14:27:32.000</td>\n",
       "      <td>2005-02-24 14:27:43.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1462482.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-01-20 14:12:09.000</td>\n",
       "      <td>2005-10-06 14:36:38.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1462483.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2005-01-20 14:12:57.000</td>\n",
       "      <td>2005-01-20 14:13:06.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 119 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          ID  DOMAIN  USERID  PROJECT            AUFNAHMEDATUM  \\\n",
       "0  1461846.0     1.0    19.0      2.0  2005-02-17 15:14:14.000   \n",
       "1  1461848.0     1.0    19.0      2.0  2005-02-28 14:04:13.000   \n",
       "2  1461648.0     1.0    19.0      2.0  2005-02-24 14:27:32.000   \n",
       "3  1462482.0     1.0    19.0      2.0  2005-01-20 14:12:09.000   \n",
       "4  1462483.0     1.0    19.0      2.0  2005-01-20 14:12:57.000   \n",
       "\n",
       "            MODIFIZIERT_AM  TITEL  INFO  INTERN_MASTER_ID  MODUL_MASTER_ID  \\\n",
       "0  2005-02-17 15:23:27.000    NaN   NaN               NaN              NaN   \n",
       "1  2005-02-28 14:04:58.000    NaN   NaN               NaN              NaN   \n",
       "2  2005-02-24 14:27:43.000    NaN   NaN               NaN              NaN   \n",
       "3  2005-10-06 14:36:38.000    NaN   NaN               NaN              NaN   \n",
       "4  2005-01-20 14:13:06.000    NaN   NaN               NaN              NaN   \n",
       "\n",
       "   ...  WEB_LITERATUR_SL  WEB_LITERATUR WEB_EKATEGORIEN_ID  \\\n",
       "0  ...               NaN           -1.0          1828627.0   \n",
       "1  ...               NaN            NaN                NaN   \n",
       "2  ...               NaN            NaN                NaN   \n",
       "3  ...               NaN            NaN                NaN   \n",
       "4  ...               NaN            NaN                NaN   \n",
       "\n",
       "  WEB_CORRIGENDA_ADDENDA_SL  WEB  WEB_WERKE WEB_FREIGABEUSER_ID  \\\n",
       "0                       NaN -1.0       -1.0           1837971.0   \n",
       "1                       NaN  NaN        NaN                 NaN   \n",
       "2                       NaN  NaN        NaN                 NaN   \n",
       "3                       NaN  NaN        NaN                 NaN   \n",
       "4                       NaN  NaN        NaN                 NaN   \n",
       "\n",
       "   WEB_TEXTANGABEN  WEB_CORRIGENDA_ADDENDA WEB_NACHLASSANGABE  \n",
       "0             -1.0                     NaN                NaN  \n",
       "1              NaN                     NaN                NaN  \n",
       "2              NaN                     NaN                NaN  \n",
       "3              NaN                     NaN                NaN  \n",
       "4              NaN                     NaN                NaN  \n",
       "\n",
       "[5 rows x 119 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bios.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_mappings = {\n",
    "    \"gender\": {\"1\": \"männlich\",\n",
    "                \"2\": \"weiblich\"\n",
    "                }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mitarbeiter = pd.read_csv(\"_MITARBEITER__202203041529.csv\")\n",
    "df_berufsgruppen = pd.read_csv(\"_K_BERUFSGRUPPEN__202203041544.csv\")\n",
    "df_berufsuntergruppen = pd.read_csv(\"_K_BERUFSUNTERGRUPPEN__202203041544.csv\")\n",
    "df_land = pd.read_csv(\"_K_LAENDER__202203041552.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "boolean_vals = [\"IST_ADELIG\", \".*_OK\", \"RELIGIONSWECHSEL\", \"BIO\", \"PUBLIZIERT\", \"RELIGIONSWECHSEL\"]\n",
    "pat_booleans = []\n",
    "for pat in boolean_vals:\n",
    "    pat_booleans.append(re.compile(pat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_value(value, col):\n",
    "    for pat in pat_booleans:\n",
    "        if pat.fullmatch(col):\n",
    "            if isinstance(value, str):\n",
    "                if value.strip() == \"n\":\n",
    "                    return True\n",
    "                elif value.strip() == \"f\":\n",
    "                    return False\n",
    "                else:\n",
    "                    return None\n",
    "            elif isinstance(value, float):\n",
    "                if np.isnan(value):\n",
    "                    return None\n",
    "                elif value == -1.0:\n",
    "                    return True\n",
    "                elif value == 0.0:\n",
    "                    return False\n",
    "                else:\n",
    "                    return None\n",
    "    if isinstance(value, float):\n",
    "        if np.isnan(value):\n",
    "            return None\n",
    "        else:\n",
    "            return value\n",
    "    elif isinstance(value, str):\n",
    "        if value == \"?\":\n",
    "            return None\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_user_gideon = {\n",
    "    13: 13,\n",
    "    11: 12,\n",
    "    52: 8,\n",
    "    8: 16,\n",
    "    9: 17,\n",
    "    40: 9,\n",
    "    33: 14\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_csv_irs = {\n",
    "    \"ADELSPRAEDIKAT\": \"nobleTitle\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude_columns = [\".*_2\", \"DUMMY_.*\", \"TEST\", \"WERK_.*\", \"WEB_.*\", \"K_.*\", \"SUCH_.*\", \".*_OK\", \n",
    "\"NACHNAME\", \"VORNAME\", \"KK\", \"WF\", \"DOMAIN\", \"ERRATA\", \"ADDENDA\", \"AKTUELL\", \"PROJECT\", \"VERWEIS\", \"WEITERE_NAMENSFORMEN\",\n",
    "\"MODIFIZIERT_VON\", \"DUMMY\", \"KOE\", \"WEB\", \"STATUS_ID\", \"STERBETAG\", \"STERBEMONAT\", \"STERBEJAHR\", \"GBEURTSTAG\", \"GEBURTSTAGMONAT\",\n",
    "\"GEBURTSTAGJAHR\", \"PNDID\"\n",
    "]\n",
    "ex_patterns = []\n",
    "for pat in exclude_columns:\n",
    "    ex_patterns.append(re.compile(pat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_col_user(col):\n",
    "    for pat in ex_patterns:\n",
    "        if pat.fullmatch(col):\n",
    "            return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error in row 33778: ['Wert „1994-11-31“ hat das korrekte Format (YYYY-MM-DD) aber ein ungültiges Datum.']\n",
      "error in row 36823: ['Wert „1982-9-31“ hat das korrekte Format (YYYY-MM-DD) aber ein ungültiges Datum.']\n",
      "error in row 45079: ['Wert „1959-11-31“ hat das korrekte Format (YYYY-MM-DD) aber ein ungültiges Datum.']\n"
     ]
    }
   ],
   "source": [
    "lists_ent = {}\n",
    "for idx, row in df_bios.iterrows():\n",
    "    if np.isnan(row[\"STERBEJAHR\"]):\n",
    "        continue\n",
    "    if int(row[\"STERBEJAHR\"]) < 1951:\n",
    "        continue\n",
    "    pers = {\n",
    "        \"name\": get_value(row[\"NACHNAME\"], \"NACHNAME\"),\n",
    "        \"first_name\": get_value(row[\"VORNAME\"], \"VORNAME\"),\n",
    "    }\n",
    "    if not np.isnan(row[\"GEBURTSJAHR\"]):\n",
    "        pers[\"date_of_birth\"] = f\"{int(row['GEBURTSJAHR'])}\"\n",
    "        if not np.isnan(row[\"GEBURTSTAGMONAT\"]):\n",
    "            pers[\"date_of_birth\"] += f'-{int(row[\"GEBURTSTAGMONAT\"])}'.strip()\n",
    "            if not np.isnan(row[\"GBEURTSTAG\"]):\n",
    "                pers[\"date_of_birth\"] += f'-{int(row[\"GBEURTSTAG\"])}'.strip()\n",
    "            else:\n",
    "                pers[\"date_of_birth\"] += '-01'\n",
    "        else:\n",
    "            pers[\"date_of_birth\"] += '-01-01'\n",
    "    if not np.isnan(row[\"STERBEJAHR\"]):\n",
    "        pers[\"date_of_death\"] = f\"{int(row['STERBEJAHR'])}\"\n",
    "        if not np.isnan(row[\"STERBETAGMONAT\"]):\n",
    "            pers[\"date_of_death\"] += f'-{int(row[\"STERBETAGMONAT\"])}'.strip()\n",
    "            if not np.isnan(row[\"STERBETAG\"]):\n",
    "                pers[\"date_of_death\"] += f'-{int(row[\"STERBETAG\"])}'.strip()\n",
    "            else:\n",
    "                pers[\"date_of_death\"] += '-01'\n",
    "        else:\n",
    "            pers[\"date_of_death\"] += '-01-01'\n",
    "    if row[\"K_GESCHLECHT_ID\"] == 1:\n",
    "        pers[\"gender\"] = \"männlich\" \n",
    "    elif row[\"K_GESCHLECHT_ID\"] == 2:\n",
    "        pers[\"gender\"] = \"weiblich\"\n",
    "    else:\n",
    "        pers[\"gender\"] = \"unbekannt\"\n",
    "    if not np.isnan(row[\"K_BERUFSGRUPPEN_ID\"]):\n",
    "        r_bg = df_berufsgruppen.loc[(df_berufsgruppen[\"ID\"] == row[\"K_BERUFSGRUPPEN_ID\"])]\n",
    "        if r_bg.shape[0] == 1:\n",
    "            bg, created = ProfessionGroup.objects.get_or_create(name=r_bg.iloc[0][\"TITEL\"])\n",
    "            pers[\"profession_group\"] = bg\n",
    "        elif r_bg.shape[0] > 1:\n",
    "            print(f\"several profession groups, row {idx}\")\n",
    "        else:\n",
    "            print(f\"no profession group specified, row {idx}\")\n",
    "    if isinstance(row[\"BERUF\"], str):\n",
    "        pers[\"profession_detail\"] = get_value(row[\"BERUF\"], \"BERUF\")\n",
    "    lit_gideon = []\n",
    "    for idx_lit, row_lit in df_literatur.loc[(df_literatur[\"BIOGRAFIE_ID\"] == row[\"ID\"])].iterrows():\n",
    "        lit_gideon.append({\n",
    "            \"id\": row_lit[\"ID\"],\n",
    "            \"value\": row_lit[\"ZITAT\"]\n",
    "        })\n",
    "    if len(lit_gideon) > 0:\n",
    "        pers[\"gideon_legacy_literature\"] = lit_gideon\n",
    "    #pers[\"alternative_names\"] = [{\"first_name\":\"test1\", \"last_name\": \"test2\"}]\n",
    "    if isinstance(row[\"LEBENSLAUF\"], str):\n",
    "        if len(row[\"LEBENSLAUF\"]) > 0:\n",
    "            pers[\"bio_note\"] = row[\"LEBENSLAUF\"]\n",
    "    if isinstance(row[\"WEITERE_NAMENSFORMEN\"], str):\n",
    "        if len(row[\"WEITERE_NAMENSFORMEN\"]) > 0:\n",
    "            pers[\"alternative_names\"] = [{\n",
    "                \"first_name\": \"-\",\n",
    "                \"last_name\": row[\"WEITERE_NAMENSFORMEN\"]\n",
    "            }]\n",
    "    pers[\"uris\"] = []\n",
    "    if isinstance(row[\"PNDID\"], str):\n",
    "        pers[\"uris\"] = [f\"https://d-nb.info/gnd/{row['PNDID']}\", ]\n",
    "    try:\n",
    "        i1 = IRSPerson.objects.create(**pers)\n",
    "    except Exception as e:\n",
    "        print(f\"error in row {idx}: {e}\")\n",
    "        continue\n",
    "    list_ent_dict = {\n",
    "        \"person\": i1, \n",
    "        \"source_id\": int(row[\"ID\"]),\n",
    "        \"columns_scrape\": {\"obv\": [], \"wikipedia\": [], \"wikidata\": []},\n",
    "        \"scrape\": {\"obv\": [], \"wikipedia\": [], \"wikidata\": []},\n",
    "        \"columns_user\": {\"selected\": False}\n",
    "        }\n",
    "    if not np.isnan(row[\"USERID\"]):\n",
    "        editor = df_mitarbeiter.loc[(df_mitarbeiter[\"INTERN_USER_ID\"] == row[\"USERID\"])]\n",
    "        if editor.shape[0] == 1:\n",
    "            editor_name = editor.iloc[0][\"NAME\"]\n",
    "            if editor_name not in lists_ent.keys():\n",
    "                d_list = {\"title\": f\"Gideon {editor_name}\"}\n",
    "                if int(row[\"USERID\"]) in map_user_gideon.keys():\n",
    "                    d_list[\"editor_id\"] = map_user_gideon[int(row[\"USERID\"])]\n",
    "                list_new, created = List.objects.get_or_create(**d_list)\n",
    "                lists_ent[editor_name] = list_new\n",
    "            list_ent_dict[\"list\"] = lists_ent[editor_name]\n",
    "    for col in df_bios.columns:\n",
    "        if not test_col_user(col):\n",
    "            continue\n",
    "        if isinstance(row[col], float):\n",
    "            if np.isnan(row[col]):\n",
    "                continue\n",
    "        col_name = col\n",
    "        if col in map_csv_irs.keys():\n",
    "            col_name = map_csv_irs[col]\n",
    "        list_ent_dict[\"columns_user\"][col_name] = get_value(row[col], col)\n",
    "\n",
    "    ListEntry.objects.create(**list_ent_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_berufsgruppen.loc[(df_berufsgruppen[\"ID\"] == 500049)].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, {})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IRSPerson.objects.all().delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, {})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ListEntry.objects.all().delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, {})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "List.objects.all().delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = IRSPerson.objects.get(name=\"Schirrmacher\")\n",
    "d2 = IRSPerson.objects.get(name=\"Löser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<IRSPerson: Schirrmacher, Frank>"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d2.alternative_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_state <django.db.models.base.ModelState object at 0x7faabbbd7490>\n",
      "id 294\n",
      "person_id 247\n",
      "last_updated 2022-03-07 15:15:40.300348+00:00\n",
      "source_id 6\n",
      "created 2022-03-07 15:14:37.073837+00:00\n",
      "selected False\n",
      "list_id 45\n",
      "deleted False\n",
      "attachements None\n",
      "works None\n",
      "references None\n"
     ]
    }
   ],
   "source": [
    "for dd in d1.listentry_set.all():\n",
    "    for d3 in dd.__dict__.keys():\n",
    "        if d3 == \"columns_scrape\" or d3 == \"columns_user\" or d3 == \"scrape\":\n",
    "            continue\n",
    "        print(d3, dd.__dict__[d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_state <django.db.models.base.ModelState object at 0x7faa796f0370>\n",
      "id 272\n",
      "person_id 225\n",
      "last_updated 2022-03-07 15:13:29.039146+00:00\n",
      "source_id 1461846\n",
      "created 2022-03-07 15:13:29.039167+00:00\n",
      "selected False\n",
      "list_id 43\n",
      "deleted False\n",
      "attachements None\n",
      "works None\n",
      "references None\n"
     ]
    }
   ],
   "source": [
    "for dd in d2.listentry_set.all():\n",
    "    for d3 in dd.__dict__.keys():\n",
    "        if d3 == \"columns_scrape\" or d3 == \"columns_user\" or d3 == \"scrape\":\n",
    "            continue\n",
    "        print(d3, dd.__dict__[d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_state <django.db.models.base.ModelState object at 0x7faab6063130>\n",
      "id 247\n",
      "irs_person_id None\n",
      "name Schirrmacher\n",
      "first_name Frank\n",
      "alternative_names []\n",
      "date_of_birth 1959-03-07\n",
      "date_of_birth_modifier None\n",
      "date_of_death 2014-03-07\n",
      "date_of_death_modifier None\n",
      "profession_group_id None\n",
      "profession_detail None\n",
      "bio_note None\n",
      "religion None\n",
      "title_of_nobility None\n",
      "gender None\n",
      "uris ['https://d-nb.info/gnd/121226948/']\n",
      "secondary_literature []\n",
      "gideon_legacy_literature []\n",
      "zotero_keys_by []\n",
      "zotero_keys_about []\n"
     ]
    }
   ],
   "source": [
    "for d3 in d1.__dict__.keys():\n",
    "    if d3 == \"columns_scrape\" or d3 == \"columns_user\":\n",
    "        continue\n",
    "    print(d3, d1.__dict__[d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_state <django.db.models.base.ModelState object at 0x7faab6063550>\n",
      "id 225\n",
      "irs_person_id None\n",
      "name Löser\n",
      "first_name Anton\n",
      "alternative_names []\n",
      "date_of_birth 1877-09-01\n",
      "date_of_birth_modifier None\n",
      "date_of_death None\n",
      "date_of_death_modifier None\n",
      "profession_group_id 1\n",
      "profession_detail Lehrer\n",
      "bio_note None\n",
      "religion None\n",
      "title_of_nobility None\n",
      "gender männlich\n",
      "uris None\n",
      "secondary_literature []\n",
      "gideon_legacy_literature [{'id': 1643106.0, 'value': 'C. Kosik, Österreich 1918 - 1934, 1935'}]\n",
      "zotero_keys_by []\n",
      "zotero_keys_about []\n"
     ]
    }
   ],
   "source": [
    "for d3 in d2.__dict__.keys():\n",
    "    if d3 == \"columns_scrape\" or d3 == \"columns_user\":\n",
    "        continue\n",
    "    print(d3, d2.__dict__[d3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2.gideon_legacy_literature = []\n",
    "d2.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Schirrmacher, Frank\n"
     ]
    }
   ],
   "source": [
    "print(d1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1.uris = []\n",
    "d1.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2.uris = ['https://d-nb.info/gnd/1212756362948/']\n",
    "d2.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2.listentry_set.all()[0].columns_use = {}\n",
    "d2.listentry_set.all()[0].save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2.alternative_names = []\n",
    "\n",
    "d2.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "d3 = IRSPerson.objects.get(name=\"Bergmann\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_state <django.db.models.base.ModelState object at 0x7fc71997eac0>\n",
      "id 288\n",
      "person_id 241\n",
      "last_updated 2022-03-07 15:30:53.839326+00:00\n",
      "source_id 0\n",
      "created 2022-03-07 15:14:37.009997+00:00\n",
      "selected False\n",
      "list_id 45\n",
      "columns_scrape {'obv': [], 'wikidata': [], 'wikipedia': []}\n",
      "columns_user {'selected': False, 'user.Beruf': 'Sportlerin'}\n",
      "scrape {'obv': [], 'wikidata': [], 'wikipedia': []}\n",
      "deleted False\n",
      "attachements None\n",
      "works None\n",
      "references None\n"
     ]
    }
   ],
   "source": [
    "for dd in d3.listentry_set.all():\n",
    "    for d4 in dd.__dict__.keys():\n",
    "        print(d4, dd.__dict__[d4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Matthias’ Germanisten 5\n",
      "testliste 2 3-21 5\n",
      "Eva testet (händisch Eingegebene) 13\n",
      "Patres 6\n",
      "test (eo) 13\n",
      "Cs in Planung (eo) 13\n",
      "Juristen 8\n",
      "Verwaltung 9\n",
      "Arnold 6\n",
      "Orgelbauer 6\n",
      "Grafs 6\n",
      "Test_Lfg_72 9\n",
      "Gideon Ulrike Rack None\n",
      "Gideon Mag. Waltraud E. Zuleger None\n",
      "Gideon Mag. Eva Offenthaler 13\n",
      "Gideon Mag. Josef Pirchner None\n",
      "Gideon Dr. Huberg Bergmann 8\n",
      "Gideon Dr. Christine Gruber 17\n",
      "Gideon Systembenutzer None\n",
      "Gideon Mag. Philipp Dittinger 9\n",
      "Gideon Leonoardo Bergmann None\n",
      "Gideon Dr. Elisabeth Lebensaft None\n",
      "Gideon Romeo Felsenreich None\n",
      "Gideon Roland Feigl 16\n",
      "Gideon Ulrike Denk None\n",
      "Gideon Marcella Stern None\n",
      "Gideon Dr. Gudrun Exner None\n",
      "Gideon Prof. Dr. Helmuth Grössing None\n",
      "Gideon Mag. Christoph Mentschl None\n",
      "Gideon Dipl.Ing. Michael Martischnig None\n",
      "Gideon Dr. Theodor Venus None\n",
      "Gideon Mag. Ágoston Bernád None\n",
      "Gideon Dr. Irene Nawrocka 14\n",
      "Gideon Dr. Elisabeth Tomasi None\n",
      "Gideon Dr. Maria Petz-Grabenbauer None\n",
      "Gideon Mag. Dr. Daniela Angetter 12\n",
      "Gideon Dr. Johannes Seidl None\n"
     ]
    }
   ],
   "source": [
    "for l in List.objects.all():\n",
    "    print(l.title, l.editor_id)\n",
    "    if l.editor_id is None:\n",
    "       l.editor_id = 24\n",
    "       l.save() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ip in IRSPerson.objects.filter(uris__isnull=False):\n",
    "    new_uris = []\n",
    "    test = False\n",
    "    for u in ip.uris:\n",
    "        if not u.endswith(\"/\"):\n",
    "            new_uris.append(f\"{u}/\")\n",
    "            test = True\n",
    "        else:\n",
    "            new_uris.append(u)\n",
    "    if test:\n",
    "        ip.uris = new_uris\n",
    "        ip.save()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}