update lab 08 example

Author: rbtzuniga

lab_08/lab_08_example.ipynb

@@ -118,100 +118,6 @@
     "    con.sql('COMMIT') # committing the transaction\n",
     "    print(con.sql('SELECT * FROM data_append').fetchdf())"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Incremental load method\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "   id     name last_update\n",
-      "0   2    Alice  2021-01-01\n",
-      "1   3  Charlie  2021-01-01\n",
-      "2   4    David  2022-01-01\n",
-      "3   5      Eve  2022-01-01\n"
-     ]
-    }
-   ],
-   "source": [
-    "def incremental_load(con, data):\n",
-    "    # creating a copy of the original data (not necessary in general)\n",
-    "    con.sql((\"CREATE OR REPLACE TABLE data_incremental AS SELECT * FROM original_data\"))\n",
-    "\n",
-    "    # deleting \"outdated\" rows\n",
-    "    new_earliest_date = data['last_update'].min().strftime('%Y-%m-%d')\n",
-    "    con.sql(f\"DELETE FROM data_incremental WHERE last_update < '{new_earliest_date}'\")\n",
-    "\n",
-    "\n",
-    "    # removing rows that will be updated\n",
-    "    ids_timestamps = con.sql('SELECT id, last_update FROM data_incremental').fetchdf()\n",
-    "    # merging new data to compoare last_update\n",
-    "    ids_timestamps = ids_timestamps.merge(data[['id','last_update']], on='id', \n",
-    "                                          suffixes=('_old', '_new'))\n",
-    "    # finding ids for which the dates are different\n",
-    "    ids_to_remove = ids_timestamps[ids_timestamps['last_update_old'] != ids_timestamps['last_update_new']]['id']\n",
-    "    # removing rows\n",
-    "    for id in ids_to_remove:\n",
-    "        con.sql(f\"DELETE FROM data_incremental WHERE id = {id}\")\n",
-    "\n",
-    "\n",
-    "    # appending the new data\n",
-    "    most_recent_date = con.sql('SELECT MAX(last_update) FROM data_incremental').fetchdf().values[0][0]\n",
-    "    data = data[data['last_update'] > most_recent_date]\n",
-    "    for _, row in data.iterrows():\n",
-    "        date_str = row['last_update'].strftime('%Y-%m-%d')\n",
-    "        con.sql(f\"INSERT INTO data_incremental VALUES ({row['id']}, '{row['name']}', '{date_str}')\")\n",
-    "\n",
-    "with duckdb.connect(file) as con:\n",
-    "    con.sql('BEGIN TRANSACTION') # starting a transaction -- changes are synced once = improves performance\n",
-    "    incremental_load(con, new_data)\n",
-    "    con.sql('COMMIT') # committing the transaction \n",
-    "    print(con.sql('SELECT * FROM data_incremental').fetchdf())\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Truncate and load method\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 62,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "   id     name last_update\n",
-      "0   2    Alice  2021-01-01\n",
-      "1   3  Charlie  2021-01-01\n",
-      "2   4    David  2022-01-01\n",
-      "3   5      Eve  2022-01-01\n"
-     ]
-    }
-   ],
-   "source": [
-    "def trunc_and_load(con,data):\n",
-    "    # we simply truncate the table and load the new data\n",
-    "    con.sql((\"CREATE OR REPLACE TABLE data_trunc AS SELECT * FROM data\"))\n",
-    "\n",
-    "with duckdb.connect(file) as con:\n",
-    "    trunc_and_load(con, new_data)\n",
-    "    print(con.sql('SELECT * FROM data_trunc').fetchdf())    "
-   ]
   }
  ],
  "metadata": {