From cc4901181687e0fd273c890f89d80492afbacca9 Mon Sep 17 00:00:00 2001 From: oscgonfer Date: Fri, 19 Jul 2024 11:22:42 +0200 Subject: [PATCH] Add Glasgow WIP --- analysis/240719_glasgow-2.ipynb | 236 ++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 analysis/240719_glasgow-2.ipynb diff --git a/analysis/240719_glasgow-2.ipynb b/analysis/240719_glasgow-2.ipynb new file mode 100644 index 0000000..0c21b59 --- /dev/null +++ b/analysis/240719_glasgow-2.ipynb @@ -0,0 +1,236 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "54d8d6f4-8a42-4f32-b65d-226d0901be7b", + "metadata": {}, + "outputs": [], + "source": [ + "import smartcitizen_connector\n", + "import scdata as sc\n", + "import time\n", + "now = time.localtime()" + ] + }, + { + "cell_type": "markdown", + "id": "557f54fd-3387-45e1-a4dc-3fafb80753c5", + "metadata": {}, + "source": [ + "# Glasgow Batch #2\n", + "\n", + "Smart Citizen Kit 2.1" + ] + }, + { + "cell_type": "markdown", + "id": "d5ac091a-dfab-4b5a-b9c4-9133230f7678", + "metadata": {}, + "source": [ + "## Get devices" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "736b9afb-6a5d-4e17-9b05-78270ea4bc91", + "metadata": {}, + "outputs": [], + "source": [ + "df = smartcitizen_connector.search_by_query(endpoint='devices',\n", + " search_items=[{\n", + " 'key': 'postprocessing_id', \n", + " 'value': 'not_null',\n", + " 'full': True\n", + " }])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b55009c-805c-4c90-991d-ce202e23d9cb", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "devices = []\n", + "for item in df['postprocessing']:\n", + " print (item['device_id'], df.loc[item['device_id'], 'name'], item['meta'])\n", + " if item['meta'] == 'delivery-glasgow 2.1-2':\n", + " devices.append(item['device_id'])\n", + "# if item['meta'] == 'delivery-glasgow-ref':\n", + "# devices.append(item['device_id'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a6b9fb1-b47a-4541-9419-d8ef1aaaf58f", + "metadata": {}, + "outputs": [], + "source": [ + "len(devices)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2e5f2da-8b6a-4646-85f8-a66e9639a084", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "test = sc.Test(name=f'GLASGOW_BATCH_2.1_2',\n", + " devices=[sc.Device(blueprint='sc_air', params=sc.APIParams(id=id)) for id in devices],\n", + " force_recreate = True)\n", + "test.options.cache=True" + ] + }, + { + "cell_type": "markdown", + "id": "208ae3a1-6f07-4849-91c0-63925e77df03", + "metadata": {}, + "source": [ + "## Load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "669a1c6e-9324-419e-ac34-decdda6fd6de", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "test = sc.Test(name=f'GLASGOW_BATCH_2.1_2')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d61138dd-7e69-47b1-beaa-a478fbe76531", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "await test.load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "638cc01c-67cb-483e-bb40-bc733472f837", + "metadata": {}, + "outputs": [], + "source": [ + "traces = {\n", + " \"1\": {\"devices\": \"all\",\n", + " \"channel\": \"TEMP\",\n", + " \"subplot\": 1}\n", + " }\n", + "\n", + "test.ts_uplot(traces = traces, \n", + " formatting = {'width':1000, \n", + " 'title': 'Temperature', \n", + " 'ylabel': {1: 'Temperature (degC)'}\n", + " }, \n", + " options = {'frequency': '5Min', 'min_date': '2024-07-10'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c1f6f99-fd7b-47ca-9434-bbc768eee636", + "metadata": {}, + "outputs": [], + "source": [ + "traces = {\n", + " \"1\": {\"devices\": 17642,\n", + " \"channel\": \"BATT\",\n", + " \"subplot\": 1}\n", + " }\n", + "\n", + "test.ts_uplot(traces = traces, \n", + " formatting = {'width':1000, \n", + " 'title': 'Battery', \n", + " 'ylabel': {1: 'Temperature (degC)'}\n", + " }, \n", + " options = {'frequency': '5Min'})" + ] + }, + { + "cell_type": "markdown", + "id": "4f335eb0-6d86-4e5e-ad3b-72c9a19ecf90", + "metadata": {}, + "source": [ + "## Data completeness" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7ae9489-7479-43b5-8083-850e2625b40f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "import datetime\n", + "\n", + "from scdata.tools.date import localise_date" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9628726a-3a64-40b6-9fee-11402f29236c", + "metadata": {}, + "outputs": [], + "source": [ + "for device in test.devices:\n", + " a = localise_date(datetime.datetime.now(), 'UTC') - device.handler.json.last_reading_at\n", + " hours_ago = round(a.total_seconds() / 3600)\n", + " print (device.id, device.handler.json.last_reading_at, hours_ago)\n", + " plt.figure(figsize=(15,6))\n", + " sns.heatmap(test.get_device(device.id).data.isna().transpose(),\n", + " cmap=\"YlGnBu\",\n", + " cbar_kws={'label': 'Missing Data'}).set(title=f'{test.name}: {device.id} - ({device.handler.json.name}) {hours_ago}h')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2af0b933-7283-46f7-b401-117299b5e40f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}