From 41b2b5a9d6d32b94fb38186ebfd3c10b999bc7e7 Mon Sep 17 00:00:00 2001 From: oscgonfer Date: Fri, 19 Jul 2024 16:08:36 +0200 Subject: [PATCH] Add sections for proof of concept --- _config.yml | 2 +- _toc.yml | 20 ++- deliveries/240719_glasgow-2.ipynb | 226 ++++++++++++++++++++++++++ development/240513_battery-test.ipynb | 201 +++++++++++++++++++++++ docs/markdown-notebooks.md | 53 ------ docs/markdown.md | 55 ------- docs/notebooks.ipynb | 122 -------------- docs/test.ipynb | 51 ------ docs/intro.md => intro.md | 0 9 files changed, 442 insertions(+), 288 deletions(-) create mode 100644 deliveries/240719_glasgow-2.ipynb create mode 100644 development/240513_battery-test.ipynb delete mode 100644 docs/markdown-notebooks.md delete mode 100644 docs/markdown.md delete mode 100644 docs/notebooks.ipynb delete mode 100644 docs/test.ipynb rename docs/intro.md => intro.md (100%) diff --git a/_config.yml b/_config.yml index 72e9c65..ed64029 100644 --- a/_config.yml +++ b/_config.yml @@ -10,7 +10,7 @@ logo: assets/logo.png # Information about where the book exists on the web repository: url: https://github.com/fablabbcn/smartcitizen-analysis-books # Online location of your book - path_to_book: docs # Optional path to your book, relative to the repository root + # path_to_book: books # Optional path to your book, relative to the repository root branch: main # Which branch of the repository should be used when creating links (optional) ###### --------------------------- diff --git a/_toc.yml b/_toc.yml index 1904802..f8410e5 100644 --- a/_toc.yml +++ b/_toc.yml @@ -3,9 +3,17 @@ # Learn more at https://jupyterbook.org/customize/toc.html format: jb-book -root: docs/intro -chapters: -- file: docs/markdown -- file: docs/markdown-notebooks -- file: docs/notebooks -- file: docs/test \ No newline at end of file +root: intro +parts: +- caption: Development + chapters: + - file: development/240513_battery-test + # - file: development/240719-liu-co2-gps + +- caption: Deliveries + chapters: + - file: deliveries/240719_glasgow-2 + +# - caption: Insights +# chapters: +# - file: deliveries/240719_glasgow-2 \ No newline at end of file diff --git a/deliveries/240719_glasgow-2.ipynb b/deliveries/240719_glasgow-2.ipynb new file mode 100644 index 0000000..da3443f --- /dev/null +++ b/deliveries/240719_glasgow-2.ipynb @@ -0,0 +1,226 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "557f54fd-3387-45e1-a4dc-3fafb80753c5", + "metadata": {}, + "source": [ + "# Glasgow Batch #2\n", + "\n", + "Smart Citizen Kit 2.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54d8d6f4-8a42-4f32-b65d-226d0901be7b", + "metadata": {}, + "outputs": [], + "source": [ + "import smartcitizen_connector\n", + "import scdata as sc\n", + "import time\n", + "now = time.localtime()" + ] + }, + { + "cell_type": "markdown", + "id": "d5ac091a-dfab-4b5a-b9c4-9133230f7678", + "metadata": {}, + "source": [ + "## Get devices" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "736b9afb-6a5d-4e17-9b05-78270ea4bc91", + "metadata": {}, + "outputs": [], + "source": [ + "df = smartcitizen_connector.search_by_query(endpoint='devices',\n", + " search_items=[{\n", + " 'key': 'postprocessing_id', \n", + " 'value': 'not_null',\n", + " 'full': True\n", + " }])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b55009c-805c-4c90-991d-ce202e23d9cb", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "devices = []\n", + "for item in df['postprocessing']:\n", + " print (item['device_id'], df.loc[item['device_id'], 'name'], item['meta'])\n", + " if item['meta'] == 'delivery-glasgow 2.1-2':\n", + " devices.append(item['device_id'])\n", + "# if item['meta'] == 'delivery-glasgow-ref':\n", + "# devices.append(item['device_id'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2e5f2da-8b6a-4646-85f8-a66e9639a084", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "test = sc.Test(name=f'GLASGOW_BATCH_2.1_2',\n", + " devices=[sc.Device(blueprint='sc_air', params=sc.APIParams(id=id)) for id in devices],\n", + " force_recreate = True)\n", + "test.options.cache=True" + ] + }, + { + "cell_type": "markdown", + "id": "208ae3a1-6f07-4849-91c0-63925e77df03", + "metadata": {}, + "source": [ + "## Load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "669a1c6e-9324-419e-ac34-decdda6fd6de", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "test = sc.Test(name=f'GLASGOW_BATCH_2.1_2')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d61138dd-7e69-47b1-beaa-a478fbe76531", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "await test.load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "638cc01c-67cb-483e-bb40-bc733472f837", + "metadata": {}, + "outputs": [], + "source": [ + "traces = {\n", + " \"1\": {\"devices\": \"all\",\n", + " \"channel\": \"TEMP\",\n", + " \"subplot\": 1}\n", + " }\n", + "\n", + "test.ts_uplot(traces = traces, \n", + " formatting = {'width':1000, \n", + " 'title': 'Temperature', \n", + " 'ylabel': {1: 'Temperature (degC)'}\n", + " }, \n", + " options = {'frequency': '5Min', 'min_date': '2024-07-10'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c1f6f99-fd7b-47ca-9434-bbc768eee636", + "metadata": {}, + "outputs": [], + "source": [ + "traces = {\n", + " \"1\": {\"devices\": 17642,\n", + " \"channel\": \"BATT\",\n", + " \"subplot\": 1}\n", + " }\n", + "\n", + "test.ts_uplot(traces = traces, \n", + " formatting = {'width':1000, \n", + " 'title': 'Battery', \n", + " 'ylabel': {1: 'Temperature (degC)'}\n", + " }, \n", + " options = {'frequency': '5Min'})" + ] + }, + { + "cell_type": "markdown", + "id": "4f335eb0-6d86-4e5e-ad3b-72c9a19ecf90", + "metadata": {}, + "source": [ + "## Data completeness" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7ae9489-7479-43b5-8083-850e2625b40f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "import datetime\n", + "\n", + "from scdata.tools.date import localise_date" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9628726a-3a64-40b6-9fee-11402f29236c", + "metadata": {}, + "outputs": [], + "source": [ + "for device in test.devices:\n", + " a = localise_date(datetime.datetime.now(), 'UTC') - device.handler.json.last_reading_at\n", + " hours_ago = round(a.total_seconds() / 3600)\n", + " print (device.id, device.handler.json.last_reading_at, hours_ago)\n", + " plt.figure(figsize=(15,6))\n", + " sns.heatmap(test.get_device(device.id).data.isna().transpose(),\n", + " cmap=\"YlGnBu\",\n", + " cbar_kws={'label': 'Missing Data'}).set(title=f'{test.name}: {device.id} - ({device.handler.json.name}) {hours_ago}h')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2af0b933-7283-46f7-b401-117299b5e40f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/development/240513_battery-test.ipynb b/development/240513_battery-test.ipynb new file mode 100644 index 0000000..4812fb9 --- /dev/null +++ b/development/240513_battery-test.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a9b9acae", + "metadata": {}, + "source": [ + "# Battery test\n", + "\n", + "SCK 2.2 Battery test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4791fa2-d0c2-4789-bbdd-218aad2f1504", + "metadata": {}, + "outputs": [], + "source": [ + "import smartcitizen_connector\n", + "import scdata as sc\n", + "import time\n", + "now = time.localtime()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14999577-1d9f-4822-8bd3-4f643dc65fea", + "metadata": {}, + "outputs": [], + "source": [ + "df = smartcitizen_connector.search_by_query(endpoint='devices',\n", + " search_items=[{\n", + " 'key': 'postprocessing_id', \n", + " 'value': 'not_null',\n", + " 'full': True\n", + " }])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "db427d48-ce30-48c7-8b0a-61dac54ced97", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "for item in df['postprocessing']:\n", + " print (item['device_id'], item['meta'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91aeb4b5-dcaa-4a23-a14f-a5cd23619c68", + "metadata": {}, + "outputs": [], + "source": [ + "test = sc.Test(name=f'BATTERY_CHECK_SCK2.2',\n", + " devices=[sc.Device(blueprint='sc_air', params=sc.APIParams(id=id)) for id in [17293, 17294]])\n", + "test.options.cache=True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45b6d190-d9e2-43aa-af35-1de3af55860a", + "metadata": {}, + "outputs": [], + "source": [ + "test = sc.Test(name=f'BATTERY_CHECK_SCK2.2')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af583e62-7fc7-4fa3-865c-75d218cf5f34", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "await test.load()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7d5c5737-f085-43c8-ad5c-da02d249ac90", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[38;20m[2024-05-21 16:24:01,329] - scdata - INFO - Preparing data for plot\u001b[0m\n", + "\u001b[38;20m[2024-05-21 16:24:01,331] - scdata - INFO - Making 1 subplots\u001b[0m\n", + "[17293, 17294]\n", + "\u001b[38;20m[2024-05-21 16:24:01,338] - scdata - INFO - Resampling at 1Min\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/oscar/.virtualenvs/scdata-python3.11/lib/python3.11/site-packages/IPython/core/display.py:431: UserWarning: Consider using IPython.display.IFrame instead\n", + " warnings.warn(\"Consider using IPython.display.IFrame instead\")\n" + ] + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "traces = {\n", + " \"1\": {\"devices\": \"all\",\n", + " \"channel\": \"BATT\",\n", + " \"subplot\": 1}\n", + " }\n", + "\n", + "test.ts_uplot(traces = traces, \n", + " formatting = {'width':1000, \n", + " 'title': 'Battery discharge test', \n", + " 'ylabel': {1: 'Battery level (%)'}\n", + " }, \n", + " options = {'frequency': '1Min'})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2903df1-33b9-4d3e-ba19-9c3fd68e8c11", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/markdown-notebooks.md b/docs/markdown-notebooks.md deleted file mode 100644 index a057a32..0000000 --- a/docs/markdown-notebooks.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -jupytext: - formats: md:myst - text_representation: - extension: .md - format_name: myst - format_version: 0.13 - jupytext_version: 1.11.5 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -# Notebooks with MyST Markdown - -Jupyter Book also lets you write text-based notebooks using MyST Markdown. -See [the Notebooks with MyST Markdown documentation](https://jupyterbook.org/file-types/myst-notebooks.html) for more detailed instructions. -This page shows off a notebook written in MyST Markdown. - -## An example cell - -With MyST Markdown, you can define code cells with a directive like so: - -```{code-cell} -print(2 + 2) -``` - -When your book is built, the contents of any `{code-cell}` blocks will be -executed with your default Jupyter kernel, and their outputs will be displayed -in-line with the rest of your content. - -```{seealso} -Jupyter Book uses [Jupytext](https://jupytext.readthedocs.io/en/latest/) to convert text-based files to notebooks, and can support [many other text-based notebook files](https://jupyterbook.org/file-types/jupytext.html). -``` - -## Create a notebook with MyST Markdown - -MyST Markdown notebooks are defined by two things: - -1. YAML metadata that is needed to understand if / how it should convert text files to notebooks (including information about the kernel needed). - See the YAML at the top of this page for example. -2. The presence of `{code-cell}` directives, which will be executed with your book. - -That's all that is needed to get started! - -## Quickly add YAML metadata for MyST Notebooks - -If you have a markdown file and you'd like to quickly add YAML metadata to it, so that Jupyter Book will treat it as a MyST Markdown Notebook, run the following command: - -``` -jupyter-book myst init path/to/markdownfile.md -``` diff --git a/docs/markdown.md b/docs/markdown.md deleted file mode 100644 index faeea60..0000000 --- a/docs/markdown.md +++ /dev/null @@ -1,55 +0,0 @@ -# Markdown Files - -Whether you write your book's content in Jupyter Notebooks (`.ipynb`) or -in regular markdown files (`.md`), you'll write in the same flavor of markdown -called **MyST Markdown**. -This is a simple file to help you get started and show off some syntax. - -## What is MyST? - -MyST stands for "Markedly Structured Text". It -is a slight variation on a flavor of markdown called "CommonMark" markdown, -with small syntax extensions to allow you to write **roles** and **directives** -in the Sphinx ecosystem. - -For more about MyST, see [the MyST Markdown Overview](https://jupyterbook.org/content/myst.html). - -## Sample Roles and Directives - -Roles and directives are two of the most powerful tools in Jupyter Book. They -are like functions, but written in a markup language. They both -serve a similar purpose, but **roles are written in one line**, whereas -**directives span many lines**. They both accept different kinds of inputs, -and what they do with those inputs depends on the specific role or directive -that is being called. - -Here is a "note" directive: - -```{note} -Here is a note -``` - -It will be rendered in a special box when you build your book. - -Here is an inline directive to refer to a document: {doc}`markdown-notebooks`. - - -## Citations - -You can also cite references that are stored in a `bibtex` file. For example, -the following syntax: `` {cite}`holdgraf_evidence_2014` `` will render like -this: {cite}`holdgraf_evidence_2014`. - -Moreover, you can insert a bibliography into your page with this syntax: -The `{bibliography}` directive must be used for all the `{cite}` roles to -render properly. -For example, if the references for your book are stored in `references.bib`, -then the bibliography is inserted with: - -```{bibliography} -``` - -## Learn more - -This is just a simple starter to get you started. -You can learn a lot more at [jupyterbook.org](https://jupyterbook.org). diff --git a/docs/notebooks.ipynb b/docs/notebooks.ipynb deleted file mode 100644 index fdb7176..0000000 --- a/docs/notebooks.ipynb +++ /dev/null @@ -1,122 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Content with notebooks\n", - "\n", - "You can also create content with Jupyter Notebooks. This means that you can include\n", - "code blocks and their outputs in your book.\n", - "\n", - "## Markdown + notebooks\n", - "\n", - "As it is markdown, you can embed images, HTML, etc into your posts!\n", - "\n", - "![](https://myst-parser.readthedocs.io/en/latest/_static/logo-wide.svg)\n", - "\n", - "You can also $add_{math}$ and\n", - "\n", - "$$\n", - "math^{blocks}\n", - "$$\n", - "\n", - "or\n", - "\n", - "$$\n", - "\\begin{aligned}\n", - "\\mbox{mean} la_{tex} \\\\ \\\\\n", - "math blocks\n", - "\\end{aligned}\n", - "$$\n", - "\n", - "But make sure you \\$Escape \\$your \\$dollar signs \\$you want to keep!\n", - "\n", - "## MyST markdown\n", - "\n", - "MyST markdown works in Jupyter Notebooks as well. For more information about MyST markdown, check\n", - "out [the MyST guide in Jupyter Book](https://jupyterbook.org/content/myst.html),\n", - "or see [the MyST markdown documentation](https://myst-parser.readthedocs.io/en/latest/).\n", - "\n", - "## Code blocks and outputs\n", - "\n", - "Jupyter Book will also embed your code blocks and output in your book.\n", - "For example, here's some sample Matplotlib code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from matplotlib import rcParams, cycler\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "plt.ion()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Fixing random state for reproducibility\n", - "np.random.seed(19680801)\n", - "\n", - "N = 10\n", - "data = [np.logspace(0, 1, 100) + np.random.randn(100) + ii for ii in range(N)]\n", - "data = np.array(data).T\n", - "cmap = plt.cm.coolwarm\n", - "rcParams['axes.prop_cycle'] = cycler(color=cmap(np.linspace(0, 1, N)))\n", - "\n", - "\n", - "from matplotlib.lines import Line2D\n", - "custom_lines = [Line2D([0], [0], color=cmap(0.), lw=4),\n", - " Line2D([0], [0], color=cmap(.5), lw=4),\n", - " Line2D([0], [0], color=cmap(1.), lw=4)]\n", - "\n", - "fig, ax = plt.subplots(figsize=(10, 5))\n", - "lines = ax.plot(data)\n", - "ax.legend(custom_lines, ['Cold', 'Medium', 'Hot']);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There is a lot more that you can do with outputs (such as including interactive outputs)\n", - "with your book. For more information about this, see [the Jupyter Book documentation](https://jupyterbook.org)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "state": {}, - "version_major": 2, - "version_minor": 0 - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/docs/test.ipynb b/docs/test.ipynb deleted file mode 100644 index 3a30045..0000000 --- a/docs/test.ipynb +++ /dev/null @@ -1,51 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "de595471", - "metadata": {}, - "source": [ - "# Example test" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "b011cb00-5cd0-420e-a87c-047257db7906", - "metadata": {}, - "outputs": [], - "source": [ - "import scdata as sc" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0ef2104-6708-4720-9acf-6b1897c761e3", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/intro.md b/intro.md similarity index 100% rename from docs/intro.md rename to intro.md