diff --git a/README.md b/README.md index 5dffa45..dab7695 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,7 @@ You can run all the data sets by following the intructions below. Keep in mind t - You can run the examples in your browser on Binder: - Data access example: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AI-multimodal/public-demos/HEAD?labpath=data-access.ipynb) - - Data access with widgets: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AI-multimodal/public-demos/HEAD?labpath=aimm-server-data-access.ipynb) - - Write arrays and dataframes: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AI-multimodal/public-demos/HEAD?labpath=array-dataframe-writer.ipynb) + - Data handling with widgets: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/AI-multimodal/public-demos/HEAD?labpath=aimm-server-data-handling.ipynb) - Or you can run the locally: diff --git a/aimm-server-data-access.ipynb b/aimm-server-data-handling.ipynb similarity index 65% rename from aimm-server-data-access.ipynb rename to aimm-server-data-handling.ipynb index bff73a7..e222033 100644 --- a/aimm-server-data-access.ipynb +++ b/aimm-server-data-handling.ipynb @@ -12,8 +12,8 @@ "\n", "import functools\n", "import ipywidgets as widgets\n", - "import numpy as np\n", - "import pandas as pd" + "import numpy\n", + "import pandas" ] }, { @@ -21,7 +21,15 @@ "id": "9f792d12", "metadata": {}, "source": [ - "# Access the online data" + "# 1. Access the online data" + ] + }, + { + "cell_type": "markdown", + "id": "4ea12a9a", + "metadata": {}, + "source": [ + "## Connect to server" ] }, { @@ -41,7 +49,17 @@ "metadata": {}, "outputs": [], "source": [ - "client = from_uri(\"https://aimm.lbl.gov/api\")" + "client = from_uri(\"https://aimm-staging.lbl.gov/api\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71847193", + "metadata": {}, + "outputs": [], + "source": [ + "client.login()" ] }, { @@ -59,7 +77,7 @@ "id": "e3672187", "metadata": {}, "source": [ - "# Browse through all the datasets by code" + "## Browse through all the datasets by code" ] }, { @@ -89,7 +107,7 @@ "metadata": {}, "outputs": [], "source": [ - "client[\"element\"][\"Cr\"]['edge']['K']" + "client[\"element\"][\"Cr\"]['edge']['K']['uid']['gZbKaqF9EdE'].read()" ] }, { @@ -97,7 +115,7 @@ "id": "8beee3ce", "metadata": {}, "source": [ - "# Browse through all the datasets with widgets" + "## Browse through all the datasets with widgets" ] }, { @@ -208,7 +226,7 @@ "id": "9cdd5383", "metadata": {}, "source": [ - "# Filtering rows with sliders" + "## Filtering rows with sliders" ] }, { @@ -306,7 +324,7 @@ "id": "bf8ddace", "metadata": {}, "source": [ - "# Filtering columns with a selection box" + "## Filtering columns with a selection box" ] }, { @@ -344,7 +362,7 @@ "id": "141b5b8e", "metadata": {}, "source": [ - "# Creating simple plots with selection boxes and dataframes" + "## Creating simple plots with selection boxes and dataframes" ] }, { @@ -387,7 +405,7 @@ "id": "24436477", "metadata": {}, "source": [ - "# More plots" + "## More plots" ] }, { @@ -440,65 +458,295 @@ "id": "b17bd33f", "metadata": {}, "source": [ - "# Search and queries" + "## Search and queries" ] }, { "cell_type": "code", "execution_count": null, - "id": "d0892fff", + "id": "9015472f", "metadata": {}, "outputs": [], "source": [ - "# register queries\n", - "\n", - "from tiled.query_registration import QueryTranslationRegistry, register\n", - "from dataclasses import dataclass\n", - "import collections\n", - "import json\n", - "\n", - "@register(name=\"raw_mongo\", overwrite=True)\n", - "@dataclass\n", - "class RawMongo:\n", - " \"\"\"\n", - " Run a MongoDB query against a given collection.\n", - " \"\"\"\n", + "client[\"uid\"].search(RawMongo({\"metadata.sample.dataset\" : \"aimm_ncm\", \"metadata.element.edge\" : \"K\", \"metadata.element.symbol\" : \"Ni\"}))['kBZPKDsZHaQ']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84800290", + "metadata": {}, + "outputs": [], + "source": [ + "client[\"dataset\"][\"aimm_ncm\"][\"sample\"][\"h6bjpUSB4Qc\"][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"].search(RawMongo({\"metadata.charge.cycle\" : 0})).values_indexer[0]" + ] + }, + { + "cell_type": "markdown", + "id": "72af46c9", + "metadata": {}, + "source": [ + "# 2. Write data" + ] + }, + { + "cell_type": "markdown", + "id": "268b9463", + "metadata": {}, + "source": [ + "## Create array" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e80d9753", + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "import tifffile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0425ac94", + "metadata": {}, + "outputs": [], + "source": [ + "#Upload a tif file(s)\n", + "array_upload = widgets.FileUpload(\n", + " accept='.tif', # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'\n", + " multiple=True # True to accept multiple files upload else False\n", + ")\n", + "display(array_upload)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5700bda", + "metadata": {}, + "outputs": [], + "source": [ + "arr_list = []\n", + "for key, value in array_upload.value.items():\n", + " tiff_array = tifffile.TiffFile(io.BytesIO(value[\"content\"])).asarray()\n", + " arr_list.append(tiff_array)\n", "\n", - " query: str # We cannot put a dict in a URL, so this a JSON str.\n", + "print(arr_list[0].shape)\n", + "print(arr_list[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24296cae", + "metadata": {}, + "outputs": [], + "source": [ + "arr_meta_list = []\n", + "for i in range(len(arr_list)):\n", + " array_metadata={\"dataset\" : \"sandbox\", \"scan_id\": i+1, \"method\": \"A\", \"element\":{\"symbol\": \"Co\", \"edge\": \"K\"}}\n", + " arr_meta_list.append(array_metadata)\n", + " \n", + "print(arr_meta_list[0])" + ] + }, + { + "cell_type": "markdown", + "id": "865f3625", + "metadata": {}, + "source": [ + "## Create DataFrames" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8487ce12", + "metadata": {}, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4590728", + "metadata": {}, + "outputs": [], + "source": [ + "#Upload a json file\n", + "df_upload = widgets.FileUpload(\n", + " accept='.json', # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'\n", + " multiple=True # True to accept multiple files upload else False\n", + ")\n", + "display(df_upload)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "307382a1", + "metadata": {}, + "outputs": [], + "source": [ + "df_list = []\n", + "df_meta_list = []\n", + "counter = 1\n", + "for key, value in df_upload.value.items():\n", + " file_data = json.loads(value[\"content\"].decode('utf8'))\n", + " dataframe_metadata = {}\n", + " data = {}\n", + " for item in file_data:\n", + " if \"Energy\" in item:\n", + " data = item.copy()\n", + " else:\n", + " if \"element\" in item:\n", + " element = {\"element\":{\"symbol\": item[\"element\"], \"edge\": item[\"edge\"]}}\n", + " item.pop(\"element\")\n", + " item.pop(\"edge\")\n", + " item.update(element)\n", + " dataframe_metadata.update(item)\n", + " \n", + " json_dataframe = pandas.DataFrame(data)\n", + " dataframe_metadata.update({\"scan_id\": counter, \"method\": \"D\"})\n", + " \n", + " df_list.append(json_dataframe)\n", + " df_meta_list.append(dataframe_metadata)\n", + " counter += 1\n", "\n", - " def __init__(self, query):\n", - " if isinstance(query, collections.abc.Mapping):\n", - " query = json.dumps(query)\n", - " self.query = query" + "print(df_list[0].shape)\n", + "df_list[0].head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "9015472f", + "id": "d918a190", "metadata": {}, "outputs": [], "source": [ - "client[\"uid\"].search(RawMongo({\"metadata.sample.dataset\" : \"aimm_ncm\", \"metadata.element.edge\" : \"K\", \"metadata.element.symbol\" : \"Ni\"}))" + "df_meta_list[0]" + ] + }, + { + "cell_type": "markdown", + "id": "85d5fc82", + "metadata": {}, + "source": [ + "## Write an array" ] }, { "cell_type": "code", "execution_count": null, - "id": "84800290", + "id": "c915e04c", "metadata": {}, "outputs": [], "source": [ - "client[\"dataset\"][\"aimm_ncm\"][\"sample\"][\"h6bjpUSB4Qc\"][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"].search(RawMongo({\"metadata.charge.cycle\" : 0})).values_indexer[0]" + "array_key = client['uid'].write_array(arr_list[0], arr_meta_list[0])\n", + "array_key" + ] + }, + { + "cell_type": "markdown", + "id": "515428b6", + "metadata": {}, + "source": [ + "## Write a dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b2e8aa3", + "metadata": {}, + "outputs": [], + "source": [ + "df_key = client[\"uid\"].write_dataframe(df_list[0], df_meta_list[0])\n", + "df_key" + ] + }, + { + "cell_type": "markdown", + "id": "8daaff4b", + "metadata": {}, + "source": [ + "## Validate write/upload" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f21ff3f5", + "metadata": {}, + "outputs": [], + "source": [ + "results = client[\"dataset\"][\"sandbox\"].search(RawMongo({\"metadata.element.symbol\":\"Co\"})) # Search for all element that meet the search criteria\n", + "list(results[\"uid\"])" + ] + }, + { + "cell_type": "markdown", + "id": "ba4fc437", + "metadata": {}, + "source": [ + "### Validate array" ] }, { "cell_type": "code", "execution_count": null, - "id": "50e6bbb1", + "id": "74e6acfb", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "array_result = results[\"uid\"][array_key].read() # Get first item of all results\n", + "results[\"uid\"][array_key].metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a346b0d", + "metadata": {}, + "outputs": [], + "source": [ + "array_result" + ] + }, + { + "cell_type": "markdown", + "id": "1cc19f64", + "metadata": {}, + "source": [ + "### Validate dataframe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1949b1f1", + "metadata": {}, + "outputs": [], + "source": [ + "dataframe_result = results[\"uid\"][df_key].read()\n", + "results[\"uid\"][df_key].metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cd6f6fc", + "metadata": {}, + "outputs": [], + "source": [ + "dataframe_result" + ] } ], "metadata": { diff --git a/array-dataframe-writer.ipynb b/array-dataframe-writer.ipynb deleted file mode 100644 index 37e80bb..0000000 --- a/array-dataframe-writer.ipynb +++ /dev/null @@ -1,271 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "375f1f2a", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy\n", - "import pandas" - ] - }, - { - "cell_type": "markdown", - "id": "18d1e891", - "metadata": {}, - "source": [ - "# Create array" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5c2cf7cb", - "metadata": {}, - "outputs": [], - "source": [ - "arr = numpy.ones((5, 5, 5))\n", - "arr" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1c96b4ad", - "metadata": {}, - "outputs": [], - "source": [ - "arr_metadata={\"dataset\" : \"sandbox\", \"element\":{\"symbol\": \"Co\", \"edge\": \"K\"}}\n", - "print(arr_metadata)" - ] - }, - { - "cell_type": "markdown", - "id": "55184fb4", - "metadata": {}, - "source": [ - "# Create DataFrames" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "db956f76", - "metadata": {}, - "outputs": [], - "source": [ - "df_data = numpy.ones((5, 5))\n", - "\n", - "data = {\n", - " \"Column1\": df_data[0],\n", - " \"Column2\": df_data[1],\n", - " \"Column3\": df_data[2],\n", - " \"Column4\": df_data[3],\n", - " \"Column5\": df_data[4],\n", - "}\n", - "\n", - "df = pandas.DataFrame(data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ecdb8fac", - "metadata": {}, - "outputs": [], - "source": [ - "df_metadata={\"dataset\" : \"sandbox\", \"element\":{\"symbol\": \"Co\", \"edge\": \"L1\"}}\n", - "df_metadata" - ] - }, - { - "cell_type": "markdown", - "id": "e87f9826", - "metadata": {}, - "source": [ - "# Start Tiled client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16217d71", - "metadata": {}, - "outputs": [], - "source": [ - "from tiled.client import from_uri" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "738ae7a9", - "metadata": {}, - "outputs": [], - "source": [ - "client = from_uri(\"https://aimm-staging.lbl.gov/api\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "447b9773", - "metadata": {}, - "outputs": [], - "source": [ - "client.login()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53a4538d", - "metadata": {}, - "outputs": [], - "source": [ - "client" - ] - }, - { - "cell_type": "markdown", - "id": "0028f196", - "metadata": {}, - "source": [ - "# Write array" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "891dee96", - "metadata": {}, - "outputs": [], - "source": [ - "array_key = client['uid'].write_array(arr, arr_metadata)\n", - "array_key" - ] - }, - { - "cell_type": "markdown", - "id": "c0924c29", - "metadata": {}, - "source": [ - "# Write dataframe" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ab57b036", - "metadata": {}, - "outputs": [], - "source": [ - "df_key = client[\"uid\"].write_dataframe(df, df_metadata)\n", - "df_key" - ] - }, - { - "cell_type": "markdown", - "id": "992772ee", - "metadata": {}, - "source": [ - "# Validate write/upload" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3980689c", - "metadata": {}, - "outputs": [], - "source": [ - "#from databroker.experimental.queries import RawMongo\n", - "from aimmdb.tree import RawMongoQuery as RawMongo" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fae5f0ee", - "metadata": {}, - "outputs": [], - "source": [ - "results = client[\"dataset\"][\"sandbox\"].search(RawMongo({\"metadata.element.symbol\":\"Co\"})) # Search for all element that meet the search criteria\n", - "list(results[\"uid\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "99f0e09b", - "metadata": {}, - "outputs": [], - "source": [ - "array_result = results[\"uid\"][array_key].read() # Get first item of all results\n", - "results[\"uid\"][array_key].metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a36f466", - "metadata": {}, - "outputs": [], - "source": [ - "array_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ee8524e", - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_result = results[\"uid\"][df_key].read()\n", - "results[\"uid\"][df_key].metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b99842a", - "metadata": {}, - "outputs": [], - "source": [ - "dataframe_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f18df17", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}