data-access.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15fb9e10-c22f-4f64-b32d-89da11114727",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tiled.client import from_uri\n",
    "from tiled.queries import Key\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import aimmdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37fb61ca-7edb-4a31-93f9-7ffe0650be2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the tiled client object represents the data on the server in the form of a tree which can be traversed using python's dictionary syntax\n",
    "# at the root level the keys correspond to available queries\n",
    "c = from_uri(\"https://aimm.lbl.gov/api\")\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "578268f9-2016-4b3c-80dd-7710d7c8cafa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data on the server is organized into datasets\n",
    "c[\"dataset\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ea85bc1-7192-4332-8993-f53f5ad4cb1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# login to see non public datasets\n",
    "c.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d83e8486-05f3-4954-b582-deb46bfa4148",
   "metadata": {},
   "outputs": [],
   "source": [
    "c[\"dataset\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd4799c4-6050-4bad-bd9e-af4e1a7f980f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choosing a query shows the distinct values that the query parameter can take\n",
    "# uid gives a flat list of all selected data\n",
    "c[\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4b9945e-8459-48af-9a65-9142db683b57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the keys can be indexed using the .keys() method\n",
    "c[\"uid\"].keys()[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "578ecdbe-d60b-442d-afd4-05e34a1df2de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# measurements are indexed by a unique identifier (uid)\n",
    "x = c[\"uid\"][\"95NqB8p5xEd\"]\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d37a916-3844-4cbc-b747-04fef8d4429c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# or by using the .values() method\n",
    "x = c[\"uid\"].values().first()\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b937376-ff6c-4328-ac91-82da1d798fca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# measurements have an associated metadata dictionary\n",
    "x.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e4670c0-9c90-4194-84ff-4957863f09e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the measurement itself is stored in a dataframe which can be obtained using the .read() method\n",
    "df = x.read()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "174257e5-0a9f-4dbd-a5ff-5f4edd2e2a2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the dataset query gives a list of datasets (groupings of data within the database)\n",
    "c[\"dataset\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3f86568-a4a5-40ab-a489-a79e220b5734",
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting a dataset yields a client object like the root client but with the dataset query removed\n",
    "c[\"dataset\"][\"newville\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a810865-92b7-422b-ab67-f5ce13b71ea4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the uid query after selecting a dataset gives only the measurements which are part of that dataset\n",
    "c[\"dataset\"][\"newville\"][\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4994fa6-9743-49d4-885c-5e0478446c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the element query gives a list of distinct absorbing elements\n",
    "c[\"element\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3395b726-52f3-40b9-b860-ae9cf0e836ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# queries may be nested\n",
    "# for example querying the list of distinct absorbing elements present in the newville dataset can be written as\n",
    "c[\"dataset\"][\"newville\"][\"element\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e63ea54f-f616-4635-baa2-59adf159d5e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# and selecting the gold spectra within the newville dataset can be written as\n",
    "c[\"dataset\"][\"newville\"][\"element\"][\"Au\"][\"uid\"].keys()[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33f71bfa-98d9-4a0c-a2ad-d867cc9952ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# measurements can also be grouped by sample\n",
    "# samples are also identified by a uid\n",
    "c[\"dataset\"][\"newville\"][\"sample\"].keys()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2383616d-e409-4258-845a-1bd6373b1dd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# samples may be selected by their uid and can have more than one measurement associated with them\n",
    "c[\"dataset\"][\"newville\"][\"sample\"][\"745QhDdAY9Y\"][\"uid\"].keys()[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42decee-51b3-40a1-819b-7f8556fb3837",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the database can also be search by edge ...\n",
    "c[\"dataset\"][\"aimm_core\"][\"edge\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88c5fbb7-72b1-42c4-ba6e-efbbd1230d77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ... and this can be nested\n",
    "c[\"dataset\"][\"aimm_core\"][\"element\"][\"O\"][\"edge\"][\"K\"][\"uid\"].keys()[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91e788a2-350d-4fad-9b8c-648a72e08449",
   "metadata": {},
   "outputs": [],
   "source": [
    "# once the uid has been found, only the uid is necessary to select the desired measurement\n",
    "x = c[\"uid\"][\"5j5ZLA6Qokx\"]\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b5b1bba-23c0-419e-a3bd-46f52ee12f38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pandas provides convenient functions for plotting dataframes\n",
    "\n",
    "pprint({**x.metadata})\n",
    "df = x.read()\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "df.plot(\"energy\", \"mu\", ax=ax, title=x.metadata[\"sample\"][\"name\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8e6bb86-6919-49b9-b9bd-a308fd018078",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the order of traversal of queries is not important\n",
    "# the following two expressions are fully equivalent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1402b55d-c8b4-4c24-b851-1f89745e9073",
   "metadata": {},
   "outputs": [],
   "source": [
    "c[\"dataset\"][\"aimm\"][\"element\"][\"Ni\"][\"edge\"][\"K\"][\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da22de21-244a-47f6-affb-49bc131ee9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "c[\"dataset\"][\"aimm\"][\"edge\"][\"K\"][\"element\"][\"Ni\"][\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6492b856-4981-4be1-93cd-ec6a8e2576bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the tree mechanism is not the only way to search the database\n",
    "# the above queries can also be written as\n",
    "\n",
    "c[\"uid\"].search(Key(\"dataset\") == \"aimm\").search(Key(\"element.edge\") == \"K\").search(Key(\"element.symbol\") == \"Ni\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f922a9e-2915-4223-b4ba-d996b296fc4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# these search mechanisms are fully composable so the following is also equivalent to the preceeding three examples\n",
    "\n",
    "c[\"dataset\"][\"aimm\"][\"uid\"].search(Key(\"element.edge\") == \"K\").search(Key(\"element.symbol\") == \"Ni\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bb1f464-c7ef-41df-b630-d964fdef392c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's look at one of the spectra in the aimm dataset\n",
    "x = c[\"dataset\"][\"aimm\"][\"uid\"].values().first()\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e74c4905-7d98-4c14-b4f4-6d9477f7ea26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the metadata for this spectra contain information on the charge state\n",
    "x.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54ce9fbd-78ca-44f4-9457-afe6a77ef7c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the .search(...) method may be used to query metadata which is not searchable using the tree traversal methods\n",
    "# for example the following query fetchs all data in the aimm dataset corresponding to the pristine condition (i.e. not charge cycled)\n",
    "\n",
    "c[\"dataset\"][\"aimm\"][\"uid\"].search(Key(\"charge.cycle\") == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2150b279-82cd-4138-8fac-90742d1683c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print aimm dataset samples\n",
    "for k in c[\"dataset\"][\"aimm\"][\"sample\"].keys():\n",
    "    print(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71715499-a024-4470-8140-1230d055d978",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose one sample\n",
    "\n",
    "sample = c[\"dataset\"][\"aimm\"][\"sample\"].keys().first()\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab560b5e-e35a-4d3c-a31b-28231ce26edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all measurements one sample (recall samples are identified by their uid not their name)\n",
    "\n",
    "c[\"dataset\"][\"aimm\"][\"sample\"][sample][\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "341f8642-9add-46f4-b24b-361076477d2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot all K edge measurements of pristine condition\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "ax.set_title(sample.name)\n",
    "\n",
    "for v in c[\"dataset\"][\"aimm\"][\"sample\"][sample][\"uid\"].search(Key(\"charge.cycle\") == 0).values():\n",
    "    df = v.read()\n",
    "    element = v.metadata[\"element\"][\"symbol\"]\n",
    "    edge = v.metadata[\"element\"][\"edge\"]\n",
    "    if \"mutrans\" in df.columns:\n",
    "        df.plot(\"energy\", \"mutrans\", ax=ax, label=f\"{element}-{edge}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b97b3d5-d300-4941-817f-320b35a95417",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot all Nickle L3 measurements\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "for v in c[\"dataset\"][\"aimm\"][\"sample\"][sample][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"].values():\n",
    "    charge = v.metadata[\"charge\"]\n",
    "    cycle, voltage, state = [charge[k] for k in [\"cycle\", \"voltage\", \"state\"]]\n",
    "    df = v.read()\n",
    "    df.plot(\"energy\", \"mu_tey\", ax=ax, label=f\"cycle {cycle} {voltage}V {state}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32e9d651-b753-4340-b5af-65b01d29224c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot Manganese K edge for pristine condition for each sample\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for v in c[\"dataset\"][\"aimm\"][\"element\"][\"Mn\"][\"edge\"][\"K\"][\"uid\"].search(Key(\"charge.cycle\") ==  0).values():\n",
    "    df = v.read()\n",
    "    sample_name = v.metadata[\"sample\"][\"name\"]\n",
    "    \n",
    "    df.plot(\"energy\", \"mutrans\", ax=ax, label=f\"{sample_name}\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10ebf54a-ac45-4857-9304-220e506c68a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find Ni-L3 edge in aimm_core dataset\n",
    "c[\"dataset\"][\"aimm_core\"][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12569cf4-d709-46ad-a73b-fbe2e9a7752c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare Ni-L3 edge in Ni metal with BM_NCM622"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a8afee2-b071-4842-b771-8e5e69631222",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = c[\"dataset\"][\"aimm_core\"][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"].values().first()\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3207bf7f-58d0-45a6-a368-acb7c6c5f030",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = c[\"dataset\"][\"aimm\"][\"sample\"][sample][\"element\"][\"Ni\"][\"edge\"][\"L3\"][\"uid\"].search(Key(\"charge.cycle\") == 0).values().first()\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6765d3cc-654f-401e-a955-366118afd84f",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "\n",
    "df = x.read()\n",
    "df.plot(\"energy\", \"mu_tey\", ax=ax, label=x.metadata[\"sample\"][\"name\"])\n",
    "\n",
    "df = y.read()\n",
    "df.plot(\"energy\", \"mu_tey\", ax=ax, label=y.metadata[\"sample\"][\"name\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:aimm]",
   "language": "python",
   "name": "conda-env-aimm-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}