databricks/utils/api_functions.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"d6393cda-a1a4-49bb-ad16-513c53e4ab52","showTitle":false,"title":""}},"outputs":[],"source":["try:\n","    DATALAKE_API_KEY = dbutils.secrets.get(scope=\"brewai-iaq-forecast\", key=\"DATALAKE-API-KEY\")\n","    API_KEY = dbutils.secrets.get(scope=\"brewai-iaq-forecast\", key=\"API-KEY\")\n","except:\n","    DATALAKE_API_KEY = None\n","    API_KEY = None"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"999e19c4-05f1-45ec-a645-cfa7964d59db","showTitle":false,"title":""}},"outputs":[{"data":{"application/vnd.databricks.v1+bamboolib_hint":"{\"pd.DataFrames\": [], \"version\": \"0.0.1\"}","text/plain":[]},"metadata":{"application/vnd.databricks.v1+output":{"addedWidgets":{},"arguments":{},"data":{"application/vnd.databricks.v1+bamboolib_hint":"{\"pd.DataFrames\": [], \"version\": \"0.0.1\"}","text/plain":""},"datasetInfos":[],"metadata":{"kernelSessionId":"5924c254-6ae91e61f11e0c65069f2500"},"removedWidgets":[],"type":"mimeBundle"}},"output_type":"display_data"}],"source":["import requests\n","import pandas as pd\n","from datetime import datetime, timedelta"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"dbdd1a57-6b24-403e-8c4d-6da8e76d4984","showTitle":false,"title":""}},"outputs":[],"source":["def get_request_header():\n","    return {\"x-api-key\": API_KEY}"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"553472f2-1c65-4740-abf5-8598850e866d","showTitle":false,"title":""}},"outputs":[],"source":["def get_device_ids(return_response=False):\n","    response = requests.get(\n","        url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/sensordata/sensors\",\n","        headers=get_request_header(),\n","    )\n","    if return_response:\n","        return response\n","    devices = [el[0] for el in response.json()['Records']]\n","    return devices"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"acd29d71-d714-463c-b4dd-ba1df7ac89da","showTitle":false,"title":""}},"outputs":[],"source":["def get_device_data(device):\n","    print(\"Handle\", device)\n","    data = requests.get(\n","        url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/sensordata?from={start_ts}&to={end_ts}&device={device}\",\n","        headers={\"x-api-key\": API_KEY},\n","    ).json()\n","    rows = data[\"Records\"]\n","    while data[\"ExecutionInfo\"].get(\"NextToken\") is not None:\n","        data = requests.get(\n","            url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/sensordata?from={start_ts}&to={end_ts}&device={device}\",\n","            headers={\"x-api-key\": API_KEY},\n","        ).json()\n","        data = data.json()\n","        rows += data[\"Records\"]\n","    df = pd.DataFrame(rows, columns=data[\"Metadata\"][\"ColumnName\"])\n","    df['device'] = device\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"c3322cb2-cd0b-49bb-9ff4-d247cc467ba8","showTitle":false,"title":""}},"outputs":[],"source":["def get_device_measures(device_id, from_ts, to_ts, return_response=False):\n","    response = requests.get(\n","        url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/sensordata?from={from_ts}&to={to_ts}&device={device_id}\",\n","        headers=get_request_header(),\n","    )\n","    n_req = 1\n","    if return_response:\n","        return response\n","    data = response.json()\n","    rows = data[\"Records\"]\n","    print(f\"REQUEST: {n_req}\")\n","    while data[\"ExecutionInfo\"].get(\"NextToken\"):\n","        token = data[\"ExecutionInfo\"].get(\"NextToken\")\n","        data = requests.get(\n","            url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/sensordata?from={from_ts}&to={to_ts}&device={device_id}&nextToken={token}\",\n","            headers=get_request_header(),\n","        ).json()\n","        n_req += 1\n","        print(f\"REQUEST: {n_req}\")\n","        rows += data[\"Records\"]\n","    \n","    df = pd.DataFrame(rows, columns=data[\"Metadata\"][\"ColumnName\"])\n","    df['device'] = device_id\n","    df = df.astype({\"temperature\": \"float64\", \"co2\": \"float64\", \"iaq\": \"float64\", \"pressure\": \"float64\", \"humidity\": \"float64\", \"voc\": \"float64\", \"device\": \"str\"})\n","    df.time = pd.to_datetime(df.time)\n","    df.time = df.time.dt.tz_localize('UTC').dt.tz_convert('Australia/ACT')\n","    return df.sort_values('time')\n","\n","def get_device_predictions(device_id, from_ts, to_ts, return_response=False):\n","    response = requests.get(\n","        url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/predictions?from={from_ts}&to={to_ts}&device={device_id}\",\n","        headers=get_request_header(),\n","    )\n","    n_req = 1\n","    print(f\"REQUEST: {n_req}\")\n","    if return_response:\n","        return response\n","    data = response.json()\n","    rows = data[\"Records\"]\n","    \n","    while data[\"ExecutionInfo\"].get(\"NextToken\"):\n","        token = data[\"ExecutionInfo\"].get(\"NextToken\")\n","        data = requests.get(\n","            url=f\"https://jov3dcr05d.execute-api.ap-southeast-2.amazonaws.com/v1/predictions?from={from_ts}&to={to_ts}&device={device_id}&nextToken={token}\",\n","            headers=get_request_header(),\n","        ).json()\n","        n_req += 1\n","        print(f\"REQUEST: {n_req}\")\n","        rows += data[\"Records\"]\n","    \n","    df = pd.DataFrame(rows, columns=data[\"Metadata\"][\"ColumnName\"])\n","    df['device'] = device_id\n","    df = df.astype({f\"iaq~{i}\": \"float64\" for i in range(1, 16)})\n","    df = df.astype({\"device\": \"str\"})\n","    df.time = pd.to_datetime(df.time)\n","    df.time = df.time.dt.tz_localize('UTC').dt.tz_convert('Australia/ACT')\n","    return df.sort_values('time')"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"c45a8544-7874-4472-912b-84f95dcde486","showTitle":false,"title":""}},"outputs":[],"source":["def get_device_measures_last(device_id, return_response=False, **kwargs):\n","    end = datetime.now().replace(second=0)\n","    start = end - timedelta(**kwargs)\n","\n","    start_ts = int(datetime.timestamp(start))\n","    end_ts = int(datetime.timestamp(end))\n","    return get_device_measures(device_id, start_ts, end_ts, return_response=return_response)\n","\n","def get_device_predictions_last(device_id, return_response=False, **kwargs):\n","    end = datetime.now().replace(second=0)\n","    start = end - timedelta(**kwargs)\n","\n","    start_ts = int(datetime.timestamp(start))\n","    end_ts = int(datetime.timestamp(end))\n","    return get_device_predictions(device_id, start_ts, end_ts, return_response=return_response)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"9090f78f-606e-453e-bad0-e7690247c760","showTitle":false,"title":""}},"outputs":[],"source":["def get_all_device_measures(from_ts, to_ts):\n","    devices = get_device_ids()\n","    df = pd.concat([get_device_measures(d, from_ts, to_ts) for d in devices])\n","    df = df.sort_values('time')\n","    return df\n","\n","def get_all_device_predictions(from_ts, to_ts):\n","    devices = get_device_ids()\n","    df = pd.concat([get_device_predictions(d, from_ts, to_ts) for d in devices])\n","    df = df.sort_values('time')\n","    return df"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{},"inputWidgets":{},"nuid":"113f6d3e-aaea-420a-8ff5-17aa5f65a480","showTitle":false,"title":""}},"outputs":[],"source":["def get_all_device_measures_last(**kwargs):\n","    devices = get_device_ids()\n","    df = pd.concat([get_device_measures_last(d, **kwargs) for d in devices])\n","    df = df.sort_values('time')\n","    return df\n","\n","def get_all_device_predictions_last(**kwargs):\n","    devices = get_device_ids()\n","    df = pd.concat([get_device_predictions_last(d, **kwargs) for d in devices])\n","    df = df.sort_values('time')\n","    return df"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"api_functions","notebookOrigID":587531220402841,"widgets":{}},"kernelspec":{"display_name":"Python 3.10.4 ('aws-tsYraiDZ')","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.4"},"vscode":{"interpreter":{"hash":"baff73f41d484e87910e2c332e636dcf571e10e117077c8a2bd319afbc8d7769"}}},"nbformat":4,"nbformat_minor":0}