From f71ffcfa1f6137e0e3a47a65d5870ef40971a429 Mon Sep 17 00:00:00 2001 From: Gilad Shaham Date: Thu, 15 Sep 2022 12:13:07 +0100 Subject: [PATCH] add example notebook --- grafana-grafwiz.ipynb | 479 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 grafana-grafwiz.ipynb diff --git a/grafana-grafwiz.ipynb b/grafana-grafwiz.ipynb new file mode 100644 index 0000000..04d12e7 --- /dev/null +++ b/grafana-grafwiz.ipynb @@ -0,0 +1,479 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generating a Grafana Dashboard with grafwiz\n", + "\n", + "This tutorial demonstrates how to use [grafwiz](https://github.com/v3io/grafwiz), Iguazio's open-source Python library for generating a Grafana dashboard programmatically." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- [Setup](#grafwiz-setup)\n", + "- [Generating Data](#grafwiz-gen-data)\n", + "- [Creating a DataFrame with the Generated Data](#grafwiz-df-create)\n", + "- [Writing the Data to the Platform's Data Store](#grafwiz-write-to-data-store)\n", + "- [Adding a Platform Data Source to Grafana](#grafwiz-add-data-source)\n", + "- [Creating a Grafana Dashboard](#grafwiz-grafana-dashboard-create)\n", + "- [Adding Dashboard Visualization Elements](#grafwiz-add-dashboard-visualization-elements)\n", + "- [Deploying the Dashboard to Grafana](#grafwiz-grafana-dashboard-deploy)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Initialize and configure your environment." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installing grafwiz\n", + "\n", + "Run the following code to ensure that the `grafwiz` Python package is installed, and the restart the Jupyter kernel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/v3io/grafwiz --upgrade" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Grafana Service\n", + "\n", + "1. Ensure that you have a running platform Grafana service.\n", + " You can create such a service from the platform dashboard's **Services** page.\n", + "2. Copy the URL of your Grafana service from the service-name link on the **Services** dashboard page." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defining Variables\n", + "\n", + "Define variables for your environment.\n", + "\n", + "> **Note:** Replace the `` placeholder with the URL of your Grafana service, as copied in the previous step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "grafana_url = '' # TODO: Replace with the API URL of your Grafana API service.\n", + "v3io_container = 'users'\n", + "stocks_kv_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_kv_table')\n", + "stocks_tsdb_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_tsdb_table')\n", + "sym = 'XYZ'\n", + "rows = 3450" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing Libraries\n", + "\n", + "Import required libraries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from grafwiz import *\n", + "import v3io_frames as v3f\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a V3IO Frames Client\n", + "\n", + "Create a V3IO Frames client object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = v3f.Client('framesd:8081',container=v3io_container)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generating Data\n", + "\n", + "Generate random data to visualize on the Grafana dashboard." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import datetime\n", + "import numpy as np\n", + "\n", + "def generate_date(rows):\n", + " \n", + " datetimes = [datetime.datetime.today() - (random.random() * datetime.timedelta(minutes=15)) for i in range(rows)]\n", + " return datetimes\n", + "\n", + "time = sorted(generate_date(rows))\n", + "volume = np.random.randint(low=100, high=10000, size=rows)\n", + "price = np.cumsum([0.0001] * rows + np.random.random(rows))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating a DataFrame with the Generated Data\n", + "\n", + "Store the generated data in a pandas DataFrame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stocks_df = pd.DataFrame(\n", + " {'last_updated': time,\n", + " 'volume': volume,\n", + " 'price': price\n", + " })\n", + "stocks_df['symbol'] = sym\n", + "stocks_df = stocks_df.sort_values('last_updated')\n", + "stocks_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the `last_updated` column (attribute) as a DataFrame index column, which will be used to identify the ingestion times of the TSDB metric samples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stocks_df_tsdb = stocks_df\n", + "stocks_df_tsdb = stocks_df.reset_index()\n", + "stocks_df_tsdb = stocks_df.set_index(['last_updated'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Writing the Data to the Platform's Data Store\n", + "\n", + "Use the V3IO Frames API to write the data from the pandas DataFrame to TSDB and NoSQL tables in the platform's persistent data store." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Writing the Data to a TSDB Table\n", + "\n", + "Write the data from the DataFrame to a new platform TSDB table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.create(backend='tsdb', table=stocks_tsdb_table, rate='1/m', if_exists=1)\n", + "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df_tsdb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Writing the Data to a NoSQL Table\n", + "\n", + "Write the data from the DataFrame to a new platform NoSQL table in order of rows arrival, to simulate real-time data consumption." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "expr_template = \"symbol='{symbol}';price='{price}';volume='{volume}';last_updated='{last_updated}'\"\n", + "# Write the stock data to a NoSQL table\n", + "for idx, record in stocks_df.iterrows():\n", + " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record['last_updated']}\n", + " expr = expr_template.format(**stock)\n", + " client.execute('kv', stocks_kv_table, 'update', args={'key': sym, 'expression': expr})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Infer the schema of the NoSQL table to verify that it can be accessed and displayed on the dashboard." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Infer the schema of the NoSQL table\n", + "client.execute(backend='kv', table=stocks_kv_table, command='infer')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding a Platform Data Source to Grafana\n", + "\n", + "Add an \"Iguazio\" data source for the platform's custom `iguazio` Grafana data source to your Grafana service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a data source\n", + "DataSource(name='Iguazio').deploy(grafana_url, use_auth=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating a Grafana Dashboard\n", + "\n", + "Create a new Grafana dashboard that uses the platform's `iguazio` data source." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create grafana dashboard\n", + "dash = Dashboard(\"stocks\", start='now-15m', dataSource='Iguazio', end='now')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding Dashboard Visualization Elements\n", + "\n", + "Create a table for the NoSQL table and graphs for each of the metrics in the TSDB table, to be used for visualizing the data on the Grafana dashboard.\n", + "\n", + "> **Note:** It might take a few minutes for the graphs to be updated with the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a table and log viewer for the NoSQL table in one row\n", + "tbl = Table('Current Stocks Value', span=12).source(table=stocks_kv_table,fields=['symbol','volume', 'price', 'last_updated'],container=v3io_container)\n", + "dash.row([tbl])\n", + "\n", + "# Create TSDB-metric graphs\n", + "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], container=v3io_container) for metric in ['price','volume']]\n", + "dash.row(metrics_row)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deploying the Dashboard to Grafana\n", + "\n", + "Deploy the new Grafana dashboard to your Grafana service." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Deploy to Grafana\n", + "dash.deploy(grafana_url)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}