Merge 0.5.2 (#224)

* Update notebook execution timestamps and fix styling in TextResult.js (#214) * Update ReactMarkdown and remark-gfm versions, add StringUtils.js utility class, and refactor CodeResult and Cell components (#216) * Refactor code to improve session handling in NotebookModel.js and StringUtils.js, and add key prop to Cell component in Notebook.js (#217) * Add dependencies for @coreui/icons, @coreui/icons-react, @coreui/react, @datalayer/icons-react, @jupyter/web-components, and @jupyterlab/ui-components in package.json (#220) Add SparkModel class for handling Spark session information Add SparkApplicationId component for displaying Spark application ID in NotebookHeader Update NotebookHeader component to include SparkApplicationId component Update NotebookKernel component to use JupyterKernelIcon from @datalayer/icons-react Refactor styling in NotebookKernel component * Refactor code to use PawMarkSparkSession instead of DataPulseSparkSession in startup.py and NotebookKernel.js (#221) * Update Notebook.js and NotebookHeader.js to include setSparkAppId prop (#222) * Refactor code to add DisplayResult component for displaying output in NotebookCell (#223)
xuwenyihust · Jun 22, 2024 · 329d80a · 329d80a
1 parent 6ed0872
commit 329d80a
Show file tree

Hide file tree

Showing 24 changed files with 1,441 additions and 149 deletions.
diff --git a/README.md b/README.md
@@ -129,6 +129,8 @@ PawMark is a platform for big data and AI. It is based on Apache Spark and Kuber
 
 
 ## Versions
+<details>
+<summary>Details</summary>
 | Component    | Version |
 |--------------|---------|
 | Scala        | 2.12    |
@@ -140,7 +142,7 @@ PawMark is a platform for big data and AI. It is based on Apache Spark and Kuber
 | Airflow      | 2.9.1   |
 | Postgres     | 13      |
 | React        | 18.3.1  |
-
+</details>
 
 ## License
 This project is licensed under the terms of the Apache-2.0 license.

diff --git a/docker/notebook/startup.py b/docker/notebook/startup.py
@@ -73,7 +73,7 @@ def set_env():
 
     display(Markdown(msg))
 
-class DataPulseSparkSession:
+class PawMarkSparkSession:
 
     def __init__(self, spark_session):
         self._spark_session = spark_session
@@ -100,7 +100,7 @@ def _repr_html_(self):
         """
 
 def create_spark_dev():
-    spark = DataPulseSparkSession(SparkSession.builder \
+    spark = PawMarkSparkSession(SparkSession.builder \
         .appName("PySpark Example") \
         .master("spark://spark-master:7077") \
         .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0") \

diff --git a/examples/222/111.ipynb b/examples/222/111.ipynb
diff --git a/examples/222/1111.ipynb b/examples/222/1111.ipynb
diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb
@@ -2,6 +2,8 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "lastExecutionResult": null,
+   "lastExecutionTime": null,
    "metadata": {},
    "source": [
     "# DataPulse\n",
@@ -17,41 +19,74 @@
     "  ```"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "lastExecutionResult": null,
+   "lastExecutionTime": null,
+   "metadata": {},
+   "source": [
+    "## Table Display\n",
+    "\n",
+    "| id| name |\n",
+    "|---|------|\n",
+    "|  0| a    |\n",
+    "|  1| b    |\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "lastExecutionResult": null,
+   "lastExecutionTime": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "lastExecutionResult": null,
+   "lastExecutionTime": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "666\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(666)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 4,
+   "lastExecutionResult": "success",
+   "lastExecutionTime": "2024-06-21 07:08:28",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
        "\n",
-       "            <div>\n",
-       "                <p><b>SparkSession - in-memory</b></p>\n",
-       "                \n",
-       "        <div>\n",
-       "            <p><b>SparkContext</b></p>\n",
-       "\n",
-       "            <p><a href=\"http://95b24b6c609c:4040\">Spark UI</a></p>\n",
-       "\n",
-       "            <dl>\n",
-       "              <dt>Version</dt>\n",
-       "                <dd><code>v3.5.0</code></dd>\n",
-       "              <dt>Master</dt>\n",
-       "                <dd><code>spark://spark-master:7077</code></dd>\n",
-       "              <dt>AppName</dt>\n",
-       "                <dd><code>PySpark Example</code></dd>\n",
-       "            </dl>\n",
+       "        <div style=\"border: 1px solid #e8e8e8; padding: 10px;\">\n",
+       "            <h3>Spark Session Information</h3>\n",
+       "            <p><strong>Application ID:</strong> app-20240621070823-0011</p>\n",
+       "            <p><strong>Spark UI:</strong> <a href=\"http://localhost:18080/history/app-20240621070823-0011\">http://localhost:18080/history/app-20240621070823-0011</a></p>\n",
        "        </div>\n",
-       "        \n",
-       "            </div>\n",
        "        "
       ],
       "text/plain": [
-       "<pyspark.sql.session.SparkSession at 0x7f4815385510>"
+       "Custom Spark Session (App ID: app-20240621070823-0011) - UI: http://f294e9875600:4040"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 1,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -63,12 +98,66 @@
   {
    "cell_type": "code",
    "execution_count": 5,
+   "lastExecutionResult": "success",
+   "lastExecutionTime": "2024-06-21 07:09:57",
    "metadata": {},
    "outputs": [],
    "source": [
     "spark.range(0, 5) \\\n",
     "    .write.format(\"delta\").mode(\"overwrite\").saveAsTable(\"test\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "lastExecutionResult": "success",
+   "lastExecutionTime": "2024-06-21 07:09:57",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---------+---------+-----------+\n",
+      "|namespace|tableName|isTemporary|\n",
+      "+---------+---------+-----------+\n",
+      "|  default|     test|      false|\n",
+      "+---------+---------+-----------+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "spark.sql(\"show tables\").show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "lastExecutionResult": "success",
+   "lastExecutionTime": "2024-06-21 07:11:12",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+---+\n",
+      "| id|\n",
+      "+---+\n",
+      "|  2|\n",
+      "|  3|\n",
+      "|  4|\n",
+      "|  0|\n",
+      "|  1|\n",
+      "+---+\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "spark.sql(\"select * from test\").show()"
+   ]
   }
  ],
  "metadata": {