SystemML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install SystemML\n",
    "! pip install --upgrade https://github.com/niketanpansare/future_of_data/raw/master/systemml-1.1.0-SNAPSHOT-python.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "! pip show systemml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initial Configuration\n",
    "import findspark \n",
    "findspark.init()\n",
    "\n",
    "from pyspark.context import SparkContext\n",
    "from pyspark.sql.session import SparkSession\n",
    "sc = SparkContext('local') # we are hosting clusters locally.\n",
    "spark = SparkSession(sc) # create an interface to the spark session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archiver-Version: Plexus Archiver\n",
      "Artifact-Id: systemml\n",
      "Build-Jdk: 1.8.0_111\n",
      "Build-Time: 2018-01-11 16:39:06 CST\n",
      "Built-By: biuser\n",
      "Created-By: Apache Maven 3.0.5\n",
      "Group-Id: org.apache.systemml\n",
      "Main-Class: org.apache.sysml.api.DMLScript\n",
      "Manifest-Version: 1.0\n",
      "Minimum-Recommended-Spark-Version: 2.1.0\n",
      "Version: 1.1.0-SNAPSHOT\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from systemml import MLContext, dml\n",
    "\n",
    "# create a MLContext object\n",
    "ml = MLContext(sc)\n",
    "print(ml.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ANTLR Tool version 4.5.3 used for code generation does not match the current runtime version 4.7ANTLR Runtime version 4.5.3 used for parser compilation does not match the current runtime version 4.7ANTLR Tool version 4.5.3 used for code generation does not match the current runtime version 4.7ANTLR Runtime version 4.5.3 used for parser compilation does not match the current runtime version 4.7\n",
      "Hello World\n",
      "SystemML Statistics:\n",
      "Total execution time:\t\t0.002 sec.\n",
      "Number of executed Spark inst:\t0.\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLResults"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "script = dml(\"\"\"\n",
    "            print('Hello World');\n",
    "            \"\"\")\n",
    "ml.execute(script)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SystemML Statistics:\n",
      "Total execution time:\t\t0.000 sec.\n",
      "Number of executed Spark inst:\t0.\n",
      "\r\n",
      "\n",
      "Hello World\n"
     ]
    }
   ],
   "source": [
    "script = dml(\"\"\"\n",
    "            s = 'Hello World'\n",
    "            \"\"\").output('s')\n",
    "output_string = ml.execute(script).get('s')\n",
    "print(output_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "plt.switch_backend('agg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example 1. Matrix Multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                         (0 + 1) / 59]\n",
      "[Stage 0:>                                                         (1 + 1) / 59]\n",
      "[Stage 0:=>                                                        (2 + 1) / 59]\n",
      "[Stage 0:==>                                                       (3 + 1) / 59]\n",
      "[Stage 0:===>                                                      (4 + 1) / 59]\n",
      "[Stage 0:====>                                                     (5 + 1) / 59]\n",
      "[Stage 0:=====>                                                    (6 + 1) / 59]\n",
      "[Stage 0:=======>                                                  (8 + 1) / 59]\n",
      "[Stage 0:========>                                                 (9 + 1) / 59]\n",
      "[Stage 0:=========>                                               (10 + 1) / 59]\n",
      "[Stage 0:==========>                                              (11 + 1) / 59]\n",
      "[Stage 0:===========>                                             (12 + 1) / 59]\n",
      "[Stage 0:============>                                            (13 + 1) / 59]\n",
      "[Stage 0:=============>                                           (14 + 1) / 59]\n",
      "[Stage 0:==============>                                          (15 + 1) / 59]\n",
      "[Stage 0:===============>                                         (16 + 1) / 59]\n",
      "[Stage 0:================>                                        (17 + 1) / 59]\n",
      "[Stage 0:=================>                                       (18 + 1) / 59]\n",
      "[Stage 0:==================>                                      (19 + 1) / 59]\n",
      "[Stage 0:===================>                                     (20 + 1) / 59]\n",
      "[Stage 0:====================>                                    (21 + 1) / 59]\n",
      "[Stage 0:=====================>                                   (22 + 1) / 59]\n",
      "[Stage 0:======================>                                  (23 + 1) / 59]\n",
      "[Stage 0:=======================>                                 (24 + 1) / 59]\n",
      "[Stage 0:========================>                                (25 + 1) / 59]\n",
      "[Stage 0:=========================>                               (26 + 1) / 59]\n",
      "[Stage 0:==========================>                              (27 + 1) / 59]\n",
      "[Stage 0:===========================>                             (28 + 1) / 59]\n",
      "[Stage 0:============================>                            (29 + 1) / 59]\n",
      "[Stage 0:============================>                            (30 + 1) / 59]\n",
      "[Stage 0:=============================>                           (31 + 1) / 59]\n",
      "[Stage 0:==============================>                          (32 + 1) / 59]\n",
      "[Stage 0:===============================>                         (33 + 1) / 59]\n",
      "[Stage 0:================================>                        (34 + 1) / 59]\n",
      "[Stage 0:=================================>                       (35 + 1) / 59]\n",
      "[Stage 0:==================================>                      (36 + 1) / 59]\n",
      "[Stage 0:===================================>                     (37 + 1) / 59]\n",
      "[Stage 0:====================================>                    (38 + 1) / 59]\n",
      "[Stage 0:=====================================>                   (39 + 1) / 59]\n",
      "[Stage 0:======================================>                  (40 + 1) / 59]\n",
      "[Stage 0:=======================================>                 (41 + 1) / 59]\n",
      "[Stage 0:========================================>                (42 + 1) / 59]\n",
      "[Stage 0:=========================================>               (43 + 1) / 59]\n",
      "[Stage 0:==========================================>              (44 + 1) / 59]\n",
      "[Stage 0:===========================================>             (45 + 1) / 59]\n",
      "[Stage 0:============================================>            (46 + 1) / 59]\n",
      "[Stage 0:=============================================>           (47 + 1) / 59]\n",
      "[Stage 0:==============================================>          (48 + 1) / 59]\n",
      "[Stage 0:===============================================>         (49 + 1) / 59]\n",
      "[Stage 0:================================================>        (50 + 1) / 59]\n",
      "[Stage 0:=================================================>       (51 + 1) / 59]\n",
      "[Stage 0:==================================================>      (52 + 1) / 59]\n",
      "[Stage 0:===================================================>     (53 + 1) / 59]\n",
      "[Stage 0:====================================================>    (54 + 1) / 59]\n",
      "[Stage 0:=====================================================>   (55 + 1) / 59]\n",
      "[Stage 0:======================================================>  (56 + 1) / 59]\n",
      "[Stage 0:=======================================================> (57 + 1) / 59]\n",
      "[Stage 0:========================================================>(58 + 1) / 59]\n",
      "SystemML Statistics:\n",
      "Total execution time:\t\t69.715 sec.\n",
      "Number of executed Spark inst:\t2.\n",
      "\n",
      "                                                                                \n",
      "\n",
      "62596128726.02343\n"
     ]
    }
   ],
   "source": [
    "script = dml(\"\"\"\n",
    "            X = rand(rows=$nr, cols=1000, sparsity=0.5)\n",
    "            A = t(X) %*% X\n",
    "            s = sum(A)\n",
    "            \"\"\").input('$nr',1e6).output('s')\n",
    "s = ml.execute(script).get('s')\n",
    "print(s)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}