standard_lib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A tour through the python standard library\n",
    "\n",
    "python comes with \"batteries included\", the standard library is extremely \n",
    "rich and powerfull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "\n",
    "$.getScript('https://kmahelona.github.io/ipython_notebook_goodies/ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 id=\"tocheading\">Table of Contents</h1>\n",
    "<div id=\"toc\"></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `os`, `os.path`: Accessing the operation system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## environment variables\n",
    "\n",
    "useful for configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.getenv('STUFF'), os.getenv('STUFF', default='foo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['STUFF'] = 'bar'\n",
    "os.getenv('STUFF', default='foo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del os.environ['STUFF']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pwd, cwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(os.getcwd())\n",
    "\n",
    "os.chdir('data')\n",
    "\n",
    "print(os.getcwd())\n",
    "\n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manipulating files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('example', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = 'example/test.txt'\n",
    "with open(fname, 'w') as f:\n",
    "    f.write('Hello World\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stat_res = os.stat(fname)\n",
    "'{:o}'.format(stat_res.st_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use an octal integer\n",
    "os.chmod(fname, 0o600) # equivalent to chmod 666 <filename>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stat_res = os.stat(fname)\n",
    "'{:o}'.format(stat_res.st_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('example/build')\n",
    "\n",
    "print(os.listdir('example/'))\n",
    "\n",
    "if os.path.exists('example/build'):\n",
    "    os.rmdir('example/build')\n",
    "\n",
    "print(os.listdir('example'))\n",
    "\n",
    "if os.path.isfile('example/test.txt'):\n",
    "    os.remove('example/test.txt')\n",
    "    \n",
    "print(os.listdir('example'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.path.join('build', 'example', 'test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.path.splitext('test.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a lot of files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import random\n",
    "\n",
    "dirnames = ['foo', 'bar', 'baz']\n",
    "extensions = ['.txt', '.dat', '.docx', '.xslx', '.fits', '.png']\n",
    "filenames = [name + ext for name in dirnames for ext in extensions]\n",
    "\n",
    "def create_files_and_subfolders(path, depth=3):\n",
    "    n_files = random.randint(1, 5)\n",
    "    for i in range(n_files):\n",
    "        open(os.path.join(path, random.choice(filenames)), 'w').close()\n",
    "    \n",
    "    if depth == 0:\n",
    "        return\n",
    "    \n",
    "    n_subdirs = random.randint(1, 3)\n",
    "    for i in range(n_subdirs):\n",
    "        subdir = os.path.join(path, random.choice(dirnames))\n",
    "        os.makedirs(subdir, exist_ok=True)\n",
    "        create_files_and_subfolders(subdir, depth=depth - 1)\n",
    "        \n",
    "create_files_and_subfolders('example', 3)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.walk goes recursivly through all directories and returns files and subdirectories\n",
    "\n",
    "for root, dirs, files in os.walk('example'):\n",
    "    print(root)\n",
    "    for d in sorted(dirs):\n",
    "        print('  ', d)\n",
    "    for f in sorted(files):\n",
    "        print('  ', f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "shutil.rmtree('example', ignore_errors=True) # rm -rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `subprocess`, calling shell commands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess as sp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = sp.check_output(['conda', 'list', 'numpy'])\n",
    "\n",
    "print(result)\n",
    "print()\n",
    "print(result.decode())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "more complex task, provide read stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/0/00/Crab_Nebula.jpg/480px-Crab_Nebula.jpg'\n",
    "    \n",
    "process = sp.run(\n",
    "    ['curl', url],\n",
    "    stdout=sp.PIPE,\n",
    "    stderr=sp.PIPE,    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from io import BytesIO # File-Like object in memory\n",
    "\n",
    "img = BytesIO(process.stdout)\n",
    "\n",
    "plt.imshow(plt.imread(img, format='jpg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `threading`, `multiprocessing`, `concurrent`: Doing stuff in parallel\n",
    "\n",
    "There are much more advanced libraries for this, e.g. `joblib`\n",
    "\n",
    "https://pythonhosted.org/joblib/\n",
    "\n",
    "Python can only run one python statement at a time through one interpreter, even\n",
    "when using multiple threads, only one thread at a time will be executed. \n",
    "This is called the **Global Interpreter Lock** (GIL).\n",
    "\n",
    "So you only gain in perfomance using threads, when:\n",
    "\n",
    "* there are I/O bound operations (Reading files, downloads, waiting on sockets)\n",
    "* When you use a lot of c-extensions (like numpy, pandas and basically all the scientific python stack)\n",
    "* sleeping\n",
    "\n",
    "For truly parallel operations, you need new python processes, this can be done with the `multiprocessing` module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import random\n",
    "import time\n",
    "\n",
    "def do_work():\n",
    "    time.sleep(random())\n",
    "    print('hello')\n",
    "    time.sleep(1)\n",
    "    print('world')\n",
    "\n",
    "for i in range(3):\n",
    "    do_work()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from threading import Thread\n",
    "\n",
    "\n",
    "threads = [Thread(target=do_work) for i in range(3)]\n",
    "\n",
    "for t in threads:\n",
    "    t.start()\n",
    "\n",
    "# block until all threads are done\n",
    "for t in threads:\n",
    "    t.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def pi_mc(n):\n",
    "    n_circle = 0\n",
    "    for i in range(n):\n",
    "        x = random.uniform(-1, 1)\n",
    "        y = random.uniform(-1, 1)\n",
    "        \n",
    "        if (x**2 + y**2) <= 1:\n",
    "            n_circle += 1\n",
    "    \n",
    "    return 4 * n_circle / n\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "\n",
    "n_jobs = 100\n",
    "n_iters = 100000\n",
    "iterable = [n_iters] * n_jobs\n",
    "\n",
    "\n",
    "with Pool(4) as pool:\n",
    "    results = pool.map(pi_mc, iterable)\n",
    "    \n",
    "print(sum(results) / len(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait, as_completed\n",
    "import time\n",
    "import os\n",
    "\n",
    "def hello(i):\n",
    "    time.sleep(random.random() * 4)  # sleep 0 to 4 seconds\n",
    "    pid = os.getpid()\n",
    "    return f'hello from job {i} with pid {pid}'\n",
    "\n",
    "\n",
    "with ThreadPoolExecutor(4) as pool:\n",
    "    futures = [pool.submit(hello, i) for i in range(20)]\n",
    "    \n",
    "    for f in as_completed(futures):\n",
    "        print(f.result())\n",
    "\n",
    "print('all done')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait, as_completed\n",
    "import time\n",
    "import os\n",
    "\n",
    "\n",
    "def square(x):\n",
    "    # heavy work\n",
    "    time.sleep(random.random() * 0.1)\n",
    "    return x * x\n",
    "    \n",
    "\n",
    "\n",
    "with ThreadPoolExecutor(4) as pool:\n",
    "    results = pool.map(square, range(20))\n",
    "\n",
    "print(list(results))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `collections`: Useful Containers\n",
    "\n",
    "example: count words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = '''Alice was beginning to get very tired of sitting by her sister on the bank,\n",
    "and of having nothing to do:\n",
    "once or twice she had peeped into the book her sister was reading,\n",
    "but it had no pictures or conversations in it,\n",
    "`and what is the use of a book,'\n",
    "thought Alice\n",
    "`without pictures or conversation?'\n",
    "So she was considering in her own mind (as well as she could,\n",
    "for the hot day made her feel very sleepy and stupid),\n",
    "whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies,\n",
    "when suddenly a White Rabbit with pink eyes ran close by her.\n",
    "\n",
    "There was nothing so very remarkable in that;\n",
    "nor did Alice think it so very much out of the way to hear the Rabbit say to itself,\n",
    "`Oh dear! Oh dear! I shall be late!' (when she thought it over afterwards,\n",
    "it occurred to her that she ought to have wondered at this, but at the time it all seemed quite natural);\n",
    "but when the Rabbit actually took a watch out of its waistcoat-pocket, and looked at it, and then hurried on,\n",
    "Alice started to her feet,\n",
    "for it flashed across her mind that she had never before seen a rabbit with either a waistcoat-pocket,\n",
    "or a watch to take out of it, and burning with curiosity, she ran across the field after it,\n",
    "and fortunately was just in time to see it pop down a large rabbit-hole under the hedge.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation\n",
    "text = text.translate({ord(c): None for c in ',`;.!:?()\\''})\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# solution one, pure python\n",
    "\n",
    "counts = {}\n",
    "for word in text.split():\n",
    "    if word not in counts:\n",
    "        counts[word] = 0\n",
    "    counts[word] += 1\n",
    "\n",
    "for name, count in sorted(counts.items(), key=lambda s: s[1], reverse=True)[:10]:\n",
    "    print(name, count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`collections.defaultdict` takes a function that initialises entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# solution 2, using default dict\n",
    "from collections import defaultdict\n",
    "\n",
    "counts = defaultdict(int)  # int() returns 0\n",
    "\n",
    "for word in text.split():\n",
    "    counts[word] += 1\n",
    "\n",
    "for name, count in sorted(counts.items(), key=lambda s: s[1], reverse=True)[:10]:\n",
    "    print(name, count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# solution 3, Counter\n",
    "from collections import Counter\n",
    "\n",
    "counts = Counter(text.split())\n",
    "for word, count in counts.most_common(10):\n",
    "    print(word, count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "color = (1.0, 0, 0, 1.0) ## what is this? RGBA? CMYK? Something else"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "\n",
    "RGBAColor = namedtuple('RGBAColor', ['r', 'g', 'b', 'a'])\n",
    "color = RGBAColor(1.0, 0, 0, 1.0)\n",
    "color"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `functools`: functional programming in python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools\n",
    "\n",
    "# reduce was a builtin in py2\n",
    "functools.reduce(lambda v1, v2: v1 + v2, range(100)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "newline_print = functools.partial(print, sep='\\n')\n",
    "\n",
    "newline_print(*range(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fib(n):\n",
    "    if n == 0:\n",
    "        return 0\n",
    "    if n in (1, 2):\n",
    "        return 1\n",
    "    else:\n",
    "        return fib(n - 1) + fib(n - 2)\n",
    "    \n",
    "\n",
    "@functools.lru_cache(maxsize=500)\n",
    "def fib_cached(n):\n",
    "    if n == 0:\n",
    "        return 0\n",
    "    if n in (1, 2):\n",
    "        return 1\n",
    "    else:\n",
    "        return fib(n - 1) + fib(n - 2)\n",
    "    \n",
    "fib(10)\n",
    "\n",
    "print('cached')\n",
    "fib_cached(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "fib(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%timeit\n",
    "fib_cached(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `re`, regular expressions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = ['img001.png', 'img002.png', 'world.txt', 'foo.txt', 'stuff.dat', 'test.xslx']\n",
    "\n",
    "\n",
    "for f in files:\n",
    "    \n",
    "    m = re.match('img([0-9]{3}).png', f)\n",
    "    if m:\n",
    "        print(f, m.groups())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# itertools, more iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "longer = [1, 2, 3, 4, 5]\n",
    "shorter = ['a', 'b', 'c']\n",
    "\n",
    "print('{:-^40}'.format(' zip '))\n",
    "for l, s in zip(longer, shorter):\n",
    "    print(l, s)\n",
    "\n",
    "print('{:-^40}'.format(' zip_longest '))\n",
    "for l, s in itertools.zip_longest(longer, shorter):\n",
    "    print(l, s)\n",
    "    \n",
    "print('{:-^40}'.format(' zip_longest, with fillvalue'))\n",
    "for l, s in itertools.zip_longest(longer, shorter, fillvalue='z'):\n",
    "    print(l, s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(itertools.permutations('ABC'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(itertools.combinations('ABC', 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `argparse`: commandline options\n",
    "\n",
    "Alternatives:\n",
    "\n",
    "- `click`: http://click.pocoo.org/6/\n",
    "- `docopt`: http://docopt.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import ArgumentParser\n",
    "\n",
    "parser = ArgumentParser()\n",
    "parser.add_argument('inputfile') # positional argument\n",
    "parser.add_argument('-o', '--output') # option\n",
    "parser.add_argument('-n', '--number', default=0, type=int) # option with default value and type\n",
    "\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = parser.parse_args(['data.csv']) # of no arguments give,  use sys.argv\n",
    "print(args.number, args.inputfile, args.output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = parser.parse_args(['data.csv', '--number=5', '-o',  'test.csv'])\n",
    "\n",
    "print(args.number, args.inputfile, args.output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `copy`, copy operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [1, 2, [4, 5]]\n",
    "b = a\n",
    "b[1] = 3\n",
    "b[2][1] = 'Hello'\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import copy\n",
    "\n",
    "a = [1, 2, [4, 5]]\n",
    "b = copy(a)\n",
    "b[1] = 3\n",
    "b[2][1] = 'Hello'\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "\n",
    "a = [1, 2, [4, 5]]\n",
    "b = deepcopy(a)\n",
    "b[1] = 3\n",
    "b[2][1] = 'Hello'\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `tempfile`, Tempory Files and Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile\n",
    "import os\n",
    "\n",
    "print(tempfile.gettempdir())\n",
    "\n",
    "# file will be deleted when exiting the with block\n",
    "with tempfile.NamedTemporaryFile(prefix='python_course_', suffix='.txt', mode='w') as f:\n",
    "    path = f.name\n",
    "    f.write('Hello World')\n",
    "\n",
    "    print('f exists:', os.path.exists(path))\n",
    "\n",
    "print('f exists:', os.path.exists(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# directory with all contents will be deleted when we exit the with block\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    print(d)\n",
    "    \n",
    "    with open(os.path.join(d, 'myfile.txt'), 'w') as f:\n",
    "        print(f.name)\n",
    "        f.write('Hello World')\n",
    "        \n",
    "        print('d exists:', os.path.exists(d))\n",
    "\n",
    "print('d exists:', os.path.exists(d))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `struct`, parsing binary data\n",
    "\n",
    "It's like every other day in the office.  \n",
    "Your supervisor does not like standardized file formats.\n",
    "Like `.fits`, or `.hdf` or, (\"Are you completely insane?\") `.json` or `.yaml`.  \n",
    "Because, you know, they are super hard to read using Fortran 77.\n",
    "\n",
    "So he sends you data in \"an easy to read\" file format, a custom, proprietary binary blob:\n",
    "\n",
    "* First 4 bytes is an unsigned integer, containing the length of the comment string\n",
    "* Then N bytes comment encoded using utf-8\n",
    "    * utf-8? Are you kidding me? ASCII!!!\n",
    "* Then triples of double, double, unsigned int for x, y, n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import struct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pack data\n",
    "struct.pack('II', 2, 1024) # pack to unsigned 32bit integers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unpack data\n",
    "struct.unpack('f', b'\\xdb\\x0f\\x49\\x40') # 32-bit float"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a binary file, let's add a comment first\n",
    "with open('letsinventourownbinaryformat.dat', 'wb') as f:\n",
    "    comment = 'Here, have this awesome data!'.encode('ascii')\n",
    "\n",
    "    f.write(struct.pack('I', len(comment)))\n",
    "    f.write(comment)\n",
    "    \n",
    "    for i in range(1000):\n",
    "        x = random.uniform(-1, 1)\n",
    "        y = random.uniform(-1, 1)\n",
    "        n = random.randint(1, 200 - int(100 * (x**2 + y**2)))\n",
    "        \n",
    "        f.write(struct.pack('ddI', x, y, n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the file back in\n",
    "\n",
    "xs, ys, ns = [], [], []\n",
    "with open('letsinventourownbinaryformat.dat', 'rb') as f:\n",
    "    comment_length, = struct.unpack('I', f.read(4))\n",
    "    comment = f.read(comment_length).decode('ascii')\n",
    "    \n",
    "    size = struct.calcsize('ddI')\n",
    "    data = f.read(size)\n",
    "    while data:\n",
    "        x, y, n = struct.unpack('ddI', data)\n",
    "        xs.append(x)\n",
    "        ys.append(y)\n",
    "        ns.append(n)\n",
    "        data = f.read(size)\n",
    "\n",
    "print(comment)\n",
    "print(len(xs))\n",
    "print(xs[0], ys[0], ns[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "numpy actually is much better for parsing homogenous binary data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "with open('letsinventourownbinaryformat.dat', 'rb') as f:\n",
    "    comment_length, = struct.unpack('I', f.read(4))\n",
    "    comment = f.read(comment_length).decode('ascii')\n",
    "    \n",
    "    blob = f.read()\n",
    "    data = np.frombuffer(blob, dtype=[('x', 'float64'), ('y', 'float64'), ('n', 'uint32')])\n",
    "    \n",
    "print(data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `email`, `smtplib`, `getpass`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import smtplib\n",
    "from email.message import EmailMessage\n",
    "from getpass import getpass\n",
    "\n",
    "text = '''Hello Participants,\n",
    "\n",
    "Thanks for attending!\n",
    "Do not forget to provide feedback\n",
    "\n",
    "Cheers,\n",
    "Max\n",
    "'''\n",
    "\n",
    "msg = EmailMessage()\n",
    "msg.set_content(text)\n",
    "\n",
    "\n",
    "msg['Subject'] = 'Email demonstration at the Python Course'\n",
    "msg['From'] = 'Firstname Surname <email address>'\n",
    "msg['To'] = 'Firstname Surname <email address>, Firstname Surname <email address>'\n",
    "\n",
    "# Send the message via our own SMTP server.\n",
    "s = smtplib.SMTP_SSL(host='server')\n",
    "\n",
    "\n",
    "\n",
    "s.login(input('Username: '), getpass('Enter password: '))\n",
    "\n",
    "s.send_message(\n",
    "    from_addr=msg['From'], \n",
    "    to_addrs=msg['To'],\n",
    "    msg=msg,\n",
    ")\n",
    "s.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}