Makes Notebooks a part of testing (Issue 74)

.gitignore

@@ -107,3 +107,4 @@ venv.bak/
 .mypy_cache/
 
 .idea/
+

examples/distance_measures.ipynb

@@ -248,6 +248,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# NBVAL_SKIP\n",
+    "# our build environment has problems with cvxpy so we skip this cell\n",
+    "\n",
     "dnorm = dm.diamond_norm_distance(choi0, choi1)\n",
     "print(\"This gate is close to the identity as the diamond norm is close to zero. Dnorm= \",dnorm)"
    ]
@@ -258,6 +261,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# NBVAL_SKIP\n",
+    "\n",
     "dnorm = dm.diamond_norm_distance(choi0, choi2)\n",
     "print(\"This gate is far from identity as diamond norm = \",dnorm)"
    ]

examples/robust_phase_estimation.ipynb

@@ -383,7 +383,7 @@
    "source": [
     "from pyquil import Program\n",
     "from pyquil.quil import DefGate, Gate\n",
-    "from forest.benchmarking.utils import sigma_x, sigma_y, sigma_z\n",
+    "from forest.benchmarking.utils import X, Y, Z\n",
     "\"\"\"\n",
     "Procedure and notation follows Sections III A, B, and C in \n",
     "[RPE]  Robust Calibration of a Universal Single-Qubit Gate-Set via Robust Phase Estimation\n",
@@ -393,7 +393,7 @@
     "\"\"\"    \n",
     "q = 0  # pick a qubit\n",
     "\n",
-    "pauli_vector = np.array([sigma_x, sigma_y, sigma_z])\n",
+    "pauli_vector = np.array([X, Y, Z])\n",
     "\n",
     "alpha = .01\n",
     "epsilon = .02\n",

examples/tomography_process.ipynb

@@ -84,7 +84,7 @@
     "# version that *only* substitutes gates to Rigetti-native gates.\n",
     "# We don't want to accidentally compile away our tomography circuit\n",
     "# or map to different qubits.\n",
-    "from forest_benchmarking.compilation import basic_compile\n",
+    "from forest.benchmarking.compilation import basic_compile\n",
     "qc.compiler.quil_to_native_quil = basic_compile\n",
     "\n",
     "results = list(measure_observables(qc=qc, tomo_experiment=experiment, n_shots=100_000))\n",
@@ -130,39 +130,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'process_choi_ideal' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mNameError\u001b[0m                                Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-7-cb314dff65b7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0max1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max2\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplots\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m12\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mplot_pauli_transfer_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchoi2pauli_liouville\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_choi_ideal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Ideal'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mplot_pauli_transfer_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchoi2pauli_liouville\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocess_choi_est\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Estimate'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtight_layout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mNameError\u001b[0m: name 'process_choi_ideal' is not defined"
-     ]
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsoAAAEzCAYAAAAo4yUMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEUpJREFUeJzt3VGIpXd5x/HfY7ap1EYtzQqS3ZhIN9VFC6ZDSBFqirYkuUgu2koCwVqCi7aRglJIsaQSr6zUgrCtLlRSBY3RC1lwJVAbCYix2RCNJiGyRms2SrNqzI1oDH16cY5lsvlv5p3Zd2ayO58PDMw552XO8+bMPnxzZs6c6u4AAADP9qLtHgAAAF6IhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwMCaoVxVH6+qJ6rqW6e4varqI1V1rKoeqKpL5x8TgKnsbYB5THlG+bYkVz7P7Vcl2bf8OJDkX09/LABOw22xtwFO25qh3N13J/nJ8xxybZJP9MI9SV5eVa+ca0AA1sfeBpjHHL+jfEGSx1ZdPr68DoAXJnsbYIJdW3lnVXUgix/z5SUvecnvv+Y1r9nKuweYxX333fej7t693XNsNjsbOFtsdG/PEcqPJ9m76vKe5XXP0d2HkhxKkpWVlT569OgMdw+wtarqv7d7htM0aW/b2cDZYqN7e45fvTic5G3LV1FfnuSp7v7hDF8XgM1hbwNMsOYzylX16SRXJDm/qo4n+Yckv5Yk3f3RJEeSXJ3kWJKfJfnLzRoWgLXZ2wDzWDOUu/v6NW7vJH8920QAnBZ7G2Ae3pkPAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMDApFCuqiur6pGqOlZVNw9uv7Cq7qqq+6vqgaq6ev5RAZjCzgaYx5qhXFXnJDmY5Kok+5NcX1X7Tzrs75Pc0d1vSHJdkn+Ze1AA1mZnA8xnyjPKlyU51t2PdvfTSW5Pcu1Jx3SSly4/f1mSH8w3IgDrYGcDzGRKKF+Q5LFVl48vr1vt/UluqKrjSY4keffoC1XVgao6WlVHT5w4sYFxAViDnQ0wk7lezHd9ktu6e0+Sq5N8sqqe87W7+1B3r3T3yu7du2e6awDWyc4GmGBKKD+eZO+qy3uW1612Y5I7kqS7v5rkxUnOn2NAANbFzgaYyZRQvjfJvqq6uKrOzeKFH4dPOub7Sd6cJFX12iyWrp/TAWw9OxtgJmuGcnc/k+SmJHcmeTiLV0o/WFW3VtU1y8Pem+QdVfWNJJ9O8vbu7s0aGoAxOxtgPrumHNTdR7J4wcfq625Z9flDSd4472gAbISdDTAP78wHAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwMCmUq+rKqnqkqo5V1c2nOOatVfVQVT1YVZ+ad0wAprKzAeaxa60DquqcJAeT/HGS40nurarD3f3QqmP2Jfm7JG/s7ier6hWbNTAAp2ZnA8xnyjPKlyU51t2PdvfTSW5Pcu1Jx7wjycHufjJJuvuJeccEYCI7G2AmU0L5giSPrbp8fHndapckuaSqvlJV91TVlXMNCMC62NkAM1nzVy/W8XX2JbkiyZ4kd1fV67v7p6sPqqoDSQ4kyYUXXjjTXQOwTnY2wARTnlF+PMneVZf3LK9b7XiSw939y+7+bpJvZ7GEn6W7D3X3Snev7N69e6MzA3BqdjbATKaE8r1J9lXVxVV1bpLrkhw+6ZjPZ/HMRKrq/Cx+rPfojHMCMI2dDTCTNUO5u59JclOSO5M8nOSO7n6wqm6tqmuWh92Z5MdV9VCSu5L8bXf/eLOGBmDMzgaYT3X3ttzxyspKHz16dFvuG+B0VNV93b2y3XNsJTsbOJNtdG97Zz4AABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAICBSaFcVVdW1SNVdayqbn6e4/60qrqqVuYbEYD1sLMB5rFmKFfVOUkOJrkqyf4k11fV/sFx5yX5myRfm3tIAKaxswHmM+UZ5cuSHOvuR7v76SS3J7l2cNwHknwwyc9nnA+A9bGzAWYyJZQvSPLYqsvHl9f9v6q6NMne7v7CjLMBsH52NsBMTvvFfFX1oiQfTvLeCcceqKqjVXX0xIkTp3vXAKyTnQ0w3ZRQfjzJ3lWX9yyv+5XzkrwuyZer6ntJLk9yePTikO4+1N0r3b2ye/fujU8NwKnY2QAzmRLK9ybZV1UXV9W5Sa5LcvhXN3b3U919fndf1N0XJbknyTXdfXRTJgbg+djZADNZM5S7+5kkNyW5M8nDSe7o7ger6taqumazBwRgOjsbYD67phzU3UeSHDnpultOcewVpz8WABtlZwPMwzvzAQDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYmBTKVXVlVT1SVceq6ubB7e+pqoeq6oGq+lJVvWr+UQGYws4GmMeaoVxV5yQ5mOSqJPuTXF9V+0867P4kK939e0k+l+Qf5x4UgLXZ2QDzmfKM8mVJjnX3o939dJLbk1y7+oDuvqu7f7a8eE+SPfOOCcBEdjbATKaE8gVJHlt1+fjyulO5MckXRzdU1YGqOlpVR0+cODF9SgCmsrMBZjLri/mq6oYkK0k+NLq9uw9190p3r+zevXvOuwZgnexsgOe3a8IxjyfZu+rynuV1z1JVb0nyviRv6u5fzDMeAOtkZwPMZMozyvcm2VdVF1fVuUmuS3J49QFV9YYkH0tyTXc/Mf+YAExkZwPMZM1Q7u5nktyU5M4kDye5o7sfrKpbq+qa5WEfSvKbST5bVV+vqsOn+HIAbCI7G2A+U371It19JMmRk667ZdXnb5l5LgA2yM4GmId35gMAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABgQygAAMCCUAQBgQCgDAMCAUAYAgAGhDAAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAAaEMAAADQhkAAAaEMgAADAhlAAAYEMoAADAglAEAYEAoAwDAgFAGAIABoQwAAANCGQAABoQyAAAMCGUAABiYFMpVdWVVPVJVx6rq5sHtv15Vn1ne/rWqumjuQQGYxs4GmMeaoVxV5yQ5mOSqJPuTXF9V+0867MYkT3b37yT55yQfnHtQANZmZwPMZ8ozypclOdbdj3b300luT3LtScdcm+Tfl59/Lsmbq6rmGxOAiexsgJlMCeULkjy26vLx5XXDY7r7mSRPJfntOQYEYF3sbICZ7NrKO6uqA0kOLC/+oqq+tZX3/wJwfpIfbfcQW8w57ww77Zx/d7sH2Ap29o77vk6c806xE895Q3t7Sig/nmTvqst7lteNjjleVbuSvCzJj0/+Qt19KMmhJKmqo929spGhz1TOeWdwzme/qjq63TM8Dzt7Js55Z3DOO8NG9/aUX724N8m+qrq4qs5Ncl2SwycdczjJXyw//7Mk/9ndvZGBADgtdjbATNZ8Rrm7n6mqm5LcmeScJB/v7ger6tYkR7v7cJJ/S/LJqjqW5CdZLGYAtpidDTCfSb+j3N1Hkhw56bpbVn3+8yR/vs77PrTO488GznlncM5nvxf0+drZs3HOO4Nz3hk2dM7lp20AAPBc3sIaAAAGNj2Ud+JbqU445/dU1UNV9UBVfamqXrUdc85prXNeddyfVlVX1Rn9atsp51tVb10+zg9W1ae2esa5Tfi+vrCq7qqq+5ff21dvx5xzqqqPV9UTp/qzaLXwkeV/kweq6tKtnnFudradfdJxZ8XOTuztnbC3N2Vnd/emfWTxQpLvJHl1knOTfCPJ/pOO+askH11+fl2Sz2zmTJv9MfGc/yjJbyw/f9dOOOflcecluTvJPUlWtnvuTX6M9yW5P8lvLS+/Yrvn3oJzPpTkXcvP9yf53nbPPcN5/2GSS5N86xS3X53ki0kqyeVJvrbdM2/B42xn74BzXh53VuzsdTzO9vYZvrc3Y2dv9jPKO/GtVNc85+6+q7t/trx4TxZ/5/RMNuVxTpIPJPlgkp9v5XCbYMr5viPJwe5+Mkm6+4ktnnFuU865k7x0+fnLkvxgC+fbFN19dxZ/FeJUrk3yiV64J8nLq+qVWzPdprCz7ezVzpadndjbO2Jvb8bO3uxQ3olvpTrlnFe7MYv/uzmTrXnOyx9v7O3uL2zlYJtkymN8SZJLquorVXVPVV25ZdNtjinn/P4kN1TV8Sz+4sK7t2a0bbXef+8vdHa2nZ3krNvZib2d2NvJBnb2lr6FNc9WVTckWUnypu2eZTNV1YuSfDjJ27d5lK20K4sf412RxbNPd1fV67v7p9s61ea6Pslt3f1PVfUHWfyd3td19/9u92AwBzv7rGdv29vPsdnPKK/nrVRTz/NWqmeQKeecqnpLkvcluaa7f7FFs22Wtc75vCSvS/LlqvpeFr8XdPgMfnHIlMf4eJLD3f3L7v5ukm9nsYDPVFPO+cYkdyRJd381yYuTnL8l022fSf/ezyB2tp2dnH07O7G3E3s72cDO3uxQ3olvpbrmOVfVG5J8LIuFe6b/DlSyxjl391PdfX53X9TdF2XxO37XdPeG3nf9BWDK9/Xns3hWIlV1fhY/0nt0K4ec2ZRz/n6SNydJVb02i4V7Ykun3HqHk7xt+Urqy5M81d0/3O6hToOdbWefjTs7sbft7YX17+wteAXi1Vn8X9l3krxved2tWfyjSxYPymeTHEvyX0levdkzvQDO+T+S/E+Sry8/Dm/3zJt9zicd++Wc+a+gXusxrix+dPlQkm8muW67Z96Cc96f5CtZvLL660n+ZLtnnuGcP53kh0l+mcWzTTcmeWeSd656nA8u/5t880z/vp74ONvZdvYZ+WFvn/17ezN2tnfmAwCAAe/MBwAAA0IZAAAGhDIAAAwIZQAAGBDKAAAwIJQBAGBAKAMAwIBQBgCAgf8DeRSHmZAsNtUAAAAASUVORK5CYII=\n",
-      "text/plain": [
-       "<matplotlib.figure.Figure at 0x12c740240>"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import matplotlib.pyplot as plt\n",
     "from forest.benchmarking.superoperator_tools import choi2pauli_liouville\n",
     "from forest.benchmarking.plotting.state_process import plot_pauli_transfer_matrix\n",
     "\n",
     "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12,5))\n",
-    "plot_pauli_transfer_matrix(choi2pauli_liouville(process_choi_ideal), ax1, title='Ideal')\n",
-    "plot_pauli_transfer_matrix(choi2pauli_liouville(process_choi_est), ax2, title='Estimate')\n",
+    "plot_pauli_transfer_matrix(np.real(choi2pauli_liouville(process_choi_ideal)), ax1, title='Ideal')\n",
+    "plot_pauli_transfer_matrix(np.real(choi2pauli_liouville(process_choi_est)), ax2, title='Estimate')\n",
     "plt.tight_layout()"
    ]
   },

forest/benchmarking/plotting/state_process.py

@@ -1,4 +1,5 @@
 import matplotlib.pyplot as plt
+import itertools
 import numpy as np
 from matplotlib.colors import LinearSegmentedColormap
 

forest/benchmarking/tests/conftest.py

@@ -92,3 +92,19 @@ def side_effect(as_dict=True):
             return {acorn, agave}
     mocked_function.side_effect = side_effect
     return mocked_function
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

forest/benchmarking/tests/test_example_notebooks.py

@@ -0,0 +1,196 @@
+import pytest
+from os.path import dirname, join, realpath
+from subprocess import PIPE, run
+
+EXAMPLES_PATH = join(dirname(realpath(__file__)), '..', '..', '..', 'examples')
+
+
+# from https://stackoverflow.com/a/36058256
+def out(command):
+    result = run(command, stdout=PIPE, stderr=PIPE, universal_newlines=True, shell=True)
+    return result.stdout
+
+
+def run_notebook(filename: str):
+    return out('pytest --nbval-lax ' + EXAMPLES_PATH + f'/{filename}')
+
+
+def _is_not_empty_str(string: str) -> bool:
+    # bool(string) tests if the string is empty
+    # bool(string.strip()) tests if the string is full of spaces
+    return bool(string and string.strip())
+
+
+def check_for_failure_and_empty_output(output: str):
+    '''
+    Searches the string for the word 'failed' and checks if the string is empty of full of spaces.
+    If any of those conditions are met False is returned.
+
+    :param output: stdout from 'pytest --nbval-lax notebook.ipynb'
+    :return: bool
+    '''
+    return (output.find('failed') is -1) and _is_not_empty_str(output)
+
+
+# NOTES:
+# (1) By using the command `out` we are accessing the command line and running the basic command:
+#
+# >pytest --nbval-lax some_notebook.ipynb
+#
+# The --nbval flag runs the notebook validation plugin (to collect and test notebook cells,
+# comparing their outputs with those saved in the file). By adding the optional -lax flag we fail
+# if there is an error in executing a cell.
+#
+# (2) We are skipping any test that is longer than 30 seconds.
+#
+# (3) Each test function has a print, if the test fails then you can see where it fails from stdout.
+
+
+# TODO: go through the slow notebooks and add the comment `# NBVAL_SKIP` on the offending slow
+#  cells. This will allow us to test imports and other functionality without sacrificing runtime.
+
+
+# ~ 6 min; passed 2019/05/13
+@pytest.mark.slow
+def test_direct_fidelity_estimation_nb():
+    # print(out('ls'))
+    output_str = run_notebook('direct_fidelity_estimation.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_distance_measures_nb():
+    output_str = run_notebook('distance_measures.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_direct_entangled_states_nb():
+    output_str = run_notebook('entangled_states.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_Hinton_Plots_nb():
+    output_str = run_notebook('Hinton-Plots.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~17 min; passed 2019/05/13
+@pytest.mark.slow
+def test_quantum_volume_nb():
+    output_str = run_notebook('quantum_volume.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_qubit_spectroscopy_cz_ramsey_nb():
+    output_str = run_notebook('qubit_spectroscopy_cz_ramsey.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_qubit_spectroscopy_rabi_nb():
+    output_str = run_notebook('qubit_spectroscopy_rabi.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_qubit_spectroscopy_t1_nb():
+    output_str = run_notebook('qubit_spectroscopy_t1.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_qubit_spectroscopy_t2_nb():
+    output_str = run_notebook('qubit_spectroscopy_t2.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 sec
+def test_random_operators_nb():
+    output_str = run_notebook('random_operators.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~5 min; passed 2019/05/13
+@pytest.mark.slow
+def test_randomized_benchmarking_interleaved_nb():
+    output_str = run_notebook('randomized_benchmarking_interleaved.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~14 sec
+def test_randomized_benchmarking_simultaneous_nb():
+    output_str = run_notebook('randomized_benchmarking_simultaneous.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~7 min; passed 2019/05/13
+@pytest.mark.slow
+def test_randomized_benchmarking_unitarity_nb():
+    output_str = run_notebook('randomized_benchmarking_unitarity.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~3 sec
+def test_readout_fidelity_nb():
+    output_str = run_notebook('readout_fidelity.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~4 sec
+def test_ripple_adder_benchmark_nb():
+    output_str = run_notebook('ripple_adder_benchmark.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~40 sec; passed 2019/05/13
+@pytest.mark.slow
+def test_robust_phase_estimation_nb():
+    output_str = run_notebook('robust_phase_estimation.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~10 sec
+def test_state_and_process_plots_nb():
+    output_str = run_notebook('state_and_process_plots.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# ~4 sec
+def test_superoperator_tools_nb():
+    output_str = run_notebook('superoperator_tools.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)
+
+
+# Notebook fails at the moment
+# def test_tomography_process_nb():
+#     output_str = run_notebook('tomography_process.ipynb')
+#     print(output_str)
+#     assert output_str.find('failed') is -1
+
+
+# ~4 sec
+def test_tomography_state_nb():
+    output_str = run_notebook('tomography_state.ipynb')
+    print(output_str)
+    assert check_for_failure_and_empty_output(output_str)

requirements.txt

@@ -24,3 +24,6 @@ scikit-learn
 # doc dependencies
 sphinx
 sphinx-autodoc-typehints
+
+# for testing examples
+nbval