{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EjbyTpwb8ThZ"
      },
      "source": [
        "# How to replicate the experiments\n",
        "\n",
        "1. Download the repository f-CMI from https://github.com/hrayrhar/f-CMI\n",
        "2. Replace the files: *scripts/fcmi_scripts.py*, *scripts/fcmi_parse_results.py*, and *modules/bound_utils.py* with the corresponding files from the supplementary material. If errors occur in the next steps, it may also be necessary to replace *nnlib/nnlib/data_utils/abstract.py* with the corresponding file from the supplementary material. Also, it may be necessary to move the files *fcmi_scripts.py* and *fcmi_parse_results.py* out of the folder *scripts*, so that they are directly in the *f-CMI* folder.\n",
        "3. Run the code in the notebook. Note that the commands generated in the second code cell must be executed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mDYp92ylLE6r",
        "outputId": "2bf08d38-8947-475b-c3c3-45bdfb639917"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "/content/drive/MyDrive/klbound/f-CMI\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%cd drive/MyDrive/klbound/f-CMI\n",
        "\n",
        "# Go to correct folder, adjust as necessary\n",
        "#%cd f-CMI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0Yy1a6WV8V8w"
      },
      "outputs": [],
      "source": [
        "# Generate commands for running all experiments. When the generated commands\n",
        "# are executed, experimental data is created\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-LD\n",
        "!python fcmi_scripts.py -E cifar10-pretrained-resnet50\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-halfnoisy\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-noisy\n",
        "!python fcmi_scripts.py -E cifar10-pretrained-resnet50-LD\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-lr\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-width\n",
        "!python fcmi_scripts.py -E fcmi-mnist-4vs9-CNN-depth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HJbvinM879qS"
      },
      "outputs": [],
      "source": [
        "# Parse the results of all experiments\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-LD\n",
        "!python fcmi_parse_results.py --exp_name cifar10-pretrained-resnet50\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-halfnoisy\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-noisy\n",
        "!python fcmi_parse_results.py --exp_name cifar10-pretrained-resnet50-LD\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-lr\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-width\n",
        "!python fcmi_parse_results.py --exp_name fcmi-mnist-4vs9-CNN-depth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T1wtR-Lpv8r1"
      },
      "outputs": [],
      "source": [
        "# Imports and utilities for plotting\n",
        "%load_ext autoreload\n",
        "%autoreload 2\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jxjshe1EwMdN"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "import os\n",
        "import argparse\n",
        "import torch\n",
        "import pickle\n",
        "import numpy as np\n",
        "from nnlib.nnlib import utils\n",
        "from nnlib.nnlib.visualizations import savefig\n",
        "from modules.bound_utils import estimate_fcmi_bound_classification\n",
        "import methods\n",
        "\n",
        "%matplotlib notebook\n",
        "import matplotlib\n",
        "from matplotlib import pyplot as plt\n",
        "from nnlib.nnlib.matplotlib_utils import set_default_configs\n",
        "import seaborn\n",
        "set_default_configs(plt, seaborn)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KZo_2SULwP92"
      },
      "outputs": [],
      "source": [
        "class NestedDict(dict):\n",
        "    def __missing__(self, key):\n",
        "        self[key] = type(self)()\n",
        "        return self[key]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4c1ZEbzswRfz"
      },
      "outputs": [],
      "source": [
        "def plot_curve(ax, xs, means, stds, marker=None, label=None):\n",
        "    means = np.array(means)\n",
        "    stds = np.array(stds)\n",
        "    ax.plot(xs, means, label=label, marker=marker)\n",
        "    plt.fill_between(xs, means-stds, means+stds, alpha=0.2)\n",
        "    return ax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z8NL8lqwwTp-"
      },
      "outputs": [],
      "source": [
        "def plot_over_n_new(results, ns, epoch):\n",
        "    fig, ax = plt.subplots(figsize=(6, 4))\n",
        "\n",
        "    fcmi_means = []\n",
        "    fcmi_stds = []\n",
        "    kl_means = []\n",
        "    kl_stds = []\n",
        "    interp_means = []\n",
        "    interp_stds = []\n",
        "    val_means = []\n",
        "    val_stds = []\n",
        "    train_means = []\n",
        "    train_stds = []\n",
        "\n",
        "    for n in ns:\n",
        "        fcmi_mean = np.nanmean([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        fcmi_std = np.nanstd([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(fcmi_std):\n",
        "            fcmi_std = 0.0\n",
        "        fcmi_means.append(fcmi_mean)\n",
        "        fcmi_stds.append(fcmi_std)\n",
        "\n",
        "        kl_mean = np.nanmean([d['kl_bound'] for d in results[n][epoch]])\n",
        "        kl_std = np.nanstd([d['kl_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(kl_std):\n",
        "            kl_std = 0.0\n",
        "        kl_means.append(kl_mean)\n",
        "        kl_stds.append(kl_std)\n",
        "\n",
        "        if 'interp_bound' in results[n][epoch][0]:\n",
        "            interp_mean = np.nanmean([d['interp_bound'] for d in results[n][epoch]])\n",
        "            interp_std = np.nanstd([d['interp_bound'] for d in results[n][epoch]])\n",
        "            if np.isnan(interp_std):\n",
        "                interp_std = 0.0\n",
        "            if interp_mean <= kl_mean:\n",
        "                interp_means.append(interp_mean)\n",
        "                interp_stds.append(interp_std)\n",
        "\n",
        "        val_mean = 1-np.nanmean([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        val_std = np.nanstd([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(val_std):\n",
        "            val_std = 0.0\n",
        "        val_means.append(val_mean)\n",
        "        val_stds.append(val_std)\n",
        "\n",
        "        train_mean = 1-np.nanmean([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        train_std = np.nanstd([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(train_std):\n",
        "            train_std = 0.0\n",
        "        train_means.append(train_mean)\n",
        "        train_stds.append(train_std)\n",
        "\n",
        "\n",
        "    plot_curve(ax, range(len(ns)), val_means, val_stds, label=f'Test error', marker='o')\n",
        "    plot_curve(ax, range(len(ns)), np.add(train_means,fcmi_means), np.add(fcmi_stds,train_stds), label=f'Square root', marker='x')#9\n",
        "    plot_curve(ax, range(len(ns)), kl_means, kl_stds, label=f'Binary KL', marker='x')#25\n",
        "    #if len(interp_means) > 1:\n",
        "    #    plot_curve(ax, range(len(interp_means)), interp_means, interp_stds, label=f'Interpolation', marker='x')#17\n",
        "    ax.set_xlabel('n')\n",
        "    ax.set_ylabel('Error')\n",
        "    ax.set_xticks(range(len(ns)))\n",
        "    ax.set_xticklabels(ns)\n",
        "    ax.legend()\n",
        "    fig.tight_layout()\n",
        "    \n",
        "    print(train_means)\n",
        "    print(val_means)\n",
        "    print(np.add(fcmi_means,train_means))\n",
        "    print(kl_means)\n",
        "    \n",
        "    return fig, ax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3YNFGxp3wXHL"
      },
      "outputs": [],
      "source": [
        "def plot_over_epochs_new(results, n, epochs):\n",
        "    fig, ax = plt.subplots(figsize=(6, 4))\n",
        "\n",
        "    fcmi_means = []\n",
        "    fcmi_stds = []\n",
        "    gap_means = []\n",
        "    gap_stds = []\n",
        "    kl_means = []\n",
        "    kl_stds = []\n",
        "    train_means = []\n",
        "    train_stds = []\n",
        "    test_means = []\n",
        "    test_stds = []\n",
        "    for epoch in epochs:\n",
        "        fcmi_mean = np.nanmean([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        fcmi_std = np.nanstd([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(fcmi_std):\n",
        "            fcmi_std = 0.0\n",
        "        gap_mean = np.nanmean([d['exp_gap'] for d in results[n][epoch]])\n",
        "        gap_std = np.nanstd([d['exp_gap'] for d in results[n][epoch]])\n",
        "        if np.isnan(gap_std):\n",
        "            gap_std = 0.0\n",
        "        kl_mean = np.nanmean([d['kl_bound'] for d in fcmi_results[n][epoch]])\n",
        "        kl_std = np.nanstd([d['kl_bound'] for d in fcmi_results[n][epoch]])\n",
        "        if np.isnan(kl_std):\n",
        "                kl_std = 0.0\n",
        "        train_mean = 1-np.nanmean([d['exp_train_acc'] for d in fcmi_results[n][epoch]])\n",
        "        train_std = np.nanstd([d['exp_train_acc'] for d in fcmi_results[n][epoch]])\n",
        "        test_mean = 1.0 - np.nanmean([d['exp_val_acc'] for d in fcmi_results[n][epoch]])\n",
        "        test_std = np.nanstd([d['exp_val_acc'] for d in fcmi_results[n][epoch]])\n",
        "        test_means.append(test_mean)\n",
        "        test_stds.append(test_std)\n",
        "        kl_means.append(kl_mean)\n",
        "        kl_stds.append(kl_std)\n",
        "        train_means.append(kl_mean)\n",
        "        train_stds.append(kl_std)\n",
        "\n",
        "        fcmi_means.append(fcmi_mean)\n",
        "        fcmi_stds.append(fcmi_std)\n",
        "        gap_means.append(gap_mean)\n",
        "        gap_stds.append(gap_std)\n",
        "    \n",
        "    plot_curve(ax, epochs, test_means, test_stds, label=f'Test error', marker='o')\n",
        "    plot_curve(ax, epochs, np.add(fcmi_means,train_means), np.add(fcmi_stds,train_stds), label=f'Square root', marker='x')#9\n",
        "    plot_curve(ax, epochs, kl_means, kl_stds, label=f'Binary KL', marker='x')#25\n",
        "    ax.set_xlabel('Epochs')\n",
        "    ax.set_ylabel('Error')\n",
        "    ax.legend()\n",
        "    fig.tight_layout()\n",
        "    return fig, ax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TE091vreL7Tw"
      },
      "outputs": [],
      "source": [
        "def plot_over_arb(results, ns, epoch, xlabel, xticks):\n",
        "    fig, ax = plt.subplots(figsize=(6, 4))\n",
        "\n",
        "    fcmi_means = []\n",
        "    fcmi_stds = []\n",
        "    kl_means = []\n",
        "    kl_stds = []\n",
        "    interp_means = []\n",
        "    interp_stds = []\n",
        "    val_means = []\n",
        "    val_stds = []\n",
        "    train_means = []\n",
        "    train_stds = []\n",
        "\n",
        "    for n in ns:\n",
        "        fcmi_mean = np.nanmean([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        fcmi_std = np.nanstd([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(fcmi_std):\n",
        "            fcmi_std = 0.0\n",
        "        fcmi_means.append(fcmi_mean)\n",
        "        fcmi_stds.append(fcmi_std)\n",
        "\n",
        "        kl_mean = np.nanmean([d['kl_bound'] for d in results[n][epoch]])\n",
        "        kl_std = np.nanstd([d['kl_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(kl_std):\n",
        "            kl_std = 0.0\n",
        "        kl_means.append(kl_mean)\n",
        "        kl_stds.append(kl_std)\n",
        "\n",
        "        if 'interp_bound' in results[n][epoch][0]:\n",
        "            interp_mean = np.nanmean([d['interp_bound'] for d in results[n][epoch]])\n",
        "            interp_std = np.nanstd([d['interp_bound'] for d in results[n][epoch]])\n",
        "            if np.isnan(interp_std):\n",
        "                interp_std = 0.0\n",
        "            if interp_mean <= kl_mean:\n",
        "                interp_means.append(interp_mean)\n",
        "                interp_stds.append(interp_std)\n",
        "\n",
        "        val_mean = 1-np.nanmean([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        val_std = np.nanstd([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(val_std):\n",
        "            val_std = 0.0\n",
        "        val_means.append(val_mean)\n",
        "        val_stds.append(val_std)\n",
        "\n",
        "        train_mean = 1-np.nanmean([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        train_std = np.nanstd([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(train_std):\n",
        "            train_std = 0.0\n",
        "        train_means.append(train_mean)\n",
        "        train_stds.append(train_std)\n",
        "\n",
        "\n",
        "    plot_curve(ax, range(len(ns)), val_means, val_stds, label=f'Test error', marker='o')\n",
        "    plot_curve(ax, range(len(ns)), np.add(train_means,fcmi_means), np.add(fcmi_stds,train_stds), label=f'Square root', marker='x')#9\n",
        "    plot_curve(ax, range(len(ns)), kl_means, kl_stds, label=f'Binary KL', marker='x')#25\n",
        "    #if len(interp_means) > 1:\n",
        "    #    plot_curve(ax, range(len(interp_means)), interp_means, interp_stds, label=f'Interpolation', marker='x')#17\n",
        "    ax.set_xlabel(xlabel)\n",
        "    ax.set_ylabel('Error')\n",
        "    ax.set_xticks(range(len(ns)))\n",
        "    ax.set_xticklabels(xticks)\n",
        "    ax.legend()\n",
        "    fig.tight_layout()\n",
        "    \n",
        "    print(train_means)\n",
        "    print(val_means)\n",
        "    print(np.add(fcmi_means,train_means))\n",
        "    print(kl_means)\n",
        "    \n",
        "    return fig, ax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5nTkTYLDMXHE"
      },
      "outputs": [],
      "source": [
        "## Plot Figure 1(a)\n",
        "\n",
        "B = np.linspace(0,np.log(2),200);\n",
        "myi = np.sqrt(2*B);\n",
        "sy = B/0.37;\n",
        "syint = B/np.log(2);\n",
        "wy = 2*(1-np.exp(-B));\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(6, 4))\n",
        "\n",
        "plot_curve(ax, B, myi, np.zeros_like(B), label=f'Square root') #10\n",
        "plot_curve(ax, B, sy, np.zeros_like(B), label=f'Linear') #16\n",
        "plot_curve(ax, B, syint, np.zeros_like(B), label=f'Interpolation') #17\n",
        "plot_curve(ax, B, wy, np.zeros_like(B), label=f'Binary KL') #23\n",
        "plt.axvline(x=0.275,color='k', linestyle='--', linewidth=0.8,alpha = 0.5)\n",
        "\n",
        "ax.set_xlabel(r'$B$')\n",
        "ax.set_ylabel(r'Bound on $L_{\\mathcal{D}}$')\n",
        "ax.set_ylim([0,1])\n",
        "ax.set_xticks([0,0.27,np.log(2)])\n",
        "ax.set_xticklabels(['0','0.27','log(2)'])\n",
        "ax.legend()\n",
        "fig.tight_layout()\n",
        "\n",
        "savefig(fig, os.path.join('figs', f'fourboundplot.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0_UmJEITxR7P"
      },
      "outputs": [],
      "source": [
        "## Plot Figure 1(b)\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import scipy.optimize as opt\n",
        "\n",
        "def kl(q,p):\n",
        "    if q>0:\n",
        "        return q*np.log(q/p) + (1-q)*np.log( (1-q)/(1-p) )\n",
        "    else:\n",
        "        return np.log( 1/(1-p) )\n",
        "\n",
        "def klbound(B, train_acc):\n",
        "    Rhat = 1-train_acc\n",
        "    # Constraints are expressions that should be non-negative\n",
        "    # Below factors guarantee R<=1, R>=0,and bound satisfied\n",
        "    def con(R):\n",
        "        return (B*np.log(2)-kl(Rhat,Rhat/2 + R/2))*R*(1-R)\n",
        "    # Minimize -R to find biggest R that satisfies constraints\n",
        "    objective = lambda R: -R\n",
        "    cons = ({'type': 'ineq', 'fun' : con})\n",
        "    results = opt.minimize(objective,x0=0.5,constraints = cons,options = {'disp':False})\n",
        "    bound = results.x[0]\n",
        "    return min(bound,1)\n",
        "\n",
        "def lgbound(B, train_acc):\n",
        "    Rhat = 1-train_acc\n",
        "    def con(x):\n",
        "        return (-x[0]*(1-x[1]) - (np.exp(x[0]) - 1 - x[0] ) * ( 1 + x[1]**2 ))\n",
        "    objective = lambda x: x[1]*Rhat + B*np.log(2)/x[0]\n",
        "    cons = ({'type': 'ineq', 'fun' : con})\n",
        "    bnds = ((0, 0.37),(1,np.inf))\n",
        "    results = opt.minimize(objective,x0=[3,2],constraints = cons,bounds = bnds,options = {'disp':False})\n",
        "    bound = results.x[1]*Rhat + B*np.log(2)/results.x[0]\n",
        "    return min(bound,1)\n",
        "\n",
        "def maxbound_int(B, train_acc):\n",
        "    Rhat = 1-train_acc\n",
        "    bound = Rhat + np.sqrt(B*2*np.log(2))\n",
        "    return min(bound,1)\n",
        "\n",
        "NN = 2000\n",
        "acc_vec = np.linspace(0.0,1.0,NN)\n",
        "B_vec = np.linspace(0.0,1.0,NN)\n",
        "data = np.zeros((NN, NN))\n",
        "\n",
        "for Ri in range(NN):\n",
        "    acc = acc_vec[Ri]\n",
        "    for Bi in range(NN):\n",
        "        B = B_vec[Bi]\n",
        "        Mi = maxbound_int(B,acc)\n",
        "        Si = lgbound(B,acc)\n",
        "        kli = klbound(B,acc)\n",
        "        index_min = np.argmin(np.array([kli,Si,Mi]))\n",
        "        if kli >= 1 and Si >= 1 and Mi >= 1:\n",
        "            index_min = 3\n",
        "        data[Ri,Bi] = index_min\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(8, 5))\n",
        "\n",
        "colorsList = ['w','b','g','r']\n",
        "CustomCmap = matplotlib.colors.ListedColormap(colorsList)\n",
        "\n",
        "data[:,-1] = 3*np.ones((1,NN))\n",
        "data[-1,:] = 3*np.ones((1,NN))\n",
        "\n",
        "data[-1,1] = 3.5\n",
        "data[0,0] = -0.5\n",
        "\n",
        "im = ax.imshow(np.flipud(data), cmap=CustomCmap, alpha = 0.3)\n",
        "\n",
        "ax.set_xlabel(r'$B$')\n",
        "ax.set_ylabel(r'$\\widehat{L}$')\n",
        "ax.set_ylim([0,1])\n",
        "ax.set_xticks(np.linspace(0,NN+1,3))\n",
        "ax.set_xticklabels([r'$0$',r'$\\log(2)/2$',r'$\\log(2)$'])\n",
        "ax.set_yticks(np.flip(np.arange(0,NN+1,NN/4)))\n",
        "ax.set_yticklabels(np.linspace(1,0,5))\n",
        "\n",
        "cbar = plt.colorbar(im, ticks = [0,1,2,3])\n",
        "cbar.set_clim(-0.5,4)\n",
        "cbar.ax.set_yticklabels([r'Binary KL', r'Linear', r'Square root','Trivial'])\n",
        "#cbar.ax.set_yticklabels([r'(23)', r'(16)', r'(10)','Trivial'])\n",
        "\n",
        "savefig(fig, os.path.join('figs', f'pixel-plot.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot functions for making one line thicker for presentation slides\n",
        "\n",
        "def plot_curve_thick(ax, xs, means, stds, marker=None, label=None):\n",
        "    means = np.array(means)\n",
        "    stds = np.array(stds)\n",
        "    ax.plot(xs, means, label=label, marker=marker, linewidth = 2.8)\n",
        "    plt.fill_between(xs, means-stds, means+stds, alpha=0.5)\n",
        "    return ax\n",
        "\n",
        "def plot_over_n_new_thick(results, ns, epoch, thick):\n",
        "    fig, ax = plt.subplots(figsize=(6, 4))\n",
        "\n",
        "    fcmi_means = []\n",
        "    fcmi_stds = []\n",
        "    kl_means = []\n",
        "    kl_stds = []\n",
        "    interp_means = []\n",
        "    interp_stds = []\n",
        "    val_means = []\n",
        "    val_stds = []\n",
        "    train_means = []\n",
        "    train_stds = []\n",
        "\n",
        "    for n in ns:\n",
        "        fcmi_mean = np.nanmean([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        fcmi_std = np.nanstd([d['fcmi_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(fcmi_std):\n",
        "            fcmi_std = 0.0\n",
        "        fcmi_means.append(fcmi_mean)\n",
        "        fcmi_stds.append(fcmi_std)\n",
        "\n",
        "        kl_mean = np.nanmean([d['kl_bound'] for d in results[n][epoch]])\n",
        "        kl_std = np.nanstd([d['kl_bound'] for d in results[n][epoch]])\n",
        "        if np.isnan(kl_std):\n",
        "            kl_std = 0.0\n",
        "        kl_means.append(kl_mean)\n",
        "        kl_stds.append(kl_std)\n",
        "\n",
        "        if 'interp_bound' in results[n][epoch][0]:\n",
        "            interp_mean = np.nanmean([d['interp_bound'] for d in results[n][epoch]])\n",
        "            interp_std = np.nanstd([d['interp_bound'] for d in results[n][epoch]])\n",
        "            if np.isnan(interp_std):\n",
        "                interp_std = 0.0\n",
        "            if interp_mean <= kl_mean:\n",
        "                interp_means.append(interp_mean)\n",
        "                interp_stds.append(interp_std)\n",
        "\n",
        "        val_mean = 1-np.nanmean([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        val_std = np.nanstd([d['exp_val_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(val_std):\n",
        "            val_std = 0.0\n",
        "        val_means.append(val_mean)\n",
        "        val_stds.append(val_std)\n",
        "\n",
        "        train_mean = 1-np.nanmean([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        train_std = np.nanstd([d['exp_train_acc'] for d in results[n][epoch]])\n",
        "        if np.isnan(train_std):\n",
        "            train_std = 0.0\n",
        "        train_means.append(train_mean)\n",
        "        train_stds.append(train_std)\n",
        "\n",
        "    if thick == 'test':\n",
        "        plot_curve_thick(ax, range(len(ns)), val_means, val_stds, label=f'Test error', marker='o')\n",
        "    else:\n",
        "        plot_curve(ax, range(len(ns)), val_means, val_stds, label=f'Test error', marker='o')\n",
        "    if thick == 'square':\n",
        "        plot_curve_thick(ax, range(len(ns)), np.add(train_means,fcmi_means), np.add(fcmi_stds,train_stds), label=f'Square root', marker='x')#9\n",
        "    else:\n",
        "        plot_curve(ax, range(len(ns)), np.add(train_means,fcmi_means), np.add(fcmi_stds,train_stds), label=f'Square root', marker='x')#9\n",
        "    if thick == 'KL':\n",
        "        plot_curve_thick(ax, range(len(ns)), kl_means, kl_stds, label=f'Binary KL', marker='x')#25\n",
        "    else:\n",
        "        plot_curve(ax, range(len(ns)), kl_means, kl_stds, label=f'Binary KL', marker='x')#25\n",
        "    if len(interp_means) > 1:\n",
        "        if thick == 'interp':\n",
        "            plot_curve_thick(ax, range(len(interp_means)), interp_means, interp_stds, label=f'Interpolation', marker='x')#17\n",
        "        else:\n",
        "            plot_curve(ax, range(len(interp_means)), interp_means, interp_stds, label=f'Interpolation', marker='x')#17\n",
        "    ax.set_xlabel('n')\n",
        "    ax.set_ylabel('Error')\n",
        "    ax.set_xticks(range(len(ns)))\n",
        "    ax.set_xticklabels(ns)\n",
        "    ax.legend()\n",
        "    fig.tight_layout()\n",
        "    \n",
        "    print(train_means)\n",
        "    print(val_means)\n",
        "    print(np.add(fcmi_means,train_means))\n",
        "    print(kl_means)\n",
        "    \n",
        "    return fig, ax"
      ],
      "metadata": {
        "id": "W3CyLSnvnogf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "olFKaEfxweZm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "outputId": "e9f003e3-91d4-413e-a642-dde0aacaac4d"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "/* Put everything inside the global mpl namespace */\n",
              "window.mpl = {};\n",
              "\n",
              "\n",
              "mpl.get_websocket_type = function() {\n",
              "    if (typeof(WebSocket) !== 'undefined') {\n",
              "        return WebSocket;\n",
              "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
              "        return MozWebSocket;\n",
              "    } else {\n",
              "        alert('Your browser does not have WebSocket support. ' +\n",
              "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
              "              'Firefox 4 and 5 are also supported but you ' +\n",
              "              'have to enable WebSockets in about:config.');\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
              "    this.id = figure_id;\n",
              "\n",
              "    this.ws = websocket;\n",
              "\n",
              "    this.supports_binary = (this.ws.binaryType != undefined);\n",
              "\n",
              "    if (!this.supports_binary) {\n",
              "        var warnings = document.getElementById(\"mpl-warnings\");\n",
              "        if (warnings) {\n",
              "            warnings.style.display = 'block';\n",
              "            warnings.textContent = (\n",
              "                \"This browser does not support binary websocket messages. \" +\n",
              "                    \"Performance may be slow.\");\n",
              "        }\n",
              "    }\n",
              "\n",
              "    this.imageObj = new Image();\n",
              "\n",
              "    this.context = undefined;\n",
              "    this.message = undefined;\n",
              "    this.canvas = undefined;\n",
              "    this.rubberband_canvas = undefined;\n",
              "    this.rubberband_context = undefined;\n",
              "    this.format_dropdown = undefined;\n",
              "\n",
              "    this.image_mode = 'full';\n",
              "\n",
              "    this.root = $('<div/>');\n",
              "    this._root_extra_style(this.root)\n",
              "    this.root.attr('style', 'display: inline-block');\n",
              "\n",
              "    $(parent_element).append(this.root);\n",
              "\n",
              "    this._init_header(this);\n",
              "    this._init_canvas(this);\n",
              "    this._init_toolbar(this);\n",
              "\n",
              "    var fig = this;\n",
              "\n",
              "    this.waiting = false;\n",
              "\n",
              "    this.ws.onopen =  function () {\n",
              "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
              "            fig.send_message(\"send_image_mode\", {});\n",
              "            if (mpl.ratio != 1) {\n",
              "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
              "            }\n",
              "            fig.send_message(\"refresh\", {});\n",
              "        }\n",
              "\n",
              "    this.imageObj.onload = function() {\n",
              "            if (fig.image_mode == 'full') {\n",
              "                // Full images could contain transparency (where diff images\n",
              "                // almost always do), so we need to clear the canvas so that\n",
              "                // there is no ghosting.\n",
              "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
              "            }\n",
              "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
              "        };\n",
              "\n",
              "    this.imageObj.onunload = function() {\n",
              "        fig.ws.close();\n",
              "    }\n",
              "\n",
              "    this.ws.onmessage = this._make_on_message_function(this);\n",
              "\n",
              "    this.ondownload = ondownload;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_header = function() {\n",
              "    var titlebar = $(\n",
              "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
              "        'ui-helper-clearfix\"/>');\n",
              "    var titletext = $(\n",
              "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
              "        'text-align: center; padding: 3px;\"/>');\n",
              "    titlebar.append(titletext)\n",
              "    this.root.append(titlebar);\n",
              "    this.header = titletext[0];\n",
              "}\n",
              "\n",
              "\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_canvas = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var canvas_div = $('<div/>');\n",
              "\n",
              "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
              "\n",
              "    function canvas_keyboard_event(event) {\n",
              "        return fig.key_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
              "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
              "    this.canvas_div = canvas_div\n",
              "    this._canvas_extra_style(canvas_div)\n",
              "    this.root.append(canvas_div);\n",
              "\n",
              "    var canvas = $('<canvas/>');\n",
              "    canvas.addClass('mpl-canvas');\n",
              "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
              "\n",
              "    this.canvas = canvas[0];\n",
              "    this.context = canvas[0].getContext(\"2d\");\n",
              "\n",
              "    var backingStore = this.context.backingStorePixelRatio ||\n",
              "\tthis.context.webkitBackingStorePixelRatio ||\n",
              "\tthis.context.mozBackingStorePixelRatio ||\n",
              "\tthis.context.msBackingStorePixelRatio ||\n",
              "\tthis.context.oBackingStorePixelRatio ||\n",
              "\tthis.context.backingStorePixelRatio || 1;\n",
              "\n",
              "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
              "\n",
              "    var rubberband = $('<canvas/>');\n",
              "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
              "\n",
              "    var pass_mouse_events = true;\n",
              "\n",
              "    canvas_div.resizable({\n",
              "        start: function(event, ui) {\n",
              "            pass_mouse_events = false;\n",
              "        },\n",
              "        resize: function(event, ui) {\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "        stop: function(event, ui) {\n",
              "            pass_mouse_events = true;\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "    });\n",
              "\n",
              "    function mouse_event_fn(event) {\n",
              "        if (pass_mouse_events)\n",
              "            return fig.mouse_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    rubberband.mousedown('button_press', mouse_event_fn);\n",
              "    rubberband.mouseup('button_release', mouse_event_fn);\n",
              "    // Throttle sequential mouse events to 1 every 20ms.\n",
              "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
              "\n",
              "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
              "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
              "\n",
              "    canvas_div.on(\"wheel\", function (event) {\n",
              "        event = event.originalEvent;\n",
              "        event['data'] = 'scroll'\n",
              "        if (event.deltaY < 0) {\n",
              "            event.step = 1;\n",
              "        } else {\n",
              "            event.step = -1;\n",
              "        }\n",
              "        mouse_event_fn(event);\n",
              "    });\n",
              "\n",
              "    canvas_div.append(canvas);\n",
              "    canvas_div.append(rubberband);\n",
              "\n",
              "    this.rubberband = rubberband;\n",
              "    this.rubberband_canvas = rubberband[0];\n",
              "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
              "    this.rubberband_context.strokeStyle = \"#000000\";\n",
              "\n",
              "    this._resize_canvas = function(width, height) {\n",
              "        // Keep the size of the canvas, canvas container, and rubber band\n",
              "        // canvas in synch.\n",
              "        canvas_div.css('width', width)\n",
              "        canvas_div.css('height', height)\n",
              "\n",
              "        canvas.attr('width', width * mpl.ratio);\n",
              "        canvas.attr('height', height * mpl.ratio);\n",
              "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
              "\n",
              "        rubberband.attr('width', width);\n",
              "        rubberband.attr('height', height);\n",
              "    }\n",
              "\n",
              "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
              "    // upon first draw.\n",
              "    this._resize_canvas(600, 600);\n",
              "\n",
              "    // Disable right mouse context menu.\n",
              "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
              "        return false;\n",
              "    });\n",
              "\n",
              "    function set_focus () {\n",
              "        canvas.focus();\n",
              "        canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    window.setTimeout(set_focus, 100);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items) {\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) {\n",
              "            // put a spacer in here.\n",
              "            continue;\n",
              "        }\n",
              "        var button = $('<button/>');\n",
              "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
              "                        'ui-button-icon-only');\n",
              "        button.attr('role', 'button');\n",
              "        button.attr('aria-disabled', 'false');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "\n",
              "        var icon_img = $('<span/>');\n",
              "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
              "        icon_img.addClass(image);\n",
              "        icon_img.addClass('ui-corner-all');\n",
              "\n",
              "        var tooltip_span = $('<span/>');\n",
              "        tooltip_span.addClass('ui-button-text');\n",
              "        tooltip_span.html(tooltip);\n",
              "\n",
              "        button.append(icon_img);\n",
              "        button.append(tooltip_span);\n",
              "\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    var fmt_picker_span = $('<span/>');\n",
              "\n",
              "    var fmt_picker = $('<select/>');\n",
              "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
              "    fmt_picker_span.append(fmt_picker);\n",
              "    nav_element.append(fmt_picker_span);\n",
              "    this.format_dropdown = fmt_picker[0];\n",
              "\n",
              "    for (var ind in mpl.extensions) {\n",
              "        var fmt = mpl.extensions[ind];\n",
              "        var option = $(\n",
              "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
              "        fmt_picker.append(option);\n",
              "    }\n",
              "\n",
              "    // Add hover states to the ui-buttons\n",
              "    $( \".ui-button\" ).hover(\n",
              "        function() { $(this).addClass(\"ui-state-hover\");},\n",
              "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
              "    );\n",
              "\n",
              "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
              "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
              "    // which will in turn request a refresh of the image.\n",
              "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_message = function(type, properties) {\n",
              "    properties['type'] = type;\n",
              "    properties['figure_id'] = this.id;\n",
              "    this.ws.send(JSON.stringify(properties));\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_draw_message = function() {\n",
              "    if (!this.waiting) {\n",
              "        this.waiting = true;\n",
              "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
              "    }\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    var format_dropdown = fig.format_dropdown;\n",
              "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
              "    fig.ondownload(fig, format);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
              "    var size = msg['size'];\n",
              "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
              "        fig._resize_canvas(size[0], size[1]);\n",
              "        fig.send_message(\"refresh\", {});\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
              "    var x0 = msg['x0'] / mpl.ratio;\n",
              "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
              "    var x1 = msg['x1'] / mpl.ratio;\n",
              "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
              "    x0 = Math.floor(x0) + 0.5;\n",
              "    y0 = Math.floor(y0) + 0.5;\n",
              "    x1 = Math.floor(x1) + 0.5;\n",
              "    y1 = Math.floor(y1) + 0.5;\n",
              "    var min_x = Math.min(x0, x1);\n",
              "    var min_y = Math.min(y0, y1);\n",
              "    var width = Math.abs(x1 - x0);\n",
              "    var height = Math.abs(y1 - y0);\n",
              "\n",
              "    fig.rubberband_context.clearRect(\n",
              "        0, 0, fig.canvas.width / mpl.ratio, fig.canvas.height / mpl.ratio);\n",
              "\n",
              "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
              "    // Updates the figure title.\n",
              "    fig.header.textContent = msg['label'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
              "    var cursor = msg['cursor'];\n",
              "    switch(cursor)\n",
              "    {\n",
              "    case 0:\n",
              "        cursor = 'pointer';\n",
              "        break;\n",
              "    case 1:\n",
              "        cursor = 'default';\n",
              "        break;\n",
              "    case 2:\n",
              "        cursor = 'crosshair';\n",
              "        break;\n",
              "    case 3:\n",
              "        cursor = 'move';\n",
              "        break;\n",
              "    }\n",
              "    fig.rubberband_canvas.style.cursor = cursor;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
              "    fig.message.textContent = msg['message'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
              "    // Request the server to send over a new figure.\n",
              "    fig.send_draw_message();\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
              "    fig.image_mode = msg['mode'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Called whenever the canvas gets updated.\n",
              "    this.send_message(\"ack\", {});\n",
              "}\n",
              "\n",
              "// A function to construct a web socket function for onmessage handling.\n",
              "// Called in the figure constructor.\n",
              "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
              "    return function socket_on_message(evt) {\n",
              "        if (evt.data instanceof Blob) {\n",
              "            /* FIXME: We get \"Resource interpreted as Image but\n",
              "             * transferred with MIME type text/plain:\" errors on\n",
              "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
              "             * to be part of the websocket stream */\n",
              "            evt.data.type = \"image/png\";\n",
              "\n",
              "            /* Free the memory for the previous frames */\n",
              "            if (fig.imageObj.src) {\n",
              "                (window.URL || window.webkitURL).revokeObjectURL(\n",
              "                    fig.imageObj.src);\n",
              "            }\n",
              "\n",
              "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
              "                evt.data);\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
              "            fig.imageObj.src = evt.data;\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        var msg = JSON.parse(evt.data);\n",
              "        var msg_type = msg['type'];\n",
              "\n",
              "        // Call the  \"handle_{type}\" callback, which takes\n",
              "        // the figure and JSON message as its only arguments.\n",
              "        try {\n",
              "            var callback = fig[\"handle_\" + msg_type];\n",
              "        } catch (e) {\n",
              "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        if (callback) {\n",
              "            try {\n",
              "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
              "                callback(fig, msg);\n",
              "            } catch (e) {\n",
              "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
              "            }\n",
              "        }\n",
              "    };\n",
              "}\n",
              "\n",
              "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
              "mpl.findpos = function(e) {\n",
              "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
              "    var targ;\n",
              "    if (!e)\n",
              "        e = window.event;\n",
              "    if (e.target)\n",
              "        targ = e.target;\n",
              "    else if (e.srcElement)\n",
              "        targ = e.srcElement;\n",
              "    if (targ.nodeType == 3) // defeat Safari bug\n",
              "        targ = targ.parentNode;\n",
              "\n",
              "    // jQuery normalizes the pageX and pageY\n",
              "    // pageX,Y are the mouse positions relative to the document\n",
              "    // offset() returns the position of the element relative to the document\n",
              "    var x = e.pageX - $(targ).offset().left;\n",
              "    var y = e.pageY - $(targ).offset().top;\n",
              "\n",
              "    return {\"x\": x, \"y\": y};\n",
              "};\n",
              "\n",
              "/*\n",
              " * return a copy of an object with only non-object keys\n",
              " * we need this to avoid circular references\n",
              " * http://stackoverflow.com/a/24161582/3208463\n",
              " */\n",
              "function simpleKeys (original) {\n",
              "  return Object.keys(original).reduce(function (obj, key) {\n",
              "    if (typeof original[key] !== 'object')\n",
              "        obj[key] = original[key]\n",
              "    return obj;\n",
              "  }, {});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.mouse_event = function(event, name) {\n",
              "    var canvas_pos = mpl.findpos(event)\n",
              "\n",
              "    if (name === 'button_press')\n",
              "    {\n",
              "        this.canvas.focus();\n",
              "        this.canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    var x = canvas_pos.x * mpl.ratio;\n",
              "    var y = canvas_pos.y * mpl.ratio;\n",
              "\n",
              "    this.send_message(name, {x: x, y: y, button: event.button,\n",
              "                             step: event.step,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "\n",
              "    /* This prevents the web browser from automatically changing to\n",
              "     * the text insertion cursor when the button is pressed.  We want\n",
              "     * to control all of the cursor setting manually through the\n",
              "     * 'cursor' event from matplotlib */\n",
              "    event.preventDefault();\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    // Handle any extra behaviour associated with a key event\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.key_event = function(event, name) {\n",
              "\n",
              "    // Prevent repeat events\n",
              "    if (name == 'key_press')\n",
              "    {\n",
              "        if (event.which === this._key)\n",
              "            return;\n",
              "        else\n",
              "            this._key = event.which;\n",
              "    }\n",
              "    if (name == 'key_release')\n",
              "        this._key = null;\n",
              "\n",
              "    var value = '';\n",
              "    if (event.ctrlKey && event.which != 17)\n",
              "        value += \"ctrl+\";\n",
              "    if (event.altKey && event.which != 18)\n",
              "        value += \"alt+\";\n",
              "    if (event.shiftKey && event.which != 16)\n",
              "        value += \"shift+\";\n",
              "\n",
              "    value += 'k';\n",
              "    value += event.which.toString();\n",
              "\n",
              "    this._key_event_extra(event, name);\n",
              "\n",
              "    this.send_message(name, {key: value,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
              "    if (name == 'download') {\n",
              "        this.handle_save(this, null);\n",
              "    } else {\n",
              "        this.send_message(\"toolbar_button\", {name: name});\n",
              "    }\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
              "    this.message.textContent = tooltip;\n",
              "};\n",
              "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
              "\n",
              "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
              "\n",
              "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
              "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
              "    // object with the appropriate methods. Currently this is a non binary\n",
              "    // socket, so there is still some room for performance tuning.\n",
              "    var ws = {};\n",
              "\n",
              "    ws.close = function() {\n",
              "        comm.close()\n",
              "    };\n",
              "    ws.send = function(m) {\n",
              "        //console.log('sending', m);\n",
              "        comm.send(m);\n",
              "    };\n",
              "    // Register the callback with on_msg.\n",
              "    comm.on_msg(function(msg) {\n",
              "        //console.log('receiving', msg['content']['data'], msg);\n",
              "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
              "        ws.onmessage(msg['content']['data'])\n",
              "    });\n",
              "    return ws;\n",
              "}\n",
              "\n",
              "mpl.mpl_figure_comm = function(comm, msg) {\n",
              "    // This is the function which gets called when the mpl process\n",
              "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
              "\n",
              "    var id = msg.content.data.id;\n",
              "    // Get hold of the div created by the display call when the Comm\n",
              "    // socket was opened in Python.\n",
              "    var element = $(\"#\" + id);\n",
              "    var ws_proxy = comm_websocket_adapter(comm)\n",
              "\n",
              "    function ondownload(figure, format) {\n",
              "        window.open(figure.imageObj.src);\n",
              "    }\n",
              "\n",
              "    var fig = new mpl.figure(id, ws_proxy,\n",
              "                           ondownload,\n",
              "                           element.get(0));\n",
              "\n",
              "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
              "    // web socket which is closed, not our websocket->open comm proxy.\n",
              "    ws_proxy.onopen();\n",
              "\n",
              "    fig.parent_element = element.get(0);\n",
              "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
              "    if (!fig.cell_info) {\n",
              "        console.error(\"Failed to find cell for figure\", id, fig);\n",
              "        return;\n",
              "    }\n",
              "\n",
              "    var output_index = fig.cell_info[2]\n",
              "    var cell = fig.cell_info[0];\n",
              "\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
              "    var width = fig.canvas.width/mpl.ratio\n",
              "    fig.root.unbind('remove')\n",
              "\n",
              "    // Update the output cell to use the data from the current canvas.\n",
              "    fig.push_to_output();\n",
              "    var dataURL = fig.canvas.toDataURL();\n",
              "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
              "    // the notebook keyboard shortcuts fail.\n",
              "    IPython.keyboard_manager.enable()\n",
              "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
              "    fig.close_ws(fig, msg);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.close_ws = function(fig, msg){\n",
              "    fig.send_message('closing', msg);\n",
              "    // fig.ws.close()\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
              "    // Turn the data on the canvas into data in the output cell.\n",
              "    var width = this.canvas.width/mpl.ratio\n",
              "    var dataURL = this.canvas.toDataURL();\n",
              "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Tell IPython that the notebook contents must change.\n",
              "    IPython.notebook.set_dirty(true);\n",
              "    this.send_message(\"ack\", {});\n",
              "    var fig = this;\n",
              "    // Wait a second, then push the new image to the DOM so\n",
              "    // that it is saved nicely (might be nice to debounce this).\n",
              "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items){\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) { continue; };\n",
              "\n",
              "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    // Add the status bar.\n",
              "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "\n",
              "    // Add the close button to the window.\n",
              "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
              "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
              "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
              "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
              "    buttongrp.append(button);\n",
              "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
              "    titlebar.prepend(buttongrp);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(el){\n",
              "    var fig = this\n",
              "    el.on(\"remove\", function(){\n",
              "\tfig.close_ws(fig, {});\n",
              "    });\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(el){\n",
              "    // this is important to make the div 'focusable\n",
              "    el.attr('tabindex', 0)\n",
              "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
              "    // off when our div gets focus\n",
              "\n",
              "    // location in version 3\n",
              "    if (IPython.notebook.keyboard_manager) {\n",
              "        IPython.notebook.keyboard_manager.register_events(el);\n",
              "    }\n",
              "    else {\n",
              "        // location in version 2\n",
              "        IPython.keyboard_manager.register_events(el);\n",
              "    }\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    var manager = IPython.notebook.keyboard_manager;\n",
              "    if (!manager)\n",
              "        manager = IPython.keyboard_manager;\n",
              "\n",
              "    // Check for shift+enter\n",
              "    if (event.shiftKey && event.which == 13) {\n",
              "        this.canvas_div.blur();\n",
              "        // select the cell after this one\n",
              "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
              "        IPython.notebook.select(index + 1);\n",
              "    }\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    fig.ondownload(fig, null);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.find_output_cell = function(html_output) {\n",
              "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
              "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
              "    // IPython event is triggered only after the cells have been serialised, which for\n",
              "    // our purposes (turning an active figure into a static one), is too late.\n",
              "    var cells = IPython.notebook.get_cells();\n",
              "    var ncells = cells.length;\n",
              "    for (var i=0; i<ncells; i++) {\n",
              "        var cell = cells[i];\n",
              "        if (cell.cell_type === 'code'){\n",
              "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
              "                var data = cell.output_area.outputs[j];\n",
              "                if (data.data) {\n",
              "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
              "                    data = data.data;\n",
              "                }\n",
              "                if (data['text/html'] == html_output) {\n",
              "                    return [cell, data, j];\n",
              "                }\n",
              "            }\n",
              "        }\n",
              "    }\n",
              "}\n",
              "\n",
              "// Register the function which deals with the matplotlib target/channel.\n",
              "// The kernel may be null if the page has been refreshed.\n",
              "if (IPython.notebook.kernel != null) {\n",
              "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
              "}\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div id='c443f3bf-2b72-41b9-9572-528d2084c2e2'></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.0, 0.0, 0.0, 6.663799285888672e-05]\n",
            "[0.08328896760940552, 0.03495997190475464, 0.014346718788146973, 0.008243381977081299]\n",
            "[0.22227421 0.09538172 0.04130252 0.02607817]\n",
            "[0.1505444583306102, 0.06676598439936836, 0.027991326709405358, 0.015554018184530444]\n"
          ]
        }
      ],
      "source": [
        "## Plot Figure 2(a) versions for presentation slides\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [75, 250, 1000, 4000]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-wo-noise\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_n_new_thick(deterministic_results, ns=ns, epoch=epochs[-1], thick = 'test')\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-testthick.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Plot Figure 2(a)\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [75, 250, 1000, 4000]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_n_new(deterministic_results, ns=ns, epoch=epochs[-1])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd.pdf'))"
      ],
      "metadata": {
        "id": "jgELdkRLuyn-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QfTQ6Tj7wlyO"
      },
      "outputs": [],
      "source": [
        "## Plot Figure 2(b)\n",
        "results_dir = \"results\"\n",
        "exp_name = \"cifar10-pretrained-resnet50\"\n",
        "n_seeds = 2\n",
        "n_S_seeds = 20\n",
        "ns = [1000, 5000, 20000]\n",
        "epochs = [40]\n",
        "num_classes = 10\n",
        "batch_size = 64\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    fcmi_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_n_new(fcmi_results, ns=ns, epoch=epochs[-1])\n",
        "ax.set_yticks(np.linspace(0.05, 0.35, 4))\n",
        "ax.set_ylim(bottom=0.0)\n",
        "savefig(fig, os.path.join('figs', f'cifar-sgd.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QSTbs74dw0yS"
      },
      "outputs": [],
      "source": [
        "## Plot Figure 2(c)\n",
        "results_dir = \"results\"\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-LD\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "n = 4000\n",
        "ns = [4000]\n",
        "epochs = np.arange(1, 11) * 4\n",
        "num_classes = 2\n",
        "batch_size = 100\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    fcmi_results = pickle.load(f)\n",
        "    \n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'sgld_results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    sgld_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_epochs_new(fcmi_results, n=ns[0], epochs=epochs)\n",
        "\n",
        "sgld_means = []\n",
        "sgld_stds = []\n",
        "train_means = []\n",
        "train_stds = []\n",
        "\n",
        "for epoch in epochs:\n",
        "    sgld_mean = np.nanmean([d['sgld_bound'] for d in sgld_results[n][epoch]])\n",
        "    sgld_std = np.nanstd([d['sgld_bound'] for d in sgld_results[n][epoch]])\n",
        "    if np.isnan(sgld_std):\n",
        "            sgld_std = 0.0\n",
        "    train_mean = 1-np.nanmean([d['exp_train_acc'] for d in fcmi_results[n][epoch]])\n",
        "    train_std = np.nanstd([d['exp_train_acc'] for d in fcmi_results[n][epoch]])\n",
        "    sgld_means.append(sgld_mean)\n",
        "    sgld_stds.append(sgld_std)\n",
        "    train_means.append(train_mean)\n",
        "    train_stds.append(train_std)\n",
        "    \n",
        "plot_curve(ax, epochs, np.add(train_means,sgld_means), np.add(train_stds,sgld_stds), label=f'SGLD [11, Eq. (6)]', marker='*')\n",
        "\n",
        "ax.legend()\n",
        "ax.set_yticks(np.linspace(0.0, 0.5, 6))\n",
        "fig.tight_layout()\n",
        "fig.show()\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgld.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a3rPuu_vRMTB"
      },
      "outputs": [],
      "source": [
        "# Plot Figure 3(a)\n",
        "\n",
        "# This is the same as Figure 2(a)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eD0Sg4v-RRzN"
      },
      "outputs": [],
      "source": [
        "# Plot Figure 3(b)\n",
        "\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [75, 250, 1000, 4000]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-halfnoisy\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_n_new(deterministic_results, ns=ns, epoch=epochs[-1])\n",
        "ax.set_ylim([0, 1])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-halfnoisy.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2zfuoniGRUPu"
      },
      "outputs": [],
      "source": [
        "# Plot Figure 3(c)\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-noisy\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_n_new(deterministic_results, ns=ns, epoch=epochs[-1])\n",
        "ax.set_ylim([0, 1])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-noisy.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B7NUMFT2WTbX"
      },
      "outputs": [],
      "source": [
        "sgld_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zb9tsMZSRiM8"
      },
      "outputs": [],
      "source": [
        "# Plot Figure 4(a)\n",
        "\n",
        "results_dir = \"results\"\n",
        "exp_name = \"cifar10-pretrained-resnet50-LD\"\n",
        "n_seeds = 2 \n",
        "n_S_seeds = 40  \n",
        "ns = [20000]\n",
        "epochs = np.arange(1, 9) * 2\n",
        "num_classes = 10\n",
        "batch_size = 64\n",
        "n = ns[0]  # 20000\n",
        "\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    fcmi_results = pickle.load(f)\n",
        "    \n",
        "print(fcmi_results)\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'sgld_results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    sgld_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_epochs_new(fcmi_results, n=ns[0], epochs=epochs)\n",
        "\n",
        "# Data from Harutyunyan et al (2021)\n",
        "sgld_means = [0.4562828011406939,\n",
        " 0.6526754635440749,\n",
        " 1.1598920611525294,\n",
        " 2.101055669357393,\n",
        " 2.624184934276673,\n",
        " 2.8768795381293772,\n",
        " 3.022861941304835,\n",
        " 3.122461987848108]\n",
        "sgld_stds = [0]*len(sgld_means)\n",
        "\n",
        "\n",
        "plot_curve(ax, epochs, np.add(train_means,sgld_means), np.add(train_stds,sgld_stds), label=f'SGLD [11, Eq. (6)]', marker='x')#\n",
        "\n",
        "ax.legend()\n",
        "#ax.set_yticks(np.linspace(0.0, 0.75, 6))\n",
        "#ax.set_ylim([0,0.75])\n",
        "fig.tight_layout()\n",
        "fig.show()\n",
        "savefig(fig, os.path.join('figs', f'cifar-sgld.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dzN--3ebRjlX"
      },
      "outputs": [],
      "source": [
        "# Plot Figure 4(b)\n",
        "\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    fcmi_results = pickle.load(f)\n",
        "\n",
        "results_file_path = os.path.join(results_dir, f'{exp_name}', 'sgld_results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    sgld_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_epochs_new(fcmi_results, n=ns[0], epochs=epochs)\n",
        "\n",
        "ax.legend()\n",
        "fig.tight_layout()\n",
        "fig.show()\n",
        "savefig(fig, os.path.join('figs', f'cifar-sgld-zoom.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "-bhNVLdXjyJ4",
        "outputId": "14d9bca9-01ea-4b9f-a851-5a588412ddcb"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "/* Put everything inside the global mpl namespace */\n",
              "window.mpl = {};\n",
              "\n",
              "\n",
              "mpl.get_websocket_type = function() {\n",
              "    if (typeof(WebSocket) !== 'undefined') {\n",
              "        return WebSocket;\n",
              "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
              "        return MozWebSocket;\n",
              "    } else {\n",
              "        alert('Your browser does not have WebSocket support. ' +\n",
              "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
              "              'Firefox 4 and 5 are also supported but you ' +\n",
              "              'have to enable WebSockets in about:config.');\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
              "    this.id = figure_id;\n",
              "\n",
              "    this.ws = websocket;\n",
              "\n",
              "    this.supports_binary = (this.ws.binaryType != undefined);\n",
              "\n",
              "    if (!this.supports_binary) {\n",
              "        var warnings = document.getElementById(\"mpl-warnings\");\n",
              "        if (warnings) {\n",
              "            warnings.style.display = 'block';\n",
              "            warnings.textContent = (\n",
              "                \"This browser does not support binary websocket messages. \" +\n",
              "                    \"Performance may be slow.\");\n",
              "        }\n",
              "    }\n",
              "\n",
              "    this.imageObj = new Image();\n",
              "\n",
              "    this.context = undefined;\n",
              "    this.message = undefined;\n",
              "    this.canvas = undefined;\n",
              "    this.rubberband_canvas = undefined;\n",
              "    this.rubberband_context = undefined;\n",
              "    this.format_dropdown = undefined;\n",
              "\n",
              "    this.image_mode = 'full';\n",
              "\n",
              "    this.root = $('<div/>');\n",
              "    this._root_extra_style(this.root)\n",
              "    this.root.attr('style', 'display: inline-block');\n",
              "\n",
              "    $(parent_element).append(this.root);\n",
              "\n",
              "    this._init_header(this);\n",
              "    this._init_canvas(this);\n",
              "    this._init_toolbar(this);\n",
              "\n",
              "    var fig = this;\n",
              "\n",
              "    this.waiting = false;\n",
              "\n",
              "    this.ws.onopen =  function () {\n",
              "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
              "            fig.send_message(\"send_image_mode\", {});\n",
              "            if (mpl.ratio != 1) {\n",
              "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
              "            }\n",
              "            fig.send_message(\"refresh\", {});\n",
              "        }\n",
              "\n",
              "    this.imageObj.onload = function() {\n",
              "            if (fig.image_mode == 'full') {\n",
              "                // Full images could contain transparency (where diff images\n",
              "                // almost always do), so we need to clear the canvas so that\n",
              "                // there is no ghosting.\n",
              "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
              "            }\n",
              "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
              "        };\n",
              "\n",
              "    this.imageObj.onunload = function() {\n",
              "        fig.ws.close();\n",
              "    }\n",
              "\n",
              "    this.ws.onmessage = this._make_on_message_function(this);\n",
              "\n",
              "    this.ondownload = ondownload;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_header = function() {\n",
              "    var titlebar = $(\n",
              "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
              "        'ui-helper-clearfix\"/>');\n",
              "    var titletext = $(\n",
              "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
              "        'text-align: center; padding: 3px;\"/>');\n",
              "    titlebar.append(titletext)\n",
              "    this.root.append(titlebar);\n",
              "    this.header = titletext[0];\n",
              "}\n",
              "\n",
              "\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_canvas = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var canvas_div = $('<div/>');\n",
              "\n",
              "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
              "\n",
              "    function canvas_keyboard_event(event) {\n",
              "        return fig.key_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
              "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
              "    this.canvas_div = canvas_div\n",
              "    this._canvas_extra_style(canvas_div)\n",
              "    this.root.append(canvas_div);\n",
              "\n",
              "    var canvas = $('<canvas/>');\n",
              "    canvas.addClass('mpl-canvas');\n",
              "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
              "\n",
              "    this.canvas = canvas[0];\n",
              "    this.context = canvas[0].getContext(\"2d\");\n",
              "\n",
              "    var backingStore = this.context.backingStorePixelRatio ||\n",
              "\tthis.context.webkitBackingStorePixelRatio ||\n",
              "\tthis.context.mozBackingStorePixelRatio ||\n",
              "\tthis.context.msBackingStorePixelRatio ||\n",
              "\tthis.context.oBackingStorePixelRatio ||\n",
              "\tthis.context.backingStorePixelRatio || 1;\n",
              "\n",
              "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
              "\n",
              "    var rubberband = $('<canvas/>');\n",
              "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
              "\n",
              "    var pass_mouse_events = true;\n",
              "\n",
              "    canvas_div.resizable({\n",
              "        start: function(event, ui) {\n",
              "            pass_mouse_events = false;\n",
              "        },\n",
              "        resize: function(event, ui) {\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "        stop: function(event, ui) {\n",
              "            pass_mouse_events = true;\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "    });\n",
              "\n",
              "    function mouse_event_fn(event) {\n",
              "        if (pass_mouse_events)\n",
              "            return fig.mouse_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    rubberband.mousedown('button_press', mouse_event_fn);\n",
              "    rubberband.mouseup('button_release', mouse_event_fn);\n",
              "    // Throttle sequential mouse events to 1 every 20ms.\n",
              "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
              "\n",
              "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
              "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
              "\n",
              "    canvas_div.on(\"wheel\", function (event) {\n",
              "        event = event.originalEvent;\n",
              "        event['data'] = 'scroll'\n",
              "        if (event.deltaY < 0) {\n",
              "            event.step = 1;\n",
              "        } else {\n",
              "            event.step = -1;\n",
              "        }\n",
              "        mouse_event_fn(event);\n",
              "    });\n",
              "\n",
              "    canvas_div.append(canvas);\n",
              "    canvas_div.append(rubberband);\n",
              "\n",
              "    this.rubberband = rubberband;\n",
              "    this.rubberband_canvas = rubberband[0];\n",
              "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
              "    this.rubberband_context.strokeStyle = \"#000000\";\n",
              "\n",
              "    this._resize_canvas = function(width, height) {\n",
              "        // Keep the size of the canvas, canvas container, and rubber band\n",
              "        // canvas in synch.\n",
              "        canvas_div.css('width', width)\n",
              "        canvas_div.css('height', height)\n",
              "\n",
              "        canvas.attr('width', width * mpl.ratio);\n",
              "        canvas.attr('height', height * mpl.ratio);\n",
              "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
              "\n",
              "        rubberband.attr('width', width);\n",
              "        rubberband.attr('height', height);\n",
              "    }\n",
              "\n",
              "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
              "    // upon first draw.\n",
              "    this._resize_canvas(600, 600);\n",
              "\n",
              "    // Disable right mouse context menu.\n",
              "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
              "        return false;\n",
              "    });\n",
              "\n",
              "    function set_focus () {\n",
              "        canvas.focus();\n",
              "        canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    window.setTimeout(set_focus, 100);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items) {\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) {\n",
              "            // put a spacer in here.\n",
              "            continue;\n",
              "        }\n",
              "        var button = $('<button/>');\n",
              "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
              "                        'ui-button-icon-only');\n",
              "        button.attr('role', 'button');\n",
              "        button.attr('aria-disabled', 'false');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "\n",
              "        var icon_img = $('<span/>');\n",
              "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
              "        icon_img.addClass(image);\n",
              "        icon_img.addClass('ui-corner-all');\n",
              "\n",
              "        var tooltip_span = $('<span/>');\n",
              "        tooltip_span.addClass('ui-button-text');\n",
              "        tooltip_span.html(tooltip);\n",
              "\n",
              "        button.append(icon_img);\n",
              "        button.append(tooltip_span);\n",
              "\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    var fmt_picker_span = $('<span/>');\n",
              "\n",
              "    var fmt_picker = $('<select/>');\n",
              "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
              "    fmt_picker_span.append(fmt_picker);\n",
              "    nav_element.append(fmt_picker_span);\n",
              "    this.format_dropdown = fmt_picker[0];\n",
              "\n",
              "    for (var ind in mpl.extensions) {\n",
              "        var fmt = mpl.extensions[ind];\n",
              "        var option = $(\n",
              "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
              "        fmt_picker.append(option);\n",
              "    }\n",
              "\n",
              "    // Add hover states to the ui-buttons\n",
              "    $( \".ui-button\" ).hover(\n",
              "        function() { $(this).addClass(\"ui-state-hover\");},\n",
              "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
              "    );\n",
              "\n",
              "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
              "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
              "    // which will in turn request a refresh of the image.\n",
              "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_message = function(type, properties) {\n",
              "    properties['type'] = type;\n",
              "    properties['figure_id'] = this.id;\n",
              "    this.ws.send(JSON.stringify(properties));\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_draw_message = function() {\n",
              "    if (!this.waiting) {\n",
              "        this.waiting = true;\n",
              "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
              "    }\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    var format_dropdown = fig.format_dropdown;\n",
              "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
              "    fig.ondownload(fig, format);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
              "    var size = msg['size'];\n",
              "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
              "        fig._resize_canvas(size[0], size[1]);\n",
              "        fig.send_message(\"refresh\", {});\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
              "    var x0 = msg['x0'] / mpl.ratio;\n",
              "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
              "    var x1 = msg['x1'] / mpl.ratio;\n",
              "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
              "    x0 = Math.floor(x0) + 0.5;\n",
              "    y0 = Math.floor(y0) + 0.5;\n",
              "    x1 = Math.floor(x1) + 0.5;\n",
              "    y1 = Math.floor(y1) + 0.5;\n",
              "    var min_x = Math.min(x0, x1);\n",
              "    var min_y = Math.min(y0, y1);\n",
              "    var width = Math.abs(x1 - x0);\n",
              "    var height = Math.abs(y1 - y0);\n",
              "\n",
              "    fig.rubberband_context.clearRect(\n",
              "        0, 0, fig.canvas.width / mpl.ratio, fig.canvas.height / mpl.ratio);\n",
              "\n",
              "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
              "    // Updates the figure title.\n",
              "    fig.header.textContent = msg['label'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
              "    var cursor = msg['cursor'];\n",
              "    switch(cursor)\n",
              "    {\n",
              "    case 0:\n",
              "        cursor = 'pointer';\n",
              "        break;\n",
              "    case 1:\n",
              "        cursor = 'default';\n",
              "        break;\n",
              "    case 2:\n",
              "        cursor = 'crosshair';\n",
              "        break;\n",
              "    case 3:\n",
              "        cursor = 'move';\n",
              "        break;\n",
              "    }\n",
              "    fig.rubberband_canvas.style.cursor = cursor;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
              "    fig.message.textContent = msg['message'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
              "    // Request the server to send over a new figure.\n",
              "    fig.send_draw_message();\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
              "    fig.image_mode = msg['mode'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Called whenever the canvas gets updated.\n",
              "    this.send_message(\"ack\", {});\n",
              "}\n",
              "\n",
              "// A function to construct a web socket function for onmessage handling.\n",
              "// Called in the figure constructor.\n",
              "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
              "    return function socket_on_message(evt) {\n",
              "        if (evt.data instanceof Blob) {\n",
              "            /* FIXME: We get \"Resource interpreted as Image but\n",
              "             * transferred with MIME type text/plain:\" errors on\n",
              "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
              "             * to be part of the websocket stream */\n",
              "            evt.data.type = \"image/png\";\n",
              "\n",
              "            /* Free the memory for the previous frames */\n",
              "            if (fig.imageObj.src) {\n",
              "                (window.URL || window.webkitURL).revokeObjectURL(\n",
              "                    fig.imageObj.src);\n",
              "            }\n",
              "\n",
              "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
              "                evt.data);\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
              "            fig.imageObj.src = evt.data;\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        var msg = JSON.parse(evt.data);\n",
              "        var msg_type = msg['type'];\n",
              "\n",
              "        // Call the  \"handle_{type}\" callback, which takes\n",
              "        // the figure and JSON message as its only arguments.\n",
              "        try {\n",
              "            var callback = fig[\"handle_\" + msg_type];\n",
              "        } catch (e) {\n",
              "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        if (callback) {\n",
              "            try {\n",
              "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
              "                callback(fig, msg);\n",
              "            } catch (e) {\n",
              "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
              "            }\n",
              "        }\n",
              "    };\n",
              "}\n",
              "\n",
              "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
              "mpl.findpos = function(e) {\n",
              "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
              "    var targ;\n",
              "    if (!e)\n",
              "        e = window.event;\n",
              "    if (e.target)\n",
              "        targ = e.target;\n",
              "    else if (e.srcElement)\n",
              "        targ = e.srcElement;\n",
              "    if (targ.nodeType == 3) // defeat Safari bug\n",
              "        targ = targ.parentNode;\n",
              "\n",
              "    // jQuery normalizes the pageX and pageY\n",
              "    // pageX,Y are the mouse positions relative to the document\n",
              "    // offset() returns the position of the element relative to the document\n",
              "    var x = e.pageX - $(targ).offset().left;\n",
              "    var y = e.pageY - $(targ).offset().top;\n",
              "\n",
              "    return {\"x\": x, \"y\": y};\n",
              "};\n",
              "\n",
              "/*\n",
              " * return a copy of an object with only non-object keys\n",
              " * we need this to avoid circular references\n",
              " * http://stackoverflow.com/a/24161582/3208463\n",
              " */\n",
              "function simpleKeys (original) {\n",
              "  return Object.keys(original).reduce(function (obj, key) {\n",
              "    if (typeof original[key] !== 'object')\n",
              "        obj[key] = original[key]\n",
              "    return obj;\n",
              "  }, {});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.mouse_event = function(event, name) {\n",
              "    var canvas_pos = mpl.findpos(event)\n",
              "\n",
              "    if (name === 'button_press')\n",
              "    {\n",
              "        this.canvas.focus();\n",
              "        this.canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    var x = canvas_pos.x * mpl.ratio;\n",
              "    var y = canvas_pos.y * mpl.ratio;\n",
              "\n",
              "    this.send_message(name, {x: x, y: y, button: event.button,\n",
              "                             step: event.step,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "\n",
              "    /* This prevents the web browser from automatically changing to\n",
              "     * the text insertion cursor when the button is pressed.  We want\n",
              "     * to control all of the cursor setting manually through the\n",
              "     * 'cursor' event from matplotlib */\n",
              "    event.preventDefault();\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    // Handle any extra behaviour associated with a key event\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.key_event = function(event, name) {\n",
              "\n",
              "    // Prevent repeat events\n",
              "    if (name == 'key_press')\n",
              "    {\n",
              "        if (event.which === this._key)\n",
              "            return;\n",
              "        else\n",
              "            this._key = event.which;\n",
              "    }\n",
              "    if (name == 'key_release')\n",
              "        this._key = null;\n",
              "\n",
              "    var value = '';\n",
              "    if (event.ctrlKey && event.which != 17)\n",
              "        value += \"ctrl+\";\n",
              "    if (event.altKey && event.which != 18)\n",
              "        value += \"alt+\";\n",
              "    if (event.shiftKey && event.which != 16)\n",
              "        value += \"shift+\";\n",
              "\n",
              "    value += 'k';\n",
              "    value += event.which.toString();\n",
              "\n",
              "    this._key_event_extra(event, name);\n",
              "\n",
              "    this.send_message(name, {key: value,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
              "    if (name == 'download') {\n",
              "        this.handle_save(this, null);\n",
              "    } else {\n",
              "        this.send_message(\"toolbar_button\", {name: name});\n",
              "    }\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
              "    this.message.textContent = tooltip;\n",
              "};\n",
              "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
              "\n",
              "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
              "\n",
              "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
              "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
              "    // object with the appropriate methods. Currently this is a non binary\n",
              "    // socket, so there is still some room for performance tuning.\n",
              "    var ws = {};\n",
              "\n",
              "    ws.close = function() {\n",
              "        comm.close()\n",
              "    };\n",
              "    ws.send = function(m) {\n",
              "        //console.log('sending', m);\n",
              "        comm.send(m);\n",
              "    };\n",
              "    // Register the callback with on_msg.\n",
              "    comm.on_msg(function(msg) {\n",
              "        //console.log('receiving', msg['content']['data'], msg);\n",
              "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
              "        ws.onmessage(msg['content']['data'])\n",
              "    });\n",
              "    return ws;\n",
              "}\n",
              "\n",
              "mpl.mpl_figure_comm = function(comm, msg) {\n",
              "    // This is the function which gets called when the mpl process\n",
              "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
              "\n",
              "    var id = msg.content.data.id;\n",
              "    // Get hold of the div created by the display call when the Comm\n",
              "    // socket was opened in Python.\n",
              "    var element = $(\"#\" + id);\n",
              "    var ws_proxy = comm_websocket_adapter(comm)\n",
              "\n",
              "    function ondownload(figure, format) {\n",
              "        window.open(figure.imageObj.src);\n",
              "    }\n",
              "\n",
              "    var fig = new mpl.figure(id, ws_proxy,\n",
              "                           ondownload,\n",
              "                           element.get(0));\n",
              "\n",
              "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
              "    // web socket which is closed, not our websocket->open comm proxy.\n",
              "    ws_proxy.onopen();\n",
              "\n",
              "    fig.parent_element = element.get(0);\n",
              "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
              "    if (!fig.cell_info) {\n",
              "        console.error(\"Failed to find cell for figure\", id, fig);\n",
              "        return;\n",
              "    }\n",
              "\n",
              "    var output_index = fig.cell_info[2]\n",
              "    var cell = fig.cell_info[0];\n",
              "\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
              "    var width = fig.canvas.width/mpl.ratio\n",
              "    fig.root.unbind('remove')\n",
              "\n",
              "    // Update the output cell to use the data from the current canvas.\n",
              "    fig.push_to_output();\n",
              "    var dataURL = fig.canvas.toDataURL();\n",
              "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
              "    // the notebook keyboard shortcuts fail.\n",
              "    IPython.keyboard_manager.enable()\n",
              "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
              "    fig.close_ws(fig, msg);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.close_ws = function(fig, msg){\n",
              "    fig.send_message('closing', msg);\n",
              "    // fig.ws.close()\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
              "    // Turn the data on the canvas into data in the output cell.\n",
              "    var width = this.canvas.width/mpl.ratio\n",
              "    var dataURL = this.canvas.toDataURL();\n",
              "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Tell IPython that the notebook contents must change.\n",
              "    IPython.notebook.set_dirty(true);\n",
              "    this.send_message(\"ack\", {});\n",
              "    var fig = this;\n",
              "    // Wait a second, then push the new image to the DOM so\n",
              "    // that it is saved nicely (might be nice to debounce this).\n",
              "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items){\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) { continue; };\n",
              "\n",
              "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    // Add the status bar.\n",
              "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "\n",
              "    // Add the close button to the window.\n",
              "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
              "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
              "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
              "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
              "    buttongrp.append(button);\n",
              "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
              "    titlebar.prepend(buttongrp);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(el){\n",
              "    var fig = this\n",
              "    el.on(\"remove\", function(){\n",
              "\tfig.close_ws(fig, {});\n",
              "    });\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(el){\n",
              "    // this is important to make the div 'focusable\n",
              "    el.attr('tabindex', 0)\n",
              "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
              "    // off when our div gets focus\n",
              "\n",
              "    // location in version 3\n",
              "    if (IPython.notebook.keyboard_manager) {\n",
              "        IPython.notebook.keyboard_manager.register_events(el);\n",
              "    }\n",
              "    else {\n",
              "        // location in version 2\n",
              "        IPython.keyboard_manager.register_events(el);\n",
              "    }\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    var manager = IPython.notebook.keyboard_manager;\n",
              "    if (!manager)\n",
              "        manager = IPython.keyboard_manager;\n",
              "\n",
              "    // Check for shift+enter\n",
              "    if (event.shiftKey && event.which == 13) {\n",
              "        this.canvas_div.blur();\n",
              "        // select the cell after this one\n",
              "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
              "        IPython.notebook.select(index + 1);\n",
              "    }\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    fig.ondownload(fig, null);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.find_output_cell = function(html_output) {\n",
              "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
              "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
              "    // IPython event is triggered only after the cells have been serialised, which for\n",
              "    // our purposes (turning an active figure into a static one), is too late.\n",
              "    var cells = IPython.notebook.get_cells();\n",
              "    var ncells = cells.length;\n",
              "    for (var i=0; i<ncells; i++) {\n",
              "        var cell = cells[i];\n",
              "        if (cell.cell_type === 'code'){\n",
              "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
              "                var data = cell.output_area.outputs[j];\n",
              "                if (data.data) {\n",
              "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
              "                    data = data.data;\n",
              "                }\n",
              "                if (data['text/html'] == html_output) {\n",
              "                    return [cell, data, j];\n",
              "                }\n",
              "            }\n",
              "        }\n",
              "    }\n",
              "}\n",
              "\n",
              "// Register the function which deals with the matplotlib target/channel.\n",
              "// The kernel may be null if the page has been refreshed.\n",
              "if (IPython.notebook.kernel != null) {\n",
              "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
              "}\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div id='340cba2c-cc1a-4981-8cbb-1dd6f2d39e38'></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[4.8279762268066406e-05, 4.8279762268066406e-05, 4.8279762268066406e-05]\n",
            "[0.008010029792785645, 0.008010029792785645, 0.008010029792785645]\n",
            "[0.0256213 0.0256213 0.0256213]\n",
            "[0.015089412027716654, 0.015089412027716654, 0.015089412027716654]\n"
          ]
        }
      ],
      "source": [
        "# # # # # Plot Figure 5(a)\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [0.1, 1e-3, 1e-5]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-lr\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_arb(deterministic_results, ns=ns, epoch=epochs[-1], \n",
        "                        xlabel = 'Learning rate', xticks = ['$10^{-1}$', '$10^{-3}$','$10^{-5}$'])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-lr-new.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "sfBBtk3LBrJO",
        "outputId": "4e0d693c-692f-4759-cf41-0b534f7ab083"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "/* Put everything inside the global mpl namespace */\n",
              "window.mpl = {};\n",
              "\n",
              "\n",
              "mpl.get_websocket_type = function() {\n",
              "    if (typeof(WebSocket) !== 'undefined') {\n",
              "        return WebSocket;\n",
              "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
              "        return MozWebSocket;\n",
              "    } else {\n",
              "        alert('Your browser does not have WebSocket support. ' +\n",
              "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
              "              'Firefox 4 and 5 are also supported but you ' +\n",
              "              'have to enable WebSockets in about:config.');\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
              "    this.id = figure_id;\n",
              "\n",
              "    this.ws = websocket;\n",
              "\n",
              "    this.supports_binary = (this.ws.binaryType != undefined);\n",
              "\n",
              "    if (!this.supports_binary) {\n",
              "        var warnings = document.getElementById(\"mpl-warnings\");\n",
              "        if (warnings) {\n",
              "            warnings.style.display = 'block';\n",
              "            warnings.textContent = (\n",
              "                \"This browser does not support binary websocket messages. \" +\n",
              "                    \"Performance may be slow.\");\n",
              "        }\n",
              "    }\n",
              "\n",
              "    this.imageObj = new Image();\n",
              "\n",
              "    this.context = undefined;\n",
              "    this.message = undefined;\n",
              "    this.canvas = undefined;\n",
              "    this.rubberband_canvas = undefined;\n",
              "    this.rubberband_context = undefined;\n",
              "    this.format_dropdown = undefined;\n",
              "\n",
              "    this.image_mode = 'full';\n",
              "\n",
              "    this.root = $('<div/>');\n",
              "    this._root_extra_style(this.root)\n",
              "    this.root.attr('style', 'display: inline-block');\n",
              "\n",
              "    $(parent_element).append(this.root);\n",
              "\n",
              "    this._init_header(this);\n",
              "    this._init_canvas(this);\n",
              "    this._init_toolbar(this);\n",
              "\n",
              "    var fig = this;\n",
              "\n",
              "    this.waiting = false;\n",
              "\n",
              "    this.ws.onopen =  function () {\n",
              "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
              "            fig.send_message(\"send_image_mode\", {});\n",
              "            if (mpl.ratio != 1) {\n",
              "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
              "            }\n",
              "            fig.send_message(\"refresh\", {});\n",
              "        }\n",
              "\n",
              "    this.imageObj.onload = function() {\n",
              "            if (fig.image_mode == 'full') {\n",
              "                // Full images could contain transparency (where diff images\n",
              "                // almost always do), so we need to clear the canvas so that\n",
              "                // there is no ghosting.\n",
              "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
              "            }\n",
              "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
              "        };\n",
              "\n",
              "    this.imageObj.onunload = function() {\n",
              "        fig.ws.close();\n",
              "    }\n",
              "\n",
              "    this.ws.onmessage = this._make_on_message_function(this);\n",
              "\n",
              "    this.ondownload = ondownload;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_header = function() {\n",
              "    var titlebar = $(\n",
              "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
              "        'ui-helper-clearfix\"/>');\n",
              "    var titletext = $(\n",
              "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
              "        'text-align: center; padding: 3px;\"/>');\n",
              "    titlebar.append(titletext)\n",
              "    this.root.append(titlebar);\n",
              "    this.header = titletext[0];\n",
              "}\n",
              "\n",
              "\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_canvas = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var canvas_div = $('<div/>');\n",
              "\n",
              "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
              "\n",
              "    function canvas_keyboard_event(event) {\n",
              "        return fig.key_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
              "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
              "    this.canvas_div = canvas_div\n",
              "    this._canvas_extra_style(canvas_div)\n",
              "    this.root.append(canvas_div);\n",
              "\n",
              "    var canvas = $('<canvas/>');\n",
              "    canvas.addClass('mpl-canvas');\n",
              "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
              "\n",
              "    this.canvas = canvas[0];\n",
              "    this.context = canvas[0].getContext(\"2d\");\n",
              "\n",
              "    var backingStore = this.context.backingStorePixelRatio ||\n",
              "\tthis.context.webkitBackingStorePixelRatio ||\n",
              "\tthis.context.mozBackingStorePixelRatio ||\n",
              "\tthis.context.msBackingStorePixelRatio ||\n",
              "\tthis.context.oBackingStorePixelRatio ||\n",
              "\tthis.context.backingStorePixelRatio || 1;\n",
              "\n",
              "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
              "\n",
              "    var rubberband = $('<canvas/>');\n",
              "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
              "\n",
              "    var pass_mouse_events = true;\n",
              "\n",
              "    canvas_div.resizable({\n",
              "        start: function(event, ui) {\n",
              "            pass_mouse_events = false;\n",
              "        },\n",
              "        resize: function(event, ui) {\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "        stop: function(event, ui) {\n",
              "            pass_mouse_events = true;\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "    });\n",
              "\n",
              "    function mouse_event_fn(event) {\n",
              "        if (pass_mouse_events)\n",
              "            return fig.mouse_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    rubberband.mousedown('button_press', mouse_event_fn);\n",
              "    rubberband.mouseup('button_release', mouse_event_fn);\n",
              "    // Throttle sequential mouse events to 1 every 20ms.\n",
              "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
              "\n",
              "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
              "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
              "\n",
              "    canvas_div.on(\"wheel\", function (event) {\n",
              "        event = event.originalEvent;\n",
              "        event['data'] = 'scroll'\n",
              "        if (event.deltaY < 0) {\n",
              "            event.step = 1;\n",
              "        } else {\n",
              "            event.step = -1;\n",
              "        }\n",
              "        mouse_event_fn(event);\n",
              "    });\n",
              "\n",
              "    canvas_div.append(canvas);\n",
              "    canvas_div.append(rubberband);\n",
              "\n",
              "    this.rubberband = rubberband;\n",
              "    this.rubberband_canvas = rubberband[0];\n",
              "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
              "    this.rubberband_context.strokeStyle = \"#000000\";\n",
              "\n",
              "    this._resize_canvas = function(width, height) {\n",
              "        // Keep the size of the canvas, canvas container, and rubber band\n",
              "        // canvas in synch.\n",
              "        canvas_div.css('width', width)\n",
              "        canvas_div.css('height', height)\n",
              "\n",
              "        canvas.attr('width', width * mpl.ratio);\n",
              "        canvas.attr('height', height * mpl.ratio);\n",
              "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
              "\n",
              "        rubberband.attr('width', width);\n",
              "        rubberband.attr('height', height);\n",
              "    }\n",
              "\n",
              "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
              "    // upon first draw.\n",
              "    this._resize_canvas(600, 600);\n",
              "\n",
              "    // Disable right mouse context menu.\n",
              "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
              "        return false;\n",
              "    });\n",
              "\n",
              "    function set_focus () {\n",
              "        canvas.focus();\n",
              "        canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    window.setTimeout(set_focus, 100);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items) {\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) {\n",
              "            // put a spacer in here.\n",
              "            continue;\n",
              "        }\n",
              "        var button = $('<button/>');\n",
              "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
              "                        'ui-button-icon-only');\n",
              "        button.attr('role', 'button');\n",
              "        button.attr('aria-disabled', 'false');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "\n",
              "        var icon_img = $('<span/>');\n",
              "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
              "        icon_img.addClass(image);\n",
              "        icon_img.addClass('ui-corner-all');\n",
              "\n",
              "        var tooltip_span = $('<span/>');\n",
              "        tooltip_span.addClass('ui-button-text');\n",
              "        tooltip_span.html(tooltip);\n",
              "\n",
              "        button.append(icon_img);\n",
              "        button.append(tooltip_span);\n",
              "\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    var fmt_picker_span = $('<span/>');\n",
              "\n",
              "    var fmt_picker = $('<select/>');\n",
              "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
              "    fmt_picker_span.append(fmt_picker);\n",
              "    nav_element.append(fmt_picker_span);\n",
              "    this.format_dropdown = fmt_picker[0];\n",
              "\n",
              "    for (var ind in mpl.extensions) {\n",
              "        var fmt = mpl.extensions[ind];\n",
              "        var option = $(\n",
              "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
              "        fmt_picker.append(option);\n",
              "    }\n",
              "\n",
              "    // Add hover states to the ui-buttons\n",
              "    $( \".ui-button\" ).hover(\n",
              "        function() { $(this).addClass(\"ui-state-hover\");},\n",
              "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
              "    );\n",
              "\n",
              "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
              "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
              "    // which will in turn request a refresh of the image.\n",
              "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_message = function(type, properties) {\n",
              "    properties['type'] = type;\n",
              "    properties['figure_id'] = this.id;\n",
              "    this.ws.send(JSON.stringify(properties));\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_draw_message = function() {\n",
              "    if (!this.waiting) {\n",
              "        this.waiting = true;\n",
              "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
              "    }\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    var format_dropdown = fig.format_dropdown;\n",
              "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
              "    fig.ondownload(fig, format);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
              "    var size = msg['size'];\n",
              "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
              "        fig._resize_canvas(size[0], size[1]);\n",
              "        fig.send_message(\"refresh\", {});\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
              "    var x0 = msg['x0'] / mpl.ratio;\n",
              "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
              "    var x1 = msg['x1'] / mpl.ratio;\n",
              "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
              "    x0 = Math.floor(x0) + 0.5;\n",
              "    y0 = Math.floor(y0) + 0.5;\n",
              "    x1 = Math.floor(x1) + 0.5;\n",
              "    y1 = Math.floor(y1) + 0.5;\n",
              "    var min_x = Math.min(x0, x1);\n",
              "    var min_y = Math.min(y0, y1);\n",
              "    var width = Math.abs(x1 - x0);\n",
              "    var height = Math.abs(y1 - y0);\n",
              "\n",
              "    fig.rubberband_context.clearRect(\n",
              "        0, 0, fig.canvas.width / mpl.ratio, fig.canvas.height / mpl.ratio);\n",
              "\n",
              "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
              "    // Updates the figure title.\n",
              "    fig.header.textContent = msg['label'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
              "    var cursor = msg['cursor'];\n",
              "    switch(cursor)\n",
              "    {\n",
              "    case 0:\n",
              "        cursor = 'pointer';\n",
              "        break;\n",
              "    case 1:\n",
              "        cursor = 'default';\n",
              "        break;\n",
              "    case 2:\n",
              "        cursor = 'crosshair';\n",
              "        break;\n",
              "    case 3:\n",
              "        cursor = 'move';\n",
              "        break;\n",
              "    }\n",
              "    fig.rubberband_canvas.style.cursor = cursor;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
              "    fig.message.textContent = msg['message'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
              "    // Request the server to send over a new figure.\n",
              "    fig.send_draw_message();\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
              "    fig.image_mode = msg['mode'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Called whenever the canvas gets updated.\n",
              "    this.send_message(\"ack\", {});\n",
              "}\n",
              "\n",
              "// A function to construct a web socket function for onmessage handling.\n",
              "// Called in the figure constructor.\n",
              "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
              "    return function socket_on_message(evt) {\n",
              "        if (evt.data instanceof Blob) {\n",
              "            /* FIXME: We get \"Resource interpreted as Image but\n",
              "             * transferred with MIME type text/plain:\" errors on\n",
              "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
              "             * to be part of the websocket stream */\n",
              "            evt.data.type = \"image/png\";\n",
              "\n",
              "            /* Free the memory for the previous frames */\n",
              "            if (fig.imageObj.src) {\n",
              "                (window.URL || window.webkitURL).revokeObjectURL(\n",
              "                    fig.imageObj.src);\n",
              "            }\n",
              "\n",
              "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
              "                evt.data);\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
              "            fig.imageObj.src = evt.data;\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        var msg = JSON.parse(evt.data);\n",
              "        var msg_type = msg['type'];\n",
              "\n",
              "        // Call the  \"handle_{type}\" callback, which takes\n",
              "        // the figure and JSON message as its only arguments.\n",
              "        try {\n",
              "            var callback = fig[\"handle_\" + msg_type];\n",
              "        } catch (e) {\n",
              "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        if (callback) {\n",
              "            try {\n",
              "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
              "                callback(fig, msg);\n",
              "            } catch (e) {\n",
              "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
              "            }\n",
              "        }\n",
              "    };\n",
              "}\n",
              "\n",
              "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
              "mpl.findpos = function(e) {\n",
              "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
              "    var targ;\n",
              "    if (!e)\n",
              "        e = window.event;\n",
              "    if (e.target)\n",
              "        targ = e.target;\n",
              "    else if (e.srcElement)\n",
              "        targ = e.srcElement;\n",
              "    if (targ.nodeType == 3) // defeat Safari bug\n",
              "        targ = targ.parentNode;\n",
              "\n",
              "    // jQuery normalizes the pageX and pageY\n",
              "    // pageX,Y are the mouse positions relative to the document\n",
              "    // offset() returns the position of the element relative to the document\n",
              "    var x = e.pageX - $(targ).offset().left;\n",
              "    var y = e.pageY - $(targ).offset().top;\n",
              "\n",
              "    return {\"x\": x, \"y\": y};\n",
              "};\n",
              "\n",
              "/*\n",
              " * return a copy of an object with only non-object keys\n",
              " * we need this to avoid circular references\n",
              " * http://stackoverflow.com/a/24161582/3208463\n",
              " */\n",
              "function simpleKeys (original) {\n",
              "  return Object.keys(original).reduce(function (obj, key) {\n",
              "    if (typeof original[key] !== 'object')\n",
              "        obj[key] = original[key]\n",
              "    return obj;\n",
              "  }, {});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.mouse_event = function(event, name) {\n",
              "    var canvas_pos = mpl.findpos(event)\n",
              "\n",
              "    if (name === 'button_press')\n",
              "    {\n",
              "        this.canvas.focus();\n",
              "        this.canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    var x = canvas_pos.x * mpl.ratio;\n",
              "    var y = canvas_pos.y * mpl.ratio;\n",
              "\n",
              "    this.send_message(name, {x: x, y: y, button: event.button,\n",
              "                             step: event.step,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "\n",
              "    /* This prevents the web browser from automatically changing to\n",
              "     * the text insertion cursor when the button is pressed.  We want\n",
              "     * to control all of the cursor setting manually through the\n",
              "     * 'cursor' event from matplotlib */\n",
              "    event.preventDefault();\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    // Handle any extra behaviour associated with a key event\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.key_event = function(event, name) {\n",
              "\n",
              "    // Prevent repeat events\n",
              "    if (name == 'key_press')\n",
              "    {\n",
              "        if (event.which === this._key)\n",
              "            return;\n",
              "        else\n",
              "            this._key = event.which;\n",
              "    }\n",
              "    if (name == 'key_release')\n",
              "        this._key = null;\n",
              "\n",
              "    var value = '';\n",
              "    if (event.ctrlKey && event.which != 17)\n",
              "        value += \"ctrl+\";\n",
              "    if (event.altKey && event.which != 18)\n",
              "        value += \"alt+\";\n",
              "    if (event.shiftKey && event.which != 16)\n",
              "        value += \"shift+\";\n",
              "\n",
              "    value += 'k';\n",
              "    value += event.which.toString();\n",
              "\n",
              "    this._key_event_extra(event, name);\n",
              "\n",
              "    this.send_message(name, {key: value,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
              "    if (name == 'download') {\n",
              "        this.handle_save(this, null);\n",
              "    } else {\n",
              "        this.send_message(\"toolbar_button\", {name: name});\n",
              "    }\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
              "    this.message.textContent = tooltip;\n",
              "};\n",
              "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
              "\n",
              "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
              "\n",
              "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
              "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
              "    // object with the appropriate methods. Currently this is a non binary\n",
              "    // socket, so there is still some room for performance tuning.\n",
              "    var ws = {};\n",
              "\n",
              "    ws.close = function() {\n",
              "        comm.close()\n",
              "    };\n",
              "    ws.send = function(m) {\n",
              "        //console.log('sending', m);\n",
              "        comm.send(m);\n",
              "    };\n",
              "    // Register the callback with on_msg.\n",
              "    comm.on_msg(function(msg) {\n",
              "        //console.log('receiving', msg['content']['data'], msg);\n",
              "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
              "        ws.onmessage(msg['content']['data'])\n",
              "    });\n",
              "    return ws;\n",
              "}\n",
              "\n",
              "mpl.mpl_figure_comm = function(comm, msg) {\n",
              "    // This is the function which gets called when the mpl process\n",
              "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
              "\n",
              "    var id = msg.content.data.id;\n",
              "    // Get hold of the div created by the display call when the Comm\n",
              "    // socket was opened in Python.\n",
              "    var element = $(\"#\" + id);\n",
              "    var ws_proxy = comm_websocket_adapter(comm)\n",
              "\n",
              "    function ondownload(figure, format) {\n",
              "        window.open(figure.imageObj.src);\n",
              "    }\n",
              "\n",
              "    var fig = new mpl.figure(id, ws_proxy,\n",
              "                           ondownload,\n",
              "                           element.get(0));\n",
              "\n",
              "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
              "    // web socket which is closed, not our websocket->open comm proxy.\n",
              "    ws_proxy.onopen();\n",
              "\n",
              "    fig.parent_element = element.get(0);\n",
              "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
              "    if (!fig.cell_info) {\n",
              "        console.error(\"Failed to find cell for figure\", id, fig);\n",
              "        return;\n",
              "    }\n",
              "\n",
              "    var output_index = fig.cell_info[2]\n",
              "    var cell = fig.cell_info[0];\n",
              "\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
              "    var width = fig.canvas.width/mpl.ratio\n",
              "    fig.root.unbind('remove')\n",
              "\n",
              "    // Update the output cell to use the data from the current canvas.\n",
              "    fig.push_to_output();\n",
              "    var dataURL = fig.canvas.toDataURL();\n",
              "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
              "    // the notebook keyboard shortcuts fail.\n",
              "    IPython.keyboard_manager.enable()\n",
              "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
              "    fig.close_ws(fig, msg);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.close_ws = function(fig, msg){\n",
              "    fig.send_message('closing', msg);\n",
              "    // fig.ws.close()\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
              "    // Turn the data on the canvas into data in the output cell.\n",
              "    var width = this.canvas.width/mpl.ratio\n",
              "    var dataURL = this.canvas.toDataURL();\n",
              "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Tell IPython that the notebook contents must change.\n",
              "    IPython.notebook.set_dirty(true);\n",
              "    this.send_message(\"ack\", {});\n",
              "    var fig = this;\n",
              "    // Wait a second, then push the new image to the DOM so\n",
              "    // that it is saved nicely (might be nice to debounce this).\n",
              "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items){\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) { continue; };\n",
              "\n",
              "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    // Add the status bar.\n",
              "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "\n",
              "    // Add the close button to the window.\n",
              "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
              "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
              "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
              "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
              "    buttongrp.append(button);\n",
              "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
              "    titlebar.prepend(buttongrp);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(el){\n",
              "    var fig = this\n",
              "    el.on(\"remove\", function(){\n",
              "\tfig.close_ws(fig, {});\n",
              "    });\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(el){\n",
              "    // this is important to make the div 'focusable\n",
              "    el.attr('tabindex', 0)\n",
              "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
              "    // off when our div gets focus\n",
              "\n",
              "    // location in version 3\n",
              "    if (IPython.notebook.keyboard_manager) {\n",
              "        IPython.notebook.keyboard_manager.register_events(el);\n",
              "    }\n",
              "    else {\n",
              "        // location in version 2\n",
              "        IPython.keyboard_manager.register_events(el);\n",
              "    }\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    var manager = IPython.notebook.keyboard_manager;\n",
              "    if (!manager)\n",
              "        manager = IPython.keyboard_manager;\n",
              "\n",
              "    // Check for shift+enter\n",
              "    if (event.shiftKey && event.which == 13) {\n",
              "        this.canvas_div.blur();\n",
              "        // select the cell after this one\n",
              "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
              "        IPython.notebook.select(index + 1);\n",
              "    }\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    fig.ondownload(fig, null);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.find_output_cell = function(html_output) {\n",
              "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
              "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
              "    // IPython event is triggered only after the cells have been serialised, which for\n",
              "    // our purposes (turning an active figure into a static one), is too late.\n",
              "    var cells = IPython.notebook.get_cells();\n",
              "    var ncells = cells.length;\n",
              "    for (var i=0; i<ncells; i++) {\n",
              "        var cell = cells[i];\n",
              "        if (cell.cell_type === 'code'){\n",
              "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
              "                var data = cell.output_area.outputs[j];\n",
              "                if (data.data) {\n",
              "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
              "                    data = data.data;\n",
              "                }\n",
              "                if (data['text/html'] == html_output) {\n",
              "                    return [cell, data, j];\n",
              "                }\n",
              "            }\n",
              "        }\n",
              "    }\n",
              "}\n",
              "\n",
              "// Register the function which deals with the matplotlib target/channel.\n",
              "// The kernel may be null if the page has been refreshed.\n",
              "if (IPython.notebook.kernel != null) {\n",
              "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
              "}\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div id='e0cc5750-4db8-4046-9859-b020ffc58e7d'></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[5.501508712768555e-05, 5.501508712768555e-05, 4.00543212890625e-05]\n",
            "[0.00807332992553711, 0.008063316345214844, 0.007841706275939941]\n",
            "[0.02515756 0.02503997 0.02460378]\n",
            "[0.01517644914178828, 0.015122383536942442, 0.014793551395221602]\n"
          ]
        }
      ],
      "source": [
        "# # # # # Plot Figure 5(b)\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [0, 1, 2]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-width\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_arb(deterministic_results, ns=ns, epoch=epochs[-1], \n",
        "                        xlabel = 'Width (normalized)', xticks = ['1', '2','4'])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-width-new.pdf'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "bL6e0-NHMuIT",
        "outputId": "68296808-0965-4d1b-8dd1-2ca32b3bd206"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "/* Put everything inside the global mpl namespace */\n",
              "window.mpl = {};\n",
              "\n",
              "\n",
              "mpl.get_websocket_type = function() {\n",
              "    if (typeof(WebSocket) !== 'undefined') {\n",
              "        return WebSocket;\n",
              "    } else if (typeof(MozWebSocket) !== 'undefined') {\n",
              "        return MozWebSocket;\n",
              "    } else {\n",
              "        alert('Your browser does not have WebSocket support. ' +\n",
              "              'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
              "              'Firefox 4 and 5 are also supported but you ' +\n",
              "              'have to enable WebSockets in about:config.');\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n",
              "    this.id = figure_id;\n",
              "\n",
              "    this.ws = websocket;\n",
              "\n",
              "    this.supports_binary = (this.ws.binaryType != undefined);\n",
              "\n",
              "    if (!this.supports_binary) {\n",
              "        var warnings = document.getElementById(\"mpl-warnings\");\n",
              "        if (warnings) {\n",
              "            warnings.style.display = 'block';\n",
              "            warnings.textContent = (\n",
              "                \"This browser does not support binary websocket messages. \" +\n",
              "                    \"Performance may be slow.\");\n",
              "        }\n",
              "    }\n",
              "\n",
              "    this.imageObj = new Image();\n",
              "\n",
              "    this.context = undefined;\n",
              "    this.message = undefined;\n",
              "    this.canvas = undefined;\n",
              "    this.rubberband_canvas = undefined;\n",
              "    this.rubberband_context = undefined;\n",
              "    this.format_dropdown = undefined;\n",
              "\n",
              "    this.image_mode = 'full';\n",
              "\n",
              "    this.root = $('<div/>');\n",
              "    this._root_extra_style(this.root)\n",
              "    this.root.attr('style', 'display: inline-block');\n",
              "\n",
              "    $(parent_element).append(this.root);\n",
              "\n",
              "    this._init_header(this);\n",
              "    this._init_canvas(this);\n",
              "    this._init_toolbar(this);\n",
              "\n",
              "    var fig = this;\n",
              "\n",
              "    this.waiting = false;\n",
              "\n",
              "    this.ws.onopen =  function () {\n",
              "            fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n",
              "            fig.send_message(\"send_image_mode\", {});\n",
              "            if (mpl.ratio != 1) {\n",
              "                fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n",
              "            }\n",
              "            fig.send_message(\"refresh\", {});\n",
              "        }\n",
              "\n",
              "    this.imageObj.onload = function() {\n",
              "            if (fig.image_mode == 'full') {\n",
              "                // Full images could contain transparency (where diff images\n",
              "                // almost always do), so we need to clear the canvas so that\n",
              "                // there is no ghosting.\n",
              "                fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
              "            }\n",
              "            fig.context.drawImage(fig.imageObj, 0, 0);\n",
              "        };\n",
              "\n",
              "    this.imageObj.onunload = function() {\n",
              "        fig.ws.close();\n",
              "    }\n",
              "\n",
              "    this.ws.onmessage = this._make_on_message_function(this);\n",
              "\n",
              "    this.ondownload = ondownload;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_header = function() {\n",
              "    var titlebar = $(\n",
              "        '<div class=\"ui-dialog-titlebar ui-widget-header ui-corner-all ' +\n",
              "        'ui-helper-clearfix\"/>');\n",
              "    var titletext = $(\n",
              "        '<div class=\"ui-dialog-title\" style=\"width: 100%; ' +\n",
              "        'text-align: center; padding: 3px;\"/>');\n",
              "    titlebar.append(titletext)\n",
              "    this.root.append(titlebar);\n",
              "    this.header = titletext[0];\n",
              "}\n",
              "\n",
              "\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_canvas = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var canvas_div = $('<div/>');\n",
              "\n",
              "    canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n",
              "\n",
              "    function canvas_keyboard_event(event) {\n",
              "        return fig.key_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    canvas_div.keydown('key_press', canvas_keyboard_event);\n",
              "    canvas_div.keyup('key_release', canvas_keyboard_event);\n",
              "    this.canvas_div = canvas_div\n",
              "    this._canvas_extra_style(canvas_div)\n",
              "    this.root.append(canvas_div);\n",
              "\n",
              "    var canvas = $('<canvas/>');\n",
              "    canvas.addClass('mpl-canvas');\n",
              "    canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n",
              "\n",
              "    this.canvas = canvas[0];\n",
              "    this.context = canvas[0].getContext(\"2d\");\n",
              "\n",
              "    var backingStore = this.context.backingStorePixelRatio ||\n",
              "\tthis.context.webkitBackingStorePixelRatio ||\n",
              "\tthis.context.mozBackingStorePixelRatio ||\n",
              "\tthis.context.msBackingStorePixelRatio ||\n",
              "\tthis.context.oBackingStorePixelRatio ||\n",
              "\tthis.context.backingStorePixelRatio || 1;\n",
              "\n",
              "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
              "\n",
              "    var rubberband = $('<canvas/>');\n",
              "    rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n",
              "\n",
              "    var pass_mouse_events = true;\n",
              "\n",
              "    canvas_div.resizable({\n",
              "        start: function(event, ui) {\n",
              "            pass_mouse_events = false;\n",
              "        },\n",
              "        resize: function(event, ui) {\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "        stop: function(event, ui) {\n",
              "            pass_mouse_events = true;\n",
              "            fig.request_resize(ui.size.width, ui.size.height);\n",
              "        },\n",
              "    });\n",
              "\n",
              "    function mouse_event_fn(event) {\n",
              "        if (pass_mouse_events)\n",
              "            return fig.mouse_event(event, event['data']);\n",
              "    }\n",
              "\n",
              "    rubberband.mousedown('button_press', mouse_event_fn);\n",
              "    rubberband.mouseup('button_release', mouse_event_fn);\n",
              "    // Throttle sequential mouse events to 1 every 20ms.\n",
              "    rubberband.mousemove('motion_notify', mouse_event_fn);\n",
              "\n",
              "    rubberband.mouseenter('figure_enter', mouse_event_fn);\n",
              "    rubberband.mouseleave('figure_leave', mouse_event_fn);\n",
              "\n",
              "    canvas_div.on(\"wheel\", function (event) {\n",
              "        event = event.originalEvent;\n",
              "        event['data'] = 'scroll'\n",
              "        if (event.deltaY < 0) {\n",
              "            event.step = 1;\n",
              "        } else {\n",
              "            event.step = -1;\n",
              "        }\n",
              "        mouse_event_fn(event);\n",
              "    });\n",
              "\n",
              "    canvas_div.append(canvas);\n",
              "    canvas_div.append(rubberband);\n",
              "\n",
              "    this.rubberband = rubberband;\n",
              "    this.rubberband_canvas = rubberband[0];\n",
              "    this.rubberband_context = rubberband[0].getContext(\"2d\");\n",
              "    this.rubberband_context.strokeStyle = \"#000000\";\n",
              "\n",
              "    this._resize_canvas = function(width, height) {\n",
              "        // Keep the size of the canvas, canvas container, and rubber band\n",
              "        // canvas in synch.\n",
              "        canvas_div.css('width', width)\n",
              "        canvas_div.css('height', height)\n",
              "\n",
              "        canvas.attr('width', width * mpl.ratio);\n",
              "        canvas.attr('height', height * mpl.ratio);\n",
              "        canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n",
              "\n",
              "        rubberband.attr('width', width);\n",
              "        rubberband.attr('height', height);\n",
              "    }\n",
              "\n",
              "    // Set the figure to an initial 600x600px, this will subsequently be updated\n",
              "    // upon first draw.\n",
              "    this._resize_canvas(600, 600);\n",
              "\n",
              "    // Disable right mouse context menu.\n",
              "    $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n",
              "        return false;\n",
              "    });\n",
              "\n",
              "    function set_focus () {\n",
              "        canvas.focus();\n",
              "        canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    window.setTimeout(set_focus, 100);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items) {\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) {\n",
              "            // put a spacer in here.\n",
              "            continue;\n",
              "        }\n",
              "        var button = $('<button/>');\n",
              "        button.addClass('ui-button ui-widget ui-state-default ui-corner-all ' +\n",
              "                        'ui-button-icon-only');\n",
              "        button.attr('role', 'button');\n",
              "        button.attr('aria-disabled', 'false');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "\n",
              "        var icon_img = $('<span/>');\n",
              "        icon_img.addClass('ui-button-icon-primary ui-icon');\n",
              "        icon_img.addClass(image);\n",
              "        icon_img.addClass('ui-corner-all');\n",
              "\n",
              "        var tooltip_span = $('<span/>');\n",
              "        tooltip_span.addClass('ui-button-text');\n",
              "        tooltip_span.html(tooltip);\n",
              "\n",
              "        button.append(icon_img);\n",
              "        button.append(tooltip_span);\n",
              "\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    var fmt_picker_span = $('<span/>');\n",
              "\n",
              "    var fmt_picker = $('<select/>');\n",
              "    fmt_picker.addClass('mpl-toolbar-option ui-widget ui-widget-content');\n",
              "    fmt_picker_span.append(fmt_picker);\n",
              "    nav_element.append(fmt_picker_span);\n",
              "    this.format_dropdown = fmt_picker[0];\n",
              "\n",
              "    for (var ind in mpl.extensions) {\n",
              "        var fmt = mpl.extensions[ind];\n",
              "        var option = $(\n",
              "            '<option/>', {selected: fmt === mpl.default_extension}).html(fmt);\n",
              "        fmt_picker.append(option);\n",
              "    }\n",
              "\n",
              "    // Add hover states to the ui-buttons\n",
              "    $( \".ui-button\" ).hover(\n",
              "        function() { $(this).addClass(\"ui-state-hover\");},\n",
              "        function() { $(this).removeClass(\"ui-state-hover\");}\n",
              "    );\n",
              "\n",
              "    var status_bar = $('<span class=\"mpl-message\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.request_resize = function(x_pixels, y_pixels) {\n",
              "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
              "    // which will in turn request a refresh of the image.\n",
              "    this.send_message('resize', {'width': x_pixels, 'height': y_pixels});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_message = function(type, properties) {\n",
              "    properties['type'] = type;\n",
              "    properties['figure_id'] = this.id;\n",
              "    this.ws.send(JSON.stringify(properties));\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.send_draw_message = function() {\n",
              "    if (!this.waiting) {\n",
              "        this.waiting = true;\n",
              "        this.ws.send(JSON.stringify({type: \"draw\", figure_id: this.id}));\n",
              "    }\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    var format_dropdown = fig.format_dropdown;\n",
              "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
              "    fig.ondownload(fig, format);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.figure.prototype.handle_resize = function(fig, msg) {\n",
              "    var size = msg['size'];\n",
              "    if (size[0] != fig.canvas.width || size[1] != fig.canvas.height) {\n",
              "        fig._resize_canvas(size[0], size[1]);\n",
              "        fig.send_message(\"refresh\", {});\n",
              "    };\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_rubberband = function(fig, msg) {\n",
              "    var x0 = msg['x0'] / mpl.ratio;\n",
              "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
              "    var x1 = msg['x1'] / mpl.ratio;\n",
              "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
              "    x0 = Math.floor(x0) + 0.5;\n",
              "    y0 = Math.floor(y0) + 0.5;\n",
              "    x1 = Math.floor(x1) + 0.5;\n",
              "    y1 = Math.floor(y1) + 0.5;\n",
              "    var min_x = Math.min(x0, x1);\n",
              "    var min_y = Math.min(y0, y1);\n",
              "    var width = Math.abs(x1 - x0);\n",
              "    var height = Math.abs(y1 - y0);\n",
              "\n",
              "    fig.rubberband_context.clearRect(\n",
              "        0, 0, fig.canvas.width / mpl.ratio, fig.canvas.height / mpl.ratio);\n",
              "\n",
              "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_figure_label = function(fig, msg) {\n",
              "    // Updates the figure title.\n",
              "    fig.header.textContent = msg['label'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_cursor = function(fig, msg) {\n",
              "    var cursor = msg['cursor'];\n",
              "    switch(cursor)\n",
              "    {\n",
              "    case 0:\n",
              "        cursor = 'pointer';\n",
              "        break;\n",
              "    case 1:\n",
              "        cursor = 'default';\n",
              "        break;\n",
              "    case 2:\n",
              "        cursor = 'crosshair';\n",
              "        break;\n",
              "    case 3:\n",
              "        cursor = 'move';\n",
              "        break;\n",
              "    }\n",
              "    fig.rubberband_canvas.style.cursor = cursor;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_message = function(fig, msg) {\n",
              "    fig.message.textContent = msg['message'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_draw = function(fig, msg) {\n",
              "    // Request the server to send over a new figure.\n",
              "    fig.send_draw_message();\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_image_mode = function(fig, msg) {\n",
              "    fig.image_mode = msg['mode'];\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Called whenever the canvas gets updated.\n",
              "    this.send_message(\"ack\", {});\n",
              "}\n",
              "\n",
              "// A function to construct a web socket function for onmessage handling.\n",
              "// Called in the figure constructor.\n",
              "mpl.figure.prototype._make_on_message_function = function(fig) {\n",
              "    return function socket_on_message(evt) {\n",
              "        if (evt.data instanceof Blob) {\n",
              "            /* FIXME: We get \"Resource interpreted as Image but\n",
              "             * transferred with MIME type text/plain:\" errors on\n",
              "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
              "             * to be part of the websocket stream */\n",
              "            evt.data.type = \"image/png\";\n",
              "\n",
              "            /* Free the memory for the previous frames */\n",
              "            if (fig.imageObj.src) {\n",
              "                (window.URL || window.webkitURL).revokeObjectURL(\n",
              "                    fig.imageObj.src);\n",
              "            }\n",
              "\n",
              "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
              "                evt.data);\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "        else if (typeof evt.data === 'string' && evt.data.slice(0, 21) == \"data:image/png;base64\") {\n",
              "            fig.imageObj.src = evt.data;\n",
              "            fig.updated_canvas_event();\n",
              "            fig.waiting = false;\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        var msg = JSON.parse(evt.data);\n",
              "        var msg_type = msg['type'];\n",
              "\n",
              "        // Call the  \"handle_{type}\" callback, which takes\n",
              "        // the figure and JSON message as its only arguments.\n",
              "        try {\n",
              "            var callback = fig[\"handle_\" + msg_type];\n",
              "        } catch (e) {\n",
              "            console.log(\"No handler for the '\" + msg_type + \"' message type: \", msg);\n",
              "            return;\n",
              "        }\n",
              "\n",
              "        if (callback) {\n",
              "            try {\n",
              "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
              "                callback(fig, msg);\n",
              "            } catch (e) {\n",
              "                console.log(\"Exception inside the 'handler_\" + msg_type + \"' callback:\", e, e.stack, msg);\n",
              "            }\n",
              "        }\n",
              "    };\n",
              "}\n",
              "\n",
              "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
              "mpl.findpos = function(e) {\n",
              "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
              "    var targ;\n",
              "    if (!e)\n",
              "        e = window.event;\n",
              "    if (e.target)\n",
              "        targ = e.target;\n",
              "    else if (e.srcElement)\n",
              "        targ = e.srcElement;\n",
              "    if (targ.nodeType == 3) // defeat Safari bug\n",
              "        targ = targ.parentNode;\n",
              "\n",
              "    // jQuery normalizes the pageX and pageY\n",
              "    // pageX,Y are the mouse positions relative to the document\n",
              "    // offset() returns the position of the element relative to the document\n",
              "    var x = e.pageX - $(targ).offset().left;\n",
              "    var y = e.pageY - $(targ).offset().top;\n",
              "\n",
              "    return {\"x\": x, \"y\": y};\n",
              "};\n",
              "\n",
              "/*\n",
              " * return a copy of an object with only non-object keys\n",
              " * we need this to avoid circular references\n",
              " * http://stackoverflow.com/a/24161582/3208463\n",
              " */\n",
              "function simpleKeys (original) {\n",
              "  return Object.keys(original).reduce(function (obj, key) {\n",
              "    if (typeof original[key] !== 'object')\n",
              "        obj[key] = original[key]\n",
              "    return obj;\n",
              "  }, {});\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.mouse_event = function(event, name) {\n",
              "    var canvas_pos = mpl.findpos(event)\n",
              "\n",
              "    if (name === 'button_press')\n",
              "    {\n",
              "        this.canvas.focus();\n",
              "        this.canvas_div.focus();\n",
              "    }\n",
              "\n",
              "    var x = canvas_pos.x * mpl.ratio;\n",
              "    var y = canvas_pos.y * mpl.ratio;\n",
              "\n",
              "    this.send_message(name, {x: x, y: y, button: event.button,\n",
              "                             step: event.step,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "\n",
              "    /* This prevents the web browser from automatically changing to\n",
              "     * the text insertion cursor when the button is pressed.  We want\n",
              "     * to control all of the cursor setting manually through the\n",
              "     * 'cursor' event from matplotlib */\n",
              "    event.preventDefault();\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    // Handle any extra behaviour associated with a key event\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.key_event = function(event, name) {\n",
              "\n",
              "    // Prevent repeat events\n",
              "    if (name == 'key_press')\n",
              "    {\n",
              "        if (event.which === this._key)\n",
              "            return;\n",
              "        else\n",
              "            this._key = event.which;\n",
              "    }\n",
              "    if (name == 'key_release')\n",
              "        this._key = null;\n",
              "\n",
              "    var value = '';\n",
              "    if (event.ctrlKey && event.which != 17)\n",
              "        value += \"ctrl+\";\n",
              "    if (event.altKey && event.which != 18)\n",
              "        value += \"alt+\";\n",
              "    if (event.shiftKey && event.which != 16)\n",
              "        value += \"shift+\";\n",
              "\n",
              "    value += 'k';\n",
              "    value += event.which.toString();\n",
              "\n",
              "    this._key_event_extra(event, name);\n",
              "\n",
              "    this.send_message(name, {key: value,\n",
              "                             guiEvent: simpleKeys(event)});\n",
              "    return false;\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onclick = function(name) {\n",
              "    if (name == 'download') {\n",
              "        this.handle_save(this, null);\n",
              "    } else {\n",
              "        this.send_message(\"toolbar_button\", {name: name});\n",
              "    }\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.toolbar_button_onmouseover = function(tooltip) {\n",
              "    this.message.textContent = tooltip;\n",
              "};\n",
              "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Pan axes with left mouse, zoom with right\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
              "\n",
              "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
              "\n",
              "mpl.default_extension = \"png\";var comm_websocket_adapter = function(comm) {\n",
              "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
              "    // object with the appropriate methods. Currently this is a non binary\n",
              "    // socket, so there is still some room for performance tuning.\n",
              "    var ws = {};\n",
              "\n",
              "    ws.close = function() {\n",
              "        comm.close()\n",
              "    };\n",
              "    ws.send = function(m) {\n",
              "        //console.log('sending', m);\n",
              "        comm.send(m);\n",
              "    };\n",
              "    // Register the callback with on_msg.\n",
              "    comm.on_msg(function(msg) {\n",
              "        //console.log('receiving', msg['content']['data'], msg);\n",
              "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
              "        ws.onmessage(msg['content']['data'])\n",
              "    });\n",
              "    return ws;\n",
              "}\n",
              "\n",
              "mpl.mpl_figure_comm = function(comm, msg) {\n",
              "    // This is the function which gets called when the mpl process\n",
              "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
              "\n",
              "    var id = msg.content.data.id;\n",
              "    // Get hold of the div created by the display call when the Comm\n",
              "    // socket was opened in Python.\n",
              "    var element = $(\"#\" + id);\n",
              "    var ws_proxy = comm_websocket_adapter(comm)\n",
              "\n",
              "    function ondownload(figure, format) {\n",
              "        window.open(figure.imageObj.src);\n",
              "    }\n",
              "\n",
              "    var fig = new mpl.figure(id, ws_proxy,\n",
              "                           ondownload,\n",
              "                           element.get(0));\n",
              "\n",
              "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
              "    // web socket which is closed, not our websocket->open comm proxy.\n",
              "    ws_proxy.onopen();\n",
              "\n",
              "    fig.parent_element = element.get(0);\n",
              "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
              "    if (!fig.cell_info) {\n",
              "        console.error(\"Failed to find cell for figure\", id, fig);\n",
              "        return;\n",
              "    }\n",
              "\n",
              "    var output_index = fig.cell_info[2]\n",
              "    var cell = fig.cell_info[0];\n",
              "\n",
              "};\n",
              "\n",
              "mpl.figure.prototype.handle_close = function(fig, msg) {\n",
              "    var width = fig.canvas.width/mpl.ratio\n",
              "    fig.root.unbind('remove')\n",
              "\n",
              "    // Update the output cell to use the data from the current canvas.\n",
              "    fig.push_to_output();\n",
              "    var dataURL = fig.canvas.toDataURL();\n",
              "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
              "    // the notebook keyboard shortcuts fail.\n",
              "    IPython.keyboard_manager.enable()\n",
              "    $(fig.parent_element).html('<img src=\"' + dataURL + '\" width=\"' + width + '\">');\n",
              "    fig.close_ws(fig, msg);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.close_ws = function(fig, msg){\n",
              "    fig.send_message('closing', msg);\n",
              "    // fig.ws.close()\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.push_to_output = function(remove_interactive) {\n",
              "    // Turn the data on the canvas into data in the output cell.\n",
              "    var width = this.canvas.width/mpl.ratio\n",
              "    var dataURL = this.canvas.toDataURL();\n",
              "    this.cell_info[1]['text/html'] = '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.updated_canvas_event = function() {\n",
              "    // Tell IPython that the notebook contents must change.\n",
              "    IPython.notebook.set_dirty(true);\n",
              "    this.send_message(\"ack\", {});\n",
              "    var fig = this;\n",
              "    // Wait a second, then push the new image to the DOM so\n",
              "    // that it is saved nicely (might be nice to debounce this).\n",
              "    setTimeout(function () { fig.push_to_output() }, 1000);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._init_toolbar = function() {\n",
              "    var fig = this;\n",
              "\n",
              "    var nav_element = $('<div/>');\n",
              "    nav_element.attr('style', 'width: 100%');\n",
              "    this.root.append(nav_element);\n",
              "\n",
              "    // Define a callback function for later on.\n",
              "    function toolbar_event(event) {\n",
              "        return fig.toolbar_button_onclick(event['data']);\n",
              "    }\n",
              "    function toolbar_mouse_event(event) {\n",
              "        return fig.toolbar_button_onmouseover(event['data']);\n",
              "    }\n",
              "\n",
              "    for(var toolbar_ind in mpl.toolbar_items){\n",
              "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
              "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
              "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
              "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
              "\n",
              "        if (!name) { continue; };\n",
              "\n",
              "        var button = $('<button class=\"btn btn-default\" href=\"#\" title=\"' + name + '\"><i class=\"fa ' + image + ' fa-lg\"></i></button>');\n",
              "        button.click(method_name, toolbar_event);\n",
              "        button.mouseover(tooltip, toolbar_mouse_event);\n",
              "        nav_element.append(button);\n",
              "    }\n",
              "\n",
              "    // Add the status bar.\n",
              "    var status_bar = $('<span class=\"mpl-message\" style=\"text-align:right; float: right;\"/>');\n",
              "    nav_element.append(status_bar);\n",
              "    this.message = status_bar[0];\n",
              "\n",
              "    // Add the close button to the window.\n",
              "    var buttongrp = $('<div class=\"btn-group inline pull-right\"></div>');\n",
              "    var button = $('<button class=\"btn btn-mini btn-primary\" href=\"#\" title=\"Stop Interaction\"><i class=\"fa fa-power-off icon-remove icon-large\"></i></button>');\n",
              "    button.click(function (evt) { fig.handle_close(fig, {}); } );\n",
              "    button.mouseover('Stop Interaction', toolbar_mouse_event);\n",
              "    buttongrp.append(button);\n",
              "    var titlebar = this.root.find($('.ui-dialog-titlebar'));\n",
              "    titlebar.prepend(buttongrp);\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._root_extra_style = function(el){\n",
              "    var fig = this\n",
              "    el.on(\"remove\", function(){\n",
              "\tfig.close_ws(fig, {});\n",
              "    });\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._canvas_extra_style = function(el){\n",
              "    // this is important to make the div 'focusable\n",
              "    el.attr('tabindex', 0)\n",
              "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
              "    // off when our div gets focus\n",
              "\n",
              "    // location in version 3\n",
              "    if (IPython.notebook.keyboard_manager) {\n",
              "        IPython.notebook.keyboard_manager.register_events(el);\n",
              "    }\n",
              "    else {\n",
              "        // location in version 2\n",
              "        IPython.keyboard_manager.register_events(el);\n",
              "    }\n",
              "\n",
              "}\n",
              "\n",
              "mpl.figure.prototype._key_event_extra = function(event, name) {\n",
              "    var manager = IPython.notebook.keyboard_manager;\n",
              "    if (!manager)\n",
              "        manager = IPython.keyboard_manager;\n",
              "\n",
              "    // Check for shift+enter\n",
              "    if (event.shiftKey && event.which == 13) {\n",
              "        this.canvas_div.blur();\n",
              "        // select the cell after this one\n",
              "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
              "        IPython.notebook.select(index + 1);\n",
              "    }\n",
              "}\n",
              "\n",
              "mpl.figure.prototype.handle_save = function(fig, msg) {\n",
              "    fig.ondownload(fig, null);\n",
              "}\n",
              "\n",
              "\n",
              "mpl.find_output_cell = function(html_output) {\n",
              "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
              "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
              "    // IPython event is triggered only after the cells have been serialised, which for\n",
              "    // our purposes (turning an active figure into a static one), is too late.\n",
              "    var cells = IPython.notebook.get_cells();\n",
              "    var ncells = cells.length;\n",
              "    for (var i=0; i<ncells; i++) {\n",
              "        var cell = cells[i];\n",
              "        if (cell.cell_type === 'code'){\n",
              "            for (var j=0; j<cell.output_area.outputs.length; j++) {\n",
              "                var data = cell.output_area.outputs[j];\n",
              "                if (data.data) {\n",
              "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
              "                    data = data.data;\n",
              "                }\n",
              "                if (data['text/html'] == html_output) {\n",
              "                    return [cell, data, j];\n",
              "                }\n",
              "            }\n",
              "        }\n",
              "    }\n",
              "}\n",
              "\n",
              "// Register the function which deals with the matplotlib target/channel.\n",
              "// The kernel may be null if the page has been refreshed.\n",
              "if (IPython.notebook.kernel != null) {\n",
              "    IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n",
              "}\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<div id='fd04b044-2323-4ad9-823c-f8dcaeedc087'></div>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1.5079975128173828e-05, 4.3272972106933594e-05, 0.0005966424942016602]\n",
            "[0.007355034351348877, 0.007998347282409668, 0.008178353309631348]\n",
            "[0.02068969 0.02529643 0.02864664]\n",
            "[0.01504993601646846, 0.014975020929010796, 0.01844365231800547]\n"
          ]
        }
      ],
      "source": [
        "# # # # # Plot Figure 5(c)\n",
        "results_dir = \"results\"\n",
        "n_seeds = 5\n",
        "n_S_seeds = 30\n",
        "ns = [0, 1, 2]\n",
        "epochs = [200]\n",
        "num_classes = 2\n",
        "\n",
        "exp_name = \"fcmi-mnist-4vs9-CNN-depth\"\n",
        "results_file_path = os.path.join(results_dir, exp_name, 'results.pkl')\n",
        "with open(results_file_path, 'rb') as f:\n",
        "    deterministic_results = pickle.load(f)\n",
        "\n",
        "fig, ax = plot_over_arb(deterministic_results, ns=ns, epoch=epochs[-1], \n",
        "                        xlabel = 'Convolutional depth', xticks = ['2', '4','6'])\n",
        "savefig(fig, os.path.join('figs', f'mnist-sgd-depth-new.pdf'))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}