{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine\n",
    "import numpy as np\n",
    "import torch\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from experiment import Experiment, MethodNest, Job\n",
    "from hyperbox import Hyperbox\n",
    "from relu_nets import ReLUNet\n",
    "from neural_nets import data_loaders as dl\n",
    "from neural_nets import train\n",
    "from lipMIP import LipProblem\n",
    "from other_methods import CLEVER, FastLip, LipLP, LipSDP, NaiveUB, RandomLB, SeqLip\n",
    "from other_methods import LOCAL_METHODS, GLOBAL_METHODS, OTHER_METHODS\n",
    "from utilities import Factory, DoEvery\n",
    "import utilities as utils\n",
    "import os\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_dim(method, val, dim=2):\n",
    "    if method == 'LipSDP':\n",
    "        return val * (dim**0.5)\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "width_files = sorted(glob.glob('../jobs/completed/width2Dv2*_GLOBAL*'))\n",
    "width_files\n",
    "\n",
    "width_to_int = lambda s: int(s[5:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "def get_mid_params(filenames, fxn=None):\n",
    "    if fxn is None:\n",
    "        fxn = lambda x: x\n",
    "    series = {}\n",
    "    for filename in filenames:\n",
    "        with open(filename, 'rb') as f:\n",
    "            basename = os.path.basename(filename)\n",
    "            param_name = (basename.split('_')[1])\n",
    "            series[param_name] = fxn(pickle.load(f))\n",
    "    return series\n",
    "\n",
    "width_objs = get_mid_params(width_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'WIDTH0020': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312d7588>,\n",
       "  'Job': <experiment.Job at 0x7fd8312dda90>},\n",
       " 'WIDTH0040': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312d7550>,\n",
       "  'Job': <experiment.Job at 0x7fd8312dda58>},\n",
       " 'WIDTH0060': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312d7cc0>,\n",
       "  'Job': <experiment.Job at 0x7fd8312ea470>},\n",
       " 'WIDTH0080': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312d7630>,\n",
       "  'Job': <experiment.Job at 0x7fd8312eae80>},\n",
       " 'WIDTH0100': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312ea160>,\n",
       "  'Job': <experiment.Job at 0x7fd8312798d0>},\n",
       " 'WIDTH0120': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312d77f0>,\n",
       "  'Job': <experiment.Job at 0x7fd83128d320>},\n",
       " 'WIDTH0140': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312eab70>,\n",
       "  'Job': <experiment.Job at 0x7fd8312d76d8>},\n",
       " 'WIDTH0160': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd8312ea7f0>,\n",
       "  'Job': <experiment.Job at 0x7fd83129ff28>},\n",
       " 'WIDTH0180': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd83128d6d8>,\n",
       "  'Job': <experiment.Job at 0x7fd8312d7b00>},\n",
       " 'WIDTH0200': {'do_unit_hypercube_eval': <experiment.Result at 0x7fd83128d5f8>,\n",
       "  'Job': <experiment.Job at 0x7fd8312acbe0>}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "width_objs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "width_dict = {}\n",
    "for k,v in width_objs.items():\n",
    "    width = width_to_int(k)\n",
    "    for method, val in v['do_unit_hypercube_eval'].values().items():\n",
    "        width_dict[method] = width_dict.get(method,[]) + [(width, scale_dim(method, val, dim=2))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Lipschitz Constants Under Changing Width')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 31908 characters]\n",
      "text/plain": [
       "<Figure size 720x439.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "long = 10\n",
    "fig = plt.figure(figsize=(long,long * 0.61))\n",
    "ax = fig.add_subplot()\n",
    "sns.set(font_scale=1.5)\n",
    "def namer(k):\n",
    "    if k == 'LipProblem':\n",
    "        return 'LipMIP'\n",
    "    else:\n",
    "        return k\n",
    "for i, (k, v) in enumerate(width_dict.items()):\n",
    "    if k not in ['LipProblem', 'FastLip', 'LipSDP', 'SeqLip', 'LipLP']:\n",
    "        continue\n",
    "    ax.scatter(*zip(*v), label=namer(k))\n",
    "ax.legend()\n",
    "ax.set_xlabel('Width of Hidden Units')\n",
    "ax.set_ylabel('Estimated Lipschitz Value')\n",
    "ax.set_title('Lipschitz Constants Under Changing Width')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plot_dict' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-bb5ed33ce204>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mcs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'rgbmyc'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplot_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'NaiveUB'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plot_dict' is not defined"
     ]
    }
   ],
   "source": [
    "cs = 'rgbmyc'\n",
    "for i, (k, v) in enumerate(plot_dict.items()):\n",
    "    if k == 'NaiveUB':\n",
    "        continue\n",
    "    print(k, cs[i])\n",
    "    plt.scatter(*zip(*v), c=cs[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEPTH SERIES \n",
    "import math\n",
    "depth_files = sorted(glob.glob('../jobs/completed/depth2Dv2*_GLOBAL*'))\n",
    "#print(depth_files)\n",
    "depth_objs = get_mid_params(depth_files)\n",
    "depth_to_int = lambda s: int(s[-4:])\n",
    "\n",
    "depth_dict = {}\n",
    "for k,v in depth_objs.items():\n",
    "\n",
    "    depth = depth_to_int(k)\n",
    "    for method, val in v['do_unit_hypercube_eval'].values().items():\n",
    "        depth_dict[method] = depth_dict.get(method,[]) + [(depth, math.log(scale_dim(method, val, dim=2)))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49616 characters]\n",
      "text/plain": [
       "<Figure size 720x439.2 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "long = 10\n",
    "fig = plt.figure(figsize=(long,long * 0.61))\n",
    "ax = fig.add_subplot()\n",
    "\n",
    "def namer(k):\n",
    "    if k == 'LipProblem':\n",
    "        return 'LipMIP'\n",
    "    else:\n",
    "        return k\n",
    "for i, (k, v) in enumerate(depth_dict.items()):\n",
    "    if k not in ['LipProblem', 'NaiveUB', 'FastLip', 'LipSDP', 'SeqLip']:\n",
    "        continue\n",
    "    ax.scatter(*zip(*v), label=namer(k))\n",
    "ax.legend()\n",
    "ax.set_xlabel('# of Hidden Units')\n",
    "ax.set_ylabel('Log of Estimated Lipschitz Value')\n",
    "ax.set_title('Lipschitz Constants Under Changing Depth')\n",
    "\n",
    "\n",
    "def plot_estimator(estimator):\n",
    "    for i, (reg, series) in enumerate(data[estimator].items()):\n",
    "        print(reg, cs[i])\n",
    "        plot_single_series(series[:40], c=cs[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "depth_dict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
