{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine\n",
    "import numpy as np\n",
    "import torch\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from experiment import Experiment, MethodNest, Job\n",
    "from hyperbox import Hyperbox\n",
    "from relu_nets import ReLUNet\n",
    "from neural_nets import data_loaders as dl\n",
    "from neural_nets import train\n",
    "from lipMIP import LipProblem\n",
    "from other_methods import CLEVER, FastLip, LipLP, LipSDP, NaiveUB, RandomLB, SeqLip\n",
    "from other_methods import LOCAL_METHODS, GLOBAL_METHODS, OTHER_METHODS\n",
    "from utilities import Factory, DoEvery\n",
    "import utilities as utils\n",
    "import os\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_dim(method, val, dim=2):\n",
    "    if method == 'LipSDP':\n",
    "        return val * (dim**0.5)\n",
    "    return val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "set([os.path.basename(_).split('_')[1][3:] for _ in glob.glob('../jobs/completed/l1_2DTRAINv2_*')])\n",
    "files = sorted(glob.glob('../jobs/completed/l1_2DTRAINv2_REGvanilla*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0000_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0025_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0050_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0075_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0100_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0125_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0150_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0175_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0200_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0225_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0250_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0275_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0300_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0325_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0350_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0375_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0400_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0425_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0450_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0475_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0500_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0525_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0550_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0575_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0600_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0625_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0650_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0675_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0700_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0725_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0750_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0775_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0800_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0825_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0850_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0875_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0900_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0925_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0950_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH0975_GLOBAL.result',\n",
       " '../jobs/completed/l1_2DTRAINv2_REGvanilla_EPOCH1000_GLOBAL.result']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "def get_series_by_method(filenames, fxn=None):\n",
    "    if fxn is None:\n",
    "        fxn = lambda x: x\n",
    "    reg_series = {}\n",
    "    for filename in filenames:\n",
    "        with open(filename, 'rb') as f:\n",
    "            obj = pickle.load(f)\n",
    "            basename = os.path.basename(filename)\n",
    "            epoch = utils.filename_to_epoch(filename)\n",
    "            for k, v in obj['do_unit_hypercube_eval'].values().items():\n",
    "                if k not in reg_series:\n",
    "                    reg_series[k] = []\n",
    "                reg_series[k].append((epoch, float(scale_dim(k, v))))\n",
    "    return reg_series\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['LipProblem', 'FastLip', 'LipLP', 'SeqLip', 'LipSDP', 'NaiveUB', 'RandomLB', 'CLEVER'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Lipschitz Estimators During Training')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 39132 characters]\n",
      "text/plain": [
       "<Figure size 720x439.2 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = get_series_by_method(files)\n",
    "print(data.keys())\n",
    "def plt_series(data, series, c):\n",
    "    plt.scatter(*zip(*data[series]), c=c,)\n",
    "markers = \".*+1\"\n",
    "\n",
    "long = 10\n",
    "fig = plt.figure(figsize=(long,long * 0.61))\n",
    "ax = fig.add_subplot()\n",
    "sns.set(font_scale=1.5)\n",
    "def namer(k):\n",
    "    if k == 'LipProblem':\n",
    "        return 'LipMIP (ours)'\n",
    "    elif k == 'LipLP':\n",
    "        return 'LipLP (ours)'\n",
    "    else:\n",
    "        return k\n",
    "for i, series in enumerate(['LipProblem', 'LipLP', 'LipSDP', 'FastLip',]):\n",
    "    ax.scatter(*zip(*data[series]), label=namer(series), marker=markers[i], s=120)\n",
    "ax.legend()\n",
    "ax.set_xlabel('Training Epoch')\n",
    "ax.set_ylabel('Reported Lispchitz Value')\n",
    "ax.set_title('Lipschitz Estimators During Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 17012 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_run(run_name, methods):\n",
    "    cs = 'rgbcymB'\n",
    "    files = sorted(glob.glob('../jobs/completed/2DTRAINv2_REG%s*.result' % run_name))\n",
    "    data = get_series_by_method(files)\n",
    "    for i, series in enumerate(methods):\n",
    "        plt_series(data, series, c=cs[i])\n",
    "    \n",
    "plot_run('fgsm', ['LipProblem', 'LipLP', 'SeqLip', 'LipSDP'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [_[0] for _ in data['LipProblem']]\n",
    "ys = [_[1] for _ in data['LipProblem']]\n",
    "sns.scatterplot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tips = sns.load_dataset('tips')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tips.__class__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
