{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine\n",
    "import numpy as np\n",
    "import torch\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from experiment import Experiment, MethodNest, Job\n",
    "from hyperbox import Hyperbox\n",
    "from relu_nets import ReLUNet\n",
    "from neural_nets import data_loaders as dl\n",
    "from neural_nets import train\n",
    "from lipMIP import LipProblem\n",
    "from other_methods import CLEVER, FastLip, LipLP, LipSDP, NaiveUB, RandomLB, SeqLip\n",
    "from other_methods import LOCAL_METHODS, GLOBAL_METHODS, OTHER_METHODS\n",
    "from utilities import Factory, DoEvery\n",
    "import utilities as utils\n",
    "import os\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2DTRAIN_REGfgsm_EPOCH0000_GLOBAL.result'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filenames = sorted(glob.glob('../jobs/completed/2DTRAIN*_GLOBAL*'))\n",
    "os.path.basename(filenames[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "def get_series_dicts(fxn):\n",
    "    series = {}\n",
    "    for filename in filenames:\n",
    "        with open(filename, 'rb') as f:\n",
    "            basename = os.path.basename(filename)\n",
    "            series_name = (basename.split('_')[1][3:])\n",
    "            series[series_name] = series.get(series_name, []) + [fxn(pickle.load(f))]\n",
    "    return series\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20.236103057861328"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obj = series['vanilla'][4]\n",
    "utils.filename_to_epoch(test_el['Job'].name)\n",
    "\n",
    "float(obj['do_unit_hypercube_eval'].values('RandomLB'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_value(k):\n",
    "    def fxn(obj, k=k):\n",
    "        return utils.filename_to_epoch(obj['Job'].name), obj['do_unit_hypercube_eval'].values(k=k)\n",
    "    return fxn\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['fgsm', 'l1LipReg', 'l1Penalty', 'l2LipReg', 'l2Penalty', 'vanilla'])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers = get_series_dicts(get_value(None))\n",
    "answers.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_series(all_series, series_name):\n",
    "    series = all_series[series_name]\n",
    "    output = {}\n",
    "    \n",
    "    for epoch, vals in series:\n",
    "        correct = vals['LipProblem']\n",
    "        for k, v in vals.items():\n",
    "            output[k] = output.get(k,[]) + [(epoch, v / correct)]\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 13164 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for v in answers.values():\n",
    "    plt.scatter(*zip(*v))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LipProblem\n",
      "FastLip\n",
      "LipLP\n",
      "SeqLip\n",
      "LipSDP\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 8428 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cs = 'rgbmy'\n",
    "for i, (k, pairs) in enumerate(collect_series(answers, 'l2Penalty').items()):\n",
    "    if k in ['NaiveUB', 'RandomLB']:\n",
    "        continue\n",
    "    print(k)\n",
    "    plt.scatter(*zip(*pairs[8:]),c=cs[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
