{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from experiment import Experiment, MethodNest, Job\n",
    "from hyperbox import Hyperbox\n",
    "from relu_nets import ReLUNet\n",
    "from neural_nets import data_loaders as dl\n",
    "from neural_nets import train\n",
    "from lipMIP import LipProblem\n",
    "from other_methods import CLEVER, FastLip, LipLP, LipSDP, NaiveUB, RandomLB, SeqLip\n",
    "from other_methods import LOCAL_METHODS, GLOBAL_METHODS, OTHER_METHODS\n",
    "from utilities import Factory, DoEvery\n",
    "import utilities as utils\n",
    "import os\n",
    "import time\n",
    "import pickle \n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('crosslip_exp_r01.pkl', 'rb') as f:\n",
    "    results = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_k = {}\n",
    "for el in results:\n",
    "    for k, v in el.items():\n",
    "        by_k[k] = by_k.get(k, []) + [v]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "times = {k: np.array([_[0] for _ in v]) for k,v in by_k.items()}\n",
    "values = {k: np.array([_[1] for _ in v]) for k, v in by_k.items()}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'targetCrossLipschitz': 2.3122949600219727, 'trueTargetCrossLipschitz': 2.8968015909194946, 'naive': 59.857210874557495}\n"
     ]
    }
   ],
   "source": [
    "print({k: np.median(v) for k, v in times.items()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'targetCrossLipschitz': 350.17590133366974,\n",
       " 'trueTargetCrossLipschitz': 350.17590371427696,\n",
       " 'naive': 72.08258921580666}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k: np.mean(v) for k,v in values.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'targetCrossLipschitz': 5.41136040323265,\n",
       "  'trueTargetCrossLipschitz': 5.411360438647332,\n",
       "  'naive': 1.0},\n",
       " {'targetCrossLipschitz': 1.8078420570574645,\n",
       "  'trueTargetCrossLipschitz': 1.8078420684721597,\n",
       "  'naive': 0.0})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k: np.mean(v) for k,v in errors.items()}, {k: np.std(v) for k,v in errors.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
