{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import sys \n",
    "import os\n",
    "sys.path.append('..')\n",
    "SCHEDULE_DIR = os.path.join(os.path.dirname(os.getcwd()), 'jobs', 'scheduled')\n",
    "import numpy as np \n",
    "import torch\n",
    "from hyperbox import Hyperbox\n",
    "from lipMIP import LipProblem\n",
    "from neural_nets import data_loaders as dl\n",
    "from neural_nets import train \n",
    "import utilities as utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['l2_penalty_EPOCH0260_LOCAL.job']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[_ for _ in os.listdir(SCHEDULE_DIR) if 'LOCAL' in _]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "job120 = [_ for _ in os.listdir(SCHEDULE_DIR) if 'LOCAL' in _][0]\n",
    "with open(os.path.join(SCHEDULE_DIR, job120), 'rb') as f:\n",
    "    job = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[other_methods.fast_lip.FastLip, other_methods.lip_lp.LipLP, lipMIP.LipProblem]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job.experiment.class_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<experiment.MethodNest at 0x7f7e6b175208>,\n",
       " <experiment.MethodNest at 0x7f7e6b175748>,\n",
       " <experiment.MethodNest at 0x7f7e6b175828>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job.method_nests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFEASIBLE\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "b\"Unable to retrieve attribute 'x'\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-894dedda3b2f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmethod_nests\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/grad/lipMIP/experiment.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, experiment, **kwargs)\u001b[0m\n\u001b[1;32m    257\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m                         \u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/grad/lipMIP/experiment.py\u001b[0m in \u001b[0;36mdo_data_evals\u001b[0;34m(self, data_points, ball_factory, num_random, **kwargs)\u001b[0m\n\u001b[1;32m    120\u001b[0m                 \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_points\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m                         \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mball_factory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mResultList\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/grad/lipMIP/experiment.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, verbose)\u001b[0m\n\u001b[1;32m    148\u001b[0m                                         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m                         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOtherResult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m                                 \u001b[0mv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m                                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/grad/lipMIP/other_methods/lip_lp.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, preact_method, tighter_relu)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStatus\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'INFEASIBLE'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetObjective\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetValue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mlinexpr.pxi\u001b[0m in \u001b[0;36mgurobipy.LinExpr.getValue (../../src/python/gurobipy.c:33377)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mvar.pxi\u001b[0m in \u001b[0;36mgurobipy.Var.__getattr__ (../../src/python/gurobipy.c:15232)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mvar.pxi\u001b[0m in \u001b[0;36mgurobipy.Var.getAttr (../../src/python/gurobipy.c:16032)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: b\"Unable to retrieve attribute 'x'\""
     ]
    }
   ],
   "source": [
    "job.method_nests[1](job.experiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_params = dl.RandomKParameters(num_points=300, k=10, radius=0.01, dimension=2)\n",
    "dataset = dl.RandomBinaryDataset(data_params, batch_size=100, random_seed=420)\n",
    "dataset.split_train_val(1.0)\n",
    "data = dataset.train_data[0][0]\n",
    "ball_factory = utils.Factory(Hyperbox.build_linf_ball, radius=0.1)\n",
    "data_points = utils.as_numpy(data).reshape((-1, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<hyperbox.Hyperbox at 0x7f7e690972b0>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for point in data_points:\n",
    "    prob = LipPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function experiment.Experiment.do_data_evals(self, data_points, ball_factory, num_random=None, **kwargs)>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "job.method_nests[1].method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_net = job.experiment.network\n",
    "c_vec = np.array([1.0, -1.0])\n",
    "domain = Hyperbox.build_linf_ball(np.array([0.5, 0.5]), 0.5)\n",
    "\n",
    "LipProblem(test_net, domain, c_vec).compute_max_lipschitz()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.031\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.025\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.086\n",
      "LipMIP Result: \n",
      "\tValue 11.116\n",
      "\tRuntime 0.097\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.108\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.025\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.096\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.082\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.016\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.030\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.037\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.200\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.026\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.048\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.111\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.049\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.023\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.027\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.051\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.024\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.050\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.045\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.020\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.055\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.083\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.033\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.143\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.032\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.019\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.021\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.100\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.054\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.124\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.134\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.123\n",
      "LipMIP Result: \n",
      "\tValue 22.045\n",
      "\tRuntime 0.026\n",
      "LipMIP Result: \n",
      "\tValue 12.573\n",
      "\tRuntime 0.093\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.129\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.043\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.017\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.019\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.044\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.037\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.040\n",
      "LipMIP Result: \n",
      "\tValue 33.091\n",
      "\tRuntime 0.019\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.027\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.018\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.031\n",
      "LipMIP Result: \n",
      "\tValue 23.805\n",
      "\tRuntime 0.061\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.020\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.035\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.089\n",
      "LipMIP Result: \n",
      "\tValue 33.091\n",
      "\tRuntime 0.023\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.107\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.020\n",
      "LipMIP Result: \n",
      "\tValue 19.372\n",
      "\tRuntime 0.075\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.023\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.026\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.041\n",
      "LipMIP Result: \n",
      "\tValue 12.312\n",
      "\tRuntime 0.055\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.027\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.017\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.038\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.028\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.035\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.024\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.049\n",
      "LipMIP Result: \n",
      "\tValue 33.091\n",
      "\tRuntime 0.028\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.018\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.027\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.044\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.035\n",
      "LipMIP Result: \n",
      "\tValue 11.577\n",
      "\tRuntime 0.050\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.028\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.037\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.136\n",
      "LipMIP Result: \n",
      "\tValue 33.091\n",
      "\tRuntime 0.035\n",
      "LipMIP Result: \n",
      "\tValue 33.091\n",
      "\tRuntime 0.018\n",
      "LipMIP Result: \n",
      "\tValue 29.289\n",
      "\tRuntime 0.019\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.023\n",
      "LipMIP Result: \n",
      "\tValue 12.573\n",
      "\tRuntime 0.048\n",
      "LipMIP Result: \n",
      "\tValue 29.210\n",
      "\tRuntime 0.050\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.032\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.039\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.030\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.027\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.043\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.018\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.024\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.117\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.044\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.108\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.037\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.016\n",
      "LipMIP Result: \n",
      "\tValue 30.055\n",
      "\tRuntime 0.041\n",
      "LipMIP Result: \n",
      "\tValue 25.838\n",
      "\tRuntime 0.019\n",
      "LipMIP Result: \n",
      "\tValue 26.542\n",
      "\tRuntime 0.031\n",
      "LipMIP Result: \n",
      "\tValue 30.171\n",
      "\tRuntime 0.046\n",
      "LipMIP Result: \n",
      "\tValue 31.076\n",
      "\tRuntime 0.043\n",
      "LipMIP Result: \n",
      "\tValue 18.083\n",
      "\tRuntime 0.025\n"
     ]
    }
   ],
   "source": [
    "bad_domains = []\n",
    "for p in data_points:\n",
    "    c_vec = np.array([1.0, -1.0])\n",
    "    domain = ball_factory(x=p)\n",
    "    try:\n",
    "        prob = LipProblem(job.experiment.network, domain, c_vec)\n",
    "        print(prob.compute_max_lipschitz())\n",
    "    except Exception as err:\n",
    "        print(err)\n",
    "        bad_domains.append(domain)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2 = Hyperbox.build_linf_ball(bad_domains[0].center, 0.12)\n",
    "LipProblem(test_net, d2, c_vec).compute_max_lipschitz()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = job.experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_nests = job.method_nests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_nests[0](exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
