{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from hyperbox import Hyperbox\n",
    "from interval_analysis import HBoxIA\n",
    "from relu_nets import ReLUNet\n",
    "from lipMIP import LipProblem\n",
    "from other_methods import CLEVER, FastLip, LipLP, LipSDP, NaiveUB, RandomLB, SeqLip\n",
    "from neural_nets import train\n",
    "from neural_nets import data_loaders as dl\n",
    "from experiment import Experiment, InstanceGroup, Result\n",
    "from utilities import Factory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic test file to demonstrate that everything works \n",
    "# 1) Build dataset and display it \n",
    "# 2) Build neural net and train it \n",
    "# 3) Run single evaluations for a particular hyperbox and multiple types of results\n",
    "# 4) Run some simple experimental series\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 68764 characters]\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1) Build a dataset and display it \n",
    "dataset_params = dl.RandomKParameters(256, 30, radius=0.02)\n",
    "dataset = dl.Random2DBinaryDataset(dataset_params, random_seed=420)\n",
    "train_data, val_data = dataset.split_train_val(1.0)\n",
    "dataset.plot_2d()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00 | Accuracy: 42.58\n",
      "Epoch 100 | Accuracy: 75.39\n",
      "Epoch 200 | Accuracy: 83.98\n",
      "Epoch 300 | Accuracy: 84.38\n",
      "Epoch 400 | Accuracy: 85.55\n",
      "Epoch 500 | Accuracy: 89.06\n",
      "Epoch 600 | Accuracy: 92.97\n",
      "Epoch 700 | Accuracy: 96.09\n",
      "Epoch 800 | Accuracy: 97.66\n",
      "Epoch 900 | Accuracy: 98.44\n"
     ]
    }
   ],
   "source": [
    "# 2) Build neural net and train it \n",
    "test_net = ReLUNet(layer_sizes=[2, 16, 32, 64, 16, 2])\n",
    "train_params = train.TrainParameters(train_data, train_data, 1000, test_after_epoch=100)\n",
    "train.training_loop(test_net, train_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3) Run single evaluations for a particular hyperbox and multiple types of results\n",
    "hbox1 = Hyperbox.build_linf_ball(np.array([0.5, 0.5]), 0.2)\n",
    "c_vec = torch.Tensor([1.0, -1.0])\n",
    "local_args = {'network': test_net, 'domain': hbox1, 'c_vector': c_vec, 'primal_norm': 'linf'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Changed value of parameter Threads to 4\n",
      "   Prev: 0  Min: 0  Max: 1024  Default: 0\n",
      "Optimize a model with 996 rows, 608 columns and 9168 nonzeros\n",
      "Variable types: 522 continuous, 86 integer (86 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [2e-05, 3e+03]\n",
      "  Objective range  [1e+00, 1e+00]\n",
      "  Bounds range     [1e-08, 1e+03]\n",
      "  RHS range        [1e-08, 3e+03]\n",
      "Presolve removed 325 rows and 194 columns\n",
      "Presolve time: 0.12s\n",
      "Presolved: 671 rows, 414 columns, 6246 nonzeros\n",
      "Variable types: 338 continuous, 76 integer (76 binary)\n",
      "\n",
      "Root relaxation: objective 1.801314e+03, 329 iterations, 0.02 seconds\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0 1801.31420    0   63          - 1801.31420      -     -    0s\n",
      "     0     0 1421.19939    0   60          - 1421.19939      -     -    0s\n",
      "     0     0 1420.77013    0   59          - 1420.77013      -     -    0s\n",
      "     0     0 1389.07954    0   57          - 1389.07954      -     -    0s\n",
      "     0     0 1382.36184    0   49          - 1382.36184      -     -    0s\n",
      "     0     0 1381.39969    0   50          - 1381.39969      -     -    0s\n",
      "     0     0 1381.34112    0   51          - 1381.34112      -     -    0s\n",
      "     0     0 1378.50764    0   50          - 1378.50764      -     -    0s\n",
      "     0     0 1376.80241    0   47          - 1376.80241      -     -    0s\n",
      "H    0     0                     239.3378831 1376.80241   475%     -    0s\n",
      "     0     0 1376.70985    0   46  239.33788 1376.70985   475%     -    0s\n",
      "     0     0 1373.20385    0   44  239.33788 1373.20385   474%     -    0s\n",
      "     0     0 1372.83095    0   49  239.33788 1372.83095   474%     -    0s\n",
      "     0     0 1372.73983    0   49  239.33788 1372.73983   474%     -    0s\n",
      "     0     0 1371.85637    0   46  239.33788 1371.85637   473%     -    0s\n",
      "     0     0 1371.28337    0   46  239.33788 1371.28337   473%     -    0s\n",
      "     0     0 1371.18903    0   47  239.33788 1371.18903   473%     -    0s\n",
      "     0     0 1370.65791    0   44  239.33788 1370.65791   473%     -    0s\n",
      "     0     0 1370.65791    0   44  239.33788 1370.65791   473%     -    0s\n",
      "     0     0 1370.65791    0   46  239.33788 1370.65791   473%     -    1s\n",
      "     0     0  903.80545    0   46  239.33788  903.80545   278%     -    1s\n",
      "     0     0  507.29013    0   45  239.33788  507.29013   112%     -    1s\n",
      "     0     0  414.23549    0   31  239.33788  414.23549  73.1%     -    1s\n",
      "     0     0  414.23549    0   29  239.33788  414.23549  73.1%     -    1s\n",
      "     0     0  401.81715    0   18  239.33788  401.81715  67.9%     -    1s\n",
      "     0     0  401.81715    0   23  239.33788  401.81715  67.9%     -    1s\n",
      "     0     0  369.29623    0   22  239.33788  369.29623  54.3%     -    1s\n",
      "     0     0  335.56697    0   18  239.33788  335.56697  40.2%     -    1s\n",
      "     0     0  335.56697    0    2  239.33788  335.56697  40.2%     -    1s\n",
      "     0     0  335.56697    0    2  239.33788  335.56697  40.2%     -    1s\n",
      "     0     2  335.56697    0    2  239.33788  335.56697  40.2%     -    1s\n",
      "\n",
      "Cutting planes:\n",
      "  Learned: 1\n",
      "  Gomory: 15\n",
      "  Cover: 3\n",
      "  Implied bound: 4\n",
      "  MIR: 86\n",
      "  Flow cover: 4\n",
      "\n",
      "Explored 570 nodes (11585 simplex iterations) in 2.05 seconds\n",
      "Thread count was 4 (of 8 available processors)\n",
      "\n",
      "Solution count 1: 239.338 \n",
      "\n",
      "Optimal solution found (tolerance 1.00e-04)\n",
      "Best objective 2.393378831031e+02, best bound 2.393378831031e+02, gap 0.0000%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LipMIP Result: \n",
       "\tValue 239.338\n",
       "\tRuntime 2.265"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.1) lipMIP\n",
    "lip_prob = LipProblem(test_net, hbox1, c_vec, verbose=True, num_threads=4)\n",
    "lip_prob.compute_max_lipschitz()\n",
    "lip_prob.result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "239.337890625"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.2) CLEVER\n",
    "clever = CLEVER(**local_args)\n",
    "clever.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2121.1284"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.3) FastLip\n",
    "fastlip = FastLip(**local_args)\n",
    "fastlip.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1942.7960521809464"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.4) LipLP\n",
    "liplp = LipLP(**local_args)\n",
    "liplp.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1044.3358311513014\n"
     ]
    }
   ],
   "source": [
    "# 3.5) LipSDP\n",
    "lipsdp = LipSDP(test_net, c_vec)\n",
    "lipsdp.compute()\n",
    "print(lipsdp.l1_value())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L1 norms of matrices: 12818.49\n",
      "L2 norms of matrices, scaled by sqrt(2) 12652.022711018659\n"
     ]
    }
   ],
   "source": [
    "# 3.6) Naive UB\n",
    "naiveub = NaiveUB(test_net, c_vec, 'linf')\n",
    "print(\"L1 norms of matrices:\", naiveub.compute())\n",
    "naiveub2 = NaiveUB(test_net, c_vec, 'l2')\n",
    "print(\"L2 norms of matrices, scaled by sqrt(2)\", naiveub2.compute() * 2**0.5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(239.3379)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.7) Random LB\n",
    "randomlb = RandomLB(**local_args)\n",
    "randomlb.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "../other_methods/seq_lip.py:31: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  c_vec = torch.tensor(self.c_vector).view(1, -1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1613.1124614533112"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3.8) SeqLip\n",
    "seqlip = SeqLip(**local_args)\n",
    "seqlip.compute() * 2 **0.5 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4) Run some simple experimental series\n",
    "exp_1 = Experiment([RandomLB, FastLip, LipProblem], network=test_net, c_vector=c_vec, \n",
    "                   primal_norm='linf', verbose=False, num_threads=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'RandomLB': tensor(239.3379), 'FastLip': 2299.7012, 'LipProblem': 277.317755669118}\n"
     ]
    }
   ],
   "source": [
    "# 4.1) Evaluate over the unit hypercube\n",
    "hypercube_result = exp_1.do_unit_hypercube_eval()\n",
    "print(hypercube_result.values())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'LipProblem': (214.28965459769287, 43.50296124510828, 10),\n",
       " 'FastLip': (1639.574, 362.4966, 10),\n",
       " 'RandomLB': (210.49165, 39.2883, 10)}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4.2) Evaluate over random Linf balls of radius 0.2\n",
    "ball_factory = Factory(Hyperbox.build_linf_ball, radius=0.2, global_lo=np.array([0,0]), global_hi=np.array([1.0, 1.0]))\n",
    "center_domain = Hyperbox.build_linf_ball(x=np.array([0.5, 0.5]), radius=0.5)\n",
    "random_results = exp_1.do_random_evals(10, center_domain, ball_factory)\n",
    "random_results.average_stdevs('value')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4.3) Evaluate over random data points \n",
    "data_results = exp_1.do_data_evals(train_data[0][0], ball_factory, num_random=10)\n",
    "data_results.average_stdevs('value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
