{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1, debug the hbox pushing works\n",
    "from relu_nets import ReLUNet\n",
    "from hyperbox import Hyperbox, BooleanHyperbox\n",
    "from interval_analysis import HBoxIA\n",
    "from lipMIP import LipProblem, GurobiSquire\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 8\n",
    "test_net = ReLUNet(layer_sizes=[n, 8, 16, 32, 16, 4], bias=True) #LRLRL \n",
    "c_vector = np.array([1., -1., 0.5, 1.0])\n",
    "input_domain = Hyperbox.build_linf_ball(np.array([0.5] * n), 0.5)\n",
    "backprop_domain = Hyperbox.from_vector(c_vector)\n",
    "ia_obj = HBoxIA(test_net, input_domain, backprop_domain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "ia_obj.compute_forward()\n",
    "ia_obj.compute_backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob = LipProblem(test_net, input_domain, c_vector, verbose=True, mip_gap=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[ 0  1  0  0  0  0  0  0  0  0  0  1  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0 -1  0]\n",
      "[ 0  0  0  0  0  0  0  1  0  0 -1  0  1  0 -1  0]\n",
      "[0 0 0 0 0 0 0 0]\n",
      "Changed value of parameter MIPGap to 1.0\n",
      "   Prev: 0.0001  Min: 0.0  Max: 1e+100  Default: 0.0001\n",
      "Changed value of parameter Threads to 4\n",
      "   Prev: 0  Min: 0  Max: 1024  Default: 0\n",
      "Optimize a model with 716 rows, 393 columns and 3950 nonzeros\n",
      "Variable types: 320 continuous, 73 integer (73 binary)\n",
      "Coefficient statistics:\n",
      "  Matrix range     [2e-05, 3e+00]\n",
      "  Objective range  [1e+00, 1e+00]\n",
      "  Bounds range     [1e-08, 2e+00]\n",
      "  RHS range        [1e-08, 3e+00]\n",
      "Presolve removed 185 rows and 70 columns\n",
      "Presolve time: 0.03s\n",
      "Presolved: 531 rows, 323 columns, 3287 nonzeros\n",
      "Variable types: 253 continuous, 70 integer (70 binary)\n",
      "\n",
      "Root relaxation: objective 9.383490e+00, 892 iterations, 0.04 seconds\n",
      "\n",
      "    Nodes    |    Current Node    |     Objective Bounds      |     Work\n",
      " Expl Unexpl |  Obj  Depth IntInf | Incumbent    BestBd   Gap | It/Node Time\n",
      "\n",
      "     0     0    9.38077    0   59          -    9.38077      -     -    0s\n",
      "     0     0    9.27940    0   57          -    9.27940      -     -    0s\n",
      "     0     0    9.27587    0   58          -    9.27587      -     -    0s\n",
      "     0     0    9.24126    0   61          -    9.24126      -     -    0s\n",
      "     0     0    9.22978    0   63          -    9.22978      -     -    0s\n",
      "     0     0    9.21780    0   60          -    9.21780      -     -    0s\n",
      "     0     0    9.21767    0   61          -    9.21767      -     -    0s\n",
      "     0     0    9.21571    0   60          -    9.21571      -     -    0s\n",
      "     0     0    9.21542    0   62          -    9.21542      -     -    0s\n",
      "     0     0    9.21501    0   64          -    9.21501      -     -    0s\n",
      "H    0     0                       0.0245965    9.21501      -     -    0s\n",
      "     0     2    9.21501    0   64    0.02460    9.21501      -     -    0s\n",
      "  2075   890    0.29799   27   33    0.02460    5.08564      -  96.4    5s\n",
      "H 2505   962                       0.0367187    4.72066      -  89.4    5s\n",
      "H 5011  2459                       0.0377720    2.61692  6828%  77.3    8s\n",
      "  6076  3081    0.16892   34   36    0.03777    2.35981  6148%  74.0   10s\n",
      "H 8703  4555                       0.0420784    2.08635  4858%  68.1   12s\n",
      " 10677  5626     cutoff   27         0.04208    1.92739  4480%  66.9   15s\n",
      "H12711  6367                       0.0541093    1.81518  3255%  65.1   19s\n",
      " 12991  6507    0.06371   30   16    0.05411    1.79111  3210%  64.9   20s\n",
      " 17296  8530    0.69833   26   38    0.05411    1.58944  2837%  63.0   25s\n",
      " 20547  9964    0.16311   26   30    0.05411    1.48394  2642%  62.0   35s\n",
      " 24329 11610    0.51099   22   37    0.05411    1.36598  2424%  61.6   40s\n",
      " 27360 12996    0.67148   23   42    0.05411    1.28835  2281%  61.5   45s\n",
      "\n",
      "Cutting planes:\n",
      "  Learned: 2\n",
      "  Gomory: 9\n",
      "  Implied bound: 23\n",
      "  MIR: 23\n",
      "  Flow cover: 29\n",
      "\n",
      "Explored 29210 nodes (1806721 simplex iterations) in 47.59 seconds\n",
      "Thread count was 4 (of 8 available processors)\n",
      "\n",
      "Solution count 5: 0.0541093 0.0420784 0.037772 ... 0.0245965\n",
      "\n",
      "Solve interrupted\n",
      "Best objective 5.410933285658e-02, best bound 1.247718397182e+00, gap 2205.9209%\n"
     ]
    }
   ],
   "source": [
    "out = prob.compute_max_lipschitz()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LipMIP Result: \n",
       "\tValue 0.292\n",
       "\tRuntime 41.688"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from other_methods.lip_sdp import LipSDPCustom as LipSDP\n",
    "from other_methods.clever import CLEVER\n",
    "from other_methods.lip_lp import LipLP\n",
    "from other_methods.fast_lip import FastLip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "build_gurobi_model() got an unexpected keyword argument 'preact_method'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-7ed5ca78b257>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msdp_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLipSDP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_net\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc_vector\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mlp_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLipLP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_net\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc_vector\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_domain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'linf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlp_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/grad/lipMIP/other_methods/lip_lp.py\u001b[0m in \u001b[0;36mcompute\u001b[0;34m(self, preact_method, tighter_relu)\u001b[0m\n\u001b[1;32m     18\u001b[0m         squire, _, _ = lm.build_gurobi_model(self.network, self.domain,\n\u001b[1;32m     19\u001b[0m                                              \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprimal_norm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_vector\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m                                              preact_method=preact_method)\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;31m# And then we'll just change any binary variables to continous ones:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: build_gurobi_model() got an unexpected keyword argument 'preact_method'"
     ]
    }
   ],
   "source": [
    "sdp_obj = LipSDP(test_net, c_vector)\n",
    "lp_obj = LipLP(test_net, c_vector, input_domain, 'linf')\n",
    "lp_obj.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8833453095723295"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdp_obj.compute()\n",
    "sdp_obj.l1_value()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0526033453643322"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clever_obj = CLEVER(test_net, c_vector, input_domain, 'linf')\n",
    "clever_obj.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([0.4, 0.5])\n",
    "print(out.squire.get_grad_at_point(x))\n",
    "test_net.get_grad_at_point(x, c_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_net.fcs[-1].weight.detach().numpy().T @ c_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ia_obj."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_net(torch.Tensor([0.4, 0.4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### for pt in input_domain.random_point(num_points=1000):\n",
    "    mip_out = out.squire.get_var_at_point(pt, 'logits')\n",
    "    nn_out = test_net(torch.Tensor(pt)).squeeze().detach().numpy()\n",
    "    \n",
    "    logit_diff = max(abs(mip_out - nn_out))\n",
    "    \n",
    "    mip_grad = out.squire.get_grad_at_point(pt)\n",
    "    nn_grad = test_net.get_grad_at_point(pt, c_vector).detach().numpy()\n",
    "    grad_diff = max(abs(mip_grad - nn_grad))\n",
    "    print(max([logit_diff, grad_diff]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.Tensor([0.4, 0.4])\n",
    "test_net.fcs[0].weight @ x + test_net.fcs[0].bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ia_obj.gradient_range.as_twocol()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_net.get_grad_at_point(torch.Tensor([0.0, 0.0]), c_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for point in input_domain.random_point(num_points=1000):\n",
    "    point = torch.Tensor(point)\n",
    "    point_out = test_net(point).squeeze().detach().numpy()\n",
    "    mip_out = out.squire.get_var_at_point(point.numpy(), 'logits')\n",
    "    print(abs(point_out - mip_out).max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gurobipy as gb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb.LinExpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pre = HBoxIA(test_net, input_domain, backprop_domain)\n",
    "test_pre.compute_forward()\n",
    "test_pre.compute_backward()\n",
    "test_pre.get_backward_box(1, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_net.num_relus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out.squire.pre_bounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
