{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import experiment_1 as exp1\n",
    "import pickle\n",
    "from experiment import ResultList, Experiment\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('l1_exp_1_random.pkl', 'rb') as f:\n",
    "    rand_results = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "def scrub_results(results):\n",
    "    output = []\n",
    "    for _ in results.results:\n",
    "        if (_.values('CLEVER')) < 10000:\n",
    "            output.append(_)\n",
    "    new_results = ResultList(output)\n",
    "    return new_results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "def csv_parser(results, dim, csvfile, no_err=False):\n",
    "    times = results.average_stdevs('time')\n",
    "    values = results.average_stdevs('value')\n",
    "    rel_errs = results.get_rel_err(dim)\n",
    "    keys = sorted(list(times.keys()))\n",
    "    max_len_k = max(len(_) for _ in keys + ['method'])\n",
    "    rows = []\n",
    "    pad = lambda s: s + ' ' * (max_len_k - len(s))\n",
    "    def dim_scale(k, val):\n",
    "        if k not in ['SeqLip', 'LipSDP']:\n",
    "            return val\n",
    "        else:\n",
    "            return math.sqrt(dim) *val\n",
    "    header_pad = lambda s: '|' + ' ' * (10 - len(s)) + s\n",
    "    row1 = ['Method', 'Time', 'Time STD', 'Value', 'Value STD', 'Err', 'ErrSTD']\n",
    "    rows.append(row1)\n",
    "    header = pad('Method') +' '+ ' '.join(header_pad(_) for _ in ['Time', 'Time STD', 'Value', 'Value STD', 'Err', 'ErrSTD'])\n",
    "    print(header + '\\n' + '-' * len(header))\n",
    "    key_order = [_[0] for _ in sorted([(k, dim_scale(k, values[k][0])) for k in keys], key=lambda p: p[1])]\n",
    "    for k in key_order:\n",
    "        elements = [pad(k),  \n",
    "                    '${:10.3f} \\pm {:10.3f}$'.format(times[k][0], times[k][1]),\n",
    "                    #'${:10.3f} \\pm {:10.3f}$'.format(dim_scale(k, values[k][0]), dim_scale(k, values[k][1]))\n",
    "                   ]\n",
    "        if not no_err:\n",
    "            elements.append(\n",
    "                    '${:10.2f}\\%$'.format(rel_errs[k][0] * 100 -100))\n",
    "        rows.append(elements)\n",
    "        print(' '.join(elements))\n",
    "    return rows\n",
    "\n",
    "\n",
    "\n",
    "def write_rows_to_csv(rows, filename):\n",
    "    with open(filename, 'w', newline='') as f:\n",
    "        csvwriter = csv.writer(f, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        for row in rows:\n",
    "            csvwriter.writerow(row)\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "crosslip_exp_r01.pkl\t      experiment_multiclass_rad01.pkl\r\n",
      "exp_1_MNIST_bin.pkl\t      experiment_multiclass_rad02.pkl\r\n",
      "exp_1_MNIST_data.pkl\t      exp_random_10-3.pkl\r\n",
      "exp_1_random.pkl\t      FIXED_experiment_multiclass_rad01.pkl\r\n",
      "exp_1_synthetic.pkl\t      FIXED_experiment_multiclass_rad02.pkl\r\n",
      "exp_2_MNIST_bin_radius01.pkl  l1_exp_1_random.pkl\r\n",
      "exp_2_random_radius01.pkl     l1_exp_1_synthetic.pkl\r\n",
      "exp_2_synthetic_radius01.pkl  l1_exp_2_MNIST_bin_radius01.pkl\r\n",
      "exp_3_random.pkl\t      l1_exp_2_MNIST_bin_radius02.pkl\r\n",
      "exp_3_synthetic.pkl\t      l1_exp_random_10-3.pkl\r\n"
     ]
    }
   ],
   "source": [
    "!ls *.pkl\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Method     |      Time |  Time STD |     Value | Value STD |       Err |    ErrSTD\n",
      "----------------------------------------------------------------------------------\n",
      "RandomLB   $     0.301 \\pm      0.004$ $    -29.27\\%$\n",
      "LipProblem $    55.847 \\pm     79.212$ $      0.00\\%$\n",
      "CLEVER     $     1.873 \\pm      0.030$ $      4.18\\%$\n",
      "LipLP      $     0.028 \\pm      0.001$ $    156.67\\%$\n",
      "FastLip    $     0.001 \\pm      0.000$ $    156.67\\%$\n",
      "LipSDP     $     2.705 \\pm      0.030$ $    432.47\\%$\n",
      "SeqLip     $     0.015 \\pm      0.002$ $    674.84\\%$\n",
      "NaiveUB    $     0.000 \\pm      0.000$ $  11979.62\\%$\n"
     ]
    }
   ],
   "source": [
    "synthetic_file = 'l1_exp_1_synthetic.pkl'\n",
    "with open(synthetic_file, 'rb') as f:\n",
    "    synthetic_results = pickle.load(f)\n",
    "synthetic_rows = csv_parser(scrub_results(synthetic_results), 10, None, no_err=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Method     |      Time |  Time STD |     Value | Value STD |       Err |    ErrSTD\n",
      "----------------------------------------------------------------------------------\n",
      "RandomLB   $     0.238 \\pm      0.004$ $    -30.43\\%$\n",
      "CLEVER     $     1.442 \\pm      0.071$ $    -13.00\\%$\n",
      "LipProblem $    18.825 \\pm     19.244$ $      0.00\\%$\n",
      "FastLip    $     0.001 \\pm      0.000$ $    167.55\\%$\n",
      "LipLP      $     0.018 \\pm      0.009$ $    167.55\\%$\n",
      "LipSDP     $     2.624 \\pm      0.026$ $    559.97\\%$\n",
      "SeqLip     $     0.007 \\pm      0.001$ $    773.38\\%$\n",
      "NaiveUB    $     0.000 \\pm      0.000$ $   3121.51\\%$\n"
     ]
    }
   ],
   "source": [
    "random_file = 'l1_exp_1_random.pkl'\n",
    "with open(random_file, 'rb') as f:\n",
    "    random_results = pickle.load(f)\n",
    "random_rows = csv_parser(scrub_results(random_results), 10, None, no_err=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'new_rand_results' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-98b081d6fffb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrandom_rows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcsv_parser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_rand_results\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'new_rand_results' is not defined"
     ]
    }
   ],
   "source": [
    "random_rows = csv_parser(new_rand_results, 16, None, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Method', 'Time', 'Time STD', 'Value', 'Value STD', 'Err', 'ErrSTD'],\n",
       " ['RandomLB  ', '$     0.238 \\\\pm      0.004$', '$     76.20\\\\%$'],\n",
       " ['CLEVER    ', '$     1.420 \\\\pm      0.061$', '$     89.89\\\\%$'],\n",
       " ['LipProblem', '$   325.337 \\\\pm    357.716$', '$    100.00\\\\%$'],\n",
       " ['LipSDP    ', '$     2.635 \\\\pm      0.025$', '$    115.16\\\\%$'],\n",
       " ['SeqLip    ', '$     0.008 \\\\pm      0.001$', '$    149.55\\\\%$'],\n",
       " ['LipLP     ', '$     0.019 \\\\pm      0.009$', '$    555.45\\\\%$'],\n",
       " ['FastLip   ', '$     0.001 \\\\pm      0.000$', '$    585.49\\\\%$'],\n",
       " ['NaiveUB   ', '$     0.000 \\\\pm      0.000$', '$   1439.31\\\\%$']]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('exp_1_MNIST_bin.pkl', 'rb') as f:\n",
    "    mnist_results = scrub_results(pickle.load(f))\n",
    "mnist_rows = csv_parser(mnist_results, 784, None, True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(random_rows), len(synthetic_rows), len(mnist_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Method', 'Time', 'Time STD', 'Value', 'Value STD', 'Err', 'ErrSTD'],\n",
       " ['RandomLB  ', '$     0.297 \\\\pm      0.004$', '$    -32.68\\\\%$'],\n",
       " ['LipProblem', '$    38.844 \\\\pm     34.906$', '$      0.00\\\\%$'],\n",
       " ['CLEVER    ', '$     1.849 \\\\pm      0.054$', '$     28.45\\\\%$'],\n",
       " ['LipSDP    ', '$     2.704 \\\\pm      0.019$', '$     39.07\\\\%$'],\n",
       " ['SeqLip    ', '$     0.016 \\\\pm      0.002$', '$     98.98\\\\%$'],\n",
       " ['LipLP     ', '$     0.030 \\\\pm      0.002$', '$    362.43\\\\%$'],\n",
       " ['FastLip   ', '$     0.001 \\\\pm      0.000$', '$    388.14\\\\%$'],\n",
       " ['NaiveUB   ', '$     0.000 \\\\pm      0.000$', '$    996.96\\\\%$']]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "synthetic_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "rand_synth_rows = []\n",
    "for i in range(1, 9):\n",
    "    rand_synth_rows.append(random_rows[i] + synthetic_rows[i][1:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['RandomLB  ',\n",
       "  '$     0.238 \\\\pm      0.004$',\n",
       "  '$    -30.43\\\\%$',\n",
       "  '$     0.301 \\\\pm      0.004$',\n",
       "  '$    -29.27\\\\%$'],\n",
       " ['CLEVER    ',\n",
       "  '$     1.442 \\\\pm      0.071$',\n",
       "  '$    -13.00\\\\%$',\n",
       "  '$    55.847 \\\\pm     79.212$',\n",
       "  '$      0.00\\\\%$'],\n",
       " ['LipProblem',\n",
       "  '$    18.825 \\\\pm     19.244$',\n",
       "  '$      0.00\\\\%$',\n",
       "  '$     1.873 \\\\pm      0.030$',\n",
       "  '$      4.18\\\\%$'],\n",
       " ['FastLip   ',\n",
       "  '$     0.001 \\\\pm      0.000$',\n",
       "  '$    167.55\\\\%$',\n",
       "  '$     0.028 \\\\pm      0.001$',\n",
       "  '$    156.67\\\\%$'],\n",
       " ['LipLP     ',\n",
       "  '$     0.018 \\\\pm      0.009$',\n",
       "  '$    167.55\\\\%$',\n",
       "  '$     0.001 \\\\pm      0.000$',\n",
       "  '$    156.67\\\\%$'],\n",
       " ['LipSDP    ',\n",
       "  '$     2.624 \\\\pm      0.026$',\n",
       "  '$    559.97\\\\%$',\n",
       "  '$     2.705 \\\\pm      0.030$',\n",
       "  '$    432.47\\\\%$'],\n",
       " ['SeqLip    ',\n",
       "  '$     0.007 \\\\pm      0.001$',\n",
       "  '$    773.38\\\\%$',\n",
       "  '$     0.015 \\\\pm      0.002$',\n",
       "  '$    674.84\\\\%$'],\n",
       " ['NaiveUB   ',\n",
       "  '$     0.000 \\\\pm      0.000$',\n",
       "  '$   3121.51\\\\%$',\n",
       "  '$     0.000 \\\\pm      0.000$',\n",
       "  '$  11979.62\\\\%$']]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rand_synth_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('l1_experiment_1_randSynth_errors.csv', 'w', newline='') as f:\n",
    "    csvwriter = csv.writer(f, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "    for row in rand_synth_rows:\n",
    "        csvwriter.writerow(row)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('experiment_1_MNIST.csv', 'w', newline='') as f:\n",
    "    csvwriter = csv.writer(f, delimiter=',',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "    for row in mnist_rows:\n",
    "        csvwriter.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_parser(new_rand_results, 16, 'exp1RANDOMCSV.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
