{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matlab.engine\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')\n",
    "import os\n",
    "import glob\n",
    "import pickle\n",
    "import utilities as utils\n",
    "import re\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "COMPLETE_DIR = os.path.join(os.path.dirname(os.getcwd()), 'jobs', 'completed')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "vanilla_results = result_getter('REGBUNDLE*vanilla*GLOBAL.result')\n",
    "l2_results = result_getter('REGBUNDLE*l2*GLOBAL.result')\n",
    "l1_results = result_getter('REGBUNDLE*l1*GLOBAL.result')\n",
    "fgsm_results = result_getter('REGBUNDLE*FGSM*GLOBAL.result')\n",
    "\n",
    "#global_results = sorted(glob.glob(os.path.join(COMPLETE_DIR, 'simple2D*GLOBAL.result')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#seq_job_outs = utils.read_result_files(wideseq_results)\n",
    "def epoch_collector(filename):\n",
    "    return int(re.search('EPOCH\\d{4}', filename).group()[-4:])\n",
    "\n",
    "\n",
    "def make_hcube_series(result_list, lip_estimator):\n",
    "    epochs = []\n",
    "    values = []\n",
    "    for filename, res_obj in result_list:\n",
    "        epochs.append(epoch_collector(filename))\n",
    "        values.append(res_obj['do_unit_hypercube_eval'].values(lip_estimator))\n",
    "    return epochs, values\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True output \n",
    "result_colors = [(vanilla_results, 'r'), (l2_results, 'b'), (l1_results, 'g'), (fgsm_results, 'y')]\n",
    "for result, c in result_colors:\n",
    "    epochs, values = make_hcube_series(result, 'LipProblem')\n",
    "    plt.scatter(epochs, values, c=c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hcube_getter = lambda k: utils.job_out_series([_[1] for _ in seq_job_outs], 'do_unit_hypercube_eval', k)\n",
    "cs = 'bgrcmy'\n",
    "xs = [20 * (_+1) for _ in range(len(seq_job_outs))]\n",
    "for i, k in enumerate(['LipProblem', 'FastLip', 'LipLP', 'SeqLip', 'LipSDP']):#, 'NaiveUB']):\n",
    "    plt.scatter(xs, hcube_getter(k), c=cs[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_job_outs = [(utils.filename_to_epoch(_[0]), _[1]) for _ in utils.read_result_files(simple_results)]\n",
    "penalty_job_outs = [(utils.filename_to_epoch(_[0]), _[1]) for _ in utils.read_result_files(l2_results)]\n",
    "simple_epochs = [_[0] for _ in simple_job_outs]\n",
    "penalty_epochs = [_[0] for _ in penalty_job_outs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_getter = lambda k: [utils.job_out_series([_[1] for _ in job_outs], 'do_random_evals', k)\n",
    "                           for job_outs in [simple_job_outs, penalty_job_outs]]\n",
    "\n",
    "hcube_getter = lambda k: [utils.job_out_series([_[1] for _ in job_outs], 'do_unit_hypercube_eval', k)\n",
    "                          for job_outs in [simple_job_outs, penalty_job_outs]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter([_[0] for _ in simple_job_outs], random_getter('LipProblem')[0], c='r')\n",
    "plt.scatter([_[0] for _ in penalty_job_outs], random_getter('LipProblem')[1], c='b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = [_[0] for _ in simple_job_outs]\n",
    "cs = ['r', 'g', 'b']\n",
    "for i, key in enumerate(['FastLip', 'LipLP', 'LipProblem']):\n",
    "    plt.scatter(epochs, random_getter(key)[1], c=cs[i])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_true, l2_true = hcube_getter('LipProblem')\n",
    "plt.scatter(simple_epochs, simple_true, c='r')\n",
    "plt.scatter(penalty_epochs, l2_true, c='b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keyc = [('FastLip', 'b'), ('LipLP', 'r'), ('LipProblem', 'g')]\n",
    "epochs = [_[0] for _ in local_job_outs]\n",
    "for key, c in keyc:\n",
    "    plt.scatter(epochs, random_getter(key), c=c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_getter = lambda k: utils.job_out_series([_[1] for _ in local_job_outs], 'do_data_evals', k)\n",
    "keyc = [('FastLip', 'b'), ('LipLP', 'r'), ('LipProblem', 'g')]\n",
    "epochs = [_[0] for _ in local_job_outs]\n",
    "for key, c in keyc:\n",
    "    plt.scatter(epochs, data_getter(key), c=c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Local vs Global LipMIP\n",
    "local_mip = random_getter('LipProblem')\n",
    "global_mip = utils.job_out_series([_[1] for _ in local_job_outs], 'do_unit_hypercube_eval', 'LipProblem')\n",
    "for key, c in [(local_mip, 'r'), (global_mip, 'b')]:\n",
    "    plt.scatter(epochs, key, c=c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_job_outs[0][1]['do_random_evals'].average_stdevs('value')['Fast']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_objs = []\n",
    "for result_file in sorted(results):\n",
    "    try:\n",
    "        with open(result_file, 'rb') as f:\n",
    "            result_objs.append((result_file, pickle.load(f)))\n",
    "    except Exception as err:\n",
    "        print(\"FAILED ON %s\" % result_file, err)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_objs[0][1]['do_random_evals'].average_stdevs('value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values_arr = [result_obj[1]['do_unit_hypercube_eval'].values() for result_obj in result_objs]\n",
    "xs = [int(_[0].split('_')[1][-4:]) for _ in result_objs]\n",
    "get_hcube_series = lambda k: [_[k] for _ in values_arr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "job_outs = [_[1] for _ in result_objs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.job_out_series(job_outs, 'do_unit_hypercube_eval', 'LipProblem')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hcube_series = lambda k: utils.job_out_series(job_outs, 'do_unit_hypercube_eval', k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_objs[0][1]['do_unit_hypercube_eval'].compute_times()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(xs, hcube_series('FastLip'), c='b')\n",
    "plt.scatter(xs, hcube_series('LipLP'), c='r')\n",
    "plt.scatter(xs, hcube_series('LipProblem'), c='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_avgs = [result_obj[1]['do_random_evals'].average_stdevs('value') for result_obj in result_objs]\n",
    "get_random_series = lambda k: [_[k][0] for _ in random_avgs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(xs, get_random_series('FastLip'), c='b')\n",
    "plt.scatter(xs, get_random_series('LipLP'), c='r')\n",
    "plt.scatter(xs, get_random_series('LipProblem'), c='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(xs, get_random_series('LipProblem'), c='b')\n",
    "plt.scatter(xs, get_hcube_series('LipProblem'), c='r')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "re.search('_EPOCH\\d+_', 'test_file_EPOCH123_').group()[6:-1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
