{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fa41c288",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rcParams['figure.dpi'] = 100\n",
    "mpl.rcParams['font.family'] = 'DeJavu Serif'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3402b469",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# observations = 10 | memory change = 100.0% | context change = 100.0%\n",
      "# observations = 20 | memory change = 37.5% | context change = 31.6%\n",
      "# observations = 30 | memory change = 21.9% | context change = 15.5%\n",
      "# observations = 40 | memory change = 15.4% | context change = 10.3%\n",
      "# observations = 50 | memory change = 12.5% | context change = 7.5%\n",
      "# observations = 60 | memory change = 9.9% | context change = 5.9%\n",
      "# observations = 70 | memory change = 8.1% | context change = 5.2%\n",
      "# observations = 80 | memory change = 6.9% | context change = 4.5%\n",
      "# observations = 90 | memory change = 5.9% | context change = 4.1%\n",
      "# observations = 100 | memory change = 5.4% | context change = 3.4%\n"
     ]
    }
   ],
   "source": [
    "sample_change = {'memory':[100], 'context':[100]}\n",
    "conflicts = {step:json.load(open(f'Data/Hyperparameter/mix=20_samples={step}0.json')) for step in range(1, 11)}\n",
    "for version in ['memory', 'context']:\n",
    "    for step in range(2, 11):\n",
    "        change = [conflicts[step-1][str(idx)][version] != conflicts[step][str(idx)][version] for idx in range(len(conflicts[1]))]\n",
    "        sample_change[version].append(np.mean(change)*100)\n",
    "        \n",
    "for idx in range(10):\n",
    "    print(f'# observations = {(idx+1)*10} | memory change = {sample_change[\"memory\"][idx]:.1f}% | context change = {sample_change[\"context\"][idx]:.1f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c15765c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# paraphrases = 1 | memory change = 100.0% | context change = 100.0%\n",
      "# paraphrases = 2 | memory change = 48.5% | context change = 45.5%\n",
      "# paraphrases = 5 | memory change = 21.7% | context change = 18.5%\n",
      "# paraphrases = 10 | memory change = 20.0% | context change = 15.8%\n",
      "# paraphrases = 20 | memory change = 19.2% | context change = 13.1%\n",
      "# paraphrases = 50 | memory change = 19.0% | context change = 13.5%\n",
      "# paraphrases = 100 | memory change = 19.1% | context change = 13.5%\n"
     ]
    }
   ],
   "source": [
    "mixes = [1, 2, 5, 10, 20, 50, 100]\n",
    "mix_change = {'memory':[100], 'context':[100]}\n",
    "conflicts = {mix:json.load(open(f'Data/Hyperparameter/mix={mix}_samples=100.json')) for mix in mixes}\n",
    "for version in ['memory', 'context']:\n",
    "    for mid in range(1, len(mixes)):\n",
    "        change = [conflicts[mixes[mid-1]][str(idx)][version] != conflicts[mixes[mid]][str(idx)][version] for idx in range(len(conflicts[1]))]\n",
    "        mix_change[version].append(np.mean(change)*100)\n",
    "\n",
    "for idx in range(len(mixes)):\n",
    "    print(f'# paraphrases = {mixes[idx]} | memory change = {mix_change[\"memory\"][idx]:.1f}% | context change = {mix_change[\"context\"][idx]:.1f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "36dd4874",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 70844 characters]",
      "text/plain": [
       "<Figure size 1000x300 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fontsize = 9\n",
    "colors = sns.color_palette('Set2', 2)\n",
    "\n",
    "num_row, num_col = 1, 2\n",
    "size_row, size_col = 3, 5\n",
    "fig, axs = plt.subplots(num_row, num_col, figsize=(num_col*size_col, num_row*size_row))\n",
    "\n",
    "sns.despine(ax=axs[0], offset=dict(left=5))\n",
    "for vid, version in enumerate(['Memory', 'Context']):\n",
    "    axs[0].plot(mixes, mix_change[version.lower()], color=colors[vid], marker='.',\n",
    "                label='Parametric' if version == 'Memory' else 'Contextual')\n",
    "axs[0].set_xscale('log')\n",
    "axs[0].set_xlabel('Number of Question Paraphrases ($M$)', fontsize=fontsize)\n",
    "axs[0].set_ylabel('Change (%) in Knowledge\\nStatus Characterization', fontsize=fontsize)\n",
    "axs[0].set_xticks(mixes)\n",
    "axs[0].set_xticklabels(mixes)\n",
    "axs[0].set_ylim(0, 100)\n",
    "axs[0].tick_params(axis='both', which='major', labelsize=fontsize-1)\n",
    "axs[0].set_title('(a) Effect of Increasing the Number of Question Paraphrases\\nWhen Sampling 100 Model Responses', fontsize=fontsize+1)\n",
    "axs[0].legend(fontsize=fontsize-1, loc=1)\n",
    "\n",
    "sns.despine(ax=axs[1], offset=dict(left=5))\n",
    "for vid, version in enumerate(['Memory', 'Context']):\n",
    "    axs[1].plot(range(1, 11), sample_change[version.lower()], color=colors[vid], marker='.',\n",
    "                label='Parametric' if version == 'Memory' else 'Contextual')\n",
    "axs[1].set_ylabel('Change (%) in Knowledge\\nStatus Characterization', fontsize=fontsize)\n",
    "axs[1].set_xlabel('Number of Model Responses ($N$)', fontsize=fontsize)\n",
    "axs[1].set_xticks(range(1, 11))\n",
    "axs[1].set_xticklabels([num*10 for num in range(1, 11)])\n",
    "axs[1].set_ylim(0, 100)\n",
    "axs[1].tick_params(axis='both', which='major', labelsize=fontsize-1)\n",
    "axs[1].set_title('(b) Effect of Increasing the Number of Model Responses\\nWhen Using 20 Question Paraphrases', fontsize=fontsize+1)\n",
    "axs[1].legend(fontsize=fontsize-1, loc=1)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(f'Figure/hyperparameter.pdf', bbox_inches='tight')\n",
    "plt.show()\n",
    "plt.close(fig)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
