{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4cb67645",
   "metadata": {},
   "source": [
    "### Hemonc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "82bdde5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from Bio import Entrez\n",
    "from urllib.error import HTTPError\n",
    "from collections import defaultdict\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d169a04a",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_ref = ['study', 'condition', 'pmid', 'pub.date']\n",
    "columns_result = ['study', 'condition', 'regimen', 'comparator', 'efficacy']\n",
    "columns_indication = ['study', 'condition', 'stage_or_status']\n",
    "\n",
    "preprocess = lambda df, col: df[col].dropna().drop_duplicates().sort_values(col).reset_index(drop=True)\n",
    "ref = preprocess(pd.read_csv(\"Data/Raw/Hemonc/ref.table.csv\"), columns_ref)\n",
    "result = preprocess(pd.read_csv(\"Data/Raw/Hemonc/study_results.csv\"), columns_result)\n",
    "indication = preprocess(pd.read_csv(\"Data/Raw/Hemonc/indications.csv\"), columns_indication)\n",
    "\n",
    "efficacy = pd.read_excel(\"Data/Raw/Hemonc/efficacy.xlsx\")\n",
    "efficacy2label = dict(zip(efficacy['efficacy_raw'], efficacy['efficacy_std']))\n",
    "efficacy2label = {k:v if 'Might Be ' not in v else v[len('Might Be '):] for k,v in efficacy2label.items()}\n",
    "stage = pd.read_excel(\"Data/Raw/Hemonc/stage.xlsx\")\n",
    "stage2label = dict(zip(stage['stage_raw'], stage['stage_std']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a53ec5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "study2pmids = defaultdict(set)\n",
    "for _, row in ref.iterrows():\n",
    "    study2pmids[(row['study'], row['condition'])].add(row['pmid'])\n",
    "    \n",
    "study2result = defaultdict(set)\n",
    "for _, row in result.iterrows():\n",
    "    if (row['study'], row['condition']) not in study2pmids or efficacy2label[row['efficacy']] == 'Other' or row['regimen'] == row['comparator']: continue\n",
    "    label = efficacy2label[row['efficacy']]\n",
    "    switched = label.replace('Inferior', 'Superior') if 'Inferior' in label else label.replace('Superior', 'Inferior')\n",
    "    study2result[(row['study'], row['condition'])].add((row['regimen'], row['comparator'], label))\n",
    "    study2result[(row['study'], row['condition'])].add((row['comparator'], row['regimen'], switched))\n",
    "    \n",
    "for study in set(study2pmids.keys()) - set(study2result.keys()):\n",
    "    del study2pmids[study]\n",
    "    \n",
    "study2stage = defaultdict(set)\n",
    "for _, row in indication.iterrows():\n",
    "    if (row['study'], row['condition']) not in study2result: continue\n",
    "    study2stage[(row['study'], row['condition'])].add(stage2label[row['stage_or_status']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "08db09d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "Entrez.email = \"\" # your email address\n",
    "def get_text(pmid, retries=50):\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            handle = Entrez.efetch(db=\"pubmed\", id=pmid, rettype=\"xml\", retmode=\"text\")\n",
    "            record = Entrez.read(handle)\n",
    "            article = record['PubmedArticle'][0]['MedlineCitation']['Article']\n",
    "            doc = article['ArticleTitle']\n",
    "            if 'Abstract' in article: doc = ' '.join([doc] + article['Abstract']['AbstractText'])\n",
    "            return pmid, doc\n",
    "        except HTTPError as e:\n",
    "            if e.code == 429: time.sleep(i)\n",
    "            else: raise Exception(f\"Failed to fetch data for PMID {pmid} after {retries} retries.\")\n",
    "\n",
    "pmid2doc = {}\n",
    "path_doc = \"Data/Raw/Hemonc/docs.csv\"\n",
    "if os.path.exists(path_doc):\n",
    "    pmid2doc = pd.read_csv(path_doc)\n",
    "    pmid2doc = dict(zip(pmid2doc['pmid'], pmid2doc['doc']))\n",
    "\n",
    "pmids_missing = set([pmid for study, pmids in study2pmids.items() for pmid in pmids]) - set(pmid2doc.keys())\n",
    "pmids_missing |= set([pmid for pmid, doc in pmid2doc.items() if not isinstance(doc, str) and np.isnan(doc)])\n",
    "if len(pmids_missing):\n",
    "    pmid2doc_missing = {}\n",
    "    with ThreadPoolExecutor(max_workers=10) as exe:\n",
    "        futures = [exe.submit(get_text, pmid) for pmid in pmids_missing]\n",
    "        for future in tqdm(as_completed(futures), total=len(futures)):\n",
    "            try:\n",
    "                pmid, doc = future.result()\n",
    "                pmid2doc_missing[pmid] = doc\n",
    "            except Exception as e:\n",
    "                print(f\"Error: {e}\")\n",
    "    for pmid in pmid2doc_missing:\n",
    "        if not isinstance(pmid2doc_missing[pmid], str) and np.isnan(pmid2doc_missing[pmid]):\n",
    "            pmid2doc_missing[pmid] = get_text(pmid)[1]\n",
    "    pmid2doc |= pmid2doc_missing\n",
    "    pmid2doc_df = pd.DataFrame(list(pmid2doc.items()), columns=['pmid', 'doc'])\n",
    "    pmid2doc_df.to_csv(path_doc, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c87328b",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n",
    "openai_model = \"gpt-4o\"\n",
    "\n",
    "def call_gpt(message_user, retries=50):\n",
    "    messages = [{\"role\": \"system\", \"content\": 'You are a helpful assistant.'}, \n",
    "                {\"role\": \"user\", \"content\": message_user}]\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            response = openai_client.chat.completions.create(\n",
    "                model = openai_model, messages = messages, max_tokens = 4096 # 512\n",
    "            )\n",
    "            return response.choices[0].message.content\n",
    "        except openai.RateLimitError as e:\n",
    "            time.sleep(i)\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {e}\")\n",
    "            raise\n",
    "    raise Exception(f\"Failed to call GPT-4 after {retries} retries.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "067ca106",
   "metadata": {},
   "outputs": [],
   "source": [
    "templates_qn = [\n",
    "    'Choose an option that best describes the efficacy of {REGIMEN} compared to {COMPARATOR} when used to treat {CONDITION}.',\n",
    "    'Select the option that most accurately reflects the effectiveness of {REGIMEN} versus {COMPARATOR} in treating {CONDITION}.',\n",
    "    'Which option best summarizes the comparative efficacy of {REGIMEN} and {COMPARATOR} for managing {CONDITION}?',\n",
    "    'Identify the option that best summarizes the effectiveness of {REGIMEN} versus {COMPARATOR} in treating {CONDITION}.',\n",
    "    'Which option most effectively illustrates the efficacy of {REGIMEN} when compared with {COMPARATOR} for {CONDITION}?'\n",
    "]\n",
    "rephrase = \\\n",
    "f'''\n",
    "### Instruction\n",
    "Do not respond to the question. \n",
    "Instead, rephrase the given question template into 20 other versions that are semantically equivalent.\n",
    "\n",
    "## Version 0: {templates_qn[0]}\n",
    "## Version 1: {templates_qn[1]}\n",
    "## Version 2: {templates_qn[2]}\n",
    "## Version 3: {templates_qn[3]}\n",
    "## Version 4: {templates_qn[4]}\n",
    "'''\n",
    "response = call_gpt(rephrase)\n",
    "matches = [re.search(fr\"## Version {i}: (.*)\", response) for i in range(5, 20)]           \n",
    "templates_qn += [each.group(1) for each in matches if each is not None and len(each.group(1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "634c0916",
   "metadata": {},
   "outputs": [],
   "source": [
    "option2idx = {'superior':1, 'inferior':2, 'no difference':3}\n",
    "dataset = []\n",
    "for key, values in study2result.items():\n",
    "    evidence = '\\n\\n'.join([pmid2doc[pmid] for pmid in study2pmids[key]])\n",
    "    stage = '' if key not in study2stage else ' ({})'.format(', '.join(sorted(study2stage[key])))\n",
    "    conditon = key[1] + stage\n",
    "    for regimen, comparator, efficacy in values:\n",
    "        if regimen == comparator: continue\n",
    "        questions = [each.format(**{'REGIMEN':regimen, 'COMPARATOR':comparator, 'CONDITION':conditon}) for each in templates_qn]\n",
    "        answer = option2idx[efficacy.lower()]\n",
    "        dataset.append((evidence, *questions, answer))\n",
    "        \n",
    "columns = ['evidence'] + [f'question {i}' for i in range(1, len(templates_qn)+1)] + ['answer']\n",
    "dataset = pd.DataFrame(dataset, columns=columns)\n",
    "dataset = dataset.dropna().drop_duplicates(subset=columns[:-1], keep=False).reset_index(drop=True)\n",
    "for option, idx in option2idx.items():\n",
    "    dataset[f'option {idx}'] = option\n",
    "dataset.to_csv('Data/Input/Hemonc.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f6a30789",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>evidence</th>\n",
       "      <th>question 1</th>\n",
       "      <th>question 2</th>\n",
       "      <th>question 3</th>\n",
       "      <th>question 4</th>\n",
       "      <th>question 5</th>\n",
       "      <th>question 6</th>\n",
       "      <th>question 7</th>\n",
       "      <th>question 8</th>\n",
       "      <th>question 9</th>\n",
       "      <th>...</th>\n",
       "      <th>question 15</th>\n",
       "      <th>question 16</th>\n",
       "      <th>question 17</th>\n",
       "      <th>question 18</th>\n",
       "      <th>question 19</th>\n",
       "      <th>question 20</th>\n",
       "      <th>answer</th>\n",
       "      <th>option 1</th>\n",
       "      <th>option 2</th>\n",
       "      <th>option 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A double-blind, placebo-controlled, randomized...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Cisplati...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Cisplatin an...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>2</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A double-blind, placebo-controlled, randomized...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Cisplati...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Cisplatin, T...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>1</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thalidomide maintenance treatment increases pr...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Interfer...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Interferon a...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>1</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thalidomide maintenance treatment increases pr...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Interfer...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Interferon a...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>2</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Randomized, Double-Blind, Placebo-Controlled P...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Placebo ...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Placebo perf...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>2</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6207</th>\n",
       "      <td>Adjuvant Abemaciclib Plus Endocrine Therapy fo...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Abemacic...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Abemaciclib ...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>1</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6208</th>\n",
       "      <td>Topotecan versus paclitaxel for the treatment ...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Paclitax...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Paclitaxel m...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>2</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6209</th>\n",
       "      <td>Topotecan versus paclitaxel for the treatment ...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how Topoteca...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how Topotecan mo...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>1</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6210</th>\n",
       "      <td>Bendamustine prolongs progression-free surviva...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how CMF comp...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how CMF performs...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>2</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6211</th>\n",
       "      <td>Bendamustine prolongs progression-free surviva...</td>\n",
       "      <td>Choose an option that best describes the effic...</td>\n",
       "      <td>Select the option that most accurately reflect...</td>\n",
       "      <td>Which option best summarizes the comparative e...</td>\n",
       "      <td>Identify the option that best summarizes the e...</td>\n",
       "      <td>Which option most effectively illustrates the ...</td>\n",
       "      <td>Pick the option that best depicts how BMF comp...</td>\n",
       "      <td>Select the statement that most clearly describ...</td>\n",
       "      <td>Which choice best compares the treatment effic...</td>\n",
       "      <td>Find the option that best outlines the compara...</td>\n",
       "      <td>...</td>\n",
       "      <td>Point out the choice that best reflects the ef...</td>\n",
       "      <td>Select the option that best indicates the effe...</td>\n",
       "      <td>Choose the answer that most accurately showcas...</td>\n",
       "      <td>Which selection best reflects how BMF performs...</td>\n",
       "      <td>Identify the option that most precisely illust...</td>\n",
       "      <td>What is the best choice that outlines the diff...</td>\n",
       "      <td>1</td>\n",
       "      <td>superior</td>\n",
       "      <td>inferior</td>\n",
       "      <td>no difference</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6212 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               evidence  \\\n",
       "0     A double-blind, placebo-controlled, randomized...   \n",
       "1     A double-blind, placebo-controlled, randomized...   \n",
       "2     Thalidomide maintenance treatment increases pr...   \n",
       "3     Thalidomide maintenance treatment increases pr...   \n",
       "4     Randomized, Double-Blind, Placebo-Controlled P...   \n",
       "...                                                 ...   \n",
       "6207  Adjuvant Abemaciclib Plus Endocrine Therapy fo...   \n",
       "6208  Topotecan versus paclitaxel for the treatment ...   \n",
       "6209  Topotecan versus paclitaxel for the treatment ...   \n",
       "6210  Bendamustine prolongs progression-free surviva...   \n",
       "6211  Bendamustine prolongs progression-free surviva...   \n",
       "\n",
       "                                             question 1  \\\n",
       "0     Choose an option that best describes the effic...   \n",
       "1     Choose an option that best describes the effic...   \n",
       "2     Choose an option that best describes the effic...   \n",
       "3     Choose an option that best describes the effic...   \n",
       "4     Choose an option that best describes the effic...   \n",
       "...                                                 ...   \n",
       "6207  Choose an option that best describes the effic...   \n",
       "6208  Choose an option that best describes the effic...   \n",
       "6209  Choose an option that best describes the effic...   \n",
       "6210  Choose an option that best describes the effic...   \n",
       "6211  Choose an option that best describes the effic...   \n",
       "\n",
       "                                             question 2  \\\n",
       "0     Select the option that most accurately reflect...   \n",
       "1     Select the option that most accurately reflect...   \n",
       "2     Select the option that most accurately reflect...   \n",
       "3     Select the option that most accurately reflect...   \n",
       "4     Select the option that most accurately reflect...   \n",
       "...                                                 ...   \n",
       "6207  Select the option that most accurately reflect...   \n",
       "6208  Select the option that most accurately reflect...   \n",
       "6209  Select the option that most accurately reflect...   \n",
       "6210  Select the option that most accurately reflect...   \n",
       "6211  Select the option that most accurately reflect...   \n",
       "\n",
       "                                             question 3  \\\n",
       "0     Which option best summarizes the comparative e...   \n",
       "1     Which option best summarizes the comparative e...   \n",
       "2     Which option best summarizes the comparative e...   \n",
       "3     Which option best summarizes the comparative e...   \n",
       "4     Which option best summarizes the comparative e...   \n",
       "...                                                 ...   \n",
       "6207  Which option best summarizes the comparative e...   \n",
       "6208  Which option best summarizes the comparative e...   \n",
       "6209  Which option best summarizes the comparative e...   \n",
       "6210  Which option best summarizes the comparative e...   \n",
       "6211  Which option best summarizes the comparative e...   \n",
       "\n",
       "                                             question 4  \\\n",
       "0     Identify the option that best summarizes the e...   \n",
       "1     Identify the option that best summarizes the e...   \n",
       "2     Identify the option that best summarizes the e...   \n",
       "3     Identify the option that best summarizes the e...   \n",
       "4     Identify the option that best summarizes the e...   \n",
       "...                                                 ...   \n",
       "6207  Identify the option that best summarizes the e...   \n",
       "6208  Identify the option that best summarizes the e...   \n",
       "6209  Identify the option that best summarizes the e...   \n",
       "6210  Identify the option that best summarizes the e...   \n",
       "6211  Identify the option that best summarizes the e...   \n",
       "\n",
       "                                             question 5  \\\n",
       "0     Which option most effectively illustrates the ...   \n",
       "1     Which option most effectively illustrates the ...   \n",
       "2     Which option most effectively illustrates the ...   \n",
       "3     Which option most effectively illustrates the ...   \n",
       "4     Which option most effectively illustrates the ...   \n",
       "...                                                 ...   \n",
       "6207  Which option most effectively illustrates the ...   \n",
       "6208  Which option most effectively illustrates the ...   \n",
       "6209  Which option most effectively illustrates the ...   \n",
       "6210  Which option most effectively illustrates the ...   \n",
       "6211  Which option most effectively illustrates the ...   \n",
       "\n",
       "                                             question 6  \\\n",
       "0     Pick the option that best depicts how Cisplati...   \n",
       "1     Pick the option that best depicts how Cisplati...   \n",
       "2     Pick the option that best depicts how Interfer...   \n",
       "3     Pick the option that best depicts how Interfer...   \n",
       "4     Pick the option that best depicts how Placebo ...   \n",
       "...                                                 ...   \n",
       "6207  Pick the option that best depicts how Abemacic...   \n",
       "6208  Pick the option that best depicts how Paclitax...   \n",
       "6209  Pick the option that best depicts how Topoteca...   \n",
       "6210  Pick the option that best depicts how CMF comp...   \n",
       "6211  Pick the option that best depicts how BMF comp...   \n",
       "\n",
       "                                             question 7  \\\n",
       "0     Select the statement that most clearly describ...   \n",
       "1     Select the statement that most clearly describ...   \n",
       "2     Select the statement that most clearly describ...   \n",
       "3     Select the statement that most clearly describ...   \n",
       "4     Select the statement that most clearly describ...   \n",
       "...                                                 ...   \n",
       "6207  Select the statement that most clearly describ...   \n",
       "6208  Select the statement that most clearly describ...   \n",
       "6209  Select the statement that most clearly describ...   \n",
       "6210  Select the statement that most clearly describ...   \n",
       "6211  Select the statement that most clearly describ...   \n",
       "\n",
       "                                             question 8  \\\n",
       "0     Which choice best compares the treatment effic...   \n",
       "1     Which choice best compares the treatment effic...   \n",
       "2     Which choice best compares the treatment effic...   \n",
       "3     Which choice best compares the treatment effic...   \n",
       "4     Which choice best compares the treatment effic...   \n",
       "...                                                 ...   \n",
       "6207  Which choice best compares the treatment effic...   \n",
       "6208  Which choice best compares the treatment effic...   \n",
       "6209  Which choice best compares the treatment effic...   \n",
       "6210  Which choice best compares the treatment effic...   \n",
       "6211  Which choice best compares the treatment effic...   \n",
       "\n",
       "                                             question 9  ...  \\\n",
       "0     Find the option that best outlines the compara...  ...   \n",
       "1     Find the option that best outlines the compara...  ...   \n",
       "2     Find the option that best outlines the compara...  ...   \n",
       "3     Find the option that best outlines the compara...  ...   \n",
       "4     Find the option that best outlines the compara...  ...   \n",
       "...                                                 ...  ...   \n",
       "6207  Find the option that best outlines the compara...  ...   \n",
       "6208  Find the option that best outlines the compara...  ...   \n",
       "6209  Find the option that best outlines the compara...  ...   \n",
       "6210  Find the option that best outlines the compara...  ...   \n",
       "6211  Find the option that best outlines the compara...  ...   \n",
       "\n",
       "                                            question 15  \\\n",
       "0     Point out the choice that best reflects the ef...   \n",
       "1     Point out the choice that best reflects the ef...   \n",
       "2     Point out the choice that best reflects the ef...   \n",
       "3     Point out the choice that best reflects the ef...   \n",
       "4     Point out the choice that best reflects the ef...   \n",
       "...                                                 ...   \n",
       "6207  Point out the choice that best reflects the ef...   \n",
       "6208  Point out the choice that best reflects the ef...   \n",
       "6209  Point out the choice that best reflects the ef...   \n",
       "6210  Point out the choice that best reflects the ef...   \n",
       "6211  Point out the choice that best reflects the ef...   \n",
       "\n",
       "                                            question 16  \\\n",
       "0     Select the option that best indicates the effe...   \n",
       "1     Select the option that best indicates the effe...   \n",
       "2     Select the option that best indicates the effe...   \n",
       "3     Select the option that best indicates the effe...   \n",
       "4     Select the option that best indicates the effe...   \n",
       "...                                                 ...   \n",
       "6207  Select the option that best indicates the effe...   \n",
       "6208  Select the option that best indicates the effe...   \n",
       "6209  Select the option that best indicates the effe...   \n",
       "6210  Select the option that best indicates the effe...   \n",
       "6211  Select the option that best indicates the effe...   \n",
       "\n",
       "                                            question 17  \\\n",
       "0     Choose the answer that most accurately showcas...   \n",
       "1     Choose the answer that most accurately showcas...   \n",
       "2     Choose the answer that most accurately showcas...   \n",
       "3     Choose the answer that most accurately showcas...   \n",
       "4     Choose the answer that most accurately showcas...   \n",
       "...                                                 ...   \n",
       "6207  Choose the answer that most accurately showcas...   \n",
       "6208  Choose the answer that most accurately showcas...   \n",
       "6209  Choose the answer that most accurately showcas...   \n",
       "6210  Choose the answer that most accurately showcas...   \n",
       "6211  Choose the answer that most accurately showcas...   \n",
       "\n",
       "                                            question 18  \\\n",
       "0     Which selection best reflects how Cisplatin an...   \n",
       "1     Which selection best reflects how Cisplatin, T...   \n",
       "2     Which selection best reflects how Interferon a...   \n",
       "3     Which selection best reflects how Interferon a...   \n",
       "4     Which selection best reflects how Placebo perf...   \n",
       "...                                                 ...   \n",
       "6207  Which selection best reflects how Abemaciclib ...   \n",
       "6208  Which selection best reflects how Paclitaxel m...   \n",
       "6209  Which selection best reflects how Topotecan mo...   \n",
       "6210  Which selection best reflects how CMF performs...   \n",
       "6211  Which selection best reflects how BMF performs...   \n",
       "\n",
       "                                            question 19  \\\n",
       "0     Identify the option that most precisely illust...   \n",
       "1     Identify the option that most precisely illust...   \n",
       "2     Identify the option that most precisely illust...   \n",
       "3     Identify the option that most precisely illust...   \n",
       "4     Identify the option that most precisely illust...   \n",
       "...                                                 ...   \n",
       "6207  Identify the option that most precisely illust...   \n",
       "6208  Identify the option that most precisely illust...   \n",
       "6209  Identify the option that most precisely illust...   \n",
       "6210  Identify the option that most precisely illust...   \n",
       "6211  Identify the option that most precisely illust...   \n",
       "\n",
       "                                            question 20 answer  option 1  \\\n",
       "0     What is the best choice that outlines the diff...      2  superior   \n",
       "1     What is the best choice that outlines the diff...      1  superior   \n",
       "2     What is the best choice that outlines the diff...      1  superior   \n",
       "3     What is the best choice that outlines the diff...      2  superior   \n",
       "4     What is the best choice that outlines the diff...      2  superior   \n",
       "...                                                 ...    ...       ...   \n",
       "6207  What is the best choice that outlines the diff...      1  superior   \n",
       "6208  What is the best choice that outlines the diff...      2  superior   \n",
       "6209  What is the best choice that outlines the diff...      1  superior   \n",
       "6210  What is the best choice that outlines the diff...      2  superior   \n",
       "6211  What is the best choice that outlines the diff...      1  superior   \n",
       "\n",
       "      option 2       option 3  \n",
       "0     inferior  no difference  \n",
       "1     inferior  no difference  \n",
       "2     inferior  no difference  \n",
       "3     inferior  no difference  \n",
       "4     inferior  no difference  \n",
       "...        ...            ...  \n",
       "6207  inferior  no difference  \n",
       "6208  inferior  no difference  \n",
       "6209  inferior  no difference  \n",
       "6210  inferior  no difference  \n",
       "6211  inferior  no difference  \n",
       "\n",
       "[6212 rows x 25 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f38b01d5",
   "metadata": {},
   "source": [
    "### PubMedQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d7dc392b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import time\n",
    "import openai\n",
    "import random\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from datasets import load_dataset\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54eb1712",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "128e3d83a74f4f73b5ac41f345acbfbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2201997"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = load_dataset(\"qiaojin/PubMedQA\", data_dir=\"pqa_labeled\", split=\"train\")\n",
    "raw.to_json('Data/Raw/PubMedQA/PubMedQA.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b42a2928",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n",
    "openai_model = \"gpt-4o\"\n",
    "\n",
    "def call_gpt(message_user, retries=50):\n",
    "    messages = [{\"role\": \"system\", \"content\": 'You are a helpful assistant.'}, \n",
    "                {\"role\": \"user\", \"content\": message_user}]\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            response = openai_client.chat.completions.create(\n",
    "                model = openai_model, messages = messages, max_tokens = 4096 # 512\n",
    "            )\n",
    "            return response.choices[0].message.content\n",
    "        except openai.RateLimitError as e:\n",
    "            time.sleep(i)\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {e}\")\n",
    "            raise\n",
    "    raise Exception(f\"Failed to call GPT-4 after {retries} retries.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0f792dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_rephrase = \\\n",
    "'''\n",
    "### Instruction\n",
    "Do not respond to the question. \n",
    "Instead, follow the provided examples and rephrase the given question into 20 other versions that are semantically equivalent.\n",
    "\n",
    "### Example\n",
    "## Question: Which option best summarizes the comparative efficacy of regimen and comparator for managing condition?\n",
    "## Version 1: Choose an option that best describes the efficacy of regimen compared to comparator when used to treat condition.\n",
    "## Version 2: Select the option that most accurately reflects the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 3: Identify the option that best summarizes the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 4: Which option most effectively illustrates the efficacy of regimen when compared with comparator for condition?\n",
    "\n",
    "### Your Task\n",
    "## Question: {QUESTION}\n",
    "'''\n",
    "\n",
    "def rephrase_qn(question, retries=50):\n",
    "    query = template_rephrase.format(**{'QUESTION':question})\n",
    "    for _ in range(retries):\n",
    "        response = call_gpt(query)\n",
    "        matches = [re.search(fr\"## Version {i}: (.*)\", response) for i in range(1, 20)]\n",
    "        if all([each is not None and len(each.group(1)) for each in matches]):            \n",
    "            questions = [question] + [each.group(1) for each in matches]\n",
    "            if len(set(questions)) == 20: return questions\n",
    "    return [question] + [None]*19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "144d0c71",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 1000/1000 [05:24<00:00,  3.08it/s]\n"
     ]
    }
   ],
   "source": [
    "def prepare_row(idx, row):\n",
    "    evidence = '\\n\\n'.join(row['context']['contexts'])\n",
    "    questions = rephrase_qn(row['question'])\n",
    "    row = (evidence, *questions, option2idx[row['final_decision']])\n",
    "    return idx, row\n",
    "\n",
    "raw = [json.loads(line) for line in open('Data/Raw/PubMedQA/PubMedQA.jsonl')]\n",
    "option2idx = {'yes':1, 'no':2, 'maybe':3}\n",
    "\n",
    "dataset, max_workers = {}, 50\n",
    "with ThreadPoolExecutor(max_workers=max_workers) as exe:\n",
    "    futures = [exe.submit(prepare_row, idx, row) for idx, row in enumerate(raw)]\n",
    "    for future in tqdm(as_completed(futures), total=len(futures)):\n",
    "        try:\n",
    "            idx, row = future.result()\n",
    "            dataset[idx] = row\n",
    "        except Exception as e:\n",
    "            print(f\"Error: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cbfdc03a",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['evidence'] + [f'question {i}' for i in range(1, 21)] + ['answer']\n",
    "dataset = [row for idx, row in sorted(dataset.items(), key = lambda x: x[0])]\n",
    "dataset = pd.DataFrame(dataset, columns=columns)\n",
    "dataset = dataset.dropna().drop_duplicates(subset=columns[:-1], keep=False).reset_index(drop=True)\n",
    "for option, idx in option2idx.items():\n",
    "    dataset[f'option {idx}'] = option\n",
    "dataset.to_csv('Data/Input/PubMedQA.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d3eebcf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>evidence</th>\n",
       "      <th>question 1</th>\n",
       "      <th>question 2</th>\n",
       "      <th>question 3</th>\n",
       "      <th>question 4</th>\n",
       "      <th>question 5</th>\n",
       "      <th>question 6</th>\n",
       "      <th>question 7</th>\n",
       "      <th>question 8</th>\n",
       "      <th>question 9</th>\n",
       "      <th>...</th>\n",
       "      <th>question 15</th>\n",
       "      <th>question 16</th>\n",
       "      <th>question 17</th>\n",
       "      <th>question 18</th>\n",
       "      <th>question 19</th>\n",
       "      <th>question 20</th>\n",
       "      <th>answer</th>\n",
       "      <th>option 1</th>\n",
       "      <th>option 2</th>\n",
       "      <th>option 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Programmed cell death (PCD) is the regulated d...</td>\n",
       "      <td>Do mitochondria play a role in remodelling lac...</td>\n",
       "      <td>Are mitochondria involved in the remodeling of...</td>\n",
       "      <td>Do mitochondria contribute to the restructurin...</td>\n",
       "      <td>Is there a role for mitochondria in the transf...</td>\n",
       "      <td>Do mitochondria influence how lace plant leave...</td>\n",
       "      <td>What role do mitochondria have in the reformat...</td>\n",
       "      <td>Are mitochondria responsible for changes in la...</td>\n",
       "      <td>Is the restructuring of lace plant leaves duri...</td>\n",
       "      <td>Do mitochondria affect the way lace plant leav...</td>\n",
       "      <td>...</td>\n",
       "      <td>Do mitochondria play a significant part in the...</td>\n",
       "      <td>Is mitochondrial involvement evident in the re...</td>\n",
       "      <td>To what extent do mitochondria contribute to l...</td>\n",
       "      <td>Are mitochondrial roles crucial in shaping lac...</td>\n",
       "      <td>Do mitochondria have an essential function in ...</td>\n",
       "      <td>Is there a mitochondrial role in the construct...</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Assessment of visual acuity depends on the opt...</td>\n",
       "      <td>Landolt C and snellen e acuity: differences in...</td>\n",
       "      <td>How do the Landolt C and Snellen E tests diffe...</td>\n",
       "      <td>What are the distinctions between Landolt C an...</td>\n",
       "      <td>In what ways do Landolt C and Snellen E acuity...</td>\n",
       "      <td>How does acuity measurement compare between La...</td>\n",
       "      <td>What are the key differences between Landolt C...</td>\n",
       "      <td>How do the acuity assessments of Landolt C and...</td>\n",
       "      <td>What differences exist between the Landolt C a...</td>\n",
       "      <td>When assessing strabismus amblyopia, how do th...</td>\n",
       "      <td>...</td>\n",
       "      <td>In cases of strabismus amblyopia, how does the...</td>\n",
       "      <td>What differences are there in assessing strabi...</td>\n",
       "      <td>How do Landolt C and Snellen E tests compare i...</td>\n",
       "      <td>How does the acuity assessment with Landolt C ...</td>\n",
       "      <td>How are the Landolt C and Snellen E tests diff...</td>\n",
       "      <td>What is the difference in acuity testing betwe...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Apparent life-threatening events in infants ar...</td>\n",
       "      <td>Syncope during bathing in infants, a pediatric...</td>\n",
       "      <td>Could syncope in infants during bathing be a p...</td>\n",
       "      <td>Is the occurrence of syncope in infants while ...</td>\n",
       "      <td>Might fainting episodes in infants during bath...</td>\n",
       "      <td>Are syncope episodes in infants during bathing...</td>\n",
       "      <td>Does syncope in infants when bathing suggest a...</td>\n",
       "      <td>Is fainting in infants during bathing a sign o...</td>\n",
       "      <td>Could fainting episodes during baths in infant...</td>\n",
       "      <td>Is there a possibility that syncope in infants...</td>\n",
       "      <td>...</td>\n",
       "      <td>Might an infant experiencing syncope during a ...</td>\n",
       "      <td>Is there a connection between bathing-induced ...</td>\n",
       "      <td>Can syncope in infants during bathing be attri...</td>\n",
       "      <td>Is syncope during bathing in infants potential...</td>\n",
       "      <td>Might the syncope observance in infants while ...</td>\n",
       "      <td>Could infant syncope during bathing sessions i...</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The transanal endorectal pull-through (TERPT) ...</td>\n",
       "      <td>Are the long-term results of the transanal pul...</td>\n",
       "      <td>Do the long-term outcomes of the transanal pul...</td>\n",
       "      <td>How do the long-term results of the transanal ...</td>\n",
       "      <td>Are the long-term effects of the transanal pul...</td>\n",
       "      <td>Do the long-term results for the transanal pul...</td>\n",
       "      <td>Are the transanal pull-through's long-term out...</td>\n",
       "      <td>How do the transanal pull-through's long-term ...</td>\n",
       "      <td>Is the transanal pull-through's long-term effe...</td>\n",
       "      <td>Do the long-term effects of transanal pull-thr...</td>\n",
       "      <td>...</td>\n",
       "      <td>Does the transanal pull-through provide long-t...</td>\n",
       "      <td>Are the sustained results of the transanal pul...</td>\n",
       "      <td>How do transanal pull-through and transabdomin...</td>\n",
       "      <td>Are the transanal pull-through's results over ...</td>\n",
       "      <td>Are the long-term results from transanal pull-...</td>\n",
       "      <td>Is the long-term effectiveness of the transana...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Telephone counseling and tailored print commun...</td>\n",
       "      <td>Can tailored interventions increase mammograph...</td>\n",
       "      <td>Are personalized interventions effective in bo...</td>\n",
       "      <td>Do customized interventions lead to increased ...</td>\n",
       "      <td>Can individualized interventions enhance the r...</td>\n",
       "      <td>Are HMO women more likely to undergo mammograp...</td>\n",
       "      <td>Do specific interventions designed for HMO wom...</td>\n",
       "      <td>Can the implementation of tailored interventio...</td>\n",
       "      <td>Are mammography usage rates among HMO women af...</td>\n",
       "      <td>Will personalized programs encourage more HMO ...</td>\n",
       "      <td>...</td>\n",
       "      <td>Are HMO women more inclined to use mammography...</td>\n",
       "      <td>Do tailored strategies for HMO women enhance t...</td>\n",
       "      <td>Can customized interventions boost mammography...</td>\n",
       "      <td>Are personalized efforts successful in raising...</td>\n",
       "      <td>Will tailored intervention efforts lead to a r...</td>\n",
       "      <td>Are interventions specifically designed for HM...</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>After 34 weeks gestation, summary measures of ...</td>\n",
       "      <td>Does gestational age misclassification explain...</td>\n",
       "      <td>Could the misclassification of gestational age...</td>\n",
       "      <td>Is the variation in birthweights between Austr...</td>\n",
       "      <td>Can the difference in birthweights between Aus...</td>\n",
       "      <td>Might the misclassification of gestational age...</td>\n",
       "      <td>Does the misclassification of gestational age ...</td>\n",
       "      <td>Is gestational age misclassification a reason ...</td>\n",
       "      <td>Can the observed birthweight disparities betwe...</td>\n",
       "      <td>Could the differences in birthweights between ...</td>\n",
       "      <td>...</td>\n",
       "      <td>Could the disparity in birthweights of Austral...</td>\n",
       "      <td>Are differences in assessing gestational age r...</td>\n",
       "      <td>Does the variation in birthweights between Aus...</td>\n",
       "      <td>Is the disparity in birthweights between Austr...</td>\n",
       "      <td>Might the classification of gestational age er...</td>\n",
       "      <td>Is the difference in birthweights between Aust...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>To evaluate the accuracy of ultrasonographic e...</td>\n",
       "      <td>Is there any interest to perform ultrasonograp...</td>\n",
       "      <td>Is performing ultrasonography on boys with und...</td>\n",
       "      <td>Does the prospect of using ultrasonography in ...</td>\n",
       "      <td>Is there any reason to consider ultrasonograph...</td>\n",
       "      <td>Is there an interest in utilizing ultrasonogra...</td>\n",
       "      <td>Are there any benefits to conducting ultrasono...</td>\n",
       "      <td>Is there any value in performing ultrasonograp...</td>\n",
       "      <td>Is ultrasonography a valued method for evaluat...</td>\n",
       "      <td>Does ultrasonography provide any advantages in...</td>\n",
       "      <td>...</td>\n",
       "      <td>What is the level of interest in the applicati...</td>\n",
       "      <td>Is performing ultrasonography deemed interesti...</td>\n",
       "      <td>Could ultrasonography be of any interest when ...</td>\n",
       "      <td>Would there be any merit in performing ultraso...</td>\n",
       "      <td>Is it worthwhile to perform ultrasonography on...</td>\n",
       "      <td>Is there any clinical interest in using ultras...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>We analyzed the pharmacokinetic-pharmacodynami...</td>\n",
       "      <td>Is peak concentration needed in therapeutic dr...</td>\n",
       "      <td>Is it necessary to measure peak concentration ...</td>\n",
       "      <td>Should peak concentration be considered in the...</td>\n",
       "      <td>Is assessing peak concentration essential in t...</td>\n",
       "      <td>Do we need to monitor peak concentration in th...</td>\n",
       "      <td>Is measuring peak concentration required in va...</td>\n",
       "      <td>Is checking peak concentration important in th...</td>\n",
       "      <td>Is it important to include peak concentration ...</td>\n",
       "      <td>Is the assessment of peak concentration necess...</td>\n",
       "      <td>...</td>\n",
       "      <td>Is the measurement of peak concentration cruci...</td>\n",
       "      <td>Does therapeutic drug monitoring of vancomycin...</td>\n",
       "      <td>Should peak concentration measurements be incl...</td>\n",
       "      <td>Is it essential to determine peak concentratio...</td>\n",
       "      <td>Is evaluating peak concentration a necessity i...</td>\n",
       "      <td>Is it critical to monitor peak concentration w...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>This investigation assesses the effect of plat...</td>\n",
       "      <td>Can autologous platelet-rich plasma gel enhanc...</td>\n",
       "      <td>Does autologous platelet-rich plasma gel impro...</td>\n",
       "      <td>Is the healing after mandibular third molar ex...</td>\n",
       "      <td>Can the use of autologous platelet-rich plasma...</td>\n",
       "      <td>Does applying autologous platelet-rich plasma ...</td>\n",
       "      <td>Is surgical healing improved by autologous pla...</td>\n",
       "      <td>Can the healing outcome be enhanced by autolog...</td>\n",
       "      <td>Does the application of autologous platelet-ri...</td>\n",
       "      <td>Is recovery after surgical extraction of mandi...</td>\n",
       "      <td>...</td>\n",
       "      <td>Is the healing period reduced after mandibular...</td>\n",
       "      <td>Does autologous platelet-rich plasma gel contr...</td>\n",
       "      <td>Can the healing process be enhanced with autol...</td>\n",
       "      <td>Does the use of autologous platelet-rich plasm...</td>\n",
       "      <td>Is healing after mandibular third molar surgic...</td>\n",
       "      <td>Are healing outcomes better with autologous pl...</td>\n",
       "      <td>1</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>The reduced use of sugars-containing (SC) liqu...</td>\n",
       "      <td>Are sugars-free medicines more erosive than su...</td>\n",
       "      <td>Do medications without sugars cause more erosi...</td>\n",
       "      <td>Are medications lacking sugars more erosive th...</td>\n",
       "      <td>Are sugar-free medications more prone to causi...</td>\n",
       "      <td>Do sugar-free drugs have more erosive effects ...</td>\n",
       "      <td>Are medicines without sugars more likely to ca...</td>\n",
       "      <td>Do sugarless medicines lead to higher erosion ...</td>\n",
       "      <td>Are medicines that do not contain sugars more ...</td>\n",
       "      <td>Is there increased erosion associated with med...</td>\n",
       "      <td>...</td>\n",
       "      <td>Do sugar-absent medications demonstrate greate...</td>\n",
       "      <td>Are sugar-free medicines more likely to lead t...</td>\n",
       "      <td>Do medications that are free of sugars exhibit...</td>\n",
       "      <td>Are sugar-free drugs more erosive than medicat...</td>\n",
       "      <td>Is the erosive potential higher in sugar-free ...</td>\n",
       "      <td>Are medications lacking sugar more erosive in ...</td>\n",
       "      <td>2</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>maybe</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              evidence  \\\n",
       "0    Programmed cell death (PCD) is the regulated d...   \n",
       "1    Assessment of visual acuity depends on the opt...   \n",
       "2    Apparent life-threatening events in infants ar...   \n",
       "3    The transanal endorectal pull-through (TERPT) ...   \n",
       "4    Telephone counseling and tailored print commun...   \n",
       "..                                                 ...   \n",
       "995  After 34 weeks gestation, summary measures of ...   \n",
       "996  To evaluate the accuracy of ultrasonographic e...   \n",
       "997  We analyzed the pharmacokinetic-pharmacodynami...   \n",
       "998  This investigation assesses the effect of plat...   \n",
       "999  The reduced use of sugars-containing (SC) liqu...   \n",
       "\n",
       "                                            question 1  \\\n",
       "0    Do mitochondria play a role in remodelling lac...   \n",
       "1    Landolt C and snellen e acuity: differences in...   \n",
       "2    Syncope during bathing in infants, a pediatric...   \n",
       "3    Are the long-term results of the transanal pul...   \n",
       "4    Can tailored interventions increase mammograph...   \n",
       "..                                                 ...   \n",
       "995  Does gestational age misclassification explain...   \n",
       "996  Is there any interest to perform ultrasonograp...   \n",
       "997  Is peak concentration needed in therapeutic dr...   \n",
       "998  Can autologous platelet-rich plasma gel enhanc...   \n",
       "999  Are sugars-free medicines more erosive than su...   \n",
       "\n",
       "                                            question 2  \\\n",
       "0    Are mitochondria involved in the remodeling of...   \n",
       "1    How do the Landolt C and Snellen E tests diffe...   \n",
       "2    Could syncope in infants during bathing be a p...   \n",
       "3    Do the long-term outcomes of the transanal pul...   \n",
       "4    Are personalized interventions effective in bo...   \n",
       "..                                                 ...   \n",
       "995  Could the misclassification of gestational age...   \n",
       "996  Is performing ultrasonography on boys with und...   \n",
       "997  Is it necessary to measure peak concentration ...   \n",
       "998  Does autologous platelet-rich plasma gel impro...   \n",
       "999  Do medications without sugars cause more erosi...   \n",
       "\n",
       "                                            question 3  \\\n",
       "0    Do mitochondria contribute to the restructurin...   \n",
       "1    What are the distinctions between Landolt C an...   \n",
       "2    Is the occurrence of syncope in infants while ...   \n",
       "3    How do the long-term results of the transanal ...   \n",
       "4    Do customized interventions lead to increased ...   \n",
       "..                                                 ...   \n",
       "995  Is the variation in birthweights between Austr...   \n",
       "996  Does the prospect of using ultrasonography in ...   \n",
       "997  Should peak concentration be considered in the...   \n",
       "998  Is the healing after mandibular third molar ex...   \n",
       "999  Are medications lacking sugars more erosive th...   \n",
       "\n",
       "                                            question 4  \\\n",
       "0    Is there a role for mitochondria in the transf...   \n",
       "1    In what ways do Landolt C and Snellen E acuity...   \n",
       "2    Might fainting episodes in infants during bath...   \n",
       "3    Are the long-term effects of the transanal pul...   \n",
       "4    Can individualized interventions enhance the r...   \n",
       "..                                                 ...   \n",
       "995  Can the difference in birthweights between Aus...   \n",
       "996  Is there any reason to consider ultrasonograph...   \n",
       "997  Is assessing peak concentration essential in t...   \n",
       "998  Can the use of autologous platelet-rich plasma...   \n",
       "999  Are sugar-free medications more prone to causi...   \n",
       "\n",
       "                                            question 5  \\\n",
       "0    Do mitochondria influence how lace plant leave...   \n",
       "1    How does acuity measurement compare between La...   \n",
       "2    Are syncope episodes in infants during bathing...   \n",
       "3    Do the long-term results for the transanal pul...   \n",
       "4    Are HMO women more likely to undergo mammograp...   \n",
       "..                                                 ...   \n",
       "995  Might the misclassification of gestational age...   \n",
       "996  Is there an interest in utilizing ultrasonogra...   \n",
       "997  Do we need to monitor peak concentration in th...   \n",
       "998  Does applying autologous platelet-rich plasma ...   \n",
       "999  Do sugar-free drugs have more erosive effects ...   \n",
       "\n",
       "                                            question 6  \\\n",
       "0    What role do mitochondria have in the reformat...   \n",
       "1    What are the key differences between Landolt C...   \n",
       "2    Does syncope in infants when bathing suggest a...   \n",
       "3    Are the transanal pull-through's long-term out...   \n",
       "4    Do specific interventions designed for HMO wom...   \n",
       "..                                                 ...   \n",
       "995  Does the misclassification of gestational age ...   \n",
       "996  Are there any benefits to conducting ultrasono...   \n",
       "997  Is measuring peak concentration required in va...   \n",
       "998  Is surgical healing improved by autologous pla...   \n",
       "999  Are medicines without sugars more likely to ca...   \n",
       "\n",
       "                                            question 7  \\\n",
       "0    Are mitochondria responsible for changes in la...   \n",
       "1    How do the acuity assessments of Landolt C and...   \n",
       "2    Is fainting in infants during bathing a sign o...   \n",
       "3    How do the transanal pull-through's long-term ...   \n",
       "4    Can the implementation of tailored interventio...   \n",
       "..                                                 ...   \n",
       "995  Is gestational age misclassification a reason ...   \n",
       "996  Is there any value in performing ultrasonograp...   \n",
       "997  Is checking peak concentration important in th...   \n",
       "998  Can the healing outcome be enhanced by autolog...   \n",
       "999  Do sugarless medicines lead to higher erosion ...   \n",
       "\n",
       "                                            question 8  \\\n",
       "0    Is the restructuring of lace plant leaves duri...   \n",
       "1    What differences exist between the Landolt C a...   \n",
       "2    Could fainting episodes during baths in infant...   \n",
       "3    Is the transanal pull-through's long-term effe...   \n",
       "4    Are mammography usage rates among HMO women af...   \n",
       "..                                                 ...   \n",
       "995  Can the observed birthweight disparities betwe...   \n",
       "996  Is ultrasonography a valued method for evaluat...   \n",
       "997  Is it important to include peak concentration ...   \n",
       "998  Does the application of autologous platelet-ri...   \n",
       "999  Are medicines that do not contain sugars more ...   \n",
       "\n",
       "                                            question 9  ...  \\\n",
       "0    Do mitochondria affect the way lace plant leav...  ...   \n",
       "1    When assessing strabismus amblyopia, how do th...  ...   \n",
       "2    Is there a possibility that syncope in infants...  ...   \n",
       "3    Do the long-term effects of transanal pull-thr...  ...   \n",
       "4    Will personalized programs encourage more HMO ...  ...   \n",
       "..                                                 ...  ...   \n",
       "995  Could the differences in birthweights between ...  ...   \n",
       "996  Does ultrasonography provide any advantages in...  ...   \n",
       "997  Is the assessment of peak concentration necess...  ...   \n",
       "998  Is recovery after surgical extraction of mandi...  ...   \n",
       "999  Is there increased erosion associated with med...  ...   \n",
       "\n",
       "                                           question 15  \\\n",
       "0    Do mitochondria play a significant part in the...   \n",
       "1    In cases of strabismus amblyopia, how does the...   \n",
       "2    Might an infant experiencing syncope during a ...   \n",
       "3    Does the transanal pull-through provide long-t...   \n",
       "4    Are HMO women more inclined to use mammography...   \n",
       "..                                                 ...   \n",
       "995  Could the disparity in birthweights of Austral...   \n",
       "996  What is the level of interest in the applicati...   \n",
       "997  Is the measurement of peak concentration cruci...   \n",
       "998  Is the healing period reduced after mandibular...   \n",
       "999  Do sugar-absent medications demonstrate greate...   \n",
       "\n",
       "                                           question 16  \\\n",
       "0    Is mitochondrial involvement evident in the re...   \n",
       "1    What differences are there in assessing strabi...   \n",
       "2    Is there a connection between bathing-induced ...   \n",
       "3    Are the sustained results of the transanal pul...   \n",
       "4    Do tailored strategies for HMO women enhance t...   \n",
       "..                                                 ...   \n",
       "995  Are differences in assessing gestational age r...   \n",
       "996  Is performing ultrasonography deemed interesti...   \n",
       "997  Does therapeutic drug monitoring of vancomycin...   \n",
       "998  Does autologous platelet-rich plasma gel contr...   \n",
       "999  Are sugar-free medicines more likely to lead t...   \n",
       "\n",
       "                                           question 17  \\\n",
       "0    To what extent do mitochondria contribute to l...   \n",
       "1    How do Landolt C and Snellen E tests compare i...   \n",
       "2    Can syncope in infants during bathing be attri...   \n",
       "3    How do transanal pull-through and transabdomin...   \n",
       "4    Can customized interventions boost mammography...   \n",
       "..                                                 ...   \n",
       "995  Does the variation in birthweights between Aus...   \n",
       "996  Could ultrasonography be of any interest when ...   \n",
       "997  Should peak concentration measurements be incl...   \n",
       "998  Can the healing process be enhanced with autol...   \n",
       "999  Do medications that are free of sugars exhibit...   \n",
       "\n",
       "                                           question 18  \\\n",
       "0    Are mitochondrial roles crucial in shaping lac...   \n",
       "1    How does the acuity assessment with Landolt C ...   \n",
       "2    Is syncope during bathing in infants potential...   \n",
       "3    Are the transanal pull-through's results over ...   \n",
       "4    Are personalized efforts successful in raising...   \n",
       "..                                                 ...   \n",
       "995  Is the disparity in birthweights between Austr...   \n",
       "996  Would there be any merit in performing ultraso...   \n",
       "997  Is it essential to determine peak concentratio...   \n",
       "998  Does the use of autologous platelet-rich plasm...   \n",
       "999  Are sugar-free drugs more erosive than medicat...   \n",
       "\n",
       "                                           question 19  \\\n",
       "0    Do mitochondria have an essential function in ...   \n",
       "1    How are the Landolt C and Snellen E tests diff...   \n",
       "2    Might the syncope observance in infants while ...   \n",
       "3    Are the long-term results from transanal pull-...   \n",
       "4    Will tailored intervention efforts lead to a r...   \n",
       "..                                                 ...   \n",
       "995  Might the classification of gestational age er...   \n",
       "996  Is it worthwhile to perform ultrasonography on...   \n",
       "997  Is evaluating peak concentration a necessity i...   \n",
       "998  Is healing after mandibular third molar surgic...   \n",
       "999  Is the erosive potential higher in sugar-free ...   \n",
       "\n",
       "                                           question 20 answer option 1  \\\n",
       "0    Is there a mitochondrial role in the construct...      1      yes   \n",
       "1    What is the difference in acuity testing betwe...      2      yes   \n",
       "2    Could infant syncope during bathing sessions i...      1      yes   \n",
       "3    Is the long-term effectiveness of the transana...      2      yes   \n",
       "4    Are interventions specifically designed for HM...      1      yes   \n",
       "..                                                 ...    ...      ...   \n",
       "995  Is the difference in birthweights between Aust...      2      yes   \n",
       "996  Is there any clinical interest in using ultras...      2      yes   \n",
       "997  Is it critical to monitor peak concentration w...      2      yes   \n",
       "998  Are healing outcomes better with autologous pl...      1      yes   \n",
       "999  Are medications lacking sugar more erosive in ...      2      yes   \n",
       "\n",
       "    option 2 option 3  \n",
       "0         no    maybe  \n",
       "1         no    maybe  \n",
       "2         no    maybe  \n",
       "3         no    maybe  \n",
       "4         no    maybe  \n",
       "..       ...      ...  \n",
       "995       no    maybe  \n",
       "996       no    maybe  \n",
       "997       no    maybe  \n",
       "998       no    maybe  \n",
       "999       no    maybe  \n",
       "\n",
       "[1000 rows x 25 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0db3b600",
   "metadata": {},
   "source": [
    "### NQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1317f50f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import time\n",
    "import openai\n",
    "import random\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from bs4 import BeautifulSoup\n",
    "from datasets import load_dataset\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "954acca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efe81985af184bb2853095294bc5930d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/8 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3712458270"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = load_dataset(\"google-research-datasets/natural_questions\", split=\"validation\")\n",
    "raw.to_json('Data/Raw/NQ/NQ.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "52eea904",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n",
    "openai_model = \"gpt-4o\"\n",
    "\n",
    "def call_gpt(message_user, retries=50):\n",
    "    messages = [{\"role\": \"system\", \"content\": 'You are a helpful assistant.'}, \n",
    "                {\"role\": \"user\", \"content\": message_user}]\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            response = openai_client.chat.completions.create(\n",
    "                model = openai_model, messages = messages, max_tokens = 512\n",
    "            )\n",
    "            return response.choices[0].message.content\n",
    "        except openai.RateLimitError as e:\n",
    "            time.sleep(i)\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {e}\")\n",
    "            raise\n",
    "    raise Exception(f\"Failed to call GPT-4 after {retries} retries.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2724d8c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_rephrase = \\\n",
    "'''\n",
    "### Instruction\n",
    "Do not respond to the question. \n",
    "Instead, follow the provided examples and rephrase the given question into 20 other versions that are semantically equivalent.\n",
    "\n",
    "### Example\n",
    "## Question: Which option best summarizes the comparative efficacy of regimen and comparator for managing condition?\n",
    "## Version 1: Choose an option that best describes the efficacy of regimen compared to comparator when used to treat condition.\n",
    "## Version 2: Select the option that most accurately reflects the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 3: Identify the option that best summarizes the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 4: Which option most effectively illustrates the efficacy of regimen when compared with comparator for condition?\n",
    "\n",
    "### Your Task\n",
    "## Question: {QUESTION}\n",
    "'''\n",
    "\n",
    "def rephrase_qn(question, retries=20):\n",
    "    query = template_rephrase.format(**{'QUESTION':question})\n",
    "    for _ in range(retries):\n",
    "        response = call_gpt(query)\n",
    "        matches = [re.search(fr\"## Version {i}: (.*)\", response) for i in range(1, 20)]\n",
    "        if all([each is not None and len(each.group(1)) for each in matches]):            \n",
    "            questions = [question] + [each.group(1) for each in matches]\n",
    "            if len(set(questions)) == 20: return questions\n",
    "    return [question] + [None]*19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "377e81df",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_mcq = \\\n",
    "'''\n",
    "### Instruction\n",
    "Using the provided examples as a guide, transform the given question with a correct answer into a multiple-choice question. \n",
    "Provide two additional incorrect options that are similar in type or category to the correct answer.\n",
    "\n",
    "# Example 1\n",
    "## Question: Which continent is the largest by land area?\n",
    "## Correct Answer: Asia\n",
    "## Incorrect Option 1: Africa\n",
    "## Incorrect Option 2: Europe\n",
    "\n",
    "# Example 2\n",
    "## Question: Is the last name scott irish or scottish?\n",
    "## Correct Answer: Scottish\n",
    "## Incorrect Option 1: Irish\n",
    "## Incorrect Option 2: English\n",
    "\n",
    "# Example 3\n",
    "## Question: Were Scott Derrickson and Ed Wood of the same nationality?\n",
    "## Correct Answer: Yes\n",
    "## Incorrect Option 1: No\n",
    "## Incorrect Option 2: Maybe\n",
    "\n",
    "# Your Task\n",
    "## Question: {QUESTION}\n",
    "## Correct Answer: {ANSWER}\n",
    "'''\n",
    "\n",
    "def convert_mcq(question, answer, retries=20):\n",
    "    query = template_mcq.format(**{'QUESTION':question, 'ANSWER':answer})\n",
    "    for _ in range(retries):\n",
    "        response = call_gpt(query)\n",
    "        match1 = re.search(r\"## Incorrect Option 1: (.*)\", response)\n",
    "        match2 = re.search(r\"## Incorrect Option 2: (.*)\", response)\n",
    "        if match1 and len(match1.group(1)) and match2 and len(match2.group(1)):            \n",
    "            options = [answer.capitalize(), match1.group(1).capitalize(), match2.group(1).capitalize()]\n",
    "            random.shuffle(options)\n",
    "            if len(set(options)) == 3: return options\n",
    "    return [answer.capitalize(), None, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f16e113c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 3610/3610 [24:17<00:00,  2.48it/s]\n"
     ]
    }
   ],
   "source": [
    "def extract_evidence(evidence):\n",
    "    soup = BeautifulSoup(evidence, \"html.parser\")\n",
    "    evidence = ' '.join([p.get_text(strip=True) for p in soup.find_all(\"p\") if p.get_text(strip=True)])\n",
    "    return evidence\n",
    "\n",
    "def prepare_row(idx, row):\n",
    "    question, answer = row['question'], row['answer'][0]\n",
    "    questions = rephrase_qn(question)\n",
    "    options = convert_mcq(question, answer)\n",
    "    answer = options.index(answer.capitalize())+1\n",
    "    evidence = extract_evidence(' '.join(raw[raw_id2idx[row['id']]]['document']['tokens']['token']))\n",
    "    row = (evidence, *questions, answer, *options)\n",
    "    return idx, row\n",
    "\n",
    "raw = [json.loads(line) for line in open('Data/Raw/NQ/NQ.jsonl')]\n",
    "raw_genread = [json.loads(line) for line in open('Data/Raw/NQ/NQ-GenRead.jsonl')]\n",
    "raw_id2idx = {each['id']:idx for idx, each in enumerate(raw)}\n",
    "\n",
    "dataset, max_workers = {}, 50\n",
    "with ThreadPoolExecutor(max_workers=max_workers) as exe:\n",
    "    futures = [exe.submit(prepare_row, idx, row) for idx, row in enumerate(raw_genread)]\n",
    "    for future in tqdm(as_completed(futures), total=len(futures)):\n",
    "        try:\n",
    "            idx, row = future.result()\n",
    "            dataset[idx] = row\n",
    "        except Exception as e:\n",
    "            print(f\"Error: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "23fc21d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['evidence'] + [f'question {i}' for i in range(1, 21)] + ['answer'] + [f'option {i}' for i in range(1, 4)]\n",
    "dataset = [row for idx, row in sorted(dataset.items(), key = lambda x: x[0])]\n",
    "dataset = pd.DataFrame(dataset, columns=columns)\n",
    "dataset = dataset.dropna().drop_duplicates(subset=columns[:21], keep=False).reset_index(drop=True)\n",
    "dataset.to_csv('Data/Input/NQ.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "21af44f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>evidence</th>\n",
       "      <th>question 1</th>\n",
       "      <th>question 2</th>\n",
       "      <th>question 3</th>\n",
       "      <th>question 4</th>\n",
       "      <th>question 5</th>\n",
       "      <th>question 6</th>\n",
       "      <th>question 7</th>\n",
       "      <th>question 8</th>\n",
       "      <th>question 9</th>\n",
       "      <th>...</th>\n",
       "      <th>question 15</th>\n",
       "      <th>question 16</th>\n",
       "      <th>question 17</th>\n",
       "      <th>question 18</th>\n",
       "      <th>question 19</th>\n",
       "      <th>question 20</th>\n",
       "      <th>answer</th>\n",
       "      <th>option 1</th>\n",
       "      <th>option 2</th>\n",
       "      <th>option 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Nobel Prize in Physics ( Swedish : Nobelpr...</td>\n",
       "      <td>who got the first nobel prize in physics?</td>\n",
       "      <td>Who was the inaugural recipient of the Nobel P...</td>\n",
       "      <td>Who was awarded the first Nobel Prize in Physics?</td>\n",
       "      <td>Can you tell me who received the first Nobel P...</td>\n",
       "      <td>Who is recognized as the first Nobel laureate ...</td>\n",
       "      <td>Who was honored with the first Nobel Prize in ...</td>\n",
       "      <td>Who was the first individual to win the Nobel ...</td>\n",
       "      <td>Can you name the first person to be awarded th...</td>\n",
       "      <td>Who was the earliest recipient of the Nobel Pr...</td>\n",
       "      <td>...</td>\n",
       "      <td>Who was the first to be honored with the Nobel...</td>\n",
       "      <td>Who achieved the first Nobel Prize award in Ph...</td>\n",
       "      <td>Who was the first laureate of the Nobel Prize ...</td>\n",
       "      <td>Who gained the inaugural Nobel Prize in Physics?</td>\n",
       "      <td>Who was the debut winner of the Nobel Prize in...</td>\n",
       "      <td>Who earned the first Nobel Prize in the Physic...</td>\n",
       "      <td>1</td>\n",
       "      <td>Wilhelm conrad röntgen</td>\n",
       "      <td>Marie curie</td>\n",
       "      <td>Albert einstein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Deadpool 2 is an upcoming American superhero f...</td>\n",
       "      <td>when is the next deadpool movie being released?</td>\n",
       "      <td>What is the release date for the next Deadpool...</td>\n",
       "      <td>Can you tell me when the upcoming Deadpool mov...</td>\n",
       "      <td>When will the next installment of the Deadpool...</td>\n",
       "      <td>Do you know the release date for the upcoming ...</td>\n",
       "      <td>When should we expect the next Deadpool movie ...</td>\n",
       "      <td>Could you provide the release date for the new...</td>\n",
       "      <td>What is the scheduled release date for the nex...</td>\n",
       "      <td>When is the anticipated release date of the up...</td>\n",
       "      <td>...</td>\n",
       "      <td>What is the release schedule for the upcoming ...</td>\n",
       "      <td>When is the official release date for the next...</td>\n",
       "      <td>Can you tell me when the next Deadpool film wi...</td>\n",
       "      <td>When is the release schedule for the next Dead...</td>\n",
       "      <td>Do you know when the next film in the Deadpool...</td>\n",
       "      <td>When will the new Deadpool movie be released t...</td>\n",
       "      <td>3</td>\n",
       "      <td>July 10, 2018</td>\n",
       "      <td>June 1, 2018</td>\n",
       "      <td>May 18 , 2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Shortwave radio is radio transmission using sh...</td>\n",
       "      <td>which mode is used for short wave broadcast se...</td>\n",
       "      <td>What is the mode employed for short wave broad...</td>\n",
       "      <td>Which mode is utilized for broadcasting short ...</td>\n",
       "      <td>Identify the mode that is applied in short wav...</td>\n",
       "      <td>What mode is adopted for short wave broadcast ...</td>\n",
       "      <td>Which mode serves as the standard for short wa...</td>\n",
       "      <td>In short wave broadcast service, which mode is...</td>\n",
       "      <td>What mode is typically used for services broad...</td>\n",
       "      <td>Determine the mode that is used for short wave...</td>\n",
       "      <td>...</td>\n",
       "      <td>Which operational mode is used in short wave b...</td>\n",
       "      <td>What is the preferred mode for short wave broa...</td>\n",
       "      <td>Specify the mode that is used for short wave b...</td>\n",
       "      <td>What mode do short wave broadcasting services ...</td>\n",
       "      <td>Which mode is designated for use in short wave...</td>\n",
       "      <td>What mode is standard for short wave broadcast...</td>\n",
       "      <td>3</td>\n",
       "      <td>Rtty</td>\n",
       "      <td>Psk31</td>\n",
       "      <td>Olivia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nigeria is a country in West Africa . Nigeria ...</td>\n",
       "      <td>the south west wind blows across nigeria between?</td>\n",
       "      <td>During which period does the southwest wind bl...</td>\n",
       "      <td>Across which time does the southwest wind make...</td>\n",
       "      <td>When does the southwest wind sweep across Nige...</td>\n",
       "      <td>At what times do the southwest winds affect Ni...</td>\n",
       "      <td>Over what span does Nigeria experience southwe...</td>\n",
       "      <td>The southwest wind crosses Nigeria at what time?</td>\n",
       "      <td>What is the time frame for the southwest wind ...</td>\n",
       "      <td>During what months do the southwest winds occu...</td>\n",
       "      <td>...</td>\n",
       "      <td>Across which season do southwest winds blow in...</td>\n",
       "      <td>At what time span does Nigeria experience the ...</td>\n",
       "      <td>When is the southwest wind active across Nigeria?</td>\n",
       "      <td>The influence of the southwest wind is felt in...</td>\n",
       "      <td>Which period marks the presence of southwest w...</td>\n",
       "      <td>During what period is Nigeria under the impact...</td>\n",
       "      <td>3</td>\n",
       "      <td>Beginning of august</td>\n",
       "      <td>End of october</td>\n",
       "      <td>Till september</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Health or vitality is an attribute assigned to...</td>\n",
       "      <td>what does hp mean in war and order?</td>\n",
       "      <td>What is the meaning of hp in the context of Wa...</td>\n",
       "      <td>In War and Order, what does the term hp refer to?</td>\n",
       "      <td>Could you explain what hp stands for in War an...</td>\n",
       "      <td>In War and Order, what does hp denote?</td>\n",
       "      <td>What does hp signify within War and Order?</td>\n",
       "      <td>How is hp defined in War and Order?</td>\n",
       "      <td>What is the interpretation of hp in War and Or...</td>\n",
       "      <td>In the game War and Order, what does hp mean?</td>\n",
       "      <td>...</td>\n",
       "      <td>What does hp imply in War and Order?</td>\n",
       "      <td>In War and Order, what does the acronym hp sta...</td>\n",
       "      <td>What does the term hp mean in War and Order ga...</td>\n",
       "      <td>In the context of War and Order, what is hp?</td>\n",
       "      <td>Can you clarify what hp means in War and Order?</td>\n",
       "      <td>What does hp stand for in the game War and Order?</td>\n",
       "      <td>1</td>\n",
       "      <td>Hit points or health points</td>\n",
       "      <td>Hero points</td>\n",
       "      <td>High power</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3591</th>\n",
       "      <td>In astronomy , the geocentric model ( also kno...</td>\n",
       "      <td>who challenged the aristotelian model of a geo...</td>\n",
       "      <td>Who questioned the Aristotelian idea of a geoc...</td>\n",
       "      <td>Who disputed the concept of a geocentric unive...</td>\n",
       "      <td>Which individual contested the Aristotelian vi...</td>\n",
       "      <td>Who opposed the traditional Aristotelian geoce...</td>\n",
       "      <td>Who contradicted the idea of an Earth-centered...</td>\n",
       "      <td>Which person challenged Aristotle's notion of ...</td>\n",
       "      <td>Who was the critic of the Earth-centered unive...</td>\n",
       "      <td>Who was responsible for challenging the Aristo...</td>\n",
       "      <td>...</td>\n",
       "      <td>Who defied the understanding of a geocentric u...</td>\n",
       "      <td>Which scholar challenged the geocentric worldv...</td>\n",
       "      <td>Who undermined the model of a universe with Ea...</td>\n",
       "      <td>Who called into question the Aristotelian asse...</td>\n",
       "      <td>Who offered a challenge to the Earth-centric c...</td>\n",
       "      <td>Who contested the notion of an Earth-centered ...</td>\n",
       "      <td>2</td>\n",
       "      <td>Ptolemy</td>\n",
       "      <td>Copernicus</td>\n",
       "      <td>Galileo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3592</th>\n",
       "      <td>The Miraculous Journey of Edward Tulane is a 2...</td>\n",
       "      <td>when was the miraculous journey of edward tula...</td>\n",
       "      <td>What is the publication date of \"The Miraculou...</td>\n",
       "      <td>In which year was \"The Miraculous Journey of E...</td>\n",
       "      <td>When did \"The Miraculous Journey of Edward Tul...</td>\n",
       "      <td>Can you tell me the year \"The Miraculous Journ...</td>\n",
       "      <td>When did the book titled \"The Miraculous Journ...</td>\n",
       "      <td>What year was \"The Miraculous Journey of Edwar...</td>\n",
       "      <td>When was \"The Miraculous Journey of Edward Tul...</td>\n",
       "      <td>Could you specify the publication year of \"The...</td>\n",
       "      <td>...</td>\n",
       "      <td>Can you provide the year that \"The Miraculous ...</td>\n",
       "      <td>When was the date of publication for \"The Mira...</td>\n",
       "      <td>Could you inform me when \"The Miraculous Journ...</td>\n",
       "      <td>When did \"The Miraculous Journey of Edward Tul...</td>\n",
       "      <td>What is the official publication year of \"The ...</td>\n",
       "      <td>In which year did \"The Miraculous Journey of E...</td>\n",
       "      <td>2</td>\n",
       "      <td>2004</td>\n",
       "      <td>2006</td>\n",
       "      <td>2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3593</th>\n",
       "      <td>Lord Banquo / ˈbæŋkwoʊ / , the Thane of Lochab...</td>\n",
       "      <td>character in macbeth who is murdered and appea...</td>\n",
       "      <td>Which character in Macbeth is killed and later...</td>\n",
       "      <td>Identify the character in Macbeth who is slain...</td>\n",
       "      <td>In Macbeth, which character is murdered and su...</td>\n",
       "      <td>Who in Macbeth is assassinated and later shows...</td>\n",
       "      <td>In Macbeth, which individual is killed and rea...</td>\n",
       "      <td>Which character meets their end in Macbeth but...</td>\n",
       "      <td>Identify the murdered character in Macbeth who...</td>\n",
       "      <td>Who is the character that is slain in Macbeth ...</td>\n",
       "      <td>...</td>\n",
       "      <td>Identify the ghostly character in Macbeth who ...</td>\n",
       "      <td>Who appears as a ghost in Macbeth after being ...</td>\n",
       "      <td>In Macbeth, whose death is followed by a ghost...</td>\n",
       "      <td>In Shakespeare's Macbeth, which character is m...</td>\n",
       "      <td>Which person is murdered in Macbeth and then s...</td>\n",
       "      <td>Who is the character in Macbeth that returns a...</td>\n",
       "      <td>3</td>\n",
       "      <td>Macduff</td>\n",
       "      <td>Duncan</td>\n",
       "      <td>Banquo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3594</th>\n",
       "      <td>As You Like It is a pastoral comedy by William...</td>\n",
       "      <td>when was as you like it first performed?</td>\n",
       "      <td>In what year did the play \"As You Like It\" hav...</td>\n",
       "      <td>Can you tell me the first time \"As You Like It...</td>\n",
       "      <td>When did the initial performance of \"As You Li...</td>\n",
       "      <td>What is the date of the premiere performance o...</td>\n",
       "      <td>What year marks the debut performance of \"As Y...</td>\n",
       "      <td>When was \"As You Like It\" staged for the first...</td>\n",
       "      <td>Could you inform me about the year \"As You Lik...</td>\n",
       "      <td>What year was the first staging of \"As You Lik...</td>\n",
       "      <td>...</td>\n",
       "      <td>When did audiences first see \"As You Like It\" ...</td>\n",
       "      <td>Can you specify the year \"As You Like It\" was ...</td>\n",
       "      <td>What year did \"As You Like It\" have its inaugu...</td>\n",
       "      <td>When did the play \"As You Like It\" first appea...</td>\n",
       "      <td>Which year saw the first performance of \"As Yo...</td>\n",
       "      <td>Can you indicate when \"As You Like It\" was fir...</td>\n",
       "      <td>2</td>\n",
       "      <td>1611</td>\n",
       "      <td>1603</td>\n",
       "      <td>1599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3595</th>\n",
       "      <td>Home computers were a class of microcomputers ...</td>\n",
       "      <td>when did computer become widespread in homes a...</td>\n",
       "      <td>In what time period did computers start to bec...</td>\n",
       "      <td>When did it become usual for homes and schools...</td>\n",
       "      <td>At what point did computers become a regular p...</td>\n",
       "      <td>During which era did computers gain popularity...</td>\n",
       "      <td>When did computers begin to be widespread in b...</td>\n",
       "      <td>In what era did computers start becoming commo...</td>\n",
       "      <td>At what time did the adoption of computers bec...</td>\n",
       "      <td>When did the use of computers become prevalent...</td>\n",
       "      <td>...</td>\n",
       "      <td>When did computers start to become a common fe...</td>\n",
       "      <td>Around which time did computers start being a ...</td>\n",
       "      <td>In what timeframe did computers become usual i...</td>\n",
       "      <td>When did the use of computers extend broadly i...</td>\n",
       "      <td>At what time did computers become extensively ...</td>\n",
       "      <td>During what timeframe were computers initially...</td>\n",
       "      <td>3</td>\n",
       "      <td>1990s</td>\n",
       "      <td>1970s</td>\n",
       "      <td>1980s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3596 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               evidence  \\\n",
       "0     The Nobel Prize in Physics ( Swedish : Nobelpr...   \n",
       "1     Deadpool 2 is an upcoming American superhero f...   \n",
       "2     Shortwave radio is radio transmission using sh...   \n",
       "3     Nigeria is a country in West Africa . Nigeria ...   \n",
       "4     Health or vitality is an attribute assigned to...   \n",
       "...                                                 ...   \n",
       "3591  In astronomy , the geocentric model ( also kno...   \n",
       "3592  The Miraculous Journey of Edward Tulane is a 2...   \n",
       "3593  Lord Banquo / ˈbæŋkwoʊ / , the Thane of Lochab...   \n",
       "3594  As You Like It is a pastoral comedy by William...   \n",
       "3595  Home computers were a class of microcomputers ...   \n",
       "\n",
       "                                             question 1  \\\n",
       "0             who got the first nobel prize in physics?   \n",
       "1       when is the next deadpool movie being released?   \n",
       "2     which mode is used for short wave broadcast se...   \n",
       "3     the south west wind blows across nigeria between?   \n",
       "4                   what does hp mean in war and order?   \n",
       "...                                                 ...   \n",
       "3591  who challenged the aristotelian model of a geo...   \n",
       "3592  when was the miraculous journey of edward tula...   \n",
       "3593  character in macbeth who is murdered and appea...   \n",
       "3594           when was as you like it first performed?   \n",
       "3595  when did computer become widespread in homes a...   \n",
       "\n",
       "                                             question 2  \\\n",
       "0     Who was the inaugural recipient of the Nobel P...   \n",
       "1     What is the release date for the next Deadpool...   \n",
       "2     What is the mode employed for short wave broad...   \n",
       "3     During which period does the southwest wind bl...   \n",
       "4     What is the meaning of hp in the context of Wa...   \n",
       "...                                                 ...   \n",
       "3591  Who questioned the Aristotelian idea of a geoc...   \n",
       "3592  What is the publication date of \"The Miraculou...   \n",
       "3593  Which character in Macbeth is killed and later...   \n",
       "3594  In what year did the play \"As You Like It\" hav...   \n",
       "3595  In what time period did computers start to bec...   \n",
       "\n",
       "                                             question 3  \\\n",
       "0     Who was awarded the first Nobel Prize in Physics?   \n",
       "1     Can you tell me when the upcoming Deadpool mov...   \n",
       "2     Which mode is utilized for broadcasting short ...   \n",
       "3     Across which time does the southwest wind make...   \n",
       "4     In War and Order, what does the term hp refer to?   \n",
       "...                                                 ...   \n",
       "3591  Who disputed the concept of a geocentric unive...   \n",
       "3592  In which year was \"The Miraculous Journey of E...   \n",
       "3593  Identify the character in Macbeth who is slain...   \n",
       "3594  Can you tell me the first time \"As You Like It...   \n",
       "3595  When did it become usual for homes and schools...   \n",
       "\n",
       "                                             question 4  \\\n",
       "0     Can you tell me who received the first Nobel P...   \n",
       "1     When will the next installment of the Deadpool...   \n",
       "2     Identify the mode that is applied in short wav...   \n",
       "3     When does the southwest wind sweep across Nige...   \n",
       "4     Could you explain what hp stands for in War an...   \n",
       "...                                                 ...   \n",
       "3591  Which individual contested the Aristotelian vi...   \n",
       "3592  When did \"The Miraculous Journey of Edward Tul...   \n",
       "3593  In Macbeth, which character is murdered and su...   \n",
       "3594  When did the initial performance of \"As You Li...   \n",
       "3595  At what point did computers become a regular p...   \n",
       "\n",
       "                                             question 5  \\\n",
       "0     Who is recognized as the first Nobel laureate ...   \n",
       "1     Do you know the release date for the upcoming ...   \n",
       "2     What mode is adopted for short wave broadcast ...   \n",
       "3     At what times do the southwest winds affect Ni...   \n",
       "4                In War and Order, what does hp denote?   \n",
       "...                                                 ...   \n",
       "3591  Who opposed the traditional Aristotelian geoce...   \n",
       "3592  Can you tell me the year \"The Miraculous Journ...   \n",
       "3593  Who in Macbeth is assassinated and later shows...   \n",
       "3594  What is the date of the premiere performance o...   \n",
       "3595  During which era did computers gain popularity...   \n",
       "\n",
       "                                             question 6  \\\n",
       "0     Who was honored with the first Nobel Prize in ...   \n",
       "1     When should we expect the next Deadpool movie ...   \n",
       "2     Which mode serves as the standard for short wa...   \n",
       "3     Over what span does Nigeria experience southwe...   \n",
       "4            What does hp signify within War and Order?   \n",
       "...                                                 ...   \n",
       "3591  Who contradicted the idea of an Earth-centered...   \n",
       "3592  When did the book titled \"The Miraculous Journ...   \n",
       "3593  In Macbeth, which individual is killed and rea...   \n",
       "3594  What year marks the debut performance of \"As Y...   \n",
       "3595  When did computers begin to be widespread in b...   \n",
       "\n",
       "                                             question 7  \\\n",
       "0     Who was the first individual to win the Nobel ...   \n",
       "1     Could you provide the release date for the new...   \n",
       "2     In short wave broadcast service, which mode is...   \n",
       "3      The southwest wind crosses Nigeria at what time?   \n",
       "4                   How is hp defined in War and Order?   \n",
       "...                                                 ...   \n",
       "3591  Which person challenged Aristotle's notion of ...   \n",
       "3592  What year was \"The Miraculous Journey of Edwar...   \n",
       "3593  Which character meets their end in Macbeth but...   \n",
       "3594  When was \"As You Like It\" staged for the first...   \n",
       "3595  In what era did computers start becoming commo...   \n",
       "\n",
       "                                             question 8  \\\n",
       "0     Can you name the first person to be awarded th...   \n",
       "1     What is the scheduled release date for the nex...   \n",
       "2     What mode is typically used for services broad...   \n",
       "3     What is the time frame for the southwest wind ...   \n",
       "4     What is the interpretation of hp in War and Or...   \n",
       "...                                                 ...   \n",
       "3591  Who was the critic of the Earth-centered unive...   \n",
       "3592  When was \"The Miraculous Journey of Edward Tul...   \n",
       "3593  Identify the murdered character in Macbeth who...   \n",
       "3594  Could you inform me about the year \"As You Lik...   \n",
       "3595  At what time did the adoption of computers bec...   \n",
       "\n",
       "                                             question 9  ...  \\\n",
       "0     Who was the earliest recipient of the Nobel Pr...  ...   \n",
       "1     When is the anticipated release date of the up...  ...   \n",
       "2     Determine the mode that is used for short wave...  ...   \n",
       "3     During what months do the southwest winds occu...  ...   \n",
       "4         In the game War and Order, what does hp mean?  ...   \n",
       "...                                                 ...  ...   \n",
       "3591  Who was responsible for challenging the Aristo...  ...   \n",
       "3592  Could you specify the publication year of \"The...  ...   \n",
       "3593  Who is the character that is slain in Macbeth ...  ...   \n",
       "3594  What year was the first staging of \"As You Lik...  ...   \n",
       "3595  When did the use of computers become prevalent...  ...   \n",
       "\n",
       "                                            question 15  \\\n",
       "0     Who was the first to be honored with the Nobel...   \n",
       "1     What is the release schedule for the upcoming ...   \n",
       "2     Which operational mode is used in short wave b...   \n",
       "3     Across which season do southwest winds blow in...   \n",
       "4                  What does hp imply in War and Order?   \n",
       "...                                                 ...   \n",
       "3591  Who defied the understanding of a geocentric u...   \n",
       "3592  Can you provide the year that \"The Miraculous ...   \n",
       "3593  Identify the ghostly character in Macbeth who ...   \n",
       "3594  When did audiences first see \"As You Like It\" ...   \n",
       "3595  When did computers start to become a common fe...   \n",
       "\n",
       "                                            question 16  \\\n",
       "0     Who achieved the first Nobel Prize award in Ph...   \n",
       "1     When is the official release date for the next...   \n",
       "2     What is the preferred mode for short wave broa...   \n",
       "3     At what time span does Nigeria experience the ...   \n",
       "4     In War and Order, what does the acronym hp sta...   \n",
       "...                                                 ...   \n",
       "3591  Which scholar challenged the geocentric worldv...   \n",
       "3592  When was the date of publication for \"The Mira...   \n",
       "3593  Who appears as a ghost in Macbeth after being ...   \n",
       "3594  Can you specify the year \"As You Like It\" was ...   \n",
       "3595  Around which time did computers start being a ...   \n",
       "\n",
       "                                            question 17  \\\n",
       "0     Who was the first laureate of the Nobel Prize ...   \n",
       "1     Can you tell me when the next Deadpool film wi...   \n",
       "2     Specify the mode that is used for short wave b...   \n",
       "3     When is the southwest wind active across Nigeria?   \n",
       "4     What does the term hp mean in War and Order ga...   \n",
       "...                                                 ...   \n",
       "3591  Who undermined the model of a universe with Ea...   \n",
       "3592  Could you inform me when \"The Miraculous Journ...   \n",
       "3593  In Macbeth, whose death is followed by a ghost...   \n",
       "3594  What year did \"As You Like It\" have its inaugu...   \n",
       "3595  In what timeframe did computers become usual i...   \n",
       "\n",
       "                                            question 18  \\\n",
       "0      Who gained the inaugural Nobel Prize in Physics?   \n",
       "1     When is the release schedule for the next Dead...   \n",
       "2     What mode do short wave broadcasting services ...   \n",
       "3     The influence of the southwest wind is felt in...   \n",
       "4          In the context of War and Order, what is hp?   \n",
       "...                                                 ...   \n",
       "3591  Who called into question the Aristotelian asse...   \n",
       "3592  When did \"The Miraculous Journey of Edward Tul...   \n",
       "3593  In Shakespeare's Macbeth, which character is m...   \n",
       "3594  When did the play \"As You Like It\" first appea...   \n",
       "3595  When did the use of computers extend broadly i...   \n",
       "\n",
       "                                            question 19  \\\n",
       "0     Who was the debut winner of the Nobel Prize in...   \n",
       "1     Do you know when the next film in the Deadpool...   \n",
       "2     Which mode is designated for use in short wave...   \n",
       "3     Which period marks the presence of southwest w...   \n",
       "4       Can you clarify what hp means in War and Order?   \n",
       "...                                                 ...   \n",
       "3591  Who offered a challenge to the Earth-centric c...   \n",
       "3592  What is the official publication year of \"The ...   \n",
       "3593  Which person is murdered in Macbeth and then s...   \n",
       "3594  Which year saw the first performance of \"As Yo...   \n",
       "3595  At what time did computers become extensively ...   \n",
       "\n",
       "                                            question 20 answer  \\\n",
       "0     Who earned the first Nobel Prize in the Physic...      1   \n",
       "1     When will the new Deadpool movie be released t...      3   \n",
       "2     What mode is standard for short wave broadcast...      3   \n",
       "3     During what period is Nigeria under the impact...      3   \n",
       "4     What does hp stand for in the game War and Order?      1   \n",
       "...                                                 ...    ...   \n",
       "3591  Who contested the notion of an Earth-centered ...      2   \n",
       "3592  In which year did \"The Miraculous Journey of E...      2   \n",
       "3593  Who is the character in Macbeth that returns a...      3   \n",
       "3594  Can you indicate when \"As You Like It\" was fir...      2   \n",
       "3595  During what timeframe were computers initially...      3   \n",
       "\n",
       "                         option 1        option 2         option 3  \n",
       "0          Wilhelm conrad röntgen     Marie curie  Albert einstein  \n",
       "1                   July 10, 2018    June 1, 2018    May 18 , 2018  \n",
       "2                            Rtty           Psk31           Olivia  \n",
       "3             Beginning of august  End of october   Till september  \n",
       "4     Hit points or health points     Hero points       High power  \n",
       "...                           ...             ...              ...  \n",
       "3591                      Ptolemy      Copernicus          Galileo  \n",
       "3592                         2004            2006             2008  \n",
       "3593                      Macduff          Duncan           Banquo  \n",
       "3594                         1611            1603             1599  \n",
       "3595                        1990s           1970s            1980s  \n",
       "\n",
       "[3596 rows x 25 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28629bf",
   "metadata": {},
   "source": [
    "### HotpotQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a0784589",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import time\n",
    "import openai\n",
    "import random\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e8fdd482",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_client = openai.OpenAI(api_key=os.environ[\"OPENAI_API_KEY\"])\n",
    "openai_model = \"gpt-4o\"\n",
    "\n",
    "def call_gpt(message_user, retries=50):\n",
    "    messages = [{\"role\": \"system\", \"content\": 'You are a helpful assistant.'}, \n",
    "                {\"role\": \"user\", \"content\": message_user}]\n",
    "    for i in range(retries):\n",
    "        try:\n",
    "            response = openai_client.chat.completions.create(\n",
    "                model = openai_model, messages = messages, max_tokens = 512\n",
    "            )\n",
    "            return response.choices[0].message.content\n",
    "        except openai.RateLimitError as e:\n",
    "            time.sleep(i)\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred: {e}\")\n",
    "            raise\n",
    "    raise Exception(f\"Failed to call GPT-4 after {retries} retries.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c196419b",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_rephrase = \\\n",
    "'''\n",
    "### Instruction\n",
    "Do not respond to the question. \n",
    "Instead, follow the provided examples and rephrase the given question into 20 other versions that are semantically equivalent.\n",
    "\n",
    "### Example\n",
    "## Question: Which option best summarizes the comparative efficacy of regimen and comparator for managing condition?\n",
    "## Version 1: Choose an option that best describes the efficacy of regimen compared to comparator when used to treat condition.\n",
    "## Version 2: Select the option that most accurately reflects the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 3: Identify the option that best summarizes the effectiveness of regimen versus comparator in treating condition.\n",
    "## Version 4: Which option most effectively illustrates the efficacy of regimen when compared with comparator for condition?\n",
    "\n",
    "### Your Task\n",
    "## Question: {QUESTION}\n",
    "'''\n",
    "\n",
    "def rephrase_qn(question, retries=20):\n",
    "    query = template_rephrase.format(**{'QUESTION':question})\n",
    "    for _ in range(retries):\n",
    "        response = call_gpt(query)\n",
    "        matches = [re.search(fr\"## Version {i}: (.*)\", response) for i in range(1, 20)]\n",
    "        if all([each is not None and len(each.group(1)) for each in matches]):            \n",
    "            questions = [question] + [each.group(1) for each in matches]\n",
    "            if len(set(questions)) == 20: return questions\n",
    "    return [question] + [None]*19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9249014",
   "metadata": {},
   "outputs": [],
   "source": [
    "template_mcq = \\\n",
    "'''\n",
    "Using the provided examples as a guide, transform the given question with a correct answer into a multiple-choice question. \n",
    "Provide two additional incorrect options that are similar in type or category to the correct answer.\n",
    "\n",
    "# Example 1\n",
    "## Question: Which continent is the largest by land area?\n",
    "## Correct Answer: Asia\n",
    "## Incorrect Option 1: Africa\n",
    "## Incorrect Option 2: Europe\n",
    "\n",
    "# Example 2\n",
    "## Question: Is the last name scott irish or scottish?\n",
    "## Correct Answer: Scottish\n",
    "## Incorrect Option 1: Irish\n",
    "## Incorrect Option 2: English\n",
    "\n",
    "# Example 3\n",
    "## Question: Were Scott Derrickson and Ed Wood of the same nationality?\n",
    "## Correct Answer: Yes\n",
    "## Incorrect Option 1: No\n",
    "## Incorrect Option 2: Maybe\n",
    "\n",
    "# Your task\n",
    "## Question: {QUESTION}\n",
    "## Correct Answer: {ANSWER}\n",
    "'''\n",
    "\n",
    "def convert_mcq(question, answer, retries=20):\n",
    "    query = template_mcq.format(**{'QUESTION':question, 'ANSWER':answer})\n",
    "    for _ in range(retries):\n",
    "        response = call_gpt(query)\n",
    "        match1 = re.search(r\"## Incorrect Option 1: (.*)\", response)\n",
    "        match2 = re.search(r\"## Incorrect Option 2: (.*)\", response)\n",
    "        if match1 and len(match1.group(1)) and match2 and len(match2.group(1)):            \n",
    "            options = [answer.capitalize(), match1.group(1).capitalize(), match2.group(1).capitalize()]\n",
    "            random.shuffle(options)\n",
    "            if len(set(options)) == 3: return options\n",
    "    return [answer.capitalize(), None, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fb502ced",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 7405/7405 [2:30:47<00:00,  1.22s/it]\n"
     ]
    }
   ],
   "source": [
    "def prepare_row(idx, row):\n",
    "    question, answer = row['question'], row['answer']\n",
    "    questions = rephrase_qn(question)\n",
    "    options = convert_mcq(question, answer)\n",
    "    answer = options.index(answer.capitalize())+1\n",
    "    name2paragraph = {each[0]:''.join(each[1]) for each in row['context']}\n",
    "    evidence = '\\n\\n'.join([name2paragraph[name] for name, _ in row['supporting_facts']])\n",
    "    row = (evidence, *questions, answer, *options)\n",
    "    return idx, row\n",
    "\n",
    "raw = json.load(open('Data/Raw/HotpotQA/hotpot_dev_distractor_v1.json'))\n",
    "dataset, max_workers = {}, 50\n",
    "with ThreadPoolExecutor(max_workers=max_workers) as exe:\n",
    "    futures = [exe.submit(prepare_row, idx, row) for idx, row in enumerate(raw)]\n",
    "    for future in tqdm(as_completed(futures), total=len(futures)):\n",
    "        try:\n",
    "            idx, row = future.result()\n",
    "            dataset[idx] = row\n",
    "        except Exception as e:\n",
    "            print(f\"Error: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0262f962",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "columns = ['evidence'] + [f'question {i}' for i in range(1, 21)] + ['answer'] + [f'option {i}' for i in range(1, 4)]\n",
    "dataset = [row for idx, row in sorted(dataset.items(), key = lambda x: x[0])]\n",
    "dataset = pd.DataFrame(dataset, columns=columns)\n",
    "dataset = dataset.dropna().drop_duplicates(subset=columns[:21], keep=False).reset_index(drop=True)\n",
    "dataset.to_csv('Data/Input/HotpotQA.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d483a75f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>evidence</th>\n",
       "      <th>question 1</th>\n",
       "      <th>question 2</th>\n",
       "      <th>question 3</th>\n",
       "      <th>question 4</th>\n",
       "      <th>question 5</th>\n",
       "      <th>question 6</th>\n",
       "      <th>question 7</th>\n",
       "      <th>question 8</th>\n",
       "      <th>question 9</th>\n",
       "      <th>...</th>\n",
       "      <th>question 15</th>\n",
       "      <th>question 16</th>\n",
       "      <th>question 17</th>\n",
       "      <th>question 18</th>\n",
       "      <th>question 19</th>\n",
       "      <th>question 20</th>\n",
       "      <th>answer</th>\n",
       "      <th>option 1</th>\n",
       "      <th>option 2</th>\n",
       "      <th>option 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Scott Derrickson (born July 16, 1966) is an Am...</td>\n",
       "      <td>Were Scott Derrickson and Ed Wood of the same ...</td>\n",
       "      <td>Are Scott Derrickson and Ed Wood from the same...</td>\n",
       "      <td>Did Scott Derrickson and Ed Wood share the sam...</td>\n",
       "      <td>Do Scott Derrickson and Ed Wood have the same ...</td>\n",
       "      <td>Is the nationality of Scott Derrickson the sam...</td>\n",
       "      <td>Are the national origins of Scott Derrickson a...</td>\n",
       "      <td>Is Scott Derrickson's nationality the same as ...</td>\n",
       "      <td>Do Scott Derrickson and Ed Wood belong to the ...</td>\n",
       "      <td>Are Scott Derrickson and Ed Wood citizens of t...</td>\n",
       "      <td>...</td>\n",
       "      <td>Do Scott Derrickson and Ed Wood have the same ...</td>\n",
       "      <td>Were Scott Derrickson and Ed Wood born in the ...</td>\n",
       "      <td>Is the national affiliation of Scott Derrickso...</td>\n",
       "      <td>Are Scott Derrickson and Ed Wood of identical ...</td>\n",
       "      <td>Do Scott Derrickson and Ed Wood share the same...</td>\n",
       "      <td>Are Scott Derrickson and Ed Wood nationals of ...</td>\n",
       "      <td>3</td>\n",
       "      <td>No</td>\n",
       "      <td>Possibly</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kiss and Tell is a 1945 American comedy film s...</td>\n",
       "      <td>What government position was held by the woman...</td>\n",
       "      <td>What official government role did the actress ...</td>\n",
       "      <td>Which public office was occupied by the woman ...</td>\n",
       "      <td>In Kiss and Tell, what governmental role was h...</td>\n",
       "      <td>The actress playing Corliss Archer in the film...</td>\n",
       "      <td>What was the official government title of the ...</td>\n",
       "      <td>What government office did the woman in the ro...</td>\n",
       "      <td>Which government position did the actress behi...</td>\n",
       "      <td>What government office did the woman portrayin...</td>\n",
       "      <td>...</td>\n",
       "      <td>What was the governmental title of the woman w...</td>\n",
       "      <td>In Kiss and Tell, what was the government role...</td>\n",
       "      <td>The actress who portrayed Corliss Archer in Ki...</td>\n",
       "      <td>What was the government role of the woman who ...</td>\n",
       "      <td>Which government office did the actress known ...</td>\n",
       "      <td>In the film Kiss and Tell, what official gover...</td>\n",
       "      <td>3</td>\n",
       "      <td>Ambassador to the united nations</td>\n",
       "      <td>Secretary of state</td>\n",
       "      <td>Chief of protocol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Laleli Mosque (Turkish: \"Laleli Camii, or ...</td>\n",
       "      <td>Are the Laleli Mosque and Esma Sultan Mansion ...</td>\n",
       "      <td>Is the Laleli Mosque situated in the same neig...</td>\n",
       "      <td>Do the Laleli Mosque and Esma Sultan Mansion s...</td>\n",
       "      <td>Can the Laleli Mosque and Esma Sultan Mansion ...</td>\n",
       "      <td>Are the Laleli Mosque and the Esma Sultan Mans...</td>\n",
       "      <td>Are the locations of the Laleli Mosque and Esm...</td>\n",
       "      <td>Do the Laleli Mosque and Esma Sultan Mansion r...</td>\n",
       "      <td>Is the neighborhood that contains the Laleli M...</td>\n",
       "      <td>Are the Laleli Mosque and Esma Sultan Mansion ...</td>\n",
       "      <td>...</td>\n",
       "      <td>Does the Laleli Mosque share the same neighbor...</td>\n",
       "      <td>Is the neighborhood of the Laleli Mosque the s...</td>\n",
       "      <td>Are the Laleli Mosque and Esma Sultan Mansion ...</td>\n",
       "      <td>Is the neighborhood for the Laleli Mosque the ...</td>\n",
       "      <td>Are both the Laleli Mosque and the Esma Sultan...</td>\n",
       "      <td>Do Laleli Mosque and Esma Sultan Mansion share...</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Maybe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Big Stone Gap is a 2014 American drama romanti...</td>\n",
       "      <td>The director of the romantic comedy \"Big Stone...</td>\n",
       "      <td>In which New York city is the director of the ...</td>\n",
       "      <td>\"Big Stone Gap\" is a romantic comedy; the dire...</td>\n",
       "      <td>Identify the New York city where the director ...</td>\n",
       "      <td>Which New York city serves as the home base fo...</td>\n",
       "      <td>The director of the movie \"Big Stone Gap,\" a r...</td>\n",
       "      <td>Locate the New York city that is the base for ...</td>\n",
       "      <td>What city in New York is the director of \"Big ...</td>\n",
       "      <td>The romantic comedy \"Big Stone Gap\" has a dire...</td>\n",
       "      <td>...</td>\n",
       "      <td>Name the New York city where the romantic come...</td>\n",
       "      <td>Which city in New York serves as the residence...</td>\n",
       "      <td>Find the New York city that is home to the dir...</td>\n",
       "      <td>What is the New York city of residence for the...</td>\n",
       "      <td>Which New York city is considered the base for...</td>\n",
       "      <td>The romantic comedy \"Big Stone Gap\" director i...</td>\n",
       "      <td>2</td>\n",
       "      <td>Tribeca, new york city</td>\n",
       "      <td>Greenwich village, new york city</td>\n",
       "      <td>Upper east side, new york city</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014 S/S is the debut album of South Korean gr...</td>\n",
       "      <td>2014 S/S is the debut album of a South Korean ...</td>\n",
       "      <td>Who is responsible for forming the South Korea...</td>\n",
       "      <td>Who founded the South Korean boy group whose f...</td>\n",
       "      <td>The debut album \"2014 S/S\" was released by a S...</td>\n",
       "      <td>The creator of the South Korean boy group that...</td>\n",
       "      <td>Who established the South Korean boy group tha...</td>\n",
       "      <td>Who was behind the formation of the South Kore...</td>\n",
       "      <td>Which individual or entity formed the South Ko...</td>\n",
       "      <td>Who is the founder of the South Korean boy gro...</td>\n",
       "      <td>...</td>\n",
       "      <td>Who was the driving force behind the South Kor...</td>\n",
       "      <td>Who played a key role in forming the South Kor...</td>\n",
       "      <td>The South Korean boy group that launched \"2014...</td>\n",
       "      <td>Who initiated the creation of the South Korean...</td>\n",
       "      <td>The South Korean boy band's debut album \"2014 ...</td>\n",
       "      <td>Who is credited with the formation of the Sout...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yg entertainment</td>\n",
       "      <td>Sm entertainment</td>\n",
       "      <td>Jyp entertainment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6114</th>\n",
       "      <td>Robert Sargent Shriver Jr. ( ; November 9, 191...</td>\n",
       "      <td>Who was the brother of the wife of the Democra...</td>\n",
       "      <td>Who was the sibling of the spouse of the Democ...</td>\n",
       "      <td>Identify the brother of the woman married to t...</td>\n",
       "      <td>Can you name the brother of the 1972 Democrati...</td>\n",
       "      <td>Who is the brother of the Democratic Party's 1...</td>\n",
       "      <td>What is the name of the brother of the woman w...</td>\n",
       "      <td>Who is identified as the brother of the spouse...</td>\n",
       "      <td>Name the brother of the Democratic Vice Presid...</td>\n",
       "      <td>Which individual was the brother of the wife o...</td>\n",
       "      <td>...</td>\n",
       "      <td>Who can be named as the brother of the spouse ...</td>\n",
       "      <td>Who was the sibling of the wife of the 1972 De...</td>\n",
       "      <td>What is the identity of the brother of the 197...</td>\n",
       "      <td>Can you provide the name of the brother of the...</td>\n",
       "      <td>Who was the brother of the partner of the 1972...</td>\n",
       "      <td>State the name of the brother of the wife of t...</td>\n",
       "      <td>1</td>\n",
       "      <td>President john f. kennedy</td>\n",
       "      <td>Senator robert f. kennedy</td>\n",
       "      <td>Attorney general robert b. kennedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6115</th>\n",
       "      <td>Volvic is a brand of mineral water. Its source...</td>\n",
       "      <td>Are both Volvic and Canfield's Diet Chocolate ...</td>\n",
       "      <td>Can we confirm that Volvic and Canfield's Diet...</td>\n",
       "      <td>Is it accurate to say that both Volvic and Can...</td>\n",
       "      <td>Are Volvic and Canfield's Diet Chocolate Fudge...</td>\n",
       "      <td>Would both Volvic and Canfield's Diet Chocolat...</td>\n",
       "      <td>Are Volvic and Canfield's Diet Chocolate Fudge...</td>\n",
       "      <td>Do both Volvic and Canfield's Diet Chocolate F...</td>\n",
       "      <td>Is it true that both Volvic and Canfield's Die...</td>\n",
       "      <td>Are both Volvic and Canfield's Diet Chocolate ...</td>\n",
       "      <td>...</td>\n",
       "      <td>Are Volvic and Canfield's Diet Chocolate Fudge...</td>\n",
       "      <td>Do the waters Volvic and Canfield's Diet Choco...</td>\n",
       "      <td>Are both Volvic and Canfield's Diet Chocolate ...</td>\n",
       "      <td>Can both Volvic and Canfield's Diet Chocolate ...</td>\n",
       "      <td>Are Volvic and Canfield's Diet Chocolate Fudge...</td>\n",
       "      <td>Is the term natural spring water applicable to...</td>\n",
       "      <td>2</td>\n",
       "      <td>Possibly</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6116</th>\n",
       "      <td>The Bosnian Coarse-haired Hound or Bosanski Oš...</td>\n",
       "      <td>Are Billy and Barak both breeds of scenthound?...</td>\n",
       "      <td>Is it true that both Billy and Barak are scent...</td>\n",
       "      <td>Can Billy and Barak be classified as breeds of...</td>\n",
       "      <td>Do both Billy and the Barak, also known as the...</td>\n",
       "      <td>Would Billy and Barak qualify as breeds within...</td>\n",
       "      <td>Are the dog breeds Billy and Barak, also calle...</td>\n",
       "      <td>Do the breeds Billy and Barak both fall under ...</td>\n",
       "      <td>Can both Billy and the Bosnian Coarse-haired H...</td>\n",
       "      <td>Are Billy and Barak recognized as breeds of sc...</td>\n",
       "      <td>...</td>\n",
       "      <td>Would both the Billy and Barak breeds be liste...</td>\n",
       "      <td>Are Billy and the Barak breed, which is also c...</td>\n",
       "      <td>Could both Billy and Barak be grouped as scent...</td>\n",
       "      <td>Do the dog breeds Billy and Barak fall within ...</td>\n",
       "      <td>Are Billy and Barak considered members of the ...</td>\n",
       "      <td>Do Billy and Barak, also known as Bosnian Coar...</td>\n",
       "      <td>3</td>\n",
       "      <td>Unsure</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6117</th>\n",
       "      <td>Dig is an American alternative rock band from ...</td>\n",
       "      <td>Were both of the following rock groups formed ...</td>\n",
       "      <td>Is it true that both the rock bands Dig and Th...</td>\n",
       "      <td>Can it be confirmed that the rock groups Dig a...</td>\n",
       "      <td>Did both the bands, Dig and Thinking Fellers U...</td>\n",
       "      <td>Are Dig and Thinking Fellers Union Local 282 b...</td>\n",
       "      <td>Were the origins of both rock groups, Dig and ...</td>\n",
       "      <td>Is California the formation site for both the ...</td>\n",
       "      <td>Were the rock groups Dig and Thinking Fellers ...</td>\n",
       "      <td>Did both bands, Dig and Thinking Fellers Union...</td>\n",
       "      <td>...</td>\n",
       "      <td>Is there evidence that both Dig and Thinking F...</td>\n",
       "      <td>Were both Dig and Thinking Fellers Union Local...</td>\n",
       "      <td>Did the two bands, Dig and Thinking Fellers Un...</td>\n",
       "      <td>Are both Dig and Thinking Fellers Union Local ...</td>\n",
       "      <td>Were the musical groups Dig and Thinking Felle...</td>\n",
       "      <td>Was the establishment of both rock groups, Dig...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Uncertain</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6118</th>\n",
       "      <td>The Blackfin is a family of 16- or 32-bit micr...</td>\n",
       "      <td>Blackfin is a family of processors developed b...</td>\n",
       "      <td>In which city is the headquarters located for ...</td>\n",
       "      <td>What city serves as the headquarters for the c...</td>\n",
       "      <td>Identify the city where the company that creat...</td>\n",
       "      <td>The company that developed Blackfin processors...</td>\n",
       "      <td>Which city is home to the headquarters of the ...</td>\n",
       "      <td>Where is the headquarters of the company that ...</td>\n",
       "      <td>What city hosts the headquarters of the compan...</td>\n",
       "      <td>Can you name the city where the company that d...</td>\n",
       "      <td>...</td>\n",
       "      <td>Which city's company developed the family of p...</td>\n",
       "      <td>What is the location of the headquarters of th...</td>\n",
       "      <td>The company that developed the Blackfin proces...</td>\n",
       "      <td>Which city serves as the base for the company ...</td>\n",
       "      <td>Identify the city where the developer of the B...</td>\n",
       "      <td>From which city does the company that launched...</td>\n",
       "      <td>2</td>\n",
       "      <td>Austin, texas</td>\n",
       "      <td>Norwood, massachusetts</td>\n",
       "      <td>San jose, california</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6119 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               evidence  \\\n",
       "0     Scott Derrickson (born July 16, 1966) is an Am...   \n",
       "1     Kiss and Tell is a 1945 American comedy film s...   \n",
       "2     The Laleli Mosque (Turkish: \"Laleli Camii, or ...   \n",
       "3     Big Stone Gap is a 2014 American drama romanti...   \n",
       "4     2014 S/S is the debut album of South Korean gr...   \n",
       "...                                                 ...   \n",
       "6114  Robert Sargent Shriver Jr. ( ; November 9, 191...   \n",
       "6115  Volvic is a brand of mineral water. Its source...   \n",
       "6116  The Bosnian Coarse-haired Hound or Bosanski Oš...   \n",
       "6117  Dig is an American alternative rock band from ...   \n",
       "6118  The Blackfin is a family of 16- or 32-bit micr...   \n",
       "\n",
       "                                             question 1  \\\n",
       "0     Were Scott Derrickson and Ed Wood of the same ...   \n",
       "1     What government position was held by the woman...   \n",
       "2     Are the Laleli Mosque and Esma Sultan Mansion ...   \n",
       "3     The director of the romantic comedy \"Big Stone...   \n",
       "4     2014 S/S is the debut album of a South Korean ...   \n",
       "...                                                 ...   \n",
       "6114  Who was the brother of the wife of the Democra...   \n",
       "6115  Are both Volvic and Canfield's Diet Chocolate ...   \n",
       "6116  Are Billy and Barak both breeds of scenthound?...   \n",
       "6117  Were both of the following rock groups formed ...   \n",
       "6118  Blackfin is a family of processors developed b...   \n",
       "\n",
       "                                             question 2  \\\n",
       "0     Are Scott Derrickson and Ed Wood from the same...   \n",
       "1     What official government role did the actress ...   \n",
       "2     Is the Laleli Mosque situated in the same neig...   \n",
       "3     In which New York city is the director of the ...   \n",
       "4     Who is responsible for forming the South Korea...   \n",
       "...                                                 ...   \n",
       "6114  Who was the sibling of the spouse of the Democ...   \n",
       "6115  Can we confirm that Volvic and Canfield's Diet...   \n",
       "6116  Is it true that both Billy and Barak are scent...   \n",
       "6117  Is it true that both the rock bands Dig and Th...   \n",
       "6118  In which city is the headquarters located for ...   \n",
       "\n",
       "                                             question 3  \\\n",
       "0     Did Scott Derrickson and Ed Wood share the sam...   \n",
       "1     Which public office was occupied by the woman ...   \n",
       "2     Do the Laleli Mosque and Esma Sultan Mansion s...   \n",
       "3     \"Big Stone Gap\" is a romantic comedy; the dire...   \n",
       "4     Who founded the South Korean boy group whose f...   \n",
       "...                                                 ...   \n",
       "6114  Identify the brother of the woman married to t...   \n",
       "6115  Is it accurate to say that both Volvic and Can...   \n",
       "6116  Can Billy and Barak be classified as breeds of...   \n",
       "6117  Can it be confirmed that the rock groups Dig a...   \n",
       "6118  What city serves as the headquarters for the c...   \n",
       "\n",
       "                                             question 4  \\\n",
       "0     Do Scott Derrickson and Ed Wood have the same ...   \n",
       "1     In Kiss and Tell, what governmental role was h...   \n",
       "2     Can the Laleli Mosque and Esma Sultan Mansion ...   \n",
       "3     Identify the New York city where the director ...   \n",
       "4     The debut album \"2014 S/S\" was released by a S...   \n",
       "...                                                 ...   \n",
       "6114  Can you name the brother of the 1972 Democrati...   \n",
       "6115  Are Volvic and Canfield's Diet Chocolate Fudge...   \n",
       "6116  Do both Billy and the Barak, also known as the...   \n",
       "6117  Did both the bands, Dig and Thinking Fellers U...   \n",
       "6118  Identify the city where the company that creat...   \n",
       "\n",
       "                                             question 5  \\\n",
       "0     Is the nationality of Scott Derrickson the sam...   \n",
       "1     The actress playing Corliss Archer in the film...   \n",
       "2     Are the Laleli Mosque and the Esma Sultan Mans...   \n",
       "3     Which New York city serves as the home base fo...   \n",
       "4     The creator of the South Korean boy group that...   \n",
       "...                                                 ...   \n",
       "6114  Who is the brother of the Democratic Party's 1...   \n",
       "6115  Would both Volvic and Canfield's Diet Chocolat...   \n",
       "6116  Would Billy and Barak qualify as breeds within...   \n",
       "6117  Are Dig and Thinking Fellers Union Local 282 b...   \n",
       "6118  The company that developed Blackfin processors...   \n",
       "\n",
       "                                             question 6  \\\n",
       "0     Are the national origins of Scott Derrickson a...   \n",
       "1     What was the official government title of the ...   \n",
       "2     Are the locations of the Laleli Mosque and Esm...   \n",
       "3     The director of the movie \"Big Stone Gap,\" a r...   \n",
       "4     Who established the South Korean boy group tha...   \n",
       "...                                                 ...   \n",
       "6114  What is the name of the brother of the woman w...   \n",
       "6115  Are Volvic and Canfield's Diet Chocolate Fudge...   \n",
       "6116  Are the dog breeds Billy and Barak, also calle...   \n",
       "6117  Were the origins of both rock groups, Dig and ...   \n",
       "6118  Which city is home to the headquarters of the ...   \n",
       "\n",
       "                                             question 7  \\\n",
       "0     Is Scott Derrickson's nationality the same as ...   \n",
       "1     What government office did the woman in the ro...   \n",
       "2     Do the Laleli Mosque and Esma Sultan Mansion r...   \n",
       "3     Locate the New York city that is the base for ...   \n",
       "4     Who was behind the formation of the South Kore...   \n",
       "...                                                 ...   \n",
       "6114  Who is identified as the brother of the spouse...   \n",
       "6115  Do both Volvic and Canfield's Diet Chocolate F...   \n",
       "6116  Do the breeds Billy and Barak both fall under ...   \n",
       "6117  Is California the formation site for both the ...   \n",
       "6118  Where is the headquarters of the company that ...   \n",
       "\n",
       "                                             question 8  \\\n",
       "0     Do Scott Derrickson and Ed Wood belong to the ...   \n",
       "1     Which government position did the actress behi...   \n",
       "2     Is the neighborhood that contains the Laleli M...   \n",
       "3     What city in New York is the director of \"Big ...   \n",
       "4     Which individual or entity formed the South Ko...   \n",
       "...                                                 ...   \n",
       "6114  Name the brother of the Democratic Vice Presid...   \n",
       "6115  Is it true that both Volvic and Canfield's Die...   \n",
       "6116  Can both Billy and the Bosnian Coarse-haired H...   \n",
       "6117  Were the rock groups Dig and Thinking Fellers ...   \n",
       "6118  What city hosts the headquarters of the compan...   \n",
       "\n",
       "                                             question 9  ...  \\\n",
       "0     Are Scott Derrickson and Ed Wood citizens of t...  ...   \n",
       "1     What government office did the woman portrayin...  ...   \n",
       "2     Are the Laleli Mosque and Esma Sultan Mansion ...  ...   \n",
       "3     The romantic comedy \"Big Stone Gap\" has a dire...  ...   \n",
       "4     Who is the founder of the South Korean boy gro...  ...   \n",
       "...                                                 ...  ...   \n",
       "6114  Which individual was the brother of the wife o...  ...   \n",
       "6115  Are both Volvic and Canfield's Diet Chocolate ...  ...   \n",
       "6116  Are Billy and Barak recognized as breeds of sc...  ...   \n",
       "6117  Did both bands, Dig and Thinking Fellers Union...  ...   \n",
       "6118  Can you name the city where the company that d...  ...   \n",
       "\n",
       "                                            question 15  \\\n",
       "0     Do Scott Derrickson and Ed Wood have the same ...   \n",
       "1     What was the governmental title of the woman w...   \n",
       "2     Does the Laleli Mosque share the same neighbor...   \n",
       "3     Name the New York city where the romantic come...   \n",
       "4     Who was the driving force behind the South Kor...   \n",
       "...                                                 ...   \n",
       "6114  Who can be named as the brother of the spouse ...   \n",
       "6115  Are Volvic and Canfield's Diet Chocolate Fudge...   \n",
       "6116  Would both the Billy and Barak breeds be liste...   \n",
       "6117  Is there evidence that both Dig and Thinking F...   \n",
       "6118  Which city's company developed the family of p...   \n",
       "\n",
       "                                            question 16  \\\n",
       "0     Were Scott Derrickson and Ed Wood born in the ...   \n",
       "1     In Kiss and Tell, what was the government role...   \n",
       "2     Is the neighborhood of the Laleli Mosque the s...   \n",
       "3     Which city in New York serves as the residence...   \n",
       "4     Who played a key role in forming the South Kor...   \n",
       "...                                                 ...   \n",
       "6114  Who was the sibling of the wife of the 1972 De...   \n",
       "6115  Do the waters Volvic and Canfield's Diet Choco...   \n",
       "6116  Are Billy and the Barak breed, which is also c...   \n",
       "6117  Were both Dig and Thinking Fellers Union Local...   \n",
       "6118  What is the location of the headquarters of th...   \n",
       "\n",
       "                                            question 17  \\\n",
       "0     Is the national affiliation of Scott Derrickso...   \n",
       "1     The actress who portrayed Corliss Archer in Ki...   \n",
       "2     Are the Laleli Mosque and Esma Sultan Mansion ...   \n",
       "3     Find the New York city that is home to the dir...   \n",
       "4     The South Korean boy group that launched \"2014...   \n",
       "...                                                 ...   \n",
       "6114  What is the identity of the brother of the 197...   \n",
       "6115  Are both Volvic and Canfield's Diet Chocolate ...   \n",
       "6116  Could both Billy and Barak be grouped as scent...   \n",
       "6117  Did the two bands, Dig and Thinking Fellers Un...   \n",
       "6118  The company that developed the Blackfin proces...   \n",
       "\n",
       "                                            question 18  \\\n",
       "0     Are Scott Derrickson and Ed Wood of identical ...   \n",
       "1     What was the government role of the woman who ...   \n",
       "2     Is the neighborhood for the Laleli Mosque the ...   \n",
       "3     What is the New York city of residence for the...   \n",
       "4     Who initiated the creation of the South Korean...   \n",
       "...                                                 ...   \n",
       "6114  Can you provide the name of the brother of the...   \n",
       "6115  Can both Volvic and Canfield's Diet Chocolate ...   \n",
       "6116  Do the dog breeds Billy and Barak fall within ...   \n",
       "6117  Are both Dig and Thinking Fellers Union Local ...   \n",
       "6118  Which city serves as the base for the company ...   \n",
       "\n",
       "                                            question 19  \\\n",
       "0     Do Scott Derrickson and Ed Wood share the same...   \n",
       "1     Which government office did the actress known ...   \n",
       "2     Are both the Laleli Mosque and the Esma Sultan...   \n",
       "3     Which New York city is considered the base for...   \n",
       "4     The South Korean boy band's debut album \"2014 ...   \n",
       "...                                                 ...   \n",
       "6114  Who was the brother of the partner of the 1972...   \n",
       "6115  Are Volvic and Canfield's Diet Chocolate Fudge...   \n",
       "6116  Are Billy and Barak considered members of the ...   \n",
       "6117  Were the musical groups Dig and Thinking Felle...   \n",
       "6118  Identify the city where the developer of the B...   \n",
       "\n",
       "                                            question 20 answer  \\\n",
       "0     Are Scott Derrickson and Ed Wood nationals of ...      3   \n",
       "1     In the film Kiss and Tell, what official gover...      3   \n",
       "2     Do Laleli Mosque and Esma Sultan Mansion share...      1   \n",
       "3     The romantic comedy \"Big Stone Gap\" director i...      2   \n",
       "4     Who is credited with the formation of the Sout...      1   \n",
       "...                                                 ...    ...   \n",
       "6114  State the name of the brother of the wife of t...      1   \n",
       "6115  Is the term natural spring water applicable to...      2   \n",
       "6116  Do Billy and Barak, also known as Bosnian Coar...      3   \n",
       "6117  Was the establishment of both rock groups, Dig...      1   \n",
       "6118  From which city does the company that launched...      2   \n",
       "\n",
       "                              option 1                          option 2  \\\n",
       "0                                   No                          Possibly   \n",
       "1     Ambassador to the united nations                Secretary of state   \n",
       "2                                   No                               Yes   \n",
       "3               Tribeca, new york city  Greenwich village, new york city   \n",
       "4                     Yg entertainment                  Sm entertainment   \n",
       "...                                ...                               ...   \n",
       "6114         President john f. kennedy         Senator robert f. kennedy   \n",
       "6115                          Possibly                                No   \n",
       "6116                            Unsure                                No   \n",
       "6117                               Yes                         Uncertain   \n",
       "6118                     Austin, texas            Norwood, massachusetts   \n",
       "\n",
       "                                option 3  \n",
       "0                                    Yes  \n",
       "1                      Chief of protocol  \n",
       "2                                  Maybe  \n",
       "3         Upper east side, new york city  \n",
       "4                      Jyp entertainment  \n",
       "...                                  ...  \n",
       "6114  Attorney general robert b. kennedy  \n",
       "6115                                 Yes  \n",
       "6116                                 Yes  \n",
       "6117                                  No  \n",
       "6118                San jose, california  \n",
       "\n",
       "[6119 rows x 25 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9253d650",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('Data/Input/HotpotQA.csv')\n",
    "raw = json.load(open('Data/Raw/HotpotQA/hotpot_dev_fullwiki_v1.json'))\n",
    "raw_ = json.load(open('Data/Raw/HotpotQA/hotpot_dev_distractor_v1.json'))\n",
    "\n",
    "retrieveds = []\n",
    "question2id = {each['question']:idx for idx, each in enumerate(raw)}\n",
    "for _, row in dataset.iterrows():\n",
    "    id = question2id[row['question 1']]\n",
    "    context = raw[id]['context'] if len(raw[id]['context']) else raw_[id]['context']\n",
    "    retrieved = '\\n\\n'.join([''.join(each[1]) for each in context])\n",
    "    retrieveds.append(retrieved)\n",
    "dataset['retrieved'] = retrieveds\n",
    "dataset.to_csv('Data/Input/HotpotQA.csv', index=False)    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
