{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Main"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Inter-family Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 127996 characters]",
      "text/plain": [
       "<Figure size 1300x1100 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "plt.rcParams['legend.fontsize'] = 16\n",
    "plt.rcParams['xtick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['axes.titlesize'] = 18\n",
    "plt.rcParams['legend.title_fontsize'] = 16\n",
    "plt.rcParams['legend.fontsize'] = 16\n",
    "plt.rcParams['ytick.labelsize'] = 16 # Increased ytick label size\n",
    "\n",
    "# Load datasets - Corrected file paths\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag each dataset\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Patch extraction logic\n",
    "def extract_patch_ratio(model_path):\n",
    "    if \"config_image\" in model_path: return 1\n",
    "    if \"config_patch/2x2\" in model_path: return 2\n",
    "    if \"config_patch/4x4\" in model_path: return 4\n",
    "    if \"config_patch/8x8\" in model_path: return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epochs\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "def model_group(model_name):\n",
    "    if model_name.startswith(\"Llama\"): return \"Llama-3.2\"\n",
    "    if model_name.startswith(\"Qwen2.5\"): return \"Qwen2.5\"\n",
    "    if model_name.startswith(\"Qwen2\"): return \"Qwen2\"\n",
    "    if model_name.startswith(\"llava-1.5\"): return \"llava-1.5\"\n",
    "    if model_name.startswith(\"llava-v1.6\"): return \"llava-v1.6\"\n",
    "    if model_name.startswith(\"gemma\"): return \"gemma\"\n",
    "    if model_name.startswith(\"InternVL3\"): return \"InternVL3\"\n",
    "    return \"unknown\"\n",
    "\n",
    "# Derive features\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "\n",
    "# Model configuration\n",
    "target_models = {\n",
    "    \"Llama-3.2-11B-Vision\": \"Llama-3.2-11B-Vision\",\n",
    "    \"Qwen2-VL-7B\": \"Qwen2-VL-7B\",\n",
    "    \"Qwen2.5-VL-7B-Instruct\": \"Qwen2.5-VL-7B-Instruct\",\n",
    "    \"llava-1.5-7b-hf\": \"llava-1.5-7b-hf\",\n",
    "    \"llava-v1.6-vicuna-7b-hf\": \"llava-v1.6-vicuna-7b-hf\",\n",
    "    \"gemma-3-12b-pt\": \"gemma-3-12b-pt\",\n",
    "    \"InternVL3-8B\": \"InternVL3-8B\"\n",
    "}\n",
    "\n",
    "alpha_map = {\n",
    "    \"Llama-3.2-11B-Vision\": 1.0,\n",
    "    \"Qwen2-VL-7B\": 0.6,\n",
    "    \"Qwen2.5-VL-7B-Instruct\": 1.0,\n",
    "    \"llava-1.5-7b-hf\": 0.6,\n",
    "    \"llava-v1.6-vicuna-7b-hf\": 1.0,\n",
    "    \"gemma-3-12b-pt\": 0.9,\n",
    "    \"InternVL3-8B\": 1.0\n",
    "}\n",
    "\n",
    "color_bases = {\n",
    "    \"Llama-3.2\": \"#1f77b4\",\n",
    "    \"Qwen2\": \"#2ca02c\",\n",
    "    \"Qwen2.5\": \"#2ca02c\",\n",
    "    \"llava-1.5\": \"#ff7f0e\",\n",
    "    \"llava-v1.6\": \"#ff7f0e\",\n",
    "    \"gemma\": \"#d62728\",\n",
    "    \"InternVL3\": \"#9467bd\"\n",
    "}\n",
    "\n",
    "# Filter valid models\n",
    "df_filtered = df_all[df_all[\"model_name\"].isin(target_models.keys())]\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "\n",
    "# Plot configuration\n",
    "datasets = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "x_ticks = [1, 2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.08\n",
    "\n",
    "fig, axes = plt.subplots(nrows=3, ncols=1, figsize=(13, 11), sharex=True)\n",
    "\n",
    "for idx, dataset in enumerate(datasets):\n",
    "    ax = axes[idx]\n",
    "    df_dataset = df_filtered[df_filtered[\"dataset\"] == dataset]\n",
    "\n",
    "    pivoted = df_dataset.pivot(index=\"patch_ratio\", columns=\"model_name\",\n",
    "                               values=[\"eval-1 mean\", \"eval-1 std\", \"eval-0 mean\", \"eval-0 std\"])\n",
    "    pivoted_filled = pivoted.reindex(index=x_ticks)\n",
    "\n",
    "    # Calculate min and max y-values for setting limits\n",
    "    min_data_y = 0\n",
    "    max_data_y = 0\n",
    "    for model in target_models.keys():\n",
    "        try:\n",
    "            upper_means = pivoted_filled[(\"eval-1 mean\", model)].values\n",
    "            upper_stds = pivoted_filled[(\"eval-1 std\", model)].values\n",
    "            lower_means = pivoted_filled[(\"eval-0 mean\", model)].values\n",
    "            lower_stds = pivoted_filled[(\"eval-0 std\", model)].values\n",
    "\n",
    "            # Update max_data_y with upper means + stds, handling NaNs\n",
    "            if not np.all(np.isnan(upper_means + upper_stds)):\n",
    "                max_data_y = max(max_data_y, np.nanmax(upper_means + upper_stds))\n",
    "\n",
    "            # Update min_data_y with lower means - stds, handling NaNs\n",
    "            if not np.all(np.isnan(lower_means) | np.isnan(lower_stds)):\n",
    "                 min_data_y = min(min_data_y, np.nanmin(lower_means - lower_stds))\n",
    "\n",
    "        except KeyError:\n",
    "            continue\n",
    "\n",
    "    # Determine y-axis limits with padding, ensuring baseline is visible\n",
    "    lower_ylim = min_data_y - 6 # Fixed padding for the lower limit\n",
    "    upper_ylim = max(max_data_y, 9.5) + 1 # Ensure upper limit is at least 9.5 + padding\n",
    "\n",
    "    # ax.set_ylim(lower_ylim, upper_ylim)\n",
    "    # Set y-ticks to [-5, 0, 5, 10]\n",
    "    # ax.set_yticks([-2, 2, 6, 10])\n",
    "    ax.set_yticks([-3, 0, 5, 10])\n",
    "\n",
    "\n",
    "    for i, model in enumerate(target_models.keys()):\n",
    "        group = model_group(model)\n",
    "        offset = (i - len(target_models) // 2) * bar_width\n",
    "        try:\n",
    "            upper_means = pivoted_filled[(\"eval-1 mean\", model)].values\n",
    "            upper_stds = pivoted_filled[(\"eval-1 std\", model)].values\n",
    "            lower_means = pivoted_filled[(\"eval-0 mean\", model)].values\n",
    "            lower_stds = pivoted_filled[(\"eval-0 std\", model)].values\n",
    "\n",
    "            base_color = color_bases[group]\n",
    "            alpha = alpha_map[model]\n",
    "\n",
    "            ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                   label=model, color=base_color, alpha=alpha, capsize=3)\n",
    "            ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                   color=base_color, alpha=alpha, capsize=3)\n",
    "        except KeyError:\n",
    "            continue\n",
    "\n",
    "    ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "    # Keep baseline text position relative to data coordinate 9.5\n",
    "    ax.text(x[len(x)//2] - 0.5, 8.3, 'Random Baseline', color='gray', fontsize=12, va='bottom', ha='center')\n",
    "    ax.axhline(0, color='black', linewidth=0.8)\n",
    "\n",
    "    # Set y-tick labels to their absolute values based on the new ticks\n",
    "    yticks = ax.get_yticks()\n",
    "    ax.set_yticklabels([f'{abs(tick)}' for tick in yticks])\n",
    "\n",
    "    # Remove the old y-axis label\n",
    "    ax.set_ylabel(\"\")\n",
    "\n",
    "    # Add new labels to the upper and lower sides of the y-axis\n",
    "    # Positioned relative to axes coordinates to be outside the plot area\n",
    "    ax.text(-0.03, 0.65, 'reference', transform=ax.transAxes,\n",
    "            rotation=90, va='center', ha='right', fontsize=18)\n",
    "    ax.text(-0.03, 0.15, 'image', transform=ax.transAxes,\n",
    "            rotation=90, va='center', ha='right', fontsize=18)\n",
    "\n",
    "\n",
    "    ax.set_title(dataset)\n",
    "\n",
    "    for spine in ax.spines.values():\n",
    "        spine.set_linewidth(2.5)\n",
    "\n",
    "    # Add subplot border\n",
    "    border = plt.Rectangle(\n",
    "        (0, 0), 1, 1,\n",
    "        transform=ax.transAxes,\n",
    "        fill=False,\n",
    "        edgecolor='black',\n",
    "        linewidth=2.5,\n",
    "        zorder=10,\n",
    "        clip_on=False\n",
    "    )\n",
    "    ax.add_patch(border)\n",
    "\n",
    "axes[-1].set_xticks(x)\n",
    "axes[-1].set_xticklabels(x_ticks)\n",
    "axes[-1].set_xlabel(r'Split Factor $(f)$', fontsize=15)\n",
    "\n",
    "for ax in axes.flatten():\n",
    "    ax.set_ylim([-3.5, 10.5])  # example y-limits\n",
    "\n",
    "# Unified legend - Adjusted bbox_to_anchor to move legend up\n",
    "handles, labels = axes[0].get_legend_handles_labels()\n",
    "fig.legend(handles, labels, loc='upper center', bbox_to_anchor=(0.5, 1.03), ncol=4, frameon=False)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.97])\n",
    "plt.savefig(\"main_comparison.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Global Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model configuration\n",
    "target_models = {\n",
    "    \"Qwen2-VL-2B\": \"Qwen2-VL-2B\",\n",
    "    \"Qwen2-VL-7B\": \"Qwen2-VL-7B\",\n",
    "    \"Qwen2-VL-72B\": \"Qwen2-VL-72B\",\n",
    "    \"Qwen2.5-VL-3B-Instruct\": \"Qwen2.5-VL-3B-Instruct\",\n",
    "    \"Qwen2.5-VL-7B-Instruct\": \"Qwen2.5-VL-7B-Instruct\",\n",
    "    \"Qwen2.5-VL-72B-Instruct\": \"Qwen2.5-VL-72B-Instruct\",\n",
    "    \"gemma-3-4b-pt\": \"gemma-3-4b-pt\",\n",
    "    \"gemma-3-12b-pt\": \"gemma-3-12b-pt\",\n",
    "    \"gemma-3-27b-pt\": \"gemma-3-27b-pt\",\n",
    "    # Added models for fourth row\n",
    "    \"InternVL3-1B\": \"InternVL3-1B\",\n",
    "    \"InternVL3-14B\": \"InternVL3-14B\",\n",
    "    \"InternVL3-8B\": \"InternVL3-8B\",\n",
    "    # Added models for fifth row\n",
    "    \"llava-1.5-7b-hf\": \"llava-1.5-7b-hf\",\n",
    "    \"llava-1.5-13b-hf\": \"llava-1.5-13b-hf\",\n",
    "    # Added models for sixth row\n",
    "    \"Llama-3.2-90B-Vision\": \"Llama-3.2-90B-Vision\",\n",
    "    \"Llama-3.2-11B-Vision\": \"Llama-3.2-11B-Vision\",\n",
    "    # Added models for seventh row\n",
    "    \"llava-v1.6-vicuna-13b-hf\": \"llava-v1.6-vicuna-13b-hf\",\n",
    "    \"llava-v1.6-vicuna-7b-hf\": \"llava-v1.6-vicuna-7b-hf\"\n",
    "}\n",
    "\n",
    "alpha_map = {\n",
    "    \"Qwen2-VL-2B\": 0.4,\n",
    "    \"Qwen2-VL-7B\": 0.7,\n",
    "    \"Qwen2-VL-72B\": 1.0,\n",
    "    \"Qwen2.5-VL-3B-Instruct\": 0.4,\n",
    "    \"Qwen2.5-VL-7B-Instruct\": 0.7,\n",
    "    \"Qwen2.5-VL-72B-Instruct\": 1.0,\n",
    "    \"gemma-3-4b-pt\": 0.4,\n",
    "    \"gemma-3-12b-pt\": 0.7,\n",
    "    \"gemma-3-27b-pt\": 1.0,\n",
    "    # Added models alpha values (example values)\n",
    "    \"InternVL3-1B\": 0.4,\n",
    "    \"InternVL3-14B\": 0.7,\n",
    "    \"InternVL3-8B\": 1.0,\n",
    "    \"llava-1.5-7b-hf\": 0.5,\n",
    "    \"llava-1.5-13b-hf\": 0.8,\n",
    "    \"Llama-3.2-11B-Vision\": 0.6, # Example alpha\n",
    "    \"Llama-3.2-90B-Vision\": 0.9, # Example alpha\n",
    "    \"llava-v1.6-vicuna-7b-hf\": 0.5, # Example alpha\n",
    "    \"llava-v1.6-vicuna-13b-hf\": 0.8 # Example alpha\n",
    "}\n",
    "\n",
    "color_bases = {\n",
    "    \"Qwen-2-VL\": \"#2ca02c\",\n",
    "    \"Qwen-2.5-VL\": \"#9467bd\",\n",
    "    \"Gemma-3\": \"#d62728\",\n",
    "    # Added families color bases\n",
    "    \"InternVL-3\": \"#ff7f0e\", # Example color (orange)\n",
    "    \"LLaVA-1.5\": \"#1f77b4\", # Example color (blue)\n",
    "    \"Llama-3.2-Vision\": \"#e377c2\", # Example color (pink)\n",
    "    \"LLaVA-1.6\": \"#7f7f7f\" # Example color (gray)\n",
    "}\n",
    "\n",
    "def model_family(model_name):\n",
    "    if model_name.startswith(\"Qwen2.5\"):\n",
    "        return \"Qwen-2.5-VL\"\n",
    "    elif model_name.startswith(\"Qwen2\"):\n",
    "        return \"Qwen-2-VL\"\n",
    "    elif model_name.startswith(\"gemma\"):\n",
    "        return \"Gemma-3\"\n",
    "    elif model_name.startswith(\"InternVL3\"):\n",
    "        return \"InternVL-3\"\n",
    "    elif model_name.startswith(\"llava-1.5\"):\n",
    "        return \"LLaVA-1.5\"\n",
    "    elif model_name.startswith(\"Llama-3.2\"):\n",
    "        return \"Llama-3.2-Vision\"\n",
    "    elif model_name.startswith(\"llava-v1.6\"):\n",
    "        return \"LLaVA-1.6\"\n",
    "    return \"Other\"\n",
    "\n",
    "param_sizes = {\n",
    "    \"Qwen2-VL-2B\": \"2B\",\n",
    "    \"Qwen2-VL-7B\": \"7B\",\n",
    "    \"Qwen2-VL-72B\": \"72B\",\n",
    "    \"Qwen2.5-VL-3B-Instruct\": \"3B\",\n",
    "    \"Qwen2.5-VL-7B-Instruct\": \"7B\",\n",
    "    \"Qwen2.5-VL-72B-Instruct\": \"72B\",\n",
    "    \"gemma-3-4b-pt\": \"4B\",\n",
    "    \"gemma-3-12b-pt\": \"12B\",\n",
    "    \"gemma-3-27b-pt\": \"27B\",\n",
    "    \"InternVL3-1B\": \"1B\",\n",
    "    \"InternVL3-14B\": \"14B\",\n",
    "    \"InternVL3-8B\": \"8B\",\n",
    "    \"llava-1.5-7b-hf\": \"7B\",\n",
    "    \"llava-1.5-13b-hf\": \"13B\",\n",
    "    \"Llama-3.2-90B-Vision\": \"90B\",\n",
    "    \"Llama-3.2-11B-Vision\": \"11B\",\n",
    "    \"llava-v1.6-vicuna-13b-hf\": \"13B\",\n",
    "    \"llava-v1.6-vicuna-7b-hf\": \"7B\"\n",
    "}\n",
    "\n",
    "# Manual order for each family - Updated to include new families\n",
    "manual_order = {\n",
    "    \"Qwen-2-VL\": [\"Qwen2-VL-2B\", \"Qwen2-VL-7B\", \"Qwen2-VL-72B\"],\n",
    "    \"Qwen-2.5-VL\": [\"Qwen2.5-VL-3B-Instruct\", \"Qwen2.5-VL-7B-Instruct\",\"Qwen2.5-VL-72B-Instruct\"],\n",
    "    \"Gemma-3\": [\"gemma-3-4b-pt\", \"gemma-3-12b-pt\", \"gemma-3-27b-pt\"],\n",
    "    # Added families manual order\n",
    "    \"InternVL-3\": [\"InternVL3-1B\", \"InternVL3-8B\", \"InternVL3-14B\"], # Example order\n",
    "    \"LLaVA-1.5\": [\"llava-1.5-7b-hf\", \"llava-1.5-13b-hf\"], # Example order\n",
    "    \"Llama-3.2-Vision\": [\"Llama-3.2-11B-Vision\", \"Llama-3.2-90B-Vision\"], # Example order\n",
    "    \"LLaVA-1.6\": [\"llava-v1.6-vicuna-7b-hf\", \"llava-v1.6-vicuna-13b-hf\"] # Example order\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 main_all_vertical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 262236 characters]",
      "text/plain": [
       "<Figure size 1200x2800 with 21 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Re-import and reload data after kernel reset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import font_manager as fm\n",
    "\n",
    "# font_prop = fm.FontProperties(fname=\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/Times New Roman.ttf\")\n",
    "# print(font_prop.get_name())\n",
    "# plt.rcParams['font.family'] = font_prop.get_name()\n",
    "plt.rcParams['xtick.labelsize'] = 14\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['axes.titlesize'] = 16\n",
    "plt.rcParams['legend.title_fontsize'] = 10\n",
    "plt.rcParams['legend.fontsize'] = 10\n",
    "\n",
    "# Load data - Corrected file paths to use uploaded files\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag datasets\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Helper functions\n",
    "def extract_patch_ratio(path):\n",
    "    if \"config_image\" in path:\n",
    "        return 1\n",
    "    elif \"config_patch/2x2\" in path:\n",
    "        return 2\n",
    "    elif \"config_patch/4x4\" in path:\n",
    "        return 4\n",
    "    elif \"config_patch/8x8\" in path:\n",
    "        return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epochs\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "\n",
    "# Filter data\n",
    "df_filtered = df_all[df_all[\"model_name\"].isin(target_models.keys())]\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "\n",
    "# Plot setup\n",
    "x_ticks = [1, 2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.20 # Adjusted bar width\n",
    "dataset_order = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "# Updated family order to include new families\n",
    "family_order = [\"Qwen-2-VL\", \"Qwen-2.5-VL\", \"Llama-3.2-Vision\", \"Gemma-3\", \"InternVL-3\", \"LLaVA-1.5\", \"LLaVA-1.6\"]\n",
    "\n",
    "subplot_width = 4    # in inches\n",
    "subplot_height = 4 # in inches\n",
    "\n",
    "# Total figure size adjusted for 7 rows\n",
    "fig_width = subplot_width * 3\n",
    "fig_height = subplot_height * 7\n",
    "\n",
    "fig, axes = plt.subplots(7, 3, figsize=(fig_width, fig_height), sharey=True)\n",
    "\n",
    "for row, family in enumerate(family_order):\n",
    "    handles_row = []\n",
    "    labels_row = []\n",
    "    for col, dataset in enumerate(dataset_order):\n",
    "        ax = axes[row, col]\n",
    "\n",
    "        # Modified filtering for Llama-3.2-Vision family\n",
    "        if family == \"Llama-3.2-Vision\":\n",
    "            df_dataset = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                     (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "        else:\n",
    "            df_dataset = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                     (df_filtered[\"model_name\"].map(model_family) == family)]\n",
    "\n",
    "\n",
    "        models = manual_order.get(family, []) # Use .get to handle families not in manual_order\n",
    "\n",
    "        for i, model in enumerate(models):\n",
    "            offset = (i - (len(models) - 1) / 2) * bar_width # Center the bars for each family\n",
    "            df_model = df_dataset[df_dataset[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "            upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "            upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "            lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "            lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "            alpha = alpha_map.get(model, 0.5) # Use .get with default alpha\n",
    "            color = color_bases.get(family, \"#000000\") # Use .get with default color (black)\n",
    "\n",
    "            # Get parameter size for legend label\n",
    "            label = param_sizes.get(model, model)\n",
    "\n",
    "            # Plot bars and capture handle\n",
    "            handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                            label=label, color=color, alpha=alpha, capsize=3)\n",
    "            ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                   color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "            # Store handle and label for row legend, ensure no duplicates\n",
    "            # Check if the label is already in labels_row to avoid duplicate legend entries per row\n",
    "            if label not in labels_row:\n",
    "                 handles_row.append(handle)\n",
    "                 labels_row.append(label)\n",
    "\n",
    "\n",
    "        # ax.set_ylim(y_min, y_max) # Keep sharey=True, so don't set limits per subplot here\n",
    "        ax.set_title(dataset if row == 0 else \"\")\n",
    "        ax.set_xticks(x)\n",
    "        ax.set_xticklabels(x_ticks)\n",
    "        ax.set_xlabel(r'Split Factor $(f)$' if (row == 6 and col == 1) else \"\") # Adjusted row index\n",
    "        ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "        ax.axhline(0, color='black', linewidth=0.8)\n",
    "        if col == 0:\n",
    "            ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "\n",
    "        # Set y-tick labels to their absolute values for each subplot\n",
    "        ax.set_ylim([-5, 10])\n",
    "        ax.set_yticks([-5, 0, 5, 10])\n",
    "        yticks = ax.get_yticks()\n",
    "        ax.set_yticklabels([f'{abs(tick)}' for tick in yticks])\n",
    "\n",
    "        # Add y-axis labels using ax.text() for the leftmost column\n",
    "        if col == 0:\n",
    "            ax.text(-0.10, 0.67, 'reference', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "            ax.text(-0.10, 0.20, 'image', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "            # Add family label on the left\n",
    "            ax.text(-0.20, 0.5, family, transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=16, fontweight='bold')\n",
    "\n",
    "\n",
    "    # After plotting all columns for the row, add the legend to the rightmost subplot\n",
    "    if handles_row: # Only add legend if there are models in the row\n",
    "        # Need to recreate handles and labels for the legend to avoid issues with duplicate labels\n",
    "        # Get handles and labels from the rightmost subplot's current artists\n",
    "        handles, labels = axes[row, 2].get_legend_handles_labels()\n",
    "        # Filter for unique labels based on the desired order\n",
    "        unique_labels = []\n",
    "        unique_handles = []\n",
    "        for model in manual_order.get(family, []):\n",
    "            param_size_label = param_sizes.get(model, model)\n",
    "            if param_size_label not in unique_labels:\n",
    "                try:\n",
    "                    # Find the handle corresponding to this label\n",
    "                    index = labels.index(param_size_label)\n",
    "                    unique_handles.append(handles[index])\n",
    "                    unique_labels.append(param_size_label)\n",
    "                except ValueError:\n",
    "                    # Handle cases where a model in manual_order might not have a corresponding handle if data was missing\n",
    "                    pass\n",
    "\n",
    "\n",
    "        if unique_handles:\n",
    "             axes[row, 2].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower right')\n",
    "\n",
    "\n",
    "\n",
    "for ax in axes.flatten():\n",
    "    for spine in ax.spines.values():\n",
    "        spine.set_linewidth(2)\n",
    "\n",
    "# Adjust layout to make space for the legends on the right\n",
    "plt.tight_layout(rect=[0, 0.03, 0.95, 0.97]) # Adjusted right value to make space for legends\n",
    "plt.savefig(\"main_all_vertical.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.4 main_selected_vertical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 156536 characters]",
      "text/plain": [
       "<Figure size 1200x1600 with 12 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Re-import and reload data after kernel reset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import font_manager as fm\n",
    "\n",
    "# font_prop = fm.FontProperties(fname=\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/Times New Roman.ttf\")\n",
    "# print(font_prop.get_name())\n",
    "# plt.rcParams['font.family'] = font_prop.get_name()\n",
    "plt.rcParams['xtick.labelsize'] = 14\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['axes.titlesize'] = 16\n",
    "plt.rcParams['legend.title_fontsize'] = 10\n",
    "plt.rcParams['legend.fontsize'] = 10\n",
    "\n",
    "# Load data - Corrected file paths to use uploaded files\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag datasets\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Helper functions\n",
    "def extract_patch_ratio(path):\n",
    "    if \"config_image\" in path:\n",
    "        return 1\n",
    "    elif \"config_patch/2x2\" in path:\n",
    "        return 2\n",
    "    elif \"config_patch/4x4\" in path:\n",
    "        return 4\n",
    "    elif \"config_patch/8x8\" in path:\n",
    "        return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epochs\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "\n",
    "# Filter data\n",
    "df_filtered = df_all[df_all[\"model_name\"].isin(target_models.keys())]\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "\n",
    "# Plot setup\n",
    "x_ticks = [1, 2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.20 # Adjusted bar width\n",
    "dataset_order = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "# Updated family order to include new families\n",
    "family_order = [\"Qwen-2-VL\", \"Llama-3.2-Vision\", \"Gemma-3\", \"InternVL-3\"]\n",
    "\n",
    "subplot_width = 4    # in inches\n",
    "subplot_height = 4 # in inches\n",
    "\n",
    "# Total figure size adjusted for 7 rows\n",
    "fig_width = subplot_width * 3\n",
    "fig_height = subplot_height * 4\n",
    "\n",
    "fig, axes = plt.subplots(4, 3, figsize=(fig_width, fig_height), sharey=True)\n",
    "\n",
    "for row, family in enumerate(family_order):\n",
    "    handles_row = []\n",
    "    labels_row = []\n",
    "    for col, dataset in enumerate(dataset_order):\n",
    "        ax = axes[row, col]\n",
    "\n",
    "        # Modified filtering for Llama-3.2-Vision family\n",
    "        if family == \"Llama-3.2-Vision\":\n",
    "            df_dataset = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                     (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "        else:\n",
    "            df_dataset = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                     (df_filtered[\"model_name\"].map(model_family) == family)]\n",
    "\n",
    "\n",
    "        models = manual_order.get(family, []) # Use .get to handle families not in manual_order\n",
    "\n",
    "        for i, model in enumerate(models):\n",
    "            offset = (i - (len(models) - 1) / 2) * bar_width # Center the bars for each family\n",
    "            df_model = df_dataset[df_dataset[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "            upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "            upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "            lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "            lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "            alpha = alpha_map.get(model, 0.5) # Use .get with default alpha\n",
    "            color = color_bases.get(family, \"#000000\") # Use .get with default color (black)\n",
    "\n",
    "            # Get parameter size for legend label\n",
    "            label = param_sizes.get(model, model)\n",
    "\n",
    "            # Plot bars and capture handle\n",
    "            handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                            label=label, color=color, alpha=alpha, capsize=3)\n",
    "            ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                   color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "            # Store handle and label for row legend, ensure no duplicates\n",
    "            # Check if the label is already in labels_row to avoid duplicate legend entries per row\n",
    "            if label not in labels_row:\n",
    "                 handles_row.append(handle)\n",
    "                 labels_row.append(label)\n",
    "\n",
    "\n",
    "        # ax.set_ylim(y_min, y_max) # Keep sharey=True, so don't set limits per subplot here\n",
    "        ax.set_title(dataset if row == 0 else \"\")\n",
    "        ax.set_xticks(x)\n",
    "        ax.set_xticklabels(x_ticks)\n",
    "        ax.set_xlabel(r'Split Factor $(f)$' if (row == 3 and col == 1) else \"\") # Adjusted row index\n",
    "        ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "        ax.axhline(0, color='black', linewidth=0.8)\n",
    "        if col == 0:\n",
    "            ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "\n",
    "        # Set y-tick labels to their absolute values for each subplot\n",
    "        ax.set_ylim([-5, 10])\n",
    "        ax.set_yticks([-5, 0, 5, 10])\n",
    "        yticks = ax.get_yticks()\n",
    "        ax.set_yticklabels([f'{abs(tick)}' for tick in yticks])\n",
    "\n",
    "        # Add y-axis labels using ax.text() for the leftmost column\n",
    "        if col == 0:\n",
    "            ax.text(-0.10, 0.67, 'reference', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "            ax.text(-0.10, 0.20, 'image', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "            # Add family label on the left\n",
    "            ax.text(-0.20, 0.5, family, transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=16, fontweight='bold')\n",
    "\n",
    "\n",
    "    # After plotting all columns for the row, add the legend to the rightmost subplot\n",
    "    if handles_row: # Only add legend if there are models in the row\n",
    "        # Need to recreate handles and labels for the legend to avoid issues with duplicate labels\n",
    "        # Get handles and labels from the rightmost subplot's current artists\n",
    "        handles, labels = axes[row, 2].get_legend_handles_labels()\n",
    "        # Filter for unique labels based on the desired order\n",
    "        unique_labels = []\n",
    "        unique_handles = []\n",
    "        for model in manual_order.get(family, []):\n",
    "            param_size_label = param_sizes.get(model, model)\n",
    "            if param_size_label not in unique_labels:\n",
    "                try:\n",
    "                    # Find the handle corresponding to this label\n",
    "                    index = labels.index(param_size_label)\n",
    "                    unique_handles.append(handles[index])\n",
    "                    unique_labels.append(param_size_label)\n",
    "                except ValueError:\n",
    "                    # Handle cases where a model in manual_order might not have a corresponding handle if data was missing\n",
    "                    pass\n",
    "\n",
    "\n",
    "        if unique_handles:\n",
    "             axes[row, 2].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower right')\n",
    "\n",
    "\n",
    "\n",
    "for ax in axes.flatten():\n",
    "    for spine in ax.spines.values():\n",
    "        spine.set_linewidth(2)\n",
    "\n",
    "# Adjust layout to make space for the legends on the right\n",
    "plt.tight_layout(rect=[0, 0.03, 0.95, 0.97]) # Adjusted right value to make space for legends\n",
    "plt.savefig(\"main_selected_vertical.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.5 main_all_horizontal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 216464 characters]",
      "text/plain": [
       "<Figure size 1800x1600 with 24 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Re-import and reload data after kernel reset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import font_manager as fm\n",
    "import math\n",
    "\n",
    "# font_prop = fm.FontProperties(fname=\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/Times New Roman.ttf\")\n",
    "# print(font_prop.get_name())\n",
    "# plt.rcParams['font.family'] = font_prop.get_name()\n",
    "plt.rcParams['xtick.labelsize'] = 14\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['axes.titlesize'] = 16\n",
    "plt.rcParams['legend.title_fontsize'] = 10\n",
    "plt.rcParams['legend.fontsize'] = 10\n",
    "\n",
    "# Load data - Corrected file paths to use uploaded files\n",
    "# PLEASE ENSURE THESE FILES ARE IN THE SAME DIRECTORY AS THE SCRIPT\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag datasets\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Helper functions\n",
    "def extract_patch_ratio(path):\n",
    "    if \"config_image\" in path:\n",
    "        return 1\n",
    "    elif \"config_patch/2x2\" in path:\n",
    "        return 2\n",
    "    elif \"config_patch/4x4\" in path:\n",
    "        return 4\n",
    "    elif \"config_patch/8x8\" in path:\n",
    "        return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epochs\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "\n",
    "\n",
    "df_filtered = df_all[df_all[\"model_name\"].isin(target_models.keys())]\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "\n",
    "x_ticks = [1, 2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.20\n",
    "dataset_order = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "# family_order = [\"Qwen2\", \"Qwen2.5\", \"gemma\", \"InternVL3\", \"llava_1.5\", \"Llama-3.2-Vision\", \"llava_1.6\"]\n",
    "family_order = [\"Qwen-2-VL\", \"Qwen-2.5-VL\", \"Llama-3.2-Vision\", \"Gemma-3\", \"LLaVA-1.5\", \"InternVL-3\", \"LLaVA-1.6\"]\n",
    "\n",
    "family_pairs = list(zip(family_order[0::2], family_order[1::2]))\n",
    "if len(family_order) % 2 != 0:\n",
    "    family_pairs.append((family_order[-1], None))\n",
    "\n",
    "\n",
    "subplot_width = 3\n",
    "subplot_height = 4\n",
    "num_rows = math.ceil(len(family_order) / 2)\n",
    "num_cols = 6\n",
    "fig_width = num_cols * subplot_width\n",
    "fig_height = num_rows * subplot_height\n",
    "\n",
    "# Disable sharey\n",
    "fig, axes = plt.subplots(num_rows, num_cols, figsize=(fig_width, fig_height))\n",
    "\n",
    "# Ensure axes is always 2D\n",
    "if num_rows == 1:\n",
    "    axes = axes.reshape(1, -1)\n",
    "\n",
    "# Define fixed y-axis limits and ticks\n",
    "y_min = -5\n",
    "y_max = 10\n",
    "y_tick_interval = 5 # Changed tick interval to 5\n",
    "yticks = np.arange(y_min, y_max + y_tick_interval/2, y_tick_interval) # Added small tolerance to include y_max\n",
    "\n",
    "\n",
    "for row_idx, family_pair in enumerate(family_pairs):\n",
    "\n",
    "    # --- Plot for family1 (left side of the row) ---\n",
    "    family1 = family_pair[0]\n",
    "    if family1:\n",
    "        handles_family1 = []\n",
    "        labels_family1 = []\n",
    "        axes_family1 = [axes[row_idx, col_idx] for col_idx in range(len(dataset_order))] # Collect axes for family1\n",
    "\n",
    "        for col_idx, dataset in enumerate(dataset_order):\n",
    "            ax = axes_family1[col_idx]\n",
    "\n",
    "            if family1 == \"Llama-3.2-Vision\":\n",
    "                df_dataset_family1 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "            else:\n",
    "                df_dataset_family1 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].map(model_family) == family1)]\n",
    "\n",
    "\n",
    "            models = manual_order.get(family1, [])\n",
    "\n",
    "            for i, model in enumerate(models):\n",
    "                offset = (i - (len(models) - 1) / 2) * bar_width\n",
    "                # Apply the filter to df_dataset_family1 before setting index\n",
    "                df_model = df_dataset_family1[df_dataset_family1[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "                upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "                upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "                lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "                lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "                alpha = alpha_map.get(model, 0.5)\n",
    "                color = color_bases.get(family1, \"#000000\")\n",
    "                label = param_sizes.get(model, model)\n",
    "\n",
    "                handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                                label=label, color=color, alpha=alpha, capsize=3)\n",
    "                ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                       color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "                if label not in labels_family1:\n",
    "                     handles_family1.append(handle)\n",
    "                     labels_family1.append(label)\n",
    "\n",
    "            ax.set_xticks(x)\n",
    "            ax.set_xticklabels(x_ticks)\n",
    "            ax.axhline(0, color='black', linewidth=0.8)\n",
    "            ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "\n",
    "\n",
    "            # Y-axis and Family label for the first family block\n",
    "            if col_idx == 0:\n",
    "                 \n",
    "                 ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "\n",
    "                 ax.tick_params(labelleft=True)\n",
    "                 ax.text(-0.15, 0.67, 'reference', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "                 ax.text(-0.15, 0.20, 'image', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "\n",
    "                 ax.text(-0.25, 0.5, family1, transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=16, fontweight='bold')\n",
    "            else:\n",
    "                 ax.tick_params(labelleft=False)\n",
    "\n",
    "        # Set fixed y-axis limits and ticks for family1 subplots\n",
    "        for ax in axes_family1:\n",
    "            ax.set_ylim(y_min, y_max)\n",
    "            ax.set_yticks(yticks)\n",
    "            ax.set_yticklabels([f'{int(abs(tick))}' for tick in yticks])\n",
    "\n",
    "\n",
    "        # Add legend for family1 to the first subplot of the row (Column 0)\n",
    "        if handles_family1:\n",
    "            unique_labels = []\n",
    "            unique_handles = []\n",
    "            for model in manual_order.get(family1, []):\n",
    "                param_size_label = param_sizes.get(model, model)\n",
    "                if param_size_label in labels_family1 and param_size_label not in unique_labels:\n",
    "                    try:\n",
    "                        index = labels_family1.index(param_size_label)\n",
    "                        unique_handles.append(handles_family1[index])\n",
    "                        unique_labels.append(param_size_label)\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "            if unique_handles:\n",
    "                axes_family1[0].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower left', frameon=False, fontsize=8)\n",
    "\n",
    "\n",
    "    # --- Plot for family2 (right side of the row) ---\n",
    "    family2 = family_pair[1]\n",
    "    if family2:\n",
    "        handles_family2 = []\n",
    "        labels_family2 = []\n",
    "        axes_family2 = [axes[row_idx, len(dataset_order) + col_idx] for col_idx in range(len(dataset_order))] # Collect axes for family2\n",
    "\n",
    "        for col_idx, dataset in enumerate(dataset_order):\n",
    "            ax = axes_family2[col_idx] # Columns 3, 4, 5\n",
    "\n",
    "            if family2 == \"Llama-3.2-Vision\":\n",
    "                df_dataset_family2 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "            else:\n",
    "                df_dataset_family2 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].map(model_family) == family2)]\n",
    "\n",
    "            models = manual_order.get(family2, [])\n",
    "\n",
    "            for i, model in enumerate(models):\n",
    "                offset = (i - (len(models) - 1) / 2) * bar_width\n",
    "                # Apply the filter to df_dataset_family2 before setting index\n",
    "                df_model = df_dataset_family2[df_dataset_family2[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "                upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "                upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "                lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "                lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "                alpha = alpha_map.get(model, 0.5)\n",
    "                color = color_bases.get(family2, \"#000000\")\n",
    "                label = param_sizes.get(model, model)\n",
    "\n",
    "                handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                                label=label, color=color, alpha=alpha, capsize=3)\n",
    "                ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                       color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "                if label not in labels_family2:\n",
    "                     handles_family2.append(handle)\n",
    "                     labels_family2.append(label)\n",
    "\n",
    "            ax.set_xticks(x)\n",
    "            ax.set_xticklabels(x_ticks)\n",
    "            ax.axhline(0, color='black', linewidth=0.8)\n",
    "\n",
    "            # Y-axis for the second family block (on the right)\n",
    "            ax.tick_params(labelleft=False)\n",
    "            ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "\n",
    "            if col_idx == len(dataset_order) - 1: # Rightmost subplot of the row (column 5)\n",
    "                 \n",
    "                 ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "                 ax2 = ax.twinx()\n",
    "                 ax2.set_ylim(y_min, y_max) # Set twin axis limits\n",
    "                 ax2.tick_params(labelright=True)\n",
    "                 # Add 'image' and 'reference' text to the rightmost subplot\n",
    "                 ax.text(1.15, 0.67, 'reference', transform=ax.transAxes,\n",
    "                         rotation=-90, va='center', ha='left', fontsize=14)\n",
    "                 ax.text(1.15, 0.20, 'image', transform=ax.transAxes,\n",
    "                         rotation=-90, va='center', ha='left', fontsize=14)\n",
    "\n",
    "                 # Set fixed y-ticks for the twin axis and label with absolute values\n",
    "                 ax2.set_yticks(yticks)\n",
    "                 ax2.set_yticklabels([f'{int(abs(tick))}' for tick in yticks])\n",
    "\n",
    "\n",
    "            else:\n",
    "                 ax.tick_params(labelright=False)\n",
    "\n",
    "        # Set fixed y-axis limits and ticks for family2 subplots\n",
    "        for ax in axes_family2:\n",
    "             ax.set_ylim(y_min, y_max)\n",
    "             ax.set_yticks(yticks)\n",
    "             ax.set_yticklabels([f'{int(abs(tick))}' for tick in yticks])\n",
    "\n",
    "\n",
    "        # Add family label for the second family block to the rightmost subplot\n",
    "        axes_family2[-1].text(1.25, 0.5, family2, transform=axes_family2[-1].transAxes,\n",
    "                              rotation=-90, va='center', ha='left', fontsize=16, fontweight='bold')\n",
    "\n",
    "        # Add legend for family2 to the rightmost subplot of its block (Column 5)\n",
    "        if handles_family2:\n",
    "            unique_labels = []\n",
    "            unique_handles = []\n",
    "            for model in manual_order.get(family2, []):\n",
    "                param_size_label = param_sizes.get(model, model)\n",
    "                if param_size_label in labels_family2 and param_size_label not in unique_labels:\n",
    "                    try:\n",
    "                        index = labels_family2.index(param_size_label)\n",
    "                        unique_handles.append(handles_family2[index])\n",
    "                        unique_labels.append(param_size_label)\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "\n",
    "            if unique_handles:\n",
    "                axes_family2[-1].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower left', frameon=False, fontsize=8) # Column 5\n",
    "\n",
    "\n",
    "# Remove the last three subplots if the last family pair has None as the second family\n",
    "if family_pairs[-1][1] is None:\n",
    "    last_row_idx = num_rows - 1\n",
    "    for col_idx in range(len(dataset_order), num_cols):\n",
    "        # Check if the axis exists before trying to delete it\n",
    "        if axes[last_row_idx, col_idx] in fig.get_axes():\n",
    "            fig.delaxes(axes[last_row_idx, col_idx])\n",
    "\n",
    "\n",
    "# Add titles to the top row subplots\n",
    "for col_idx in range(num_cols):\n",
    "    # Check if this column index corresponds to a dataset column in the first row\n",
    "    # Also ensure the axis still exists after potential deletion\n",
    "    if (col_idx < len(dataset_order) or (col_idx >= len(dataset_order) and col_idx < 2 * len(dataset_order))) and axes[0, col_idx] in fig.get_axes():\n",
    "        # Determine the dataset index based on the column index\n",
    "        dataset_idx = col_idx % len(dataset_order)\n",
    "        axes[0, col_idx].set_title(dataset_order[dataset_idx])\n",
    "\n",
    "# Add a single xlabel centered at the bottom of the figure\n",
    "fig.text(0.5, 0.04, r'Split Factor $(f)$', ha='center', va='top', fontsize=16)\n",
    "\n",
    "\n",
    "for ax_row in axes:\n",
    "    for ax in ax_row:\n",
    "        # Check if the axis is not one of the deleted ones\n",
    "        if ax in fig.get_axes():\n",
    "            for spine in ax.spines.values():\n",
    "                spine.set_linewidth(2)\n",
    "        else:\n",
    "            # If the axis was deleted, remove its spines as well (though fig.delaxes should handle this)\n",
    "            for spine in ax.spines.values():\n",
    "                 spine.set_visible(False)\n",
    "\n",
    "\n",
    "# Adjust layout\n",
    "fig.subplots_adjust(left=0.05, right=0.95, bottom=0.08, top=0.88, wspace=0.05, hspace=0.2) # Reduced hspace\n",
    "plt.savefig(\"main_all_horizontal.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.6 main_selected_horizontal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 124336 characters]",
      "text/plain": [
       "<Figure size 1800x700 with 14 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Re-import and reload data after kernel reset\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import font_manager as fm\n",
    "import math\n",
    "\n",
    "# font_prop = fm.FontProperties(fname=\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/Times New Roman.ttf\")\n",
    "# print(font_prop.get_name())\n",
    "# plt.rcParams['font.family'] = font_prop.get_name()\n",
    "plt.rcParams['xtick.labelsize'] = 14\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['axes.titlesize'] = 16\n",
    "plt.rcParams['legend.title_fontsize'] = 10\n",
    "plt.rcParams['legend.fontsize'] = 10\n",
    "\n",
    "# Load data - Corrected file paths to use uploaded files\n",
    "# PLEASE ENSURE THESE FILES ARE IN THE SAME DIRECTORY AS THE SCRIPT\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag datasets\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Helper functions\n",
    "def extract_patch_ratio(path):\n",
    "    if \"config_image\" in path:\n",
    "        return 1\n",
    "    elif \"config_patch/2x2\" in path:\n",
    "        return 2\n",
    "    elif \"config_patch/4x4\" in path:\n",
    "        return 4\n",
    "    elif \"config_patch/8x8\" in path:\n",
    "        return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epochs\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "\n",
    "\n",
    "df_filtered = df_all[df_all[\"model_name\"].isin(target_models.keys())]\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "\n",
    "x_ticks = [1, 2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.20\n",
    "dataset_order = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "family_order = [\"Qwen-2-VL\", \"InternVL-3\", \"Gemma-3\",\"Llama-3.2-Vision\"]\n",
    "\n",
    "family_pairs = list(zip(family_order[0::2], family_order[1::2]))\n",
    "if len(family_order) % 2 != 0:\n",
    "    family_pairs.append((family_order[-1], None))\n",
    "\n",
    "\n",
    "subplot_width = 3\n",
    "subplot_height = 3.5\n",
    "num_rows = math.ceil(len(family_order) / 2)\n",
    "num_cols = 6\n",
    "fig_width = num_cols * subplot_width\n",
    "fig_height = num_rows * subplot_height\n",
    "\n",
    "# Disable sharey\n",
    "fig, axes = plt.subplots(num_rows, num_cols, figsize=(fig_width, fig_height))\n",
    "\n",
    "# Ensure axes is always 2D\n",
    "if num_rows == 1:\n",
    "    axes = axes.reshape(1, -1)\n",
    "\n",
    "# Define fixed y-axis limits and ticks\n",
    "y_min = -5\n",
    "y_max = 10\n",
    "y_tick_interval = 5 # Changed tick interval to 5\n",
    "yticks = np.arange(y_min, y_max + y_tick_interval/2, y_tick_interval) # Added small tolerance to include y_max\n",
    "\n",
    "\n",
    "for row_idx, family_pair in enumerate(family_pairs):\n",
    "\n",
    "    # --- Plot for family1 (left side of the row) ---\n",
    "    family1 = family_pair[0]\n",
    "    if family1:\n",
    "        handles_family1 = []\n",
    "        labels_family1 = []\n",
    "        axes_family1 = [axes[row_idx, col_idx] for col_idx in range(len(dataset_order))] # Collect axes for family1\n",
    "\n",
    "        for col_idx, dataset in enumerate(dataset_order):\n",
    "            ax = axes_family1[col_idx]\n",
    "\n",
    "            if family1 == \"Llama-3.2-Vision\":\n",
    "                df_dataset_family1 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "            else:\n",
    "                df_dataset_family1 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].map(model_family) == family1)]\n",
    "\n",
    "\n",
    "            models = manual_order.get(family1, [])\n",
    "\n",
    "            for i, model in enumerate(models):\n",
    "                offset = (i - (len(models) - 1) / 2) * bar_width\n",
    "                # Apply the filter to df_dataset_family1 before setting index\n",
    "                df_model = df_dataset_family1[df_dataset_family1[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "                upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "                upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "                lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "                lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "                alpha = alpha_map.get(model, 0.5)\n",
    "                color = color_bases.get(family1, \"#000000\")\n",
    "                label = param_sizes.get(model, model)\n",
    "\n",
    "                handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                                label=label, color=color, alpha=alpha, capsize=3)\n",
    "                ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                       color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "                if label not in labels_family1:\n",
    "                     handles_family1.append(handle)\n",
    "                     labels_family1.append(label)\n",
    "\n",
    "            ax.set_xticks(x)\n",
    "            ax.set_xticklabels(x_ticks)\n",
    "            ax.axhline(0, color='black', linewidth=0.8)\n",
    "            ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "\n",
    "            # Y-axis and Family label for the first family block\n",
    "            if col_idx == 0:\n",
    "                 \n",
    "                 ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "\n",
    "                 ax.tick_params(labelleft=True)\n",
    "                 ax.text(-0.15, 0.67, 'reference', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "                 ax.text(-0.15, 0.15, 'image', transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=14)\n",
    "\n",
    "                 ax.text(-0.25, 0.5, family1, transform=ax.transAxes,\n",
    "                     rotation=90, va='center', ha='right', fontsize=16, fontweight='bold')\n",
    "            else:\n",
    "                 ax.tick_params(labelleft=False)\n",
    "\n",
    "        # Set fixed y-axis limits and ticks for family1 subplots\n",
    "        for ax in axes_family1:\n",
    "            ax.set_ylim(y_min, y_max)\n",
    "            ax.set_yticks(yticks)\n",
    "            ax.set_yticklabels([f'{int(abs(tick))}' for tick in yticks])\n",
    "\n",
    "\n",
    "        # Add legend for family1 to the first subplot of the row (Column 0)\n",
    "        if handles_family1:\n",
    "            unique_labels = []\n",
    "            unique_handles = []\n",
    "            for model in manual_order.get(family1, []):\n",
    "                param_size_label = param_sizes.get(model, model)\n",
    "                if param_size_label in labels_family1 and param_size_label not in unique_labels:\n",
    "                    try:\n",
    "                        index = labels_family1.index(param_size_label)\n",
    "                        unique_handles.append(handles_family1[index])\n",
    "                        unique_labels.append(param_size_label)\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "            if unique_handles:\n",
    "                axes_family1[0].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower left', frameon=False, fontsize=8)\n",
    "\n",
    "\n",
    "    # --- Plot for family2 (right side of the row) ---\n",
    "    family2 = family_pair[1]\n",
    "    if family2:\n",
    "        handles_family2 = []\n",
    "        labels_family2 = []\n",
    "        axes_family2 = [axes[row_idx, len(dataset_order) + col_idx] for col_idx in range(len(dataset_order))] # Collect axes for family2\n",
    "\n",
    "        for col_idx, dataset in enumerate(dataset_order):\n",
    "            ax = axes_family2[col_idx] # Columns 3, 4, 5\n",
    "\n",
    "            if family2 == \"Llama-3.2-Vision\":\n",
    "                df_dataset_family2 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].isin(manual_order[\"Llama-3.2-Vision\"]))]\n",
    "            else:\n",
    "                df_dataset_family2 = df_filtered[(df_filtered[\"dataset\"] == dataset) &\n",
    "                                         (df_filtered[\"model_name\"].map(model_family) == family2)]\n",
    "\n",
    "            models = manual_order.get(family2, [])\n",
    "\n",
    "            for i, model in enumerate(models):\n",
    "                offset = (i - (len(models) - 1) / 2) * bar_width\n",
    "                # Apply the filter to df_dataset_family2 before setting index\n",
    "                df_model = df_dataset_family2[df_dataset_family2[\"model_name\"] == model].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "                upper_means = df_model[\"eval-1 mean\"].fillna(0).values\n",
    "                upper_stds = df_model[\"eval-1 std\"].fillna(0).values\n",
    "                lower_means = df_model[\"eval-0 mean\"].fillna(0).values\n",
    "                lower_stds = df_model[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "                alpha = alpha_map.get(model, 0.5)\n",
    "                color = color_bases.get(family2, \"#000000\")\n",
    "                label = param_sizes.get(model, model)\n",
    "\n",
    "                handle = ax.bar(x + offset, upper_means, bar_width, yerr=upper_stds,\n",
    "                                label=label, color=color, alpha=alpha, capsize=3)\n",
    "                ax.bar(x + offset, -lower_means, bar_width, yerr=lower_stds,\n",
    "                       color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "                if label not in labels_family2:\n",
    "                     handles_family2.append(handle)\n",
    "                     labels_family2.append(label)\n",
    "\n",
    "            ax.set_xticks(x)\n",
    "            ax.set_xticklabels(x_ticks)\n",
    "            ax.axhline(0, color='black', linewidth=0.8)\n",
    "\n",
    "            # Y-axis for the second family block (on the right)\n",
    "            ax.tick_params(labelleft=False)\n",
    "            ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "\n",
    "            if col_idx == len(dataset_order) - 1: # Rightmost subplot of the row (column 5)\n",
    "                 \n",
    "                 ax.text(x[len(x)//2] - 0.5, 8.5, \"Random Baseline\", color='gray', ha='center', fontsize=12)\n",
    "                 ax2 = ax.twinx()\n",
    "                 ax2.set_ylim(y_min, y_max) # Set twin axis limits\n",
    "                 ax2.tick_params(labelright=True)\n",
    "                 # Add 'image' and 'reference' text to the rightmost subplot\n",
    "                 ax.text(1.15, 0.67, 'reference', transform=ax.transAxes,\n",
    "                         rotation=-90, va='center', ha='left', fontsize=14)\n",
    "                 ax.text(1.15, 0.15, 'image', transform=ax.transAxes,\n",
    "                         rotation=-90, va='center', ha='left', fontsize=14)\n",
    "\n",
    "                 # Set fixed y-ticks for the twin axis and label with absolute values\n",
    "                 ax2.set_yticks(yticks)\n",
    "                 ax2.set_yticklabels([f'{int(abs(tick))}' for tick in yticks])\n",
    "\n",
    "\n",
    "            else:\n",
    "                 ax.tick_params(labelright=False)\n",
    "\n",
    "        # Set fixed y-axis limits and ticks for family2 subplots\n",
    "        for ax in axes_family2:\n",
    "             ax.set_ylim(y_min, y_max)\n",
    "             ax.set_yticks(yticks)\n",
    "             ax.set_yticklabels([f'{abs(tick)}' for tick in yticks])\n",
    "\n",
    "\n",
    "        # Add family label for the second family block to the rightmost subplot\n",
    "        axes_family2[-1].text(1.25, 0.5, family2, transform=axes_family2[-1].transAxes,\n",
    "                              rotation=-90, va='center', ha='left', fontsize=16, fontweight='bold')\n",
    "\n",
    "        # Add legend for family2 to the rightmost subplot of its block (Column 5)\n",
    "        if handles_family2:\n",
    "            unique_labels = []\n",
    "            unique_handles = []\n",
    "            for model in manual_order.get(family2, []):\n",
    "                param_size_label = param_sizes.get(model, model)\n",
    "                if param_size_label in labels_family2 and param_size_label not in unique_labels:\n",
    "                    try:\n",
    "                        index = labels_family2.index(param_size_label)\n",
    "                        unique_handles.append(handles_family2[index])\n",
    "                        unique_labels.append(param_size_label)\n",
    "                    except ValueError:\n",
    "                        pass\n",
    "\n",
    "            if unique_handles:\n",
    "                axes_family2[-1].legend(unique_handles, unique_labels, title=\"Parameter Size\", loc='lower left', frameon=False, fontsize=8) # Column 5\n",
    "\n",
    "\n",
    "# Remove the last three subplots if the last family pair has None as the second family\n",
    "if family_pairs[-1][1] is None:\n",
    "    last_row_idx = num_rows - 1\n",
    "    for col_idx in range(len(dataset_order), num_cols):\n",
    "        # Check if the axis exists before trying to delete it\n",
    "        if axes[last_row_idx, col_idx] in fig.get_axes():\n",
    "            fig.delaxes(axes[last_row_idx, col_idx])\n",
    "\n",
    "\n",
    "# Add titles to the top row subplots\n",
    "for col_idx in range(num_cols):\n",
    "    # Check if this column index corresponds to a dataset column in the first row\n",
    "    # Also ensure the axis still exists after potential deletion\n",
    "    if (col_idx < len(dataset_order) or (col_idx >= len(dataset_order) and col_idx < 2 * len(dataset_order))) and axes[0, col_idx] in fig.get_axes():\n",
    "        # Determine the dataset index based on the column index\n",
    "        dataset_idx = col_idx % len(dataset_order)\n",
    "        axes[0, col_idx].set_title(dataset_order[dataset_idx])\n",
    "\n",
    "# Add a single xlabel centered at the bottom of the figure\n",
    "fig.text(0.5, 0.04, r'Split Factor $(f)$', ha='center', va='top', fontsize=16)\n",
    "\n",
    "\n",
    "for ax_row in axes:\n",
    "    for ax in ax_row:\n",
    "        # Check if the axis is not one of the deleted ones\n",
    "        if ax in fig.get_axes():\n",
    "            for spine in ax.spines.values():\n",
    "                spine.set_linewidth(2)\n",
    "        else:\n",
    "            # If the axis was deleted, remove its spines as well (though fig.delaxes should handle this)\n",
    "            for spine in ax.spines.values():\n",
    "                 spine.set_visible(False)\n",
    "\n",
    "\n",
    "# Adjust layout\n",
    "fig.subplots_adjust(left=0.05, right=0.95, bottom=0.1, top=0.90, wspace=0.05, hspace=0.2) # Reduced hspace\n",
    "plt.savefig(\"main_selected_horizontal.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.7 main_ablation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 81784 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 6 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import font_manager as fm\n",
    "import re\n",
    "from matplotlib.patches import Patch\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "# Matplotlib config\n",
    "plt.rcParams['legend.fontsize'] = 10\n",
    "plt.rcParams['xtick.labelsize'] = 16\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 16\n",
    "plt.rcParams['axes.titlesize'] = 18\n",
    "plt.rcParams['legend.title_fontsize'] = 12\n",
    "\n",
    "# Load data\n",
    "df_food = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_food_v3.csv\")\n",
    "df_animals = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_imagenet_animals_v2.csv\")\n",
    "df_landmarks = pd.read_csv(\"/mnt/lustrenew/mllm_safety-shared/tmp/lingjie/v-oocr/visualization/data/data_landmarks.csv\")\n",
    "\n",
    "# Tag datasets\n",
    "df_food[\"dataset\"] = \"Food\"\n",
    "df_animals[\"dataset\"] = \"Animal\"\n",
    "df_landmarks[\"dataset\"] = \"Landmark\"\n",
    "df_all = pd.concat([df_food, df_animals, df_landmarks], ignore_index=True)\n",
    "\n",
    "# Extract info from path\n",
    "def extract_patch_ratio(path):\n",
    "    match = re.search(r'/sfilter/(\\d+)x\\d+/', path)\n",
    "    return int(match.group(1)) if match else None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    match = re.search(r'/sfilter/\\d+x\\d+/([^/]+)/unrecognizable/', path)\n",
    "    return match.group(1) if match else \"unknown\"\n",
    "\n",
    "def extract_threshold(path):\n",
    "    match = re.search(r'/threshold(\\d+)/', path)\n",
    "    return int(match.group(1)) if match else None\n",
    "\n",
    "# def model_family(model_name):\n",
    "#     if \"Qwen2.5\" in model_name:\n",
    "#         return \"Qwen2.5\"\n",
    "#     elif \"Qwen2\" in model_name:\n",
    "#         return \"Qwen2\"\n",
    "#     elif \"gemma\" in model_name:\n",
    "#         return \"gemma\"\n",
    "#     elif \"InternVL3\" in model_name:\n",
    "#         return \"InternVL3\"\n",
    "#     elif \"llava-1.5\" in model_name:\n",
    "#         return \"llava_1.5\"\n",
    "#     elif \"Llama-3.2-Vision\" in model_name:\n",
    "#         return \"Llama-3.2-Vision\"\n",
    "#     elif \"llava-1.6\" in model_name:\n",
    "#         return \"llava_1.6\"\n",
    "#     else:\n",
    "#         return \"Other\"\n",
    "\n",
    "# Apply parsing\n",
    "df_all[\"patch_ratio\"] = df_all[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df_all[\"model_name\"] = df_all[\"Model Path\"].apply(extract_model_name)\n",
    "df_all[\"threshold\"] = df_all[\"Model Path\"].apply(extract_threshold)\n",
    "\n",
    "# Filter and aggregate\n",
    "df_filtered = df_all[df_all[\"Model Path\"].str.contains(\"/sfilter/\", na=False)].copy()\n",
    "df_filtered = df_filtered[df_filtered[\"patch_ratio\"].notna()]\n",
    "df_filtered = df_filtered[df_filtered[\"model_name\"] != \"unknown\"]\n",
    "df_filtered = df_filtered[df_filtered[\"threshold\"].notna()]\n",
    "df_aggregated = df_filtered.groupby(\n",
    "    [\"model_name\", \"dataset\", \"patch_ratio\", \"threshold\"]\n",
    ")[[\"eval-1 mean\", \"eval-1 std\", \"eval-0 mean\", \"eval-0 std\"]].mean().reset_index()\n",
    "df_aggregated[\"model_family\"] = df_aggregated[\"model_name\"].apply(model_family)\n",
    "\n",
    "# Organize plotting info\n",
    "unique_families = df_aggregated[\"model_family\"].unique()\n",
    "family_order = [f for f in [\"Qwen-2-VL\", \"Qwen-2.5-VL\", \"Gemma-3\", \"InternVL-3\", \"LLaVA-1.5\", \"Llama-3.2-Vision\", \"LLaVA-1.6\", \"Other\"] if f in unique_families]\n",
    "manual_order = {fam: sorted(df_aggregated[df_aggregated[\"model_family\"] == fam][\"model_name\"].unique()) for fam in family_order}\n",
    "unique_thresholds = sorted(df_aggregated[\"threshold\"].dropna().unique())\n",
    "alpha_map_thresholds = dict(zip(unique_thresholds, np.linspace(0.4, 1.0, len(unique_thresholds))))\n",
    "\n",
    "num_families = len(family_order)\n",
    "subplot_width = 4\n",
    "subplot_height = 4\n",
    "fig_width = subplot_width * 3\n",
    "fig_height = subplot_height * num_families\n",
    "\n",
    "fig, axes = plt.subplots(num_families, 3, figsize=(fig_width, fig_height), sharey=True)\n",
    "axes = np.array([axes]).reshape(1, 3) if num_families == 1 else axes\n",
    "dataset_order = [\"Food\", \"Animal\", \"Landmark\"]\n",
    "\n",
    "# color_bases = {\n",
    "#     \"Qwen2\": \"#2ca02c\", \"Qwen2.5\": \"#9467bd\", \"gemma\": \"#d62728\", \"InternVL3\": \"#ff7f0e\",\n",
    "#     \"llava_1.5\": \"#1f77b4\", \"Llama-3.2-Vision\": \"#e377c2\", \"llava_1.6\": \"#7f7f7f\", \"Other\": \"#8c564b\"\n",
    "# }\n",
    "\n",
    "bar_width_individual = 0.1\n",
    "bar_width_model_group = bar_width_individual * len(unique_thresholds)\n",
    "spacing_between_models = bar_width_individual * len(unique_thresholds) / 2\n",
    "total_model_group_width = bar_width_model_group + spacing_between_models\n",
    "\n",
    "for row, family in enumerate(family_order):\n",
    "    for col, dataset in enumerate(dataset_order):\n",
    "        ax = axes[row, col]\n",
    "        df_dataset = df_aggregated[(df_aggregated[\"dataset\"] == dataset) & (df_aggregated[\"model_family\"] == family)]\n",
    "        models = manual_order.get(family, [])\n",
    "        unique_patch_ratios = sorted(df_dataset['patch_ratio'].dropna().unique())\n",
    "        x_positions = np.arange(len(unique_patch_ratios))\n",
    "\n",
    "        for i, model in enumerate(models):\n",
    "            model_offset_base = (i - (len(models) - 1) / 2) * total_model_group_width\n",
    "            for j, threshold in enumerate(unique_thresholds):\n",
    "                threshold_offset = (j - (len(unique_thresholds) - 1) / 2) * bar_width_individual\n",
    "                bar_positions = x_positions + model_offset_base + threshold_offset\n",
    "                df_plot_data = df_dataset[(df_dataset[\"model_name\"] == model) & (df_dataset[\"threshold\"] == threshold)]\n",
    "                df_plot_data = df_plot_data.set_index(\"patch_ratio\").reindex(unique_patch_ratios).fillna(0)\n",
    "\n",
    "                upper_means = df_plot_data[\"eval-1 mean\"].values\n",
    "                upper_stds = df_plot_data[\"eval-1 std\"].values\n",
    "                lower_means = df_plot_data[\"eval-0 mean\"].values\n",
    "                lower_stds = df_plot_data[\"eval-0 std\"].values\n",
    "\n",
    "                color = color_bases.get(family, \"#000000\")\n",
    "                alpha = alpha_map_thresholds.get(threshold, 0.5)\n",
    "\n",
    "                ax.bar(bar_positions, upper_means, bar_width_individual, yerr=upper_stds, color=color, alpha=alpha, capsize=3)\n",
    "                ax.bar(bar_positions, -lower_means, bar_width_individual, yerr=lower_stds, color=color, alpha=alpha, capsize=3)\n",
    "\n",
    "        ax.set_title(dataset if row == 0 else \"\")\n",
    "        ax.set_xticks(x_positions)\n",
    "        ax.set_xticklabels(unique_patch_ratios)\n",
    "        ax.set_xlim(-0.5, len(unique_patch_ratios) - 0.5)\n",
    "        if row == num_families - 1 and col == 1:\n",
    "            ax.set_xlabel(r'Split Factor $(f)$')\n",
    "        ax.axhline(9.5, color='gray', linestyle='--', linewidth=1)\n",
    "        ax.axhline(0, color='black', linewidth=0.8)\n",
    "        ax.text(x_positions[len(x_positions)//2]-0.5, 9.6, \"Random Baseline\", color='gray', ha='center', fontsize=10)\n",
    "        ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda y, _: f'{int(abs(y))}'))\n",
    "\n",
    "        if col == 0:\n",
    "            ax.text(-0.08, 0.75, 'reference', transform=ax.transAxes, rotation=90, va='center', ha='right', fontsize=12)\n",
    "            ax.text(-0.08, 0.25, 'image', transform=ax.transAxes, rotation=90, va='center', ha='right', fontsize=12)\n",
    "\n",
    "    # Row title with reduced spacing\n",
    "    row_model_names = ', '.join(manual_order[family])\n",
    "    axes[row, 0].text(-0.18, 0.5, row_model_names, transform=axes[row, 0].transAxes,\n",
    "                      rotation=90, va='center', ha='right', fontsize=13, fontweight='bold')\n",
    "\n",
    "    # Add legend in lower right\n",
    "    legend_handles = [\n",
    "        Patch(facecolor=color_bases.get(family, \"#000000\"), alpha=alpha_map_thresholds[t], label=f'Threshold {t}')\n",
    "        for t in unique_thresholds\n",
    "    ]\n",
    "    if legend_handles:\n",
    "        axes[row, 2].legend(\n",
    "            handles=legend_handles,\n",
    "            loc='lower right',\n",
    "            fontsize=8,\n",
    "            title=\"Threshold\",\n",
    "            title_fontsize=9,\n",
    "            frameon=True,\n",
    "            edgecolor='black'\n",
    "        )\n",
    "\n",
    "# Clean up and finalize\n",
    "for row in range(axes.shape[0]):\n",
    "    for col in range(axes.shape[1]):\n",
    "        if not axes[row, col].has_data() and not axes[row, col].texts:\n",
    "            if col != 2 or not axes[row, col].get_legend():\n",
    "                fig.delaxes(axes[row, col])\n",
    "\n",
    "for ax in fig.get_axes():\n",
    "    for spine in ax.spines.values():\n",
    "        spine.set_linewidth(2)\n",
    "        spine.set_visible(True)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0.03, 0.98, 0.96])\n",
    "plt.savefig(\"main_ablation.pdf\", dpi=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 Moderation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 patch_evasion_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 44724 characters]",
      "text/plain": [
       "<Figure size 800x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Set plot style\n",
    "plt.rcParams['xtick.labelsize'] = 16\n",
    "plt.rcParams['ytick.labelsize'] = 16\n",
    "plt.rcParams['axes.labelsize'] = 18\n",
    "\n",
    "# Data\n",
    "patch_sizes = ['1', '2', '4', '8']\n",
    "x = np.arange(len(patch_sizes)) * 1.5  # Increase spacing between groups\n",
    "\n",
    "# OpenAI Moderation\n",
    "unfiltered_openai = [0, 17, 217, 1161]\n",
    "total = [20, 80, 320, 1280]\n",
    "evasion_openai = [u / t * 100 for u, t in zip(unfiltered_openai, total)]\n",
    "\n",
    "# ShieldGemma\n",
    "unfiltered_shieldgemma = [1, 17, 173, 891]  # Note: corrected value for split factor = 1\n",
    "total_shieldgemma = [20, 80, 320, 1280]\n",
    "evasion_shieldgemma = [u / t * 100 for u, t in zip(unfiltered_shieldgemma, total_shieldgemma)]\n",
    "\n",
    "# Plot configuration\n",
    "bar_width = 0.5  # Wider bars\n",
    "fig, ax = plt.subplots(figsize=(8, 5))  # Wider figure\n",
    "\n",
    "# Plot bars\n",
    "bars1 = ax.bar(x - bar_width / 2, evasion_openai, width=bar_width, color='tomato', edgecolor='black', label='OpenAI-Moderation')\n",
    "bars2 = ax.bar(x + bar_width / 2, evasion_shieldgemma, width=bar_width, color='steelblue', edgecolor='black', label='ShieldGemma-2')\n",
    "\n",
    "# Annotate bars\n",
    "for i in range(len(x)):\n",
    "    ax.text(x[i] - bar_width / 2, evasion_openai[i] + 2, f'{unfiltered_openai[i]} / {total[i]}',\n",
    "            ha='center', fontsize=7)\n",
    "    ax.text(x[i] + bar_width / 2, evasion_shieldgemma[i] + 2, f'{unfiltered_shieldgemma[i]} / {total_shieldgemma[i]}',\n",
    "            ha='center', fontsize=7)\n",
    "\n",
    "# Axis labels and formatting\n",
    "ax.set_ylabel('Patch Evasion Rate (%)')\n",
    "ax.set_xlabel(r'Split Factor $(f)$')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(patch_sizes)\n",
    "ax.set_ylim(0, 100)\n",
    "ax.set_xlim(x[0] - 1, x[-1] + 1)\n",
    "ax.legend(fontsize=12)\n",
    "\n",
    "# Thicker axis spines\n",
    "for spine in ax.spines.values():\n",
    "    spine.set_linewidth(2)\n",
    "\n",
    "# Finalize and show\n",
    "plt.tight_layout()\n",
    "# plt.show()\n",
    "plt.savefig(\"patch_evasion_rate.pdf\", dpi=300, bbox_inches=\"tight\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 moderation_qwen2_7b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_74773/2516951083.py:316: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n",
      "/tmp/ipykernel_74773/2516951083.py:316: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n",
      "/tmp/ipykernel_74773/2516951083.py:316: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 74868 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# model evasion rank (qwen2_7b)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# === Load data ===\n",
    "df = pd.read_csv(\"data/moderation_v2.csv\")        # Adjust path if the CSV sits elsewhere\n",
    "\n",
    "# ---------------------------------------------------------------------------\n",
    "# Helper functions to parse information from the “Model Path” column\n",
    "# ---------------------------------------------------------------------------\n",
    "def extract_setting(path: str):\n",
    "    if \"config_violence_sex\" in path:\n",
    "        return \"config_violence_sex\"\n",
    "    if \"config_sex_violence\" in path:\n",
    "        return \"config_sex_violence\"\n",
    "    if \"config\" in path:\n",
    "        return \"config\"\n",
    "    return None\n",
    "\n",
    "def extract_patch_ratio(path: str):\n",
    "    if \"/2x2/\" in path: return 2\n",
    "    if \"/4x4/\" in path: return 4\n",
    "    if \"/8x8/\" in path: return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path: str):\n",
    "    parts = path.split(\"/\")\n",
    "    for idx, part in enumerate(parts):\n",
    "        if part.startswith(\"epoch\"):\n",
    "            return parts[idx - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "def extract_safety(path: str):\n",
    "    if \"/safe/\"   in path: return \"safe\"\n",
    "    if \"/unsafe/\" in path: return \"unsafe\"\n",
    "    return \"unknown\"\n",
    "\n",
    "# ---------------------------------------------------------------------------\n",
    "# Parse columns\n",
    "# ---------------------------------------------------------------------------\n",
    "df[\"setting\"]     = df[\"Model Path\"].apply(extract_setting)\n",
    "df[\"patch_ratio\"] = df[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df[\"model_name\"]  = df[\"Model Path\"].apply(extract_model_name)\n",
    "df[\"safety\"]      = df[\"Model Path\"].apply(extract_safety)\n",
    "\n",
    "# Keep only Qwen2‑VL‑7B rows with Split Ratios 2, 4, 8\n",
    "df_qwen = df[\n",
    "    (df[\"model_name\"] == \"Qwen2-VL-7B\") &\n",
    "    (df[\"patch_ratio\"].isin([2, 4, 8]))\n",
    "]\n",
    "\n",
    "# ---------------------------------------------------------------------------\n",
    "# Matplotlib global style (matches the reference script)\n",
    "# ---------------------------------------------------------------------------\n",
    "plt.rcParams['legend.fontsize']       = 16\n",
    "plt.rcParams['legend.title_fontsize'] = 18\n",
    "plt.rcParams['xtick.labelsize']       = 16\n",
    "plt.rcParams['ytick.labelsize']       = 16\n",
    "plt.rcParams['axes.labelsize']        = 18\n",
    "plt.rcParams['axes.titlesize']        = 18\n",
    "\n",
    "# ---------------------------------------------------------------------------\n",
    "# Plot configuration\n",
    "# ---------------------------------------------------------------------------\n",
    "settings    = [\"config\", \"config_violence_sex\", \"config_sex_violence\"]\n",
    "titles      = [\"sex & violence (safe)\\nanimal (unsafe)\", \"violence (safe)\\nsex (unsafe)\", \"sex (safe)\\nviolence (unsafe)\"]\n",
    "x_ticks     = [2, 4, 8]\n",
    "x           = np.arange(len(x_ticks))\n",
    "bar_width   = 0.20\n",
    "\n",
    "colors = {\"safe\": \"#1f77b4\", \"unsafe\": \"#ff7f0e\"}   # blue / orange\n",
    "alphas = {\"safe\": 0.8,        \"unsafe\": 0.8}\n",
    "\n",
    "fig, axes = plt.subplots(1, 3, figsize=(12, 5), sharey=True)\n",
    "\n",
    "for idx, setting in enumerate(settings):\n",
    "    ax = axes[idx]\n",
    "    df_setting = df_qwen[df_qwen[\"setting\"] == setting]\n",
    "\n",
    "    # Draw bars for “safe” then “unsafe”\n",
    "    for j, safety in enumerate([\"safe\", \"unsafe\"]):\n",
    "        offset   = (j - 0.5) * bar_width\n",
    "        df_slice = (\n",
    "            df_setting[df_setting[\"safety\"] == safety]\n",
    "            .set_index(\"patch_ratio\")\n",
    "            .reindex(x_ticks)\n",
    "        )\n",
    "\n",
    "        upper_mean = df_slice[\"eval-1 mean\"].fillna(0).values\n",
    "        upper_std  = df_slice[\"eval-1 std\"].fillna(0).values\n",
    "        lower_mean = df_slice[\"eval-0 mean\"].fillna(0).values\n",
    "        lower_std  = df_slice[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "        # Upper bars (level‑1)\n",
    "        ax.bar(\n",
    "            x + offset,\n",
    "            upper_mean,\n",
    "            bar_width,\n",
    "            yerr=upper_std,\n",
    "            color=colors[safety],\n",
    "            alpha=alphas[safety],\n",
    "            capsize=3,\n",
    "            label = {\"safe\": \"filtered\", \"unsafe\": \"unfiltered\"}[safety] if idx == 0 else None\n",
    "        )\n",
    "\n",
    "        # Lower bars (level‑2)\n",
    "        ax.bar(\n",
    "            x + offset,\n",
    "            -lower_mean,\n",
    "            bar_width,\n",
    "            yerr=lower_std,\n",
    "            color=colors[safety],\n",
    "            alpha=alphas[safety],\n",
    "            capsize=3\n",
    "        )\n",
    "\n",
    "    # --- Axes cosmetics ---\n",
    "    ax.set_title(titles[idx])\n",
    "    # plt.rcParams['text.usetex'] = True\n",
    "    # if \"unsafe\" in titles[idx]:\n",
    "    #     title_colored = titles[idx].replace(\"unsafe\", r\"$\\bf{\\color{red}{unsafe}}$\").replace(\"safe\", r\"$\\bf{\\color{green}{safe}}$\")\n",
    "    #     ax.set_title(title_colored, fontsize=18)\n",
    "    # else:\n",
    "    #     ax.set_title(titles[idx], fontsize=18)\n",
    "    ax.set_xticks(x)\n",
    "    ax.set_xticklabels(x_ticks)\n",
    "    ax.set_xlabel(r'Split Factor $(f)$')\n",
    "    ax.axhline(0, color=\"black\", linewidth=0.8)\n",
    "\n",
    "    # Random baseline at ±0.5\n",
    "    ax.axhline( 0.5, color=\"gray\", linestyle=\"--\", linewidth=1)\n",
    "    ax.axhline(-0.5, color=\"gray\", linestyle=\"--\", linewidth=1)\n",
    "    if idx == 0:\n",
    "        ax.text(x[len(x)//2],  0.50, \"Random Baseline\",\n",
    "                color=\"gray\", ha=\"center\", va=\"bottom\", fontsize=14)\n",
    "        ax.text(x[len(x)//2], -0.43, \"Random Baseline\",\n",
    "                color=\"gray\", ha=\"center\", va=\"top\",    fontsize=14)\n",
    "\n",
    "    # Y‑tick labels as absolute values\n",
    "    \n",
    "    # ax.set_ylim(-1, 1) # Set twin axis limits\n",
    "    # ax.set_yticks([-1, 0, 1])\n",
    "    ax.set_ylim(-0.6, 0.6)\n",
    "    ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n",
    "\n",
    "    # Left‑margin labels on the first subplot only\n",
    "    if idx == 0:\n",
    "        ax.text(-0.15, 0.75, 'reference',\n",
    "                transform=ax.transAxes,\n",
    "                rotation=90, va='center', ha='right', fontsize=16)\n",
    "        ax.text(-0.15, 0.25, 'image',\n",
    "                transform=ax.transAxes,\n",
    "                rotation=90, va='center', ha='right', fontsize=16)\n",
    "        ax.legend(\n",
    "            loc=\"lower left\",\n",
    "            bbox_to_anchor=(0, 0.18),  # (x, y); y > 0 moves it up\n",
    "            fontsize=14,\n",
    "            frameon=True,\n",
    "            edgecolor='black'\n",
    "        )\n",
    "\n",
    "# Thicker spines, no grid\n",
    "for ax in axes:\n",
    "    for spine in ax.spines.values():\n",
    "        spine.set_linewidth(2)\n",
    "    ax.grid(False)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.98])\n",
    "\n",
    "# Save or show\n",
    "plt.savefig(\"moderation_qwen2_7b.pdf\", dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_74773/1229240438.py:122: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n",
      "/tmp/ipykernel_74773/1229240438.py:122: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 179628 characters]",
      "text/plain": [
       "<Figure size 1400x1000 with 8 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# # model evasion rank (all)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "# === Load CSV ===\n",
    "df = pd.read_csv(\"data/moderation_v2.csv\")  # Adjust if necessary\n",
    "\n",
    "# === Metadata extraction functions ===\n",
    "def extract_setting(path):\n",
    "    if \"config_violence_sex\" in path:\n",
    "        return \"config_violence_sex\"\n",
    "    elif \"config_sex_violence\" in path:\n",
    "        return \"config_sex_violence\"\n",
    "    return None\n",
    "\n",
    "def extract_patch_ratio(path):\n",
    "    if \"/2x2/\" in path: return 2\n",
    "    if \"/4x4/\" in path: return 4\n",
    "    if \"/8x8/\" in path: return 8\n",
    "    return None\n",
    "\n",
    "def extract_model_name(path):\n",
    "    parts = path.split(\"/\")\n",
    "    for i in range(len(parts)):\n",
    "        if parts[i].startswith(\"epoch\"):\n",
    "            return parts[i - 1]\n",
    "    return \"unknown\"\n",
    "\n",
    "def extract_safety(path):\n",
    "    if \"/safe/\" in path: return \"safe\"\n",
    "    elif \"/unsafe/\" in path: return \"unsafe\"\n",
    "    return \"unknown\"\n",
    "\n",
    "# === Parse dataframe ===\n",
    "df[\"setting\"] = df[\"Model Path\"].apply(extract_setting)\n",
    "df[\"patch_ratio\"] = df[\"Model Path\"].apply(extract_patch_ratio)\n",
    "df[\"model_name\"] = df[\"Model Path\"].apply(extract_model_name)\n",
    "df[\"safety\"] = df[\"Model Path\"].apply(extract_safety)\n",
    "\n",
    "# === Filter models and settings ===\n",
    "selected_models = [\"Qwen2-VL-7B\", \"InternVL3-8B\", \"Llama-3.2-11B-Vision\", \"gemma-3-12b-pt\"]\n",
    "df_filtered = df[\n",
    "    df[\"model_name\"].isin(selected_models) &\n",
    "    df[\"patch_ratio\"].isin([2, 4, 8])\n",
    "]\n",
    "\n",
    "# === Plot settings ===\n",
    "plt.rcParams.update({\n",
    "    'xtick.labelsize': 14,\n",
    "    'ytick.labelsize': 16,\n",
    "    'axes.labelsize': 16,\n",
    "    'axes.titlesize': 16,\n",
    "    'legend.title_fontsize': 10,\n",
    "    'legend.fontsize': 10\n",
    "})\n",
    "\n",
    "settings = [\"config_violence_sex\", \"config_sex_violence\"]\n",
    "titles = [\"sex (unsafe), violence (safe)\", \"sex (safe), violence (unsafe)\"]\n",
    "x_ticks = [2, 4, 8]\n",
    "x = np.arange(len(x_ticks))\n",
    "bar_width = 0.20\n",
    "pattern_styles = {\"safe\": \"\", \"unsafe\": \"//\"}\n",
    "\n",
    "# Color and alpha for model families\n",
    "model_colors = {\n",
    "    \"Qwen2-VL-7B\": \"#2ca02c\", \"InternVL3-8B\": \"#ff7f0e\", \"Llama-3.2-11B-Vision\": \"#e377c2\",\n",
    "    \"Qwen2-VL-2B\": \"#2ca02c\", \"Qwen2-VL-72B\": \"#2ca02c\",\n",
    "    \"Qwen2.5-VL-3B-Instruct\": \"#9467bd\", \"Qwen2.5-VL-7B-Instruct\": \"#9467bd\", \"Qwen2.5-VL-72B-Instruct\": \"#9467bd\",\n",
    "    \"gemma-3-4b-pt\": \"#d62728\", \"gemma-3-12b-pt\": \"#d62728\", \"gemma-3-27b-pt\": \"#d62728\",\n",
    "    \"InternVL3-1B\": \"#ff7f0e\", \"InternVL3-14B\": \"#ff7f0e\",\n",
    "    \"llava-1.5-7b-hf\": \"#1f77b4\", \"llava-1.5-13b-hf\": \"#1f77b4\",\n",
    "    \"Llama-3.2-90B-Vision\": \"#e377c2\",\n",
    "    \"llava-v1.6-vicuna-7b-hf\": \"#7f7f7f\", \"llava-v1.6-vicuna-13b-hf\": \"#7f7f7f\"\n",
    "}\n",
    "\n",
    "model_alphas = {\n",
    "    \"Qwen2-VL-7B\": 0.7, \"InternVL3-8B\": 1.0, \"Llama-3.2-11B-Vision\": 0.6,\n",
    "    \"Qwen2-VL-2B\": 0.4, \"Qwen2-VL-72B\": 1.0,\n",
    "    \"Qwen2.5-VL-3B-Instruct\": 0.4, \"Qwen2.5-VL-7B-Instruct\": 0.7, \"Qwen2.5-VL-72B-Instruct\": 1.0,\n",
    "    \"gemma-3-4b-pt\": 0.4, \"gemma-3-12b-pt\": 0.7, \"gemma-3-27b-pt\": 1.0,\n",
    "    \"InternVL3-1B\": 0.4, \"InternVL3-14B\": 0.7,\n",
    "    \"llava-1.5-7b-hf\": 0.5, \"llava-1.5-13b-hf\": 0.8,\n",
    "    \"Llama-3.2-90B-Vision\": 0.9,\n",
    "    \"llava-v1.6-vicuna-7b-hf\": 0.5, \"llava-v1.6-vicuna-13b-hf\": 0.8\n",
    "}\n",
    "\n",
    "# === Plot ===\n",
    "fig, axes = plt.subplots(2, 4, figsize=(14, 10), sharex=True)\n",
    "\n",
    "for i, setting in enumerate(settings):\n",
    "    for j, model in enumerate(selected_models):\n",
    "        ax = axes[i][j]\n",
    "        df_setting_model = df_filtered[(df_filtered[\"setting\"] == setting) & (df_filtered[\"model_name\"] == model)]\n",
    "\n",
    "        for k, safety in enumerate([\"safe\", \"unsafe\"]):\n",
    "            offset = (k - 0.5) * bar_width\n",
    "            df_slice = df_setting_model[df_setting_model[\"safety\"] == safety].set_index(\"patch_ratio\").reindex(x_ticks)\n",
    "\n",
    "            upper_mean = df_slice[\"eval-1 mean\"].fillna(0).values\n",
    "            upper_std = df_slice[\"eval-1 std\"].fillna(0).values\n",
    "            lower_mean = df_slice[\"eval-0 mean\"].fillna(0).values\n",
    "            lower_std = df_slice[\"eval-0 std\"].fillna(0).values\n",
    "\n",
    "            ax.bar(x + offset, upper_mean, bar_width, yerr=upper_std,\n",
    "                   color=model_colors[model], alpha=model_alphas[model],\n",
    "                   hatch=pattern_styles[safety], edgecolor=\"black\", capsize=3)\n",
    "\n",
    "            ax.bar(x + offset, -lower_mean, bar_width, yerr=lower_std,\n",
    "                   color=model_colors[model], alpha=model_alphas[model],\n",
    "                   hatch=pattern_styles[safety], edgecolor=\"black\", capsize=3)\n",
    "\n",
    "        ax.set_xticks(x)\n",
    "        ax.set_xticklabels(x_ticks)\n",
    "        ax.axhline(0, color=\"black\", linewidth=0.8)\n",
    "        ax.axhline(0.5, color=\"gray\", linestyle=\"--\", linewidth=1)\n",
    "        ax.axhline(-0.5, color=\"gray\", linestyle=\"--\", linewidth=1)\n",
    "        ax.set_ylim(-0.6, 0.6)\n",
    "\n",
    "        if j == 0:\n",
    "            ax.set_yticklabels([f\"{abs(t):.1f}\" for t in ax.get_yticks()])\n",
    "            ax.text(-0.12, 0.75, 'image', transform=ax.transAxes, rotation=90,\n",
    "                    va='center', ha='right', fontsize=14)\n",
    "            ax.text(-0.12, 0.25, 'reference', transform=ax.transAxes, rotation=90,\n",
    "                    va='center', ha='right', fontsize=14)\n",
    "            ax.text(-0.20, 0.5, titles[i], transform=ax.transAxes,\n",
    "                    fontsize=13, ha='center', va='center', rotation=90,\n",
    "                    fontweight='bold')\n",
    "        else:\n",
    "            ax.set_yticklabels([])\n",
    "            ax.set_yticks([])\n",
    "\n",
    "        if i == 0:\n",
    "            ax.set_title(model)\n",
    "        if i == 1:\n",
    "            ax.set_xlabel(r'Split Factor $(f)$')\n",
    "\n",
    "        for spine in ax.spines.values():\n",
    "            spine.set_linewidth(2)\n",
    "        ax.grid(False)\n",
    "\n",
    "# Final legend\n",
    "final_legend_handles = [\n",
    "    mpatches.Patch(facecolor='white', edgecolor='black', hatch=pattern_styles[\"safe\"], label='filtered'),\n",
    "    mpatches.Patch(facecolor='white', edgecolor='black', hatch=pattern_styles[\"unsafe\"], label='unfiltered'),\n",
    "]\n",
    "fig.legend(handles=final_legend_handles, loc=\"lower center\", bbox_to_anchor=(0.5, -0.02), ncol=2, frameon=False)\n",
    "\n",
    "# Layout\n",
    "plt.tight_layout(rect=[0, 0.04, 1, 1])\n",
    "plt.savefig(\"moderation_all.pdf\", dpi=300, bbox_inches=\"tight\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Moderation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original\n",
      "2x2\n",
      "4x4\n",
      "8x8\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 64124 characters]",
      "text/plain": [
       "<Figure size 800x800 with 16 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from PIL import Image, ImageFilter, ImageDraw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Parameters\n",
    "resize_to = 32\n",
    "blur_radius = 3\n",
    "patch_splits = [1, 2, 4, 8]\n",
    "titles = ['Original', '2x2', '4x4', '8x8']\n",
    "image_ids = [f\"sex0{i}\" for i in range(4)]\n",
    "\n",
    "def mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=3):\n",
    "    original_path = f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/data/moderation_v2/files/{image_id}.jpg\"\n",
    "    img = Image.open(original_path).convert(\"RGB\").resize((resize_to, resize_to))\n",
    "    patch_size = resize_to // patch_split\n",
    "    white_patch_coords = []\n",
    "\n",
    "    if patch_dir is not None:\n",
    "        existing_patches = set(os.listdir(patch_dir))\n",
    "        for i in range(patch_split):\n",
    "            for j in range(patch_split):\n",
    "                patch_name = f\"{image_id}-{i}_{j}.jpg\"\n",
    "                if patch_name not in existing_patches:\n",
    "                    left = j * patch_size\n",
    "                    top = i * patch_size\n",
    "                    right = left + patch_size\n",
    "                    bottom = top + patch_size\n",
    "                    white_patch_coords.append((left, top, right, bottom))\n",
    "\n",
    "    masked_img = img.filter(ImageFilter.GaussianBlur(radius=radius)) if radius > 0 else img.copy()\n",
    "    draw = ImageDraw.Draw(masked_img)\n",
    "\n",
    "    # Mask white patches\n",
    "    for box in white_patch_coords:\n",
    "        draw.rectangle(box, fill=(255, 255, 255))\n",
    "\n",
    "    # Draw outer gray border\n",
    "    draw.rectangle([0, 0, resize_to - 1, resize_to - 1], outline=(160, 160, 160), width=1)\n",
    "\n",
    "    return masked_img\n",
    "\n",
    "# Create plot\n",
    "fig, axs = plt.subplots(4, len(image_ids), figsize=(len(image_ids) * 2, 8))\n",
    "\n",
    "for col_idx, image_id in enumerate(image_ids):\n",
    "    patch_dirs = [\n",
    "        None,\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/2x2/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/4x4/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/8x8/OpenAI_Moderation_Filter/safe/{image_id}\"\n",
    "    ]\n",
    "\n",
    "    for row_idx, (patch_dir, patch_split, title) in enumerate(zip(patch_dirs, patch_splits, titles)):\n",
    "        img = mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=blur_radius)\n",
    "        ax = axs[row_idx, col_idx] if len(image_ids) > 1 else axs[row_idx]\n",
    "        ax.imshow(img)\n",
    "        if row_idx == 0:\n",
    "            ax.set_title(image_id, fontsize=16)\n",
    "        if col_idx == 0:\n",
    "            print(title)\n",
    "            ax.set_ylabel(title, fontsize=16)\n",
    "        # ax.axis('off')\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "        for spine in ax.spines.values():\n",
    "            spine.set_visible(False)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "plt.savefig(\"moderation_visualization_sex_selected.pdf\", dpi=200, bbox_inches=\"tight\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original\n",
      "2x2\n",
      "4x4\n",
      "8x8\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 130896 characters]",
      "text/plain": [
       "<Figure size 2000x800 with 40 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from PIL import Image, ImageFilter, ImageDraw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Parameters\n",
    "resize_to = 32\n",
    "blur_radius = 3\n",
    "patch_splits = [1, 2, 4, 8]\n",
    "titles = ['Original', '2x2', '4x4', '8x8']\n",
    "image_ids = [f\"sex0{i}\" for i in range(10)]\n",
    "\n",
    "def mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=3):\n",
    "    original_path = f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/data/moderation_v2/files/{image_id}.jpg\"\n",
    "    img = Image.open(original_path).convert(\"RGB\").resize((resize_to, resize_to))\n",
    "    patch_size = resize_to // patch_split\n",
    "    white_patch_coords = []\n",
    "\n",
    "    if patch_dir is not None:\n",
    "        existing_patches = set(os.listdir(patch_dir))\n",
    "        for i in range(patch_split):\n",
    "            for j in range(patch_split):\n",
    "                patch_name = f\"{image_id}-{i}_{j}.jpg\"\n",
    "                if patch_name not in existing_patches:\n",
    "                    left = j * patch_size\n",
    "                    top = i * patch_size\n",
    "                    right = left + patch_size\n",
    "                    bottom = top + patch_size\n",
    "                    white_patch_coords.append((left, top, right, bottom))\n",
    "\n",
    "    masked_img = img.filter(ImageFilter.GaussianBlur(radius=radius)) if radius > 0 else img.copy()\n",
    "    draw = ImageDraw.Draw(masked_img)\n",
    "\n",
    "    # Mask white patches\n",
    "    for box in white_patch_coords:\n",
    "        draw.rectangle(box, fill=(255, 255, 255))\n",
    "\n",
    "    # Draw outer gray border\n",
    "    draw.rectangle([0, 0, resize_to - 1, resize_to - 1], outline=(160, 160, 160), width=1)\n",
    "\n",
    "    return masked_img\n",
    "\n",
    "# Create plot\n",
    "fig, axs = plt.subplots(4, len(image_ids), figsize=(len(image_ids) * 2, 8))\n",
    "\n",
    "for col_idx, image_id in enumerate(image_ids):\n",
    "    patch_dirs = [\n",
    "        None,\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/2x2/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/4x4/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/8x8/OpenAI_Moderation_Filter/safe/{image_id}\"\n",
    "    ]\n",
    "\n",
    "    for row_idx, (patch_dir, patch_split, title) in enumerate(zip(patch_dirs, patch_splits, titles)):\n",
    "        img = mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=blur_radius)\n",
    "        ax = axs[row_idx, col_idx] if len(image_ids) > 1 else axs[row_idx]\n",
    "        ax.imshow(img)\n",
    "        if row_idx == 0:\n",
    "            ax.set_title(image_id, fontsize=16)\n",
    "        if col_idx == 0:\n",
    "            print(title)\n",
    "            ax.set_ylabel(title, fontsize=16)\n",
    "        # ax.axis('off')\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "        for spine in ax.spines.values():\n",
    "            spine.set_visible(False)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "plt.savefig(\"moderation_visualization_sex_all.pdf\", dpi=200, bbox_inches=\"tight\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original\n",
      "2x2\n",
      "4x4\n",
      "8x8\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 116692 characters]",
      "text/plain": [
       "<Figure size 2000x800 with 40 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from PIL import Image, ImageFilter, ImageDraw\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Parameters\n",
    "resize_to = 32\n",
    "blur_radius = 3\n",
    "patch_splits = [1, 2, 4, 8]\n",
    "titles = ['Original', '2x2', '4x4', '8x8']\n",
    "image_ids = [f\"violence0{i}\" for i in range(10)]\n",
    "\n",
    "def mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=3):\n",
    "    original_path = f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/data/moderation_v2/files/{image_id}.jpg\"\n",
    "    img = Image.open(original_path).convert(\"RGB\").resize((resize_to, resize_to))\n",
    "    patch_size = resize_to // patch_split\n",
    "    white_patch_coords = []\n",
    "\n",
    "    if patch_dir is not None:\n",
    "        existing_patches = set(os.listdir(patch_dir))\n",
    "        for i in range(patch_split):\n",
    "            for j in range(patch_split):\n",
    "                patch_name = f\"{image_id}-{i}_{j}.jpg\"\n",
    "                if patch_name not in existing_patches:\n",
    "                    left = j * patch_size\n",
    "                    top = i * patch_size\n",
    "                    right = left + patch_size\n",
    "                    bottom = top + patch_size\n",
    "                    white_patch_coords.append((left, top, right, bottom))\n",
    "\n",
    "    masked_img = img.filter(ImageFilter.GaussianBlur(radius=radius)) if radius > 0 else img.copy()\n",
    "    draw = ImageDraw.Draw(masked_img)\n",
    "\n",
    "    # Mask white patches\n",
    "    for box in white_patch_coords:\n",
    "        draw.rectangle(box, fill=(255, 255, 255))\n",
    "\n",
    "    # Draw outer gray border\n",
    "    draw.rectangle([0, 0, resize_to - 1, resize_to - 1], outline=(160, 160, 160), width=1)\n",
    "\n",
    "    return masked_img\n",
    "\n",
    "# Create plot\n",
    "fig, axs = plt.subplots(4, len(image_ids), figsize=(len(image_ids) * 2, 8))\n",
    "\n",
    "for col_idx, image_id in enumerate(image_ids):\n",
    "    patch_dirs = [\n",
    "        None,\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/2x2/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/4x4/OpenAI_Moderation_Filter/safe/{image_id}\",\n",
    "        f\"/mnt/petrelfs/zhouzhanhui/projects/v-oocr/tmp/data/moderation_v2/files/others/filter/8x8/OpenAI_Moderation_Filter/safe/{image_id}\"\n",
    "    ]\n",
    "\n",
    "    for row_idx, (patch_dir, patch_split, title) in enumerate(zip(patch_dirs, patch_splits, titles)):\n",
    "        img = mask_image_from_patch_directory(image_id, patch_dir, patch_split, radius=blur_radius)\n",
    "        ax = axs[row_idx, col_idx] if len(image_ids) > 1 else axs[row_idx]\n",
    "        ax.imshow(img)\n",
    "        if row_idx == 0:\n",
    "            ax.set_title(image_id, fontsize=16)\n",
    "        if col_idx == 0:\n",
    "            print(title)\n",
    "            ax.set_ylabel(title, fontsize=16)\n",
    "        # ax.axis('off')\n",
    "        ax.set_xticks([])\n",
    "        ax.set_yticks([])\n",
    "        for spine in ax.spines.values():\n",
    "            spine.set_visible(False)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "plt.savefig(\"moderation_visualization_violence_all.pdf\", dpi=200, bbox_inches=\"tight\")\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "0229e0ccfe771600e38d892a03c989b0b471272c7a17d82a8b9bd878ddb98269"
  },
  "kernelspec": {
   "display_name": "Python 3.10.16 64-bit ('trl-vlm': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
