{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Test models with the xAI evaluatin metrics of th ReVel framework\n", "================================================================" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded the pretrained model.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_93584/3587458112.py:46: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n", " state_dict = torch.load(\n" ] } ], "source": [ "import torch\n", "\n", "torch.set_num_threads(1)\n", "from torch.utils.data.dataloader import DataLoader\n", "import tqdm\n", "import numpy as np\n", "import json\n", "import argparse\n", "import pandas as pd\n", "import os\n", "\n", "from ReVel.LLEs import get_xai_model\n", "from ReVel.perturbations import get_perturbation\n", "from ReVel.load_data import load_data\n", "from ReVel.revel.revel import ReVel\n", "import TSHIELD.procedures as procedures\n", "device = \"cpu\"\n", "\n", "n_class = 102\n", "ds = \"Flowers\"\n", "iterations = 5\n", "batch_size = 32\n", "max_examples = 500\n", "samples = 10\n", "sigma = 12\n", "xai_model = \"LIME\"\n", "dim = 8\n", "\n", "csv_file = \"./metrics.csv\"\n", "perturbation = get_perturbation(\n", " name=\"square\", dim=dim, num_classes=n_class, final_size=(224, 224)\n", ")\n", "Test = load_data(ds, perturbation=perturbation, train=False, dir=\"./data\")\n", "# Hacer que Test tenga solo las primeras 'samples' de Test\n", "if isinstance(samples, int):\n", " indices = np.random.choice(\n", " [i for i in range(len(Test))], size=samples, replace=False\n", " )\n", " Test = torch.utils.data.Subset(Test, indices)\n", "\n", "TestLoader = iter(DataLoader(Test, batch_size=1, shuffle=False))\n", "classifier = procedures.classifier(\"efficientnet-b2\", n_class)\n", "classifier.to(device)\n", "\n", "\n", "state_dict = torch.load(\n", " f'{os.getcwd()}/model.pth',\n", " map_location=device,\n", ")\n", "\n", "classifier.load_state_dict(state_dict)\n", "classifier.to(device)\n", "classifier.eval()\n", "print(\"Loaded the pretrained model.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "REVEL metrics calculation for a model trained on the Flowers dataset\n", "======================================================================" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "index = 0\n", "for data in tqdm.tqdm(TestLoader, total=len(TestLoader)):#len(TestLoader)):\n", " inputs, labels = data\n", " for k, inp in enumerate(inputs):\n", " inp = inp.to(device)\n", "\n", " # inp dims: (C,H,W) -> (H,W,C)\n", " inp = np.transpose(inp, (1, 2, 0))\n", "\n", " labels = labels[k].to(device)\n", " explainer = get_xai_model(\n", " name=xai_model,\n", " perturbation=perturbation,\n", " max_examples=max_examples,\n", " dim=dim,\n", " sigma=sigma,\n", " )\n", " def classify(image, model=classifier):\n", " \"\"\"\n", " This function takes an image and returns the predicted probabilities.\n", " :param image: A tensor of shape HxWxC\n", " :return: A tensor of shape Cx1\n", " \"\"\"\n", " if isinstance(image, np.ndarray):\n", " image = np.expand_dims(image, 0)\n", "\n", " image = torch.Tensor(image).to(device)\n", "\n", " else:\n", " image = torch.unsqueeze(image, 0)\n", "\n", " # image dims: (N,H,W,C) -> (N,C,H,W)\n", "\n", " image = torch.transpose(image, 3, 2).transpose(2, 1)\n", "\n", " result = model(image)\n", " return result\n", " def model_fordward(\n", " X: np.array, explainator=explainer, model=classify, img=inp\n", " ):\n", " neutral = explainator.perturbation.fn_neutral_image(img)\n", "\n", " avoid = [i for i in range(len(X)) if X[i] == 0]\n", "\n", " segments = explainator.perturbation.segmentation_fn(img.numpy())\n", " perturbation = explainator.perturbation.perturbation(\n", " img, neutral, segments=segments, indexes=avoid\n", " )\n", " return model(perturbation)\n", " segments = explainer.perturbation.segmentation_fn(inp.numpy())\n", "\n", " revel = ReVel(\n", " model_f=classify,\n", " model_g=model_fordward,\n", " instance=inp,\n", " lle=explainer,\n", " n_classes=n_class,\n", " segments=segments,\n", " )\n", " df = revel.evaluate(times=iterations)\n", " df.loc[:, \"dataset\"] = ds\n", " df.loc[:, \"name\"] = \"SHIELD\"\n", " df.loc[:, \"index\"] = index\n", " index+=1\n", " if os.path.exists(csv_file):\n", " bigDF = pd.read_csv(csv_file)\n", " bigDF = pd.concat([bigDF, df])\n", " else:\n", " bigDF = df\n", " bigDF.to_csv(csv_file, index=False)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
concisenesslocal_fidelitylocal_concordanceprescriptivityrobustnessdatasetnameindex
00.5923740.9688210.9483010.9860140.900298FlowersSHIELD0
10.5752380.9700560.9502140.9861200.900298FlowersSHIELD0
20.6214030.9716540.9535860.9828340.900298FlowersSHIELD0
30.6562000.9685530.9485880.9868660.900298FlowersSHIELD0
40.5552710.9667480.9443050.9885160.900298FlowersSHIELD0
50.7182550.9742890.9556350.9839460.905916FlowersSHIELD1
60.6973980.9707100.9509230.9809300.905916FlowersSHIELD1
70.6906740.9756970.9579710.9801830.905916FlowersSHIELD1
80.6876140.9636200.9384030.9784760.905916FlowersSHIELD1
90.6683010.9743790.9566230.9818300.905916FlowersSHIELD1
\n", "
" ], "text/plain": [ " conciseness local_fidelity local_concordance prescriptivity robustness \\\n", "0 0.592374 0.968821 0.948301 0.986014 0.900298 \n", "1 0.575238 0.970056 0.950214 0.986120 0.900298 \n", "2 0.621403 0.971654 0.953586 0.982834 0.900298 \n", "3 0.656200 0.968553 0.948588 0.986866 0.900298 \n", "4 0.555271 0.966748 0.944305 0.988516 0.900298 \n", "5 0.718255 0.974289 0.955635 0.983946 0.905916 \n", "6 0.697398 0.970710 0.950923 0.980930 0.905916 \n", "7 0.690674 0.975697 0.957971 0.980183 0.905916 \n", "8 0.687614 0.963620 0.938403 0.978476 0.905916 \n", "9 0.668301 0.974379 0.956623 0.981830 0.905916 \n", "\n", " dataset name index \n", "0 Flowers SHIELD 0 \n", "1 Flowers SHIELD 0 \n", "2 Flowers SHIELD 0 \n", "3 Flowers SHIELD 0 \n", "4 Flowers SHIELD 0 \n", "5 Flowers SHIELD 1 \n", "6 Flowers SHIELD 1 \n", "7 Flowers SHIELD 1 \n", "8 Flowers SHIELD 1 \n", "9 Flowers SHIELD 1 " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "display(bigDF.head(10))\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 2 }