9.  GradCAM - onnx - fastai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "documented-picture",
   "metadata": {
    "id": "based-therapy"
   },
   "source": [
    "# Exercise 09: fastai_onnx_gradcam\n",
    "\n",
    "***search for # TASK XX in the code and fill missing lines***"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eight-looking",
   "metadata": {
    "id": "subject-weekend"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Miro001/advanced-computer-vision/blob/main/9.%20%20GradCAM%20-%20onnx%20-%20fastai.ipynb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pressing-driver",
   "metadata": {
    "id": "frank-faculty",
    "tags": []
   },
   "source": [
    "## Install necessary dependencies\n",
    "###  on Google Colab\n",
    "1. Use GPU. Runtime -> Change runtime type -> Hardware accelerator -> GPU\n",
    "2. Install dependencies from the following cell\n",
    "3. Runtime -> Restart runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hired-repeat",
   "metadata": {
    "id": "vw637lwoFqUh"
   },
   "outputs": [],
   "source": [
    "!pip install light-the-torch >> /.tmp\n",
    "!ltt install torch torchvision >> /.tmp\n",
    "!pip install fastai --upgrade >> /.tmp\n",
    "!pip install wandb\n",
    "!pip install onnxruntime"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stunning-duration",
   "metadata": {},
   "source": [
    "### on local comp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dangerous-passion",
   "metadata": {},
   "outputs": [],
   "source": [
    "conda install -c fastai -c pytorch fastai\n",
    "pip install wandb\n",
    "pip install onnxruntime\n",
    "pip install ipywidgets\n",
    "pip install matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "laughing-green",
   "metadata": {
    "id": "municipal-jewel",
    "tags": []
   },
   "source": [
    "## Download dataset and import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "minus-lightweight",
   "metadata": {
    "id": "prepared-median"
   },
   "outputs": [],
   "source": [
    "from ipywidgets import IntSlider\n",
    "from matplotlib.backends.backend_agg import FigureCanvasAgg\n",
    "from matplotlib.figure import Figure\n",
    "from fastai.vision.widgets import *\n",
    "from fastai.vision.all import *\n",
    "from fastai.callback.wandb import *\n",
    "import wandb\n",
    "import onnxruntime as ort\n",
    "wandb.init(anonymous='allow') # use wandb without an account \n",
    "# wandb.init() # uncomment if you are a registered user"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "democratic-arena",
   "metadata": {
    "id": "removable-violation"
   },
   "source": [
    "## Create Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "manufactured-creativity",
   "metadata": {
    "id": "absent-auditor"
   },
   "outputs": [],
   "source": [
    "# DOWNLOAD DATASET\n",
    "path = untar_data(URLs.PETS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "artistic-liverpool",
   "metadata": {
    "id": "genetic-christopher"
   },
   "outputs": [],
   "source": [
    "files = get_image_files(path/\"images\")\n",
    "SIZE = 224\n",
    "files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "electoral-observer",
   "metadata": {
    "id": "starting-client"
   },
   "source": [
    "### TASK 01: name of the file is a class. fix a get_y oneliner method to transform string e.g. 'Abyssinian_106.jpg' into string(class) 'abyssinian' \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "informal-destruction",
   "metadata": {
    "id": "interim-northeast"
   },
   "outputs": [],
   "source": [
    "def get_y(f): return ...\n",
    "dls = ImageDataLoaders.from_name_func(path, fnames=files, label_func=get_y, item_tfms=Resize(SIZE)) # add num_workers=0 when running on windows\n",
    "classes = dls.vocab\n",
    "dls.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "christian-egyptian",
   "metadata": {
    "id": "suitable-brunswick"
   },
   "outputs": [],
   "source": [
    "assert len(dls.train_ds.vocab) == 37, \"Not correct number of classes\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "collected-stupid",
   "metadata": {
    "id": "returning-criterion"
   },
   "source": [
    "### (OPTIONAL TASK) change resnet18 to other arch. Is the model improving with a change of the arch ? \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "infrared-holiday",
   "metadata": {
    "id": "zWqec3m3hzCK"
   },
   "outputs": [],
   "source": [
    "dir(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "extreme-immune",
   "metadata": {
    "id": "s-Hr-1JBij1y"
   },
   "source": [
    "## TRAIN on pretrained model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "velvet-matthew",
   "metadata": {
    "id": "faced-appraisal"
   },
   "outputs": [],
   "source": [
    "learn = cnn_learner(dls, resnet18, metrics=[error_rate, accuracy], cbs=[WandbCallback(), SaveModelCallback(monitor='error_rate', comp=np.less)])\n",
    "# learn = cnn_learner(dls, ..., metrics=[error_rate, accuracy], cbs=[WandbCallback(), SaveModelCallback(monitor='error_rate', comp=np.less)]) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "classical-parameter",
   "metadata": {
    "id": "legal-diameter"
   },
   "outputs": [],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "grand-guide",
   "metadata": {
    "id": "decimal-outdoors"
   },
   "outputs": [],
   "source": [
    "learn.fine_tune(5, 3e-2, freeze_epochs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eligible-cylinder",
   "metadata": {
    "id": "lonely-license"
   },
   "outputs": [],
   "source": [
    "# INFERENCE\n",
    "learn.show_results()\n",
    "# If you experiment\n",
    "learn.export(fname='classifier.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "common-deputy",
   "metadata": {
    "id": "geological-operation"
   },
   "outputs": [],
   "source": [
    "interp = ClassificationInterpretation.from_learner(learn)\n",
    "losses,idxs = interp.top_losses()\n",
    "len(dls.valid_ds)==len(losses)==len(idxs)\n",
    "interp.plot_confusion_matrix(figsize=(7,7))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "prescription-toolbox",
   "metadata": {
    "id": "rising-international"
   },
   "source": [
    "### TASK 02: based on the confusion matrix compute sensitivity and specificity of EACH class\n",
    "***\n",
    "Sensitivity (True Positive rate) measures the proportion of positives that are correctly identified. It is defined as follows:\n",
    "$$sensitivity = \\frac{true\\_positive}{true\\_positive + false\\_negative}$$\n",
    "Specificity (True Negative rate) measures the proportion of negatives that are correctly identified. It is defined as follows:\n",
    "$$specificity = \\frac{true\\_negative}{true\\_negative + false\\_positive}$$\n",
    "***\n",
    "https://en.wikipedia.org/wiki/Sensitivity_and_specificity\n",
    "\n",
    "HINT: https://stackoverflow.com/questions/55635406/how-to-calculate-multiclass-overall-accuracy-sensitivity-and-specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "preliminary-printing",
   "metadata": {
    "id": "statutory-teaching"
   },
   "outputs": [],
   "source": [
    "interp.confusion_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "practical-painting",
   "metadata": {
    "id": "aw4UDtlWhHsd"
   },
   "outputs": [],
   "source": [
    "..."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "indie-southeast",
   "metadata": {
    "id": "essential-court"
   },
   "source": [
    "### TASK 03: compare inference times of the learn.predict and the onnx one. You can also use jupyter's %timeit, see: https://ipython.readthedocs.io/en/stable/interactive/magics.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "elementary-cheese",
   "metadata": {
    "id": "northern-compilation"
   },
   "outputs": [],
   "source": [
    "# ORIGINAL INFERENCE\n",
    "learn.predict(files[0])\n",
    "..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "curious-victim",
   "metadata": {
    "id": "consolidated-mason"
   },
   "outputs": [],
   "source": [
    "torch.onnx.export(\n",
    "    learn.model,\n",
    "    torch.randn(1, 3, SIZE, SIZE).cuda(), # The content of the tensor does not matter, but it needs to have the correct shape.\n",
    "                                          # In particular, the first axis must be the batch size, even if it is 1. \n",
    "    \"classifier.onnx\",\n",
    "    input_names=[\"image\"],\n",
    "    output_names=[\"diagnosis\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "narrow-hartford",
   "metadata": {
    "id": "dirty-grill"
   },
   "outputs": [],
   "source": [
    "# ONNX INFERENCE\n",
    "session = ort.InferenceSession('classifier.onnx')\n",
    "input_name = session.get_inputs()[0].name\n",
    "img = PILImage.create(files[0])\n",
    "session.run(None, {input_name: np.array(img.resize((SIZE,SIZE))).reshape(1,3,SIZE,SIZE).astype(np.float32)})\n",
    "..."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hawaiian-edinburgh",
   "metadata": {
    "id": "classified-testimony"
   },
   "source": [
    "## Grad CAM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "armed-track",
   "metadata": {
    "id": "rubber-train"
   },
   "source": [
    "### TASK 04: Debug get_cam_map method with jupyter's %debug\n",
    "* 'up', 'down' is for moving in the call stack,\n",
    "* 'list' shows code\n",
    "* 'help' for more options\n",
    "* try to print also some variables in the current call context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fitting-patent",
   "metadata": {
    "id": "utility-assets"
   },
   "outputs": [],
   "source": [
    "# grad cam\n",
    "def get_cam_map(model, x, cls, layer=-2):\n",
    "    with Hook(model[0][layer], lambda m, i, o: o[10].detach().clone(), is_forward=False) as hookg:\n",
    "        with Hook(model[0][layer], lambda m, i, o: o.detach().clone(), is_forward=True) as hook:\n",
    "            output = model.eval()(x.cuda())\n",
    "            act = hook.stored\n",
    "        output[0, cls].backward()\n",
    "        grad = hookg.stored\n",
    "    w = grad[0].mean(dim=[1,2], keepdim=True)\n",
    "    cam_map = (w * act[0]).sum(0)\n",
    "    return cam_map.detach().cpu()\n",
    "\n",
    "# merge image with heatmap\n",
    "def merge_img_cam(image, cam_map):\n",
    "    image = image.resize((SIZE, SIZE))\n",
    "    # make a Figure and attach it to a canvas.\n",
    "    fig = Figure(figsize=(SIZE/100, SIZE/100), dpi=100)\n",
    "    canvas = FigureCanvasAgg(fig)\n",
    "    ax = fig.subplots()\n",
    "    # Do some plotting here\n",
    "    show_image(image, ctx=ax)\n",
    "    ax.axis('off')\n",
    "    ax.imshow(cam_map, alpha=0.6, extent=(0,SIZE,SIZE,0), interpolation='bilinear', cmap='magma')\n",
    "\n",
    "    # Retrieve a view on the renderer buffer\n",
    "    canvas.draw()\n",
    "    buf = canvas.buffer_rgba()\n",
    "    # convert to a NumPy array\n",
    "    X = np.asarray(buf)\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sophisticated-dragon",
   "metadata": {
    "id": "cALLb9tzi5Ii"
   },
   "source": [
    "## Simple site with widgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "warming-bracelet",
   "metadata": {
    "id": "filled-forward"
   },
   "outputs": [],
   "source": [
    "btn_upload = widgets.FileUpload()\n",
    "out_pl = widgets.Output()\n",
    "grad_pl = widgets.Output()\n",
    "lbl_pred = widgets.Label('Prediction: -; Probability: -')\n",
    "grad_classes = Dropdown(options=classes, index=0)\n",
    "grad_layer_slider = IntSlider(min=-5, max=-2, step=1, value=-2)\n",
    "\n",
    "def prepare_image(img):\n",
    "    return PILImage.create(np.array(PILImage.create(img).resize((SIZE,SIZE))))\n",
    "\n",
    "def on_class_slider_change(change):\n",
    "    grad_pl.clear_output()\n",
    "    img = prepare_image(btn_upload.data[-1])\n",
    "    x, = first(learn.dls.test_dl([img], rm_type_tfms=None, num_workers=0)) \n",
    "    with grad_pl:\n",
    "        cam_map = get_cam_map(model=learn.model, x=x, cls=grad_classes.index, layer=grad_layer_slider.value)\n",
    "        display(Image.fromarray(merge_img_cam(img, cam_map)))\n",
    "\n",
    "\n",
    "def on_click(change):\n",
    "    out_pl.clear_output()\n",
    "    grad_pl.clear_output()\n",
    "    img = prepare_image(btn_upload.data[-1])\n",
    "    with out_pl: \n",
    "        display(img)\n",
    "    pred,pred_idx,probs = learn.predict(img)\n",
    "    lbl_pred.value = f'Prediction: {pred}; Probability: {str(probs[pred_idx].numpy())}'\n",
    "    x, = first(learn.dls.test_dl([img], rm_type_tfms=None, num_workers=0))\n",
    "    \n",
    "    with grad_pl:\n",
    "        cam_map = get_cam_map(model=learn.model, x=x, cls=grad_classes.index)\n",
    "        display(Image.fromarray(merge_img_cam(img, cam_map)))\n",
    "\n",
    "btn_upload.observe(on_click, names=['data'])\n",
    "grad_classes.observe(on_class_slider_change)\n",
    "grad_layer_slider.observe(on_class_slider_change)\n",
    "\n",
    "display(VBox([widgets.Label('Classify your cat or dog breed!'), \n",
    "              btn_upload, \n",
    "              out_pl, \n",
    "              lbl_pred, \n",
    "              HBox([widgets.Label('Grad Class Activation Map for class: '), grad_classes]), \n",
    "              HBox([widgets.Label('Grad Class Activation Map for model layer: '), grad_layer_slider]), \n",
    "              grad_pl]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "korean-graphics",
   "metadata": {
    "id": "zOIV4C51gQqh"
   },
   "outputs": [],
   "source": [
    "%debug"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "8_fastai_onnx_gradcam.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}