notebooks/explaining/cam-gradcam-score.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_AbalcTKgGgw"
   },
   "source": [
    "# Score-CAM\n",
    "\n",
    "Implement the following saliency algorithm described in the article \"Score-CAM:Score-Weighted Visual Explanations for Convolutional Neural Networks\", which is summarized in the following equation:\n",
    "\n",
    "$$\n",
    "L^c_\\text{Score-CAM} = ReLU(∑_k C(A_l^k) A_l^k)\n",
    "$$\n",
    "Where\n",
    "$$\n",
    "C(A_l^k) = f(X_b \\odot s(Up(A_l^k))) - f(X_b)\n",
    "$$\n",
    "\n",
    "And $s$ is the standardization function and $Up$ the upscaling function.\n",
    "\n",
    "\n",
    "This approach main idea is to build the saliency map by weighting the activation signal of the last convolutional layer with its local contribution factor, which will focus solely on the regions that do contribute to the output\n",
    "classification maximization while nullifying unrelated regions.\n",
    "\n",
    "References:\n",
    "\n",
    "- Wang H, Du M, Yang F, Zhang Z. Score-cam: Improved visual explanations via score-weighted class activation mapping. [arXiv preprint arXiv:1910.01279](https://arxiv.org/pdf/1910.01279.pdf), 2019."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LAST_CONV_LAYER = 'block14_sepconv2_act'\n",
    "\n",
    "NUM_SAMPLES = 50\n",
    "NOISE = .2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "from config import config, data, model, utils\n",
    "from config.utils import to_image, normalize\n",
    "\n",
    "utils.setup_clean_image_plotting()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = data.load_images()\n",
    "x = model.preprocess(images)\n",
    "\n",
    "utils.plot(to_image(images),\n",
    "           titles=data.class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2jz1ZKnmi3AX"
   },
   "outputs": [],
   "source": [
    "nn = model.build()\n",
    "\n",
    "logits = nn(x, training=False)\n",
    "preds = tf.argmax(logits, axis=1)\n",
    "probs = tf.nn.softmax(logits)\n",
    "\n",
    "utils.print_predictions(model, images, top=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t65qnU0pMxO3"
   },
   "source": [
    "## Score-CAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 11293,
     "status": "ok",
     "timestamp": 1613687900476,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "Q8aUsyopNnEh"
   },
   "outputs": [],
   "source": [
    "def activation_loss(y, units):\n",
    "    return tf.gather(y, units, axis=1, batch_dims=1)\n",
    "\n",
    "def standardize(x, axis=(1, 2)):\n",
    "    x -= tf.reduce_min(x, axis=axis, keepdims=True)\n",
    "    x /= tf.reduce_max(x, axis=axis, keepdims=True) + 1e-07\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 11290,
     "status": "ok",
     "timestamp": 1613687900478,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "z-6J1ndQ8Gl4"
   },
   "outputs": [],
   "source": [
    "model_s = Model(model.inputs, model.outputs + [model.get_layer(LAST_CONV_LAYER).output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 11286,
     "status": "ok",
     "timestamp": 1613687900478,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "v7uqI-4v2m9E"
   },
   "outputs": [],
   "source": [
    "def scorecam(inputs, units, activations=None):\n",
    "    fb, a = model_s(inputs, training=False)\n",
    "    fb = tf.nn.softmax(fb)\n",
    "    fb = activation_loss(fb, units)\n",
    "\n",
    "    if not activations:\n",
    "        activations = a.shape[-1]\n",
    "\n",
    "    # Always sort so we can break on the following loop.\n",
    "    std = tf.math.reduce_std(a, axis=(1, 2))\n",
    "    s = tf.argsort(std, axis=-1, direction='DESCENDING')[:, :activations]\n",
    "    a = tf.gather(a, s, axis=-1, batch_dims=-1)\n",
    "\n",
    "    a = tf.map_fn(lambda i: tf.image.resize(i, INPUT_SHAPE[:2]), a)\n",
    "    an = standardize(a)\n",
    "\n",
    "    s = tf.zeros(a.shape[:-1])\n",
    "\n",
    "    for i in range(activations):\n",
    "        c = an[..., i:i+1]\n",
    "\n",
    "        an_min = tf.reduce_min(an[..., i:i+1], axis=(1, 2))\n",
    "        an_max = tf.reduce_max(an[..., i:i+1], axis=(1, 2))\n",
    "\n",
    "        if (an_min == an_max).numpy().all():\n",
    "            break  # The NEXT filters have 0 var as well\n",
    "\n",
    "        x = inputs * c # (B, H, W, C) x (B, H, W, 1)\n",
    "\n",
    "        y = model(x, training=False)\n",
    "        y = tf.nn.softmax(y)\n",
    "        y = activation_loss(y, units)\n",
    "        y = tf.reshape(y, (-1, 1, 1))\n",
    "\n",
    "        y = y * a[..., i]\n",
    "\n",
    "        s += y\n",
    "\n",
    "    s = tf.nn.relu(s)\n",
    "    return s / (tf.reduce_max(s, axis=(1, 2), keepdims=True) + 1e-07)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 212395,
     "status": "ok",
     "timestamp": 1613688101592,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "mJBMFNTA4Ni_"
   },
   "outputs": [],
   "source": [
    "SAMPLES = 10\n",
    "\n",
    "maps = tf.concat(\n",
    "    [scorecam(images[ix:ix+1], preds[ix:ix+1], activations=512)\n",
    "     for ix in range(min(SAMPLES, len(images)))],\n",
    "    axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "cellView": "form",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "output_embedded_package_id": "14t4M11B06G4w1HiVlzdtc2xK3AOxNZHl"
    },
    "executionInfo": {
     "elapsed": 11729,
     "status": "ok",
     "timestamp": 1613688443196,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiQTUp6domdv59RQ9rxuJo49Q7CXVKQp2RuAgkgeQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "ugsqYBdr4RDP",
    "outputId": "47299f0a-410f-4c21-ee1d-aa5c35078304"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Output hidden; open in https://colab.research.google.com to view."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#@title\n",
    "\n",
    "def plot_saliency(image, map, rows=1, cols=4, i0=0):\n",
    "    plot([image, map, (image*map[..., np.newaxis]).astype('uint8')],\n",
    "         rows=rows, cols=cols, i0=i0)\n",
    "    plt.subplot(rows, cols, i0+4)\n",
    "    draw_heatmap(image, map)\n",
    "\n",
    "\n",
    "plt.figure(figsize=(12, 30))\n",
    "for ix, (i, m) in enumerate(zip(as_image_vector(images[:SAMPLES]), maps.numpy())):\n",
    "    plot_saliency(i, m, rows=SAMPLES, i0=ix*4)\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SgrB2b8H_RsI"
   },
   "source": [
    "### Vectorized Implementation\n",
    "\n",
    "I have also written a vectorized version of the method.\n",
    "Unfortunatelly, it's not very useful as the statement `inputs * b` will quickly consume all memory resources.\n",
    "\n",
    "P.s.: you can probably use it if passing `activations=256`.\n",
    "\n",
    "```python\n",
    "@tf.function\n",
    "def scorecam(inputs, units, activations=None):\n",
    "    fb, a = model_s(inputs, training=False)\n",
    "    fb = tf.nn.softmax(fb)\n",
    "    fb = activation_loss(fb, units)\n",
    "\n",
    "    if activations:\n",
    "        std = tf.math.reduce_std(a, axis=(1, 2))\n",
    "        s = tf.argsort(std, axis=-1, direction='DESCENDING')[:, :activations]\n",
    "        a = tf.gather(a, s, axis=-1, batch_dims=-1)\n",
    "    else:\n",
    "        activations = a.shape[-1]\n",
    "\n",
    "    a = tf.map_fn(lambda i: tf.image.resize(i, INPUT_SHAPE[:2]), a)\n",
    "    b = standardize(a)\n",
    "    b = tf.transpose(b, (0, 3, 1, 2))\n",
    "    b = b[..., tf.newaxis]               # (B, A, H, W, 1)\n",
    "    inputs = inputs[:, tf.newaxis, ...]  # (B, 1, H, W, C)\n",
    "\n",
    "    b = tf.reshape(inputs * b, (-1, *INPUT_SHAPE))  # batchify (B*A, H, W, C)\n",
    "    fm = model(b, training=False)\n",
    "    fm = tf.nn.softmax(fm)\n",
    "    fm = activation_loss(fm, tf.repeat(units, activations, axis=0))\n",
    "\n",
    "    fm = tf.reshape(fm, (-1, 1, 1, activations)) # restore (B, A)\n",
    "    s = tf.reduce_sum(fm*a, axis=-1)\n",
    "    s = tf.nn.relu(s)\n",
    "\n",
    "    return s / (tf.reduce_max(s, axis=(1, 2), keepdims=True) + 1e-07)\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyN2pdt/SkEKhEhdXdazhBMM",
   "collapsed_sections": [
    "DJogAqXDk0hB",
    "jZOvw6Vvkthh"
   ],
   "name": "score-cam.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}