main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torchvision.utils import save_image\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision import datasets, models, transforms\n",
    "import torchvision\n",
    "\n",
    "import os\n",
    "import sys\n",
    "sys.path.insert(0, \"../utils\")\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "from tqdm import tqdm\n",
    "\n",
    "import src.datasets.cityscapes_loader as cityscapes_loader\n",
    "import utils.train_eval as train_eval\n",
    "import importlib\n",
    "import visualizations as vis\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2975 train images\n",
      "Found 500 val images\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(cityscapes_loader)\n",
    "\n",
    "is_sequence = True\n",
    "\n",
    "dataset_root_dir = \"/home/nfs/inf6/data/datasets/cityscapes/\"\n",
    "\n",
    "train_ds = cityscapes_loader.cityscapesLoader(root=dataset_root_dir, split='train', img_size=(512, 1024), is_transform=True, is_sequence=is_sequence)\n",
    "val_ds = cityscapes_loader.cityscapesLoader(root=dataset_root_dir, split='val', img_size=(1024, 2048), is_transform=True, is_sequence=is_sequence)\n",
    "#val_ds = cityscapes_loader.cityscapesLoader(root=dataset_root_dir, split='val', img_size=(512, 1024), is_transform=True, is_sequence=is_sequence)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_ds, batch_size=2, shuffle=True, drop_last=True)\n",
    "valid_loader = torch.utils.data.DataLoader(val_ds, batch_size=1, shuffle=False, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/user/sheludzk/CudaLab_Project/tboard_logs/Temporal_ConvUNextConfig_Conv2dGRUCell/Layers4_InitDim16\n"
     ]
    }
   ],
   "source": [
    "from src.architectures.architecture_configs import *\n",
    "import src.architectures.Temporal_UNET_Template as Temporal_UNET_Template\n",
    "import utils.utils\n",
    "\"\"\"\n",
    "encoder_blocks = SmallDeep_NetworkSize.encoder_blocks\n",
    "decoder_blocks = SmallDeep_NetworkSize.decoder_blocks\n",
    "\n",
    "config = Temporal_ResUNetConfig(\n",
    "    encoder_blocks=encoder_blocks,\n",
    "    decoder_blocks=decoder_blocks,\n",
    "    temporal_cell= Conv2dGRUCell\n",
    "    )\n",
    "\n",
    "temp_unet = Temporal_UNET_Template.Temporal_UNet(config)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "encoder_blocks = SmallDeep_NetworkSize.encoder_blocks\n",
    "decoder_blocks = SmallDeep_NetworkSize.decoder_blocks\n",
    "\n",
    "config = Temporal_ConvUNextConfig(\n",
    "        encoder_blocks=encoder_blocks,\n",
    "        decoder_blocks=decoder_blocks,\n",
    "        temporal_cell= Conv2dGRUCell\n",
    "        )\n",
    "\n",
    "\"\"\"\n",
    "encoder_blocks = MediumDeep_NetworkSize.encoder_blocks\n",
    "decoder_blocks = MediumDeep_NetworkSize.decoder_blocks\n",
    "\n",
    "config = Temporal_ConvUNextConfig(\n",
    "        encoder_blocks=encoder_blocks,\n",
    "        decoder_blocks=decoder_blocks,\n",
    "        temporal_cell= Conv2dGRUCell\n",
    "        )\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "\n",
    "temp_unet = Temporal_UNET_Template.Temporal_UNet(config)\n",
    "\n",
    "temp_unet_optim = torch.optim.Adam(temp_unet.parameters(), lr=3e-4)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "epochs=100\n",
    "temp_unet_trainer = utils.train_eval.Trainer(\n",
    "            temp_unet, temp_unet_optim, criterion,\n",
    "            train_loader, valid_loader, \"cityscapes\", epochs,\n",
    "            sequence=True, all_labels=20, start_epoch=62)\n",
    "\n",
    "load_model = True\n",
    "if load_model:\n",
    "    temp_unet_trainer.load_model(\"cityscapes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: torch.Size([1, 12, 20, 512, 1024])\n",
      "1: torch.Size([1, 12, 20, 512, 1024])\n",
      "2: torch.Size([1, 12, 20, 512, 1024])\n",
      "3: torch.Size([1, 12, 20, 512, 1024])\n",
      "4: torch.Size([1, 12, 20, 512, 1024])\n",
      "5: torch.Size([1, 12, 20, 512, 1024])\n",
      "6: torch.Size([1, 12, 20, 512, 1024])\n",
      "7: torch.Size([1, 12, 20, 512, 1024])\n",
      "8: torch.Size([1, 12, 20, 512, 1024])\n",
      "9: torch.Size([1, 12, 20, 512, 1024])\n",
      "10: torch.Size([1, 12, 20, 512, 1024])\n",
      "11: torch.Size([1, 12, 20, 512, 1024])\n",
      "12: torch.Size([1, 12, 20, 512, 1024])\n",
      "13: torch.Size([1, 12, 20, 512, 1024])\n",
      "14: torch.Size([1, 12, 20, 512, 1024])\n",
      "15: torch.Size([1, 12, 20, 512, 1024])\n",
      "16: torch.Size([1, 12, 20, 512, 1024])\n",
      "17: torch.Size([1, 12, 20, 512, 1024])\n",
      "18: torch.Size([1, 12, 20, 512, 1024])\n",
      "19: torch.Size([1, 12, 20, 512, 1024])\n",
      "20: torch.Size([1, 12, 20, 512, 1024])\n",
      "21: torch.Size([1, 12, 20, 512, 1024])\n"
     ]
    }
   ],
   "source": [
    "vis.save_vis_seq(temp_unet, valid_loader, model_name=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vis.create_gifs(\"test\", mode=\"overlay\", transparency=0.45, fps=8)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizations \n",
    "\n",
    "### Baselines\n",
    "\n",
    "Vanilla Original size:\n",
    "\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Baselines/BaselineVanillaOriginalSizes_high_res/1.gif\"></center>\n",
    "\n",
    "VanillaSmallDeep:\n",
    "\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Baselines/BaselineVanillaSmallDeep_high_res/1.gif\"></center>\n",
    "\n",
    "BaselineVanillaSmallShallow:\n",
    "\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Baselines/BaselineVanillaSmallShallow_high_res/1.gif\"></center>\n",
    "\n",
    "According to the validation metrics (mIoU and mAcc) \"Vanilla Original size\"-model should perform betterthen other 2. But visually we can see that VanillaSmallDeep is the leader. All the predictions (even by VanillaSmallDeep) have some chaotic flickering by changing frames.\n",
    "\n",
    "You can find more visualizations for this 3 models here: resources/gifs/Baselines\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Temporal models\n",
    "\n",
    "ResUNet with Conv2dGRUCell SmallDeep:\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Temporal/Temporal_ResUNetConfig_Conv2dGRUCell_SmallDeep_high_res/1.gif\"></center>\n",
    "\n",
    "ResUNet with Conv2dGRUCell SmallShallow:\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Temporal/Temporal_ResUNetConfig_Conv2dGRUCell_SmallShallow_high_res/1.gif\"></center>\n",
    "\n",
    "ConvUNext with Conv2dGRUCell SmallDeep:\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Temporal/Temporal_ConvUNextConfig_Conv2dGRUCell_SmallDeep_high_res/1.gif\"></center>\n",
    "\n",
    "VanillaUNet with Conv2dGRUCell SmallShallow:\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Temporal/Temporal_VanillaUNetConfig_Conv2dGRUCell_SmallShallow_high_res/1.gif\"></center>\n",
    "\n",
    "VanillaUNet  with Conv2dRNNCell SmallShallow:\n",
    "<center><img style=\"width: 70%\" src=\"resources/gifs/Temporal/Temporal_VanillaUNetConfig_Conv2dRNNCell_SmallShallow_high_res/1.gif\"></center>\n",
    "\n",
    "\n",
    "According to the validation metrics (mAcc and mIoU) the leader in the group of temporal models has to be \"ResUNet with Conv2dGRUCell SmallDeep\". That corresponds to the observed results.<br>\n",
    "\n",
    "In general we can observe the following pattern: <br>\n",
    "* first image in the sequence has some level of false predicted regions\n",
    "* every following frame has better prediction\n",
    "\n",
    "This results proove the advantage of usage of recurrent modules as part of U-Net\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Side remark\n",
    "\n",
    "During the training we have observed some interesting detail.<br>\n",
    "\n",
    "Training of all the models was done using 512x1024 frames. According to the conditions of the final project the evaluation should be done using 1024x2048 frames.\n",
    "Some examples of this evaluation can be found here: /resources/gifs <br>\n",
    "\n",
    "We have also done the evaluation of the models using 512x1024 frames.<br>\n",
    "The results of these predictions looks better. Here is an example:\n",
    "\n",
    "ResUNet with Conv2dGRUCell SmallDeep:\n",
    "left-1024x2048, right-512x1024\n",
    "<p float=\"left\">\n",
    "  <img style=\"width: 49%\" src=\"resources/gifs/Temporal/Temporal_ResUNetConfig_Conv2dGRUCell_SmallDeep_high_res/10.gif\" />\n",
    "  <img style=\"width: 49%\" src=\"resources/gifs/Temporal_low_res/Temporal_ResUNetConfig_Conv2dGRUCell_SmallDeep_low_res/10.gif\"  /> \n",
    "</p>\n",
    "\n",
    "We have used random crop augmentation to compensate this effect, but could not completry get rid of it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}