changes from ada

Author: Suman Michael

conf/config.yaml

@@ -1,6 +1,6 @@
 defaults:
   - trainer: default
-  - data: default
+  - data: ada
   - model: default
   - optim: default
   - logger: default
@@ -10,4 +10,15 @@ defaults:
 
 hydra:
   run:
-    dir: .
+    dir: .
+
+trainer:
+  gpus: 1
+  profiler: null
+  progress_bar_refresh_rate: 10
+  benchmark: true
+
+data:
+  batch_size: 32
+  num_workers: 1
+  random_sampler: false

conf/data/ada.yaml

@@ -0,0 +1,5 @@
+defaults:
+  - default
+
+train_list: /ssd_scratch/cvit/sumanmichael/annotation_txt_files/c1_10l.txt
+val_list: /ssd_scratch/cvit/sumanmichael/annotation_txt_files/val.txt

conf/optim/default.yaml

@@ -1,3 +1,4 @@
 lr: 1.0
+rho: 0.9
 eps: 1.0e-06
 weight_decay: 0

conf/trainer/default.yaml

@@ -39,7 +39,7 @@ weights_save_path: null
 num_sanity_val_steps: 2
 truncated_bptt_steps: null
 resume_from_checkpoint: null
-profiler: pytorch
+profiler: null
 benchmark: false
 deterministic: false
 reload_dataloaders_every_n_epochs: 0
@@ -55,4 +55,4 @@ amp_level: O2
 distributed_backend: null
 move_metrics_to_cpu: false
 multiple_trainloader_mode: max_size_cycle
-stochastic_weight_avg: false
+stochastic_weight_avg: false

run.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+#SBATCH --time=6-00:00:00
+#SBATCH -A sumanmichael
+#SBATCH --partition=long
+#SBATCH --gres=gpu:2
+#SBATCH --mem-per-cpu=2G
+#SBATCH -c 20
+#SBATCH --mail-user=sumanmichael01@gmail.com
+#SBATCH --mail-type=ALL
+
+echo "Running on $(hostname)"
+
+#echo "Loading Cuda Modules" &&
+module load cuda/10.2 &&
+module load cudnn/7.6.5-cuda-10.2 &&
+#echo "Loaded cuda modules" &&
+
+echo "Starting Data Transfer"
+echo "-----------------------------------"
+echo "Annotations Transfer:"
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/share3/sumanmichael/annotation_txt_files /ssd_scratch/cvit/sumanmichael/ &&
+echo "Images Transfer:"
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/share3/bvk_cvit/img_data /ssd_scratch/cvit/sumanmichael/ &&
+echo "Codebase Transfer:"
+rm -rf /ssd_scratch/cvit/sumanmichael/seq2seq-attention-ocr-pytorch
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/home2/sumanmichael/seq2seq-attention-ocr-pytorch /ssd_scratch/cvit/sumanmichael/ &&
+echo "Data Transfer successful"
+
+echo "-----------------------------------"
+
+source /home2/sumanmichael/miniconda3/etc/profile.d/conda.sh &&
+conda activate pl &&
+echo "conda env Activated!" &&
+
+cd /ssd_scratch/cvit/sumanmichael/seq2seq-attention-ocr-pytorch &&
+echo "-------------------------------"
+echo " CUDA VERSIONS "
+echo "--------nvidia-smi-------------"
+nvidia-smi
+echo "--------nvcc-------------------"
+nvcc --version
+echo "---------pytorch-cuda----------"
+python -c "import torch; print('TORCH_CUDA_V:',torch.version.cuda)"
+echo "-------------------------------"
+
+CUDA_LAUNCH_BLOCKING=1;HYDRA_FULL_ERROR=1 python train.py logger.name="$1"

run_pt.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+#SBATCH --time=6-00:00:00
+#SBATCH -A sumanmichael
+#SBATCH --partition=long
+#SBATCH --gres=gpu:1
+#SBATCH --mem-per-cpu=2G
+#SBATCH -c 20
+#SBATCH --mail-user=sumanmichael01@gmail.com
+#SBATCH --mail-type=ALL
+
+echo "Running on $(hostname)"
+
+#echo "Loading Cuda Modules" &&
+module load cuda/10.2 &&
+module load cudnn/7.6.5-cuda-10.2 &&
+#echo "Loaded cuda modules" &&
+
+echo "Starting Data Transfer"
+echo "-----------------------------------"
+echo "Annotations Transfer:"
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/share3/sumanmichael/annotation_txt_files /ssd_scratch/cvit/sumanmichael/ &&
+echo "Images Transfer:"
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/share3/bvk_cvit/img_data /ssd_scratch/cvit/sumanmichael/ &&
+echo "Codebase Transfer:"
+rm -rf /ssd_scratch/cvit/sumanmichael/seq2seq-attention-ocr-pytorch
+rsync -az --info=progress2 sumanmichael@ada.iiit.ac.in:/home2/sumanmichael/seq2seq-attention-ocr-pytorch /ssd_scratch/cvit/sumanmichael/ &&
+echo "Data Transfer successful"
+
+echo "-----------------------------------"
+
+source /home2/sumanmichael/miniconda3/etc/profile.d/conda.sh &&
+conda activate pl &&
+echo "conda env Activated!" &&
+
+cd /ssd_scratch/cvit/sumanmichael/seq2seq-attention-ocr-pytorch &&
+echo "-------------------------------"
+echo " CUDA VERSIONS "
+echo "--------nvidia-smi-------------"
+nvidia-smi
+echo "--------nvcc-------------------"
+nvcc --version
+echo "---------pytorch-cuda----------"
+python -c "import torch; print('TORCH_CUDA_V:',torch.version.cuda)"
+echo "-------------------------------"
+
+UDA_LAUNCH_BLOCKING=1;HYDRA_FULL_ERROR=1 python train_pt.py --train_list /ssd_scratch/cvit/sumanmichael/annotation_txt_files/c1_10l.txt --eval_list  /ssd_scratch/cvit/sumanmichael/annotation_txt_files/val.txt --num_epochs 100 --learning_rate 0.1

src/aocr.py

@@ -145,8 +145,6 @@ def training_step(self, train_batch, batch_idx, optimizer_idx=None):
             'train/loss': loss,
             # 'train_wer': self.wer(decoder_outputs, target_variable)   # batch WER?
         }
-        for k, v in log_dict.items():
-            self.logger.experiment[k].log(v)
         self.log_dict(log_dict)
         return loss
 
@@ -165,8 +163,6 @@ def validation_step(self, val_batch, batch_idx, optimizer_idx=None):
             'val/wer': self.wer(decoder_outputs, target_variable),
             'val/cer': self.cer(decoder_outputs, target_variable)
         }
-        for k,v in log_dict.items():
-            self.logger.experiment[k].log(v)
         self.log_dict(log_dict)
         return loss
 

src/modules/decoder.py

@@ -94,7 +94,7 @@ def forward(self, prev_output, attention_context, state):
         # TODO: COMPLETED
         attention_context, attention_weights = self.attention(state_flat)
         # moving to cpu to prevent bloating in GPU
-        self.attention_weights_history.append(attention_weights.detach().to("cpu"))
+        # self.attention_weights_history.append(attention_weights.detach().to("cpu"))
 
         cell_output = cell_output.permute(1, 0, 2)
         out_proj_inp = torch.cat((cell_output, attention_context), dim=2)

train_pt.py

@@ -11,6 +11,7 @@
 from src.utils import utils, dataset
 from src.utils.utils import get_alphabet
 
+import dotenv; dotenv.load_dotenv()
 import neptune.new as neptune
 
 alphabet = get_alphabet()
@@ -37,7 +38,6 @@ def train(train_loader, encoder, decoder, criterion, logger, teach_forcing_prob=
 
     for epoch in range(cfg.num_epochs):
         train_iter = iter(train_loader)
-
         for i in range(len(train_loader)):
             cpu_images, cpu_texts = train_iter.next()
             batch_size = cpu_images.size(0)
@@ -56,11 +56,8 @@ def train(train_loader, encoder, decoder, criterion, logger, teach_forcing_prob=
 
             target_variable = target_variable.cuda()
 
-            decoder_outputs = []
-
             for di in range(1, max_length):
                 decoder_output, attention_context, state = decoder(decoder_input, attention_context, state)
-                decoder_outputs.append(decoder_output)
                 if di == 1:
                     loss = criterion(decoder_output, target_variable[di])
                 else:
@@ -69,6 +66,7 @@ def train(train_loader, encoder, decoder, criterion, logger, teach_forcing_prob=
                     decoder_input = utils.get_one_hot(target_variable[di], num_classes)
                 else:
                     _, topi = decoder_output.data.topk(1)
+                    del decoder_output
                     topi = topi.detach()
                     ni = topi.T[0]
                     decoder_input = utils.get_one_hot(ni, num_classes)
@@ -77,8 +75,7 @@ def train(train_loader, encoder, decoder, criterion, logger, teach_forcing_prob=
             decoder.zero_grad()
             loss.backward()
 
-            logger["train/loss"].log(loss)
-            del loss
+            logger["train/loss"].log(loss.item())
             encoder_optimizer.step()
             decoder_optimizer.step()