[fix] Fix flake8 errors (#42)

* [fix] Fix flake8 errors

* [enhancement] Add .flake8

Author: vedanuj

.flake8

@@ -0,0 +1,7 @@
+# This is an example .flake8 config used when developing *Black* itself.
+
+[flake8]
+max-line-length = 88
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+ignore = E203, E266, C901, C408, W503

projects/M4C/scripts/extract_ocr_frcn_feature.py

@@ -159,7 +159,7 @@ def main():
     print("Faster R-CNN OCR features")
     print("\textracting from", IMDB_FILE)
     print("\tsaving to", SAVE_DIR)
-    for n, info in enumerate(tqdm.tqdm(imdb)):
+    for _, info in enumerate(tqdm.tqdm(imdb)):
         image_path = os.path.join(IMAGE_DIR, info["image_path"])
         save_feat_path = os.path.join(SAVE_DIR, info["feature_path"])
         save_info_path = save_feat_path.replace(".npy", "_info.npy")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.isort]
 skip_glob = ["*/build/*"]
-known_third_party = ["PIL", "caffe2", "coco_caption_eval", "common", "cv2", "datasets", "demjson", "git", "h5py", "lmdb", "maskrcnn_benchmark", "numpy", "omegaconf", "pycocoevalcap", "pytorch_pretrained_bert", "recommonmark", "requests", "setuptools", "sklearn", "sphinx_rtd_theme", "torch", "torchtext", "torchvision", "tqdm", "transformers", "utils", "yaml"]
+known_third_party = ["PIL", "caffe2", "coco_caption_eval", "common", "cv2", "demjson", "git", "h5py", "lmdb", "maskrcnn_benchmark", "numpy", "omegaconf", "pycocoevalcap", "pytorch_pretrained_bert", "recommonmark", "requests", "setuptools", "sklearn", "sphinx_rtd_theme", "torch", "torchtext", "torchvision", "tqdm", "transformers", "utils", "yaml"]
 multi_line_output = 3
 line_length = 88
 include_trailing_comma = true

pythia/common/constants.py

@@ -1,5 +1,4 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
-import os
 
 imdb_version = 1
 FASTTEXT_WIKI_URL = (

pythia/common/dataset_loader.py

@@ -1,15 +1,8 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
-import os
 
-import yaml
-from torch.utils.data import DataLoader
-
-from pythia.common.batch_collator import BatchCollator
 from pythia.common.sample import SampleList
 from pythia.common.test_reporter import TestReporter
 from pythia.datasets.multi_dataset import MultiDataset
-from pythia.datasets.samplers import DistributedSampler
-from pythia.utils.general import get_batch_size
 
 
 class DatasetLoader:

pythia/common/report.py

@@ -7,8 +7,10 @@
 
 
 class Report(OrderedDict):
-    def __init__(self, batch, model_output={}, *args):
+    def __init__(self, batch, model_output=None, *args):
         super().__init__(self)
+        if model_output is None:
+            model_output = {}
         if self._check_and_load_tuple(batch):
             return
 

pythia/common/sample.py

@@ -13,7 +13,6 @@
 
 import collections
 from collections import OrderedDict
-from copy import deepcopy
 
 import torch
 
@@ -33,7 +32,9 @@ class Sample(OrderedDict):
         >>> sample.context = torch.tensor(4)
     """
 
-    def __init__(self, init_dict={}):
+    def __init__(self, init_dict=None):
+        if init_dict is None:
+            init_dict = {}
         super().__init__(init_dict)
 
     def __setattr__(self, key, value):
@@ -82,8 +83,10 @@ class SampleList(OrderedDict):
 
     _TENSOR_FIELD_ = "_tensor_field"
 
-    def __init__(self, samples=[]):
+    def __init__(self, samples=None):
         super().__init__(self)
+        if samples is None:
+            samples = []
 
         if len(samples) == 0:
             return

pythia/common/test_reporter.py

@@ -99,7 +99,9 @@ def get_dataloader(self):
             **other_args
         )
 
-    def _add_extra_args_for_dataloader(self, other_args={}):
+    def _add_extra_args_for_dataloader(self, other_args=None):
+        if other_args is None:
+            other_args = {}
         training = self.config.training
 
         if training.local_rank is not None and training.distributed:

pythia/datasets/base_dataset.py

@@ -18,8 +18,10 @@ class BaseDataset(Dataset):
         config (ConfigNode): Configuration for the current dataset
     """
 
-    def __init__(self, name, dataset_type, config={}):
+    def __init__(self, name, dataset_type, config=None):
         super(BaseDataset, self).__init__()
+        if config is None:
+            config = {}
         self.config = config
         self._name = name
         self._dataset_type = dataset_type

pythia/datasets/base_dataset_builder.py

@@ -36,7 +36,6 @@ def _build(self, dataset_type, config, *args, **kwargs):
 """
 
 from pythia.utils.distributed_utils import is_master, synchronize
-from pythia.common.registry import registry
 
 
 class BaseDatasetBuilder:

pythia/datasets/captioning/coco/masked_dataset.py

@@ -1,8 +1,5 @@
 import random
 
-import numpy as np
-import torch
-
 from pythia.common.sample import Sample
 from pythia.datasets.captioning.coco import COCODataset
 
@@ -23,7 +20,6 @@ def load_item(self, idx):
         if self._use_features is True:
             features = self.features_db[idx]
             image_labels = []
-            overlaps = np.ones((features["image_feature_0"].shape[0]))
 
             for i in range(features["image_feature_0"].shape[0]):
                 prob = random.random()

pythia/datasets/dialog/visual_dialog/database.py

@@ -1,4 +1,5 @@
 import json
+
 import torch
 
 
@@ -40,7 +41,6 @@ def __getitem__(self, idx):
         data["id"] = idx
         data["dialog_id"] = dialog_id
         data["round_id"] = round_id
-        caption = dialog["caption"]
         round = dialog["dialog"][round_id]
         data["question"] = self._questions[round["question"]]
         # data["answers"] = [self.]

pythia/datasets/dialog/visual_dialog/scripts/build_imdb.py

@@ -90,7 +90,7 @@ def get_tokens(self, sentences):
         if not isinstance(sentences, list):
             sentences = [sentences]
         final_sentences = []
-        for idx, sentence in enumerate(sentences):
+        for _, sentence in enumerate(sentences):
             tokens = text_tokenize(sentence)
             final_sentences.append(tokens)
 

pythia/datasets/multi_dataset.py

@@ -7,12 +7,12 @@
 import sys
 
 import numpy as np
-from torch.utils.data import DataLoader, Dataset
+from torch.utils.data import DataLoader
 
 from pythia.common.batch_collator import BatchCollator
 from pythia.common.registry import registry
 from pythia.datasets.samplers import DistributedSampler
-from pythia.utils.distributed_utils import broadcast_scalar, is_master, synchronize
+from pythia.utils.distributed_utils import broadcast_scalar, is_master
 from pythia.utils.general import get_batch_size
 
 # from torch.utils.data.distributed import DistributedSampler
@@ -241,7 +241,9 @@ def build_dataloader(self, dataset, opts):
 
         return loader, other_args.get("sampler", None)
 
-    def _add_extra_args_for_dataloader(self, dataset, opts, other_args={}):
+    def _add_extra_args_for_dataloader(self, dataset, opts, other_args=None):
+        if other_args is None:
+            other_args = {}
         training = self._global_config.training
         dataset_type = self._dataset_type
 

pythia/datasets/processors/processors.py

@@ -69,15 +69,13 @@ def __call__(self, item, *args, **kwargs):
             text = [t.strip() for t in text.split(" ")]
             return {"text": text}
 """
-import multiprocessing
 import os
 import re
 import warnings
 from collections import Counter, defaultdict
 
 import numpy as np
 import torch
-from transformers.tokenization_bert import BertTokenizer
 
 from pythia.common.registry import registry
 from pythia.utils.configuration import ConfigNode
@@ -745,7 +743,6 @@ def __call__(self, item):
         length = min(len(tokens), self.max_length)
 
         gt_answers = list(enumerate(answers))
-        unique_answers = set(answers)
 
         if self.context_preprocessor is not None:
             tokens = [

pythia/datasets/reasoning/mmimdb/masked_dataset.py

@@ -1,7 +1,5 @@
 import random
 
-import numpy as np
-
 from pythia.common.sample import Sample
 from pythia.datasets.vqa.vqa2.dataset import VQA2Dataset
 
@@ -20,7 +18,6 @@ def load_item(self, idx):
             features = self.features_db[idx]
             current_sample.update(features)
             image_labels = []
-            overlaps = np.ones((features["image_feature_0"].shape[0]))
 
             for i in range(features["image_feature_0"].shape[0]):
                 prob = random.random()

pythia/datasets/vqa/vqa2/masked_dataset.py

@@ -1,7 +1,5 @@
 import random
 
-import numpy as np
-
 from pythia.common.sample import Sample
 from pythia.datasets.vqa.vqa2.dataset import VQA2Dataset
 
@@ -20,7 +18,6 @@ def load_item(self, idx):
             features = self.features_db[idx]
             current_sample.update(features)
             image_labels = []
-            overlaps = np.ones((features["image_feature_0"].shape[0]))
 
             for i in range(features["image_feature_0"].shape[0]):
                 prob = random.random()

pythia/models/ban.py

@@ -49,7 +49,7 @@ def _init_bilinear_attention(self):
         b_net = []
         q_prj = []
 
-        for i in range(module_config["gamma"]):
+        for _ in range(module_config["gamma"]):
             b_net.append(
                 BCNet(
                     v_dim, num_hidden, num_hidden, None, k=module_config["bc_net"]["k"]

pythia/models/base_model.py

@@ -43,14 +43,11 @@ def forward(self, sample_list):
 import warnings
 from copy import deepcopy
 
-from omegaconf import OmegaConf
 from torch import nn
 
 from pythia.common.registry import registry
-from pythia.common.report import Report
 from pythia.modules.losses import Losses
 from pythia.modules.metrics import Metrics
-from pythia.utils.configuration import load_yaml
 
 
 class BaseModel(nn.Module):

pythia/models/butd.py

@@ -27,7 +27,7 @@ def _build_word_embedding(self):
         self.word_embedding = self.vocab.get_embedding(
             torch.nn.Embedding, embedding_dim=self.config["embedding_dim"]
         )
-        setattr(self, "text_embeddings_out_dim", self.config["embedding_dim"])
+        self.text_embeddings_out_dim = self.config["embedding_dim"]
 
     def _init_classifier(self):
         self.classifier = ClassifierLayer(

pythia/models/lorra.py

@@ -1,9 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
-import torch
 
 from pythia.common.registry import registry
 from pythia.models.pythia import Pythia
-from pythia.modules.layers import ClassifierLayer
 
 
 @registry.register_model("lorra")

pythia/models/m4c.py

@@ -277,7 +277,7 @@ def _forward_mmt_and_output(self, sample_list, fwd_results):
             fwd_results["prev_inds"][:, 0] = self.answer_processor.BOS_IDX
 
             # greedy decoding at test time
-            for t in range(dec_step_num):
+            for _ in range(dec_step_num):
                 self._forward_mmt(sample_list, fwd_results)
                 self._forward_output(sample_list, fwd_results)
 

pythia/models/pythia.py

@@ -13,11 +13,7 @@
     TextEmbedding,
 )
 from pythia.modules.encoders import ImageEncoder
-from pythia.modules.layers import (
-    ClassifierLayer,
-    ModalCombineLayer,
-    ReLUWithWeightNormFC,
-)
+from pythia.modules.layers import ClassifierLayer, ModalCombineLayer
 from pythia.utils.configuration import ConfigNode
 
 
@@ -208,8 +204,10 @@ def process_text_embedding(
         return text_embeddding_total
 
     def process_feature_embedding(
-        self, attr, sample_list, text_embedding_total, extra=[], batch_size_t=None
+        self, attr, sample_list, text_embedding_total, extra=None, batch_size_t=None
     ):
+        if extra is None:
+            extra = []
         feature_embeddings = []
         feature_attentions = []
         features = []
@@ -427,8 +425,10 @@ def _init_feature_embeddings(self, attr):
         )
 
     def process_feature_embedding(
-        self, attr, sample_list, text_embedding_total, extra=[], batch_size_t=None
+        self, attr, sample_list, text_embedding_total, extra=None, batch_size_t=None
     ):
+        if extra is None:
+            extra = []
         feature_embeddings = []
         feature_attentions = []
         features = []

pythia/models/pythia_bert.py

@@ -3,9 +3,7 @@
 from transformers.modeling_bert import (
     BertConfig,
     BertEmbeddings,
-    BertEncoder,
     BertForPreTraining,
-    BertLayer,
     BertLayerNorm,
     BertPooler,
     BertPredictionHeadTransform,
@@ -184,9 +182,11 @@ def process_feature_embedding(
         text_embedding_total,
         key_padding_mask=None,
         attn_mask=None,
-        extra=[],
+        extra=None,
         batch_size_t=None,
     ):
+        if extra is None:
+            extra = []
         feature_embeddings = []
         feature_attentions = []
         features = []
@@ -332,7 +332,6 @@ def forward(self, sample_list):
         else:
             joint_embedding = image_embedding_total
 
-        dataset_name = sample_list.dataset_name
         output_dict = {}
 
         pooled_output = self.pooler(joint_embedding)

pythia/models/top_down_bottom_up.py

@@ -1,21 +1,9 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
 import torch
-from torch import nn
 
 from pythia.common.registry import registry
 from pythia.models.base_model import BaseModel
-from pythia.modules.embeddings import (
-    ImageEmbedding,
-    PreExtractedEmbedding,
-    TextEmbedding,
-)
-from pythia.modules.encoders import ImageEncoder
-from pythia.modules.layers import (
-    ClassifierLayer,
-    Identity,
-    ModalCombineLayer,
-    ReLUWithWeightNormFC,
-)
+from pythia.modules.layers import ReLUWithWeightNormFC
 
 
 # Note: Doesn't work currently. Needs to be migrated to new API