[refactor] Remove args totally from the picture (#41)

Author: apsdehal

pythia/common/defaults/configs/base.yml

@@ -2,20 +2,65 @@
 training:
     # Name of the trainer class used to define the training/evalution loop
     trainer: 'base_trainer'
+    # Seed to be used for training. -1 means random seed between 1 and 100000.
+    # Either pass fixed through your config or command line arguments
+    seed: null
     # Name of the experiment, will be used while saving checkpoints
     # and generating reports
     experiment_name: run
     # Maximum number of iterations the training will run
     max_updates: 22000
-    # Maximum epochs in case you don't want to use iterations
+    # Maximum epochs in case you don't want to use max_updates
     # Can be mixed with max iterations, so it will stop whichever is
     # completed first. Default: null means epochs won't be used
     max_epochs: null
+
+    # Type of run, train+inference by default means both training and inference
+    # (test) stage will be run, if run_type contains 'val',
+    # inference will be run on val set also.
+    run_type: train_inference
+
+    # Directory for saving checkpoints and other metadata
+    save_dir: "./save"
+
     # After `log_interval` iterations, current iteration's training loss and
     # metrics will be reported. This will also report validation
     # loss and metrics on a single batch from validation set
     # to provide an estimate on validation side
     log_interval: 100
+    # Directory for saving logs, default is "logs" inside the save folder
+    # If log_dir is specifically passed, logs will be written inside that folder
+    log_dir: null
+    # Level of logging, only logs which are >= to current level will be logged
+    logger_level: info
+    # Log format: json, simple
+    log_format: simple
+    # Whether to log detailed final configuration parameters
+    log_detailed_config: false
+    # Whether Pythia should log or not, Default: False, which means
+    # pythia will log by default
+    should_not_log: false
+
+    # Tensorboard control, by default tensorboard is disabled
+    tensorboard: false
+    # Log directory for tensorboard, default points to same as logs
+    tensorboard_logdir: null
+
+    # Size of each batch. If distributed or data_parallel
+    # is used, this will be divided equally among GPUs
+    batch_size: 512
+    # Number of workers to be used in dataloaders
+    num_workers: 4
+    # Some datasets allow fast reading by loading everything in the memory
+    # Use this to enable it
+    fast_read: false
+    # Whether JSON files for evalai evaluation should be generated
+    evalai_inference: false
+    # Use in multi-tasking, when you want to sample tasks proportional to their sizes
+    dataset_size_proportional_sampling: true
+    # Whether to pin memory in dataloader
+    pin_memory: false
+
     # After `checkpoint_interval` iterations, pythia will make a snapshot
     # which will involve creating a checkpoint for current training scenarios
     checkpoint_interval: 1000
@@ -25,18 +70,6 @@ training:
     clip_gradients: false
     # Mode for clip norm
     clip_norm_mode: all
-    # Tensorboard control
-    tensorboard: false
-    tensorboard_logdir: null
-
-    # Seed to be used for training. -1 means random seed.
-    # Either pass fixed through your config or command line arguments
-    seed: null
-    # Size of each batch. If distributed or data_parallel
-    # is used, this will be divided equally among GPUs
-    batch_size: 512
-    # Number of workers to be used in dataloaders
-    num_workers: 4
 
     # Whether to use early stopping, (Default: false)
     should_early_stop: false
@@ -66,44 +99,20 @@ training:
     # Iteration until which warnup should be done
     warmup_iterations: 1000
 
-    # Type of run, train+inference by default means both training and inference
-    # (test) stage will be run, if run_type contains 'val',
-    # inference will be run on val set also.
-    run_type: train+inference
-    # Level of logging, only logs which are >= to current level will be logged
-    logger_level: info
-
-    device: cuda
-
     # Local rank of the GPU device
+    device: cuda
     local_rank: null
 
-    # Whether JSON files for evalai evaluation should be generated
-    evalai_inference: false
     # Use to load specific modules from checkpoint to your model,
     # this is helpful in finetuning. for e.g. you can specify
     # text_embeddings: text_embedding_pythia
     # for loading `text_embedding` module of your model
     # from `text_embedding_pythia`
     pretrained_mapping: {}
-    # Whether the above mentioned pretrained mapping should be loaded or not
+    # If using a pretrained model. Must be used with --resume_file parameter
+    # to specify pretrained model checkpoint. Will load only specific layers if
+    # pretrained mapping is specified in config
     load_pretrained: false
-
-    # Directory for saving checkpoints and other metadata
-    save_dir: "./save"
-    # Directory for saving logs
-    log_dir: "./logs"
-    # Log format: json, simple
-    log_format: simple
-    # Whether to log detailed final configuration parameters
-    log_detailed_config: false
-    # Whether Pythia should log or not, Default: False, which means
-    # pythia will log by default
-    should_not_log: false
-
-    # If verbose dump is active, pythia will dump dataset, model specific
-    # information which can be useful in debugging
-    verbose_dump: false
     # If resume is true, pythia will try to load automatically load
     # last of same parameters from save_dir
     resume: false
@@ -112,14 +121,15 @@ training:
     # `resume_best` will load the best checkpoint according to monitored metric instead of
     # the last saved ckpt
     resume_best: false
-    # Whether to pin memory in dataloader
-    pin_memory: false
 
-    # Use in multi-tasking, when you want to sample tasks proportional to their sizes
-    dataset_size_proportional_sampling: true
+    # If verbose dump is active, pythia will dump dataset, model specific
+    # information which can be useful in debugging
+    verbose_dump: false
 
+    # Turn on if you want to ignore unused parameters in case of DDP
     find_unused_parameters: false
 
+
 # Configuration for models, default configuration files for various models
 # included in pythia can be found under configs directory in root folder
 model_config: {}

pythia/datasets/multi_dataset.py

@@ -209,40 +209,6 @@ def update_registry_for_model(self, config):
         for builder in self._builders:
             builder.update_registry_for_model(config)
 
-    def init_args(self, parser):
-        parser.add_argument_group("General MultiDataset Arguments")
-        parser.add_argument(
-            "-dsp",
-            "--dataset_size_proportional_sampling",
-            type=bool,
-            default=0,
-            help="Pass if you want to sample from"
-            " dataset according to its size. Default: Equal "
-            " weighted sampling",
-        )
-
-        # TODO: Figure out later if we want to init args from datasets
-        # self._init_args(parser)
-
-    def _init_args(self, parser):
-        """Override this function to add extra parameters to
-        parser in your child task class.
-
-        Parameters
-        ----------
-        parser : ArgumentParser
-            Original parser object passed from the higher level classes like
-            trainer
-
-        Returns
-        -------
-        type
-            Description of returned object.
-
-        """
-        for builder in self._builders:
-            builder.init_args(parser)
-
     def clean_config(self, config):
         """
         Override this in case you want to clean the config you updated earlier

pythia/datasets/vqa/vqa2/builder.py

@@ -49,22 +49,6 @@ def update_registry_for_model(self, config):
                 self.dataset.answer_processor.get_vocab_size(),
             )
 
-    def init_args(self, parser):
-        parser.add_argument_group("VQA2 task specific arguments")
-        parser.add_argument(
-            "--data_root_dir",
-            type=str,
-            default="../data",
-            help="Root directory for data",
-        )
-        parser.add_argument(
-            "-nfr",
-            "--fast_read",
-            type=bool,
-            default=None,
-            help="Disable fast read and load features on fly",
-        )
-
     def set_dataset_class(self, cls):
         self.dataset_class = cls
 

pythia/models/base_model.py

@@ -105,10 +105,6 @@ def init_losses_and_metrics(self):
     def config_path(cls):
         return None
 
-    @classmethod
-    def init_args(cls, parser):
-        return parser
-
     @classmethod
     def format_state_key(cls, key):
         """Can be implemented if something special needs to be done

pythia/utils/configuration.py

@@ -134,17 +134,14 @@ def __init__(self, args):
         user_config = self._build_user_config(opts_config)
         model_config = self._build_model_config(opts_config)
         dataset_config = self._build_dataset_config(opts_config)
-        args_overrides = self._build_args_overrides(args)
+        args_overrides = self._build_demjson_config(args.config_override)
 
         self._default_config = default_config
         self._user_config = user_config
         self.config = OmegaConf.merge(
             default_config, model_config, dataset_config, user_config, args_overrides
         )
 
-        # TODO: Remove in next iteration
-        self.config = self._update_with_args(self.config, args)
-
         self.config = self._merge_with_dotlist(self.config, args.opts)
         self._update_specific(self.config)
 
@@ -167,29 +164,24 @@ def _build_user_config(self, opts):
 
         return user_config
 
-    def _build_args_overrides(self, args):
-        # Update with demjson if passed
-        demjson_config = self._get_demjson_config(args.config_override)
-        # TODO: Remove in next iteration
-        args_config = self._get_args_config(args)
-        return OmegaConf.merge(demjson_config, args_config)
-
     def _build_model_config(self, config):
         model = config.model
         if model is None:
             raise KeyError("Required argument 'model' not passed")
         model_cls = registry.get_model_class(model)
 
         if model_cls is None:
-            warnings.warn("No model named '{}' has been registered".format(model))
+            warning = "No model named '{}' has been registered".format(model)
+            warnings.warn(warning)
             return OmegaConf.create()
 
         default_model_config_path = model_cls.config_path()
 
         if default_model_config_path is None:
-            warnings.warn(
-                "Model {}'s class has no default configuration provided".format(model)
+            warning = "Model {}'s class has no default configuration provided".format(
+                model
             )
+            warnings.warn(warning)
             return OmegaConf.create()
 
         return load_yaml(default_model_config_path)
@@ -213,17 +205,15 @@ def _build_dataset_config(self, config):
             builder_cls = registry.get_builder_class(dataset)
 
             if builder_cls is None:
-                warnings.warn(
-                    "No dataset named '{}' has been registered".format(dataset)
-                )
+                warning = "No dataset named '{}' has been registered".format(dataset)
+                warnings.warn(warning)
                 continue
             default_dataset_config_path = builder_cls.config_path()
             if default_dataset_config_path is None:
-                warnings.warn(
-                    "Dataset {}'s builder class has no default configuration provided".format(
-                        dataset
-                    )
+                warning = "Dataset {}'s builder class has no default configuration provided".format(
+                    dataset
                 )
+                warnings.warn(warning)
                 continue
             dataset_config = OmegaConf.merge(
                 dataset_config, load_yaml(default_dataset_config_path)
@@ -235,16 +225,7 @@ def get_config(self):
         self._register_resolvers()
         return self.config
 
-    def _update_with_args(self, config, args, force=False):
-        args_dict = vars(args)
-
-        self._update_key(config, args_dict)
-        if force is True:
-            config.update(args_dict)
-
-        return config
-
-    def _get_demjson_config(self, demjson_string):
+    def _build_demjson_config(self, demjson_string):
         if demjson_string is None:
             return OmegaConf.create()
 
@@ -350,6 +331,9 @@ def freeze(self):
         # self.config = ConfigNode(self.config)
         OmegaConf.set_struct(self.config, True)
 
+    def defrost(self):
+        OmegaConf.set_struct(self.config, False)
+
     def _convert_to_dot_list(self, opts):
         if opts is None:
             opts = []
@@ -414,21 +398,6 @@ def _get_default_config_path(self):
             directory, "..", "common", "defaults", "configs", "base.yml"
         )
 
-    def _update_key(self, dictionary, update_dict):
-        """
-        Takes a single depth dictionary update_dict and uses it to
-        update 'dictionary' whenever key in 'update_dict' is found at
-        any level in 'dictionary'
-        """
-        for key, value in dictionary.items():
-            if not isinstance(value, collections.abc.Mapping):
-                if key in update_dict and update_dict[key] is not None:
-                    dictionary[key] = update_dict[key]
-            else:
-                dictionary[key] = self._update_key(value, update_dict)
-
-        return dictionary
-
     def _update_specific(self, config):
         self.writer = registry.get("writer")
         tp = self.config.training

pythia/utils/flags.py

@@ -11,255 +11,25 @@ class Flags:
     def __init__(self):
         self.parser = argparse.ArgumentParser()
         self.add_core_args()
-        self.update_model_args()
 
     def get_parser(self):
         return self.parser
 
     def add_core_args(self):
-        # TODO: Update default values
         self.parser.add_argument_group("Core Arguments")
-
-        self.parser.add_argument(
-            "--config", type=str, default=None, required=False, help="config yaml file"
-        )
-
-        self.parser.add_argument(
-            "--tasks", type=str, default="", help="Tasks for training"
-        )
-        self.parser.add_argument(
-            "--datasets",
-            type=str,
-            required=False,
-            default="all",
-            help="Datasets to be used for required task",
-        )
-        # self.parser.add_argument(
-        #     "--model", type=str, default="", help="Model for training"
-        # )
-        self.parser.add_argument(
-            "--run_type",
-            type=str,
-            default=None,
-            help="Type of run. Default=train+predict",
-        )
-        self.parser.add_argument(
-            "-exp",
-            "--experiment_name",
-            type=str,
-            default=None,
-            help="Name of the experiment",
-        )
-
-        self.parser.add_argument(
-            "--seed",
-            type=int,
-            default=None,
-            help="random seed, default None, meaning nothing will be seeded"
-            " set seed to -1 if need a random seed"
-            " between 1 and 100000",
-        )
-        self.parser.add_argument(
-            "--config_overwrite",
-            type=str,
-            help="a json string to update yaml config file",
-            default=None,
-        )
-
-        self.parser.add_argument(
-            "--force_restart",
-            action="store_true",
-            help="flag to force clean previous result and restart training",
-        )
-        self.parser.add_argument(
-            "--log_interval",
-            type=int,
-            default=None,
-            help="Number of iterations after which we should log validation results",
-        )
-        self.parser.add_argument(
-            "--checkpoint_interval",
-            type=int,
-            default=None,
-            help="Number of iterations after which we should save snapshots",
-        )
-        self.parser.add_argument(
-            "--evaluation_interval",
-            type=int,
-            default=None,
-            help="Number of iterations after which we should save snapshots",
-        )
-        self.parser.add_argument(
-            "--max_updates",
-            type=int,
-            default=None,
-            help="Number of iterations after which  we should stop training",
-        )
-        self.parser.add_argument(
-            "--max_epochs",
-            type=int,
-            default=None,
-            help="Number of epochs after which "
-            " we should stop training"
-            " (mutually exclusive with max_updates)",
-        )
-        self.parser.add_argument(
-            "--batch_size",
-            type=int,
-            default=None,
-            help="Batch size to be used for training "
-            "If not passed it will default to config one",
-        )
-        self.parser.add_argument(
-            "--save_dir",
-            type=str,
-            default="./save",
-            help="Location for saving model checkpoint",
-        )
-        self.parser.add_argument(
-            "--log_dir", type=str, default=None, help="Location for saving logs"
-        )
-        self.parser.add_argument(
-            "--logger_level", type=str, default=None, help="Level of logging"
-        )
-        self.parser.add_argument(
-            "--log_detailed_config",
-            type=int,
-            default=None,
-            help="Log detailed final configuration parameters",
-        )
-
-        self.parser.add_argument(
-            "--should_not_log",
-            action="store_true",
-            default=False,
-            help="Set when you don't want logging to happen",
-        )
         self.parser.add_argument(
             "-co",
             "--config_override",
             type=str,
             default=None,
             help="Use to override config from command line directly",
         )
-        self.parser.add_argument(
-            "--resume_file",
-            type=str,
-            default=None,
-            help="File from which to resume checkpoint",
-        )
-        self.parser.add_argument(
-            "--resume",
-            type=bool,
-            default=None,
-            help="Use when you want to restore from automatic checkpoint",
-        )
-        self.parser.add_argument(
-            "--resume_best",
-            type=bool,
-            default=None,
-            help="Use when you want to restore from last best checkpoint instead of last ckpt",
-        )
-        self.parser.add_argument(
-            "--evalai_inference",
-            type=bool,
-            default=None,
-            help="Whether predictions should be made for EvalAI.",
-        )
-        self.parser.add_argument(
-            "--verbose_dump",
-            type=bool,
-            default=None,
-            help="Whether to do verbose dump of dataset"
-            " samples, predictions and other things.",
-        )
-        self.parser.add_argument(
-            "--lr_scheduler",
-            type=bool,
-            default=None,
-            help="Use when you want to use lr scheduler",
-        )
-        self.parser.add_argument(
-            "--clip_gradients",
-            type=bool,
-            default=None,
-            help="Use when you want to clip gradients",
-        )
-        self.parser.add_argument(
-            "--tensorboard", type=bool, default=False, help="Enable tensorboard"
-        )
-        self.parser.add_argument(
-            "--tensorboard_logdir",
-            type=str,
-            default=None,
-            help="Default logdir for tensorboard",
-        )
-
-        self.parser.add_argument(
-            "-dev",
-            "--device_id",
-            type=str,
-            default=None,
-            help="Specify device to be used for training",
-        )
-        self.parser.add_argument(
-            "-p", "--patience", type=int, default=None, help="Patience for early stop"
-        )
-        self.parser.add_argument(
-            "-fr",
-            "--fast_read",
-            type=bool,
-            default=None,
-            help="If fast read should be activated",
-        )
-        self.parser.add_argument(
-            "-pt",
-            "--load_pretrained",
-            type=int,
-            default=None,
-            help="If using a pretrained model. "
-            "Must be used with --resume_file parameter "
-            "to specify pretrained model checkpoint. "
-            "Will load only specific layers if "
-            "pretrained mapping is specified in config",
-        )
-
-        self.parser.add_argument(
-            "-nw",
-            "--num_workers",
-            type=int,
-            default=None,
-            help="Number of workers for dataloaders",
-        )
-        self.parser.add_argument(
-            "-lr",
-            "--local_rank",
-            type=int,
-            default=None,
-            help="Local rank of the current node",
-        )
         self.parser.add_argument(
             "opts",
             default=None,
             nargs=argparse.REMAINDER,
             help="Modify config options from command line",
         )
 
-    def update_model_args(self):
-        args = sys.argv
-        model_name = None
-        for index, item in enumerate(args):
-            if item == "--model":
-                model_name = args[index + 1]
-
-        if model_name is None:
-            return
-
-        model_class = registry.get_model_class(model_name)
-        if model_class is None:
-            return
-
-        model_class.init_args(self.parser)
-
 
 flags = Flags()

tests/datasets/test_base_dataset.py

@@ -18,7 +18,8 @@ def test_init_processors(self):
         args = dummy_args()
         args.opts.append("config={}".format(path))
         configuration = Configuration(args)
-        answer_processor = configuration.get_config()["dataset_attributes"]["vqa2"][
+        print(configuration.get_config())
+        answer_processor = configuration.get_config()["dataset_config"]["vqa2"][
             "processors"
         ]["answer_processor"]
         vocab_path = os.path.join(