[feat] Inference CLI (#830)

Summary:
Created script to allow users to run
inference using various models, text
inputs, and image inputs.

All that is required is using
mmf_inference and passing some
command line args.

Here is how you use it:
```
mmf_interactive checkpoint_path=/checkpoint/brettallen/mmft/model
```

This will initiate an interactive inference
script that will ask for image URL and text
input to run on. It will keep prompting the user
for more images and text until they say `exit`.

I also added the ability to just say `same` to use
the same image because I figured it was inconvenient
for the user to constantly have to copy over
the image URL every time they want to ask a new
question.

Pull Request resolved: #830

Reviewed By: apsdehal

Differential Revision: D27339291

Pulled By: brettallenyo

fbshipit-source-id: db190c40626229900a3530da75b61d6e6b63cdd0

Author: Brett Zachary Allen

mmf/utils/configuration.py

@@ -163,6 +163,116 @@ def get_mmf_env(key=None):
         return config.env
 
 
+def _merge_with_dotlist(
+    config: DictConfig,
+    opts: List[str],
+    skip_missing: bool = False,
+    log_info: bool = True,
+):
+    # TODO: To remove technical debt, a possible solution is to use
+    # struct mode to update with dotlist OmegaConf node. Look into this
+    # in next iteration
+    # TODO: Simplify this function
+    if opts is None:
+        opts = []
+
+    if len(opts) == 0:
+        return config
+
+    # Support equal e.g. model=visual_bert for better future hydra support
+    has_equal = opts[0].find("=") != -1
+    if has_equal:
+        opt_values = [opt.split("=", maxsplit=1) for opt in opts]
+        if not all(len(opt) == 2 for opt in opt_values):
+            for opt in opt_values:
+                assert len(opt) == 2, f"{opt} has no value"
+    else:
+        assert len(opts) % 2 == 0, "Number of opts should be multiple of 2"
+        opt_values = zip(opts[0::2], opts[1::2])
+
+    for opt, value in opt_values:
+        if opt == "dataset":
+            opt = "datasets"
+
+        splits = opt.split(".")
+        current = config
+        for idx, field in enumerate(splits):
+            array_index = -1
+            if field.find("[") != -1 and field.find("]") != -1:
+                stripped_field = field[: field.find("[")]
+                array_index = int(field[field.find("[") + 1 : field.find("]")])
+            else:
+                stripped_field = field
+            if stripped_field not in current:
+                if skip_missing is True:
+                    break
+                raise AttributeError(
+                    "While updating configuration"
+                    " option {} is missing from"
+                    " configuration at field {}".format(opt, stripped_field)
+                )
+            if isinstance(current[stripped_field], collections.abc.Mapping):
+                current = current[stripped_field]
+            elif (
+                isinstance(current[stripped_field], collections.abc.Sequence)
+                and array_index != -1
+            ):
+                try:
+                    current_value = current[stripped_field][array_index]
+                except OCErrors.ConfigIndexError:
+                    if skip_missing:
+                        break
+                    raise
+
+                # Case where array element to be updated is last element
+                if (
+                    not isinstance(
+                        current_value,
+                        (collections.abc.Mapping, collections.abc.Sequence),
+                    )
+                    or idx == len(splits) - 1
+                ):
+                    if log_info:
+                        logger.info(f"Overriding option {opt} to {value}")
+                    current[stripped_field][array_index] = _decode_value(value)
+                else:
+                    # Otherwise move on down the chain
+                    current = current_value
+            else:
+                if idx == len(splits) - 1:
+                    if log_info:
+                        logger.info(f"Overriding option {opt} to {value}")
+                    current[stripped_field] = _decode_value(value)
+                else:
+                    if skip_missing:
+                        break
+
+                    raise AttributeError(
+                        "While updating configuration",
+                        "option {} is not present "
+                        "after field {}".format(opt, stripped_field),
+                    )
+
+    return config
+
+
+def _decode_value(value):
+    # https://github.com/rbgirshick/yacs/blob/master/yacs/config.py#L400
+    if not isinstance(value, str):
+        return value
+
+    if value == "None":
+        value = None
+
+    try:
+        value = literal_eval(value)
+    except ValueError:
+        pass
+    except SyntaxError:
+        pass
+    return value
+
+
 def resolve_cache_dir(env_variable="MMF_CACHE_DIR", default="mmf"):
     # Some of this follow what "transformers" does for there cache resolving
     try:
@@ -217,7 +327,7 @@ def __init__(self, args=None, default_only=False):
 
         # Initially, silently add opts so that some of the overrides for the defaults
         # from command line required for setup can be honored
-        self._default_config = self._merge_with_dotlist(
+        self._default_config = _merge_with_dotlist(
             self._default_config, args.opts, skip_missing=True, log_info=False
         )
         # Register the config and configuration for setup
@@ -231,7 +341,7 @@ def __init__(self, args=None, default_only=False):
 
         self.config = OmegaConf.merge(self._default_config, other_configs)
 
-        self.config = self._merge_with_dotlist(self.config, args.opts)
+        self.config = _merge_with_dotlist(self.config, args.opts)
         self._update_specific(self.config)
         self.upgrade(self.config)
         # Resolve the config here itself after full creation so that spawned workers
@@ -382,115 +492,6 @@ def _register_resolvers(self):
         OmegaConf.register_resolver("resolve_cache_dir", resolve_cache_dir)
         OmegaConf.register_resolver("resolve_dir", resolve_dir)
 
-    def _merge_with_dotlist(
-        self,
-        config: DictConfig,
-        opts: List[str],
-        skip_missing: bool = False,
-        log_info: bool = True,
-    ):
-        # TODO: To remove technical debt, a possible solution is to use
-        # struct mode to update with dotlist OmegaConf node. Look into this
-        # in next iteration
-        # TODO: Simplify this function
-        if opts is None:
-            opts = []
-
-        if len(opts) == 0:
-            return config
-
-        # Support equal e.g. model=visual_bert for better future hydra support
-        has_equal = opts[0].find("=") != -1
-        if has_equal:
-            opt_values = [opt.split("=", maxsplit=1) for opt in opts]
-            if not all(len(opt) == 2 for opt in opt_values):
-                for opt in opt_values:
-                    assert len(opt) == 2, "{} has no value".format(opt)
-        else:
-            assert len(opts) % 2 == 0, "Number of opts should be multiple of 2"
-            opt_values = zip(opts[0::2], opts[1::2])
-
-        for opt, value in opt_values:
-            if opt == "dataset":
-                opt = "datasets"
-
-            splits = opt.split(".")
-            current = config
-            for idx, field in enumerate(splits):
-                array_index = -1
-                if field.find("[") != -1 and field.find("]") != -1:
-                    stripped_field = field[: field.find("[")]
-                    array_index = int(field[field.find("[") + 1 : field.find("]")])
-                else:
-                    stripped_field = field
-                if stripped_field not in current:
-                    if skip_missing is True:
-                        break
-                    raise AttributeError(
-                        "While updating configuration"
-                        " option {} is missing from"
-                        " configuration at field {}".format(opt, stripped_field)
-                    )
-                if isinstance(current[stripped_field], collections.abc.Mapping):
-                    current = current[stripped_field]
-                elif (
-                    isinstance(current[stripped_field], collections.abc.Sequence)
-                    and array_index != -1
-                ):
-                    try:
-                        current_value = current[stripped_field][array_index]
-                    except OCErrors.ConfigIndexError:
-                        if skip_missing:
-                            break
-                        raise
-
-                    # Case where array element to be updated is last element
-                    if (
-                        not isinstance(
-                            current_value,
-                            (collections.abc.Mapping, collections.abc.Sequence),
-                        )
-                        or idx == len(splits) - 1
-                    ):
-                        if log_info:
-                            logger.info(f"Overriding option {opt} to {value}")
-                        current[stripped_field][array_index] = self._decode_value(value)
-                    else:
-                        # Otherwise move on down the chain
-                        current = current_value
-                else:
-                    if idx == len(splits) - 1:
-                        if log_info:
-                            logger.info(f"Overriding option {opt} to {value}")
-                        current[stripped_field] = self._decode_value(value)
-                    else:
-                        if skip_missing:
-                            break
-
-                        raise AttributeError(
-                            "While updating configuration",
-                            "option {} is not present "
-                            "after field {}".format(opt, stripped_field),
-                        )
-
-        return config
-
-    def _decode_value(self, value):
-        # https://github.com/rbgirshick/yacs/blob/master/yacs/config.py#L400
-        if not isinstance(value, str):
-            return value
-
-        if value == "None":
-            value = None
-
-        try:
-            value = literal_eval(value)
-        except ValueError:
-            pass
-        except SyntaxError:
-            pass
-        return value
-
     def freeze(self):
         OmegaConf.set_struct(self.config, True)
 

mmf_cli/interactive.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3 -u
+# Copyright (c) Facebook, Inc. and its affiliates.
+import argparse
+import logging
+import typing
+
+from mmf.utils.configuration import _merge_with_dotlist
+from mmf.utils.flags import flags
+from mmf.utils.inference import Inference
+from mmf.utils.logger import setup_logger
+from omegaconf import OmegaConf
+
+
+def construct_config(opts: typing.List[str]):
+    config = OmegaConf.create({"checkpoint_path": ""})
+    return _merge_with_dotlist(config, opts)
+
+
+def interactive(opts: typing.Optional[typing.List[str]] = None):
+    """Inference runs inference on an image and text provided by the user.
+    You can optionally run inference programmatically by passing an optlist as opts.
+
+    Args:
+        opts (typing.Optional[typing.List[str]], optional): Optlist which can be used.
+            to override opts programmatically. For e.g. if you pass
+            opts = ["checkpoint_path=my/directory"], this will set the checkpoint.
+    """
+    if opts is None:
+        parser = flags.get_parser()
+        args = parser.parse_args()
+    else:
+        args = argparse.Namespace(config_override=None)
+        args.opts = opts
+
+    setup_logger()
+    logger = logging.getLogger("mmf_cli.interactive")
+
+    config = construct_config(args.opts)
+    inference = Inference(checkpoint_path=config.checkpoint_path)
+    logger.info("Enter 'exit' at any point to terminate.")
+    logger.info("Enter an image URL:")
+    image_url = input()
+    logger.info("Got image URL.")
+    logger.info("Enter text:")
+    text = input()
+    logger.info("Got text input.")
+    while text != "exit":
+        logger.info("Running inference on image and text input.")
+        answer = inference.forward(image_url, {"text": text}, image_format="url")
+        logger.info("Model response: " + answer)
+        logger.info(
+            f"Enter another image URL or leave it blank to continue using {image_url}:"
+        )
+        new_image_url = input()
+        if new_image_url != "":
+            image_url = new_image_url
+        if new_image_url == "exit":
+            break
+        logger.info("Enter another text input:")
+        text = input()
+
+
+if __name__ == "__main__":
+    interactive()

setup.py

@@ -156,6 +156,7 @@ def fetch_package_data():
                 "mmf_run = mmf_cli.run:run",
                 "mmf_predict = mmf_cli.predict:predict",
                 "mmf_convert_hm = mmf_cli.hm_convert:main",
+                "mmf_interactive = mmf_cli.interactive:interactive",
             ]
         },
     )