report progress of delegated operations

Author: brimoor

plugins/annotation/__init__.py

@@ -6,6 +6,7 @@
 |
 """
 import contextlib
+import inspect
 import json
 import threading
 
@@ -893,8 +894,17 @@ def execute(self, ctx):
 
         _inject_annotation_secrets(ctx)
 
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         ctx.dataset.load_annotations(
-            anno_key, unexpected=unexpected, cleanup=cleanup
+            anno_key,
+            unexpected=unexpected,
+            cleanup=cleanup,
+            **kwargs,
         )
 
         if not ctx.delegated:

plugins/annotation/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/annotation"
 description: Utilities for integrating FiftyOne with annotation tools
 version: 1.0.1
 fiftyone:
-  version: ">=0.22"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/annotation
 license: Apache 2.0
 operators:

plugins/brain/__init__.py

@@ -8,6 +8,7 @@
 import base64
 from collections import defaultdict
 from datetime import datetime
+import inspect
 import json
 
 from bson import json_util
@@ -72,6 +73,13 @@ def execute(self, ctx):
             num_workers = 0
 
         target_view = _get_target_view(ctx, target)
+
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         fob.compute_visualization(
             target_view,
             patches_field=patches_field,
@@ -82,6 +90,7 @@ def execute(self, ctx):
             batch_size=batch_size,
             num_workers=num_workers,
             skip_failures=skip_failures,
+            **kwargs,
         )
 
 
@@ -176,6 +185,11 @@ def execute(self, ctx):
             num_workers = 0
 
         target_view = _get_target_view(ctx, target)
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         fob.compute_similarity(
             target_view,
             patches_field=patches_field,
@@ -1057,6 +1071,13 @@ def execute(self, ctx):
             num_workers = 0
 
         target_view = _get_target_view(ctx, target)
+
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         fob.compute_uniqueness(
             target_view,
             uniqueness_field=uniqueness_field,
@@ -1066,6 +1087,7 @@ def execute(self, ctx):
             batch_size=batch_size,
             num_workers=num_workers,
             skip_failures=skip_failures,
+            **kwargs,
         )
 
         if not ctx.delegated:
@@ -1142,6 +1164,11 @@ def execute(self, ctx):
         mistakenness_field = kwargs.pop("mistakenness_field")
 
         target_view = _get_target_view(ctx, target)
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         fob.compute_mistakenness(
             target_view,
             pred_field,
@@ -1320,12 +1347,21 @@ def execute(self, ctx):
         target = ctx.params.get("target", None)
         label_field = ctx.params.get("label_field")
         hardness_field = ctx.params.get("hardness_field")
+        delegate = ctx.params.get("delegate", False)
 
         target_view = _get_target_view(ctx, target)
+
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         fob.compute_hardness(
             target_view,
             label_field,
             hardness_field=hardness_field,
+            **kwargs,
         )
 
         if not ctx.delegated:

plugins/brain/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/brain"
 description: Utilities for working with the FiftyOne Brain
 version: 1.1.1
 fiftyone:
-  version: ">=0.22"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/brain
 license: Apache 2.0
 operators:

plugins/evaluation/__init__.py

@@ -5,6 +5,7 @@
 | `voxel51.com <https://voxel51.com/>`_
 |
 """
+import inspect
 import json
 
 from bson import json_util
@@ -61,6 +62,10 @@ def execute(self, ctx):
         elif eval_type == "segmentation":
             eval_fcn = target_view.evaluate_segmentations
 
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         eval_fcn(
             pred_field,
             gt_field=gt_field,

plugins/evaluation/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/evaluation"
 description: Utilities for evaluating models with FiftyOne
 version: 1.0.1
 fiftyone:
-  version: ">=0.22"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/evaluation
 license: Apache 2.0
 operators:

plugins/io/__init__.py

@@ -7,6 +7,7 @@
 """
 import base64
 import contextlib
+import inspect
 import multiprocessing.dummy
 import os
 
@@ -820,8 +821,13 @@ def _import_media_only(ctx):
     make_sample = lambda f: fo.Sample(filepath=f, tags=tags)
 
     if ctx.delegated:
+        kwargs = {}
+
+        progress = lambda pb: ctx.set_progress(progress=pb.progress)
+        kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         samples = map(make_sample, filepaths)
-        ctx.dataset.add_samples(samples, num_samples=len(filepaths))
+        ctx.dataset.add_samples(samples, num_samples=len(filepaths), **kwargs)
         return
 
     batcher = fou.DynamicBatcher(
@@ -859,6 +865,10 @@ def _import_media_and_labels(ctx):
     if label_types is not None:
         kwargs["label_types"] = label_types
 
+    if ctx.delegated:
+        progress = lambda pb: ctx.set_progress(progress=pb.progress)
+        kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
     ctx.dataset.add_dir(
         dataset_dir=dataset_dir,
         dataset_type=dataset_type,
@@ -899,6 +909,10 @@ def _import_labels_only(ctx):
     if label_types is not None:
         kwargs["label_types"] = label_types
 
+    if ctx.delegated:
+        progress = lambda pb: ctx.set_progress(progress=pb.progress)
+        kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
     with contextlib.ExitStack() as exit_context:
         if labels_file is not None:
             tmp_dir = exit_context.enter_context(fos.TempDir())
@@ -2062,6 +2076,10 @@ def _export_samples(ctx):
         if "abs_paths" not in kwargs:
             kwargs["abs_paths"] = abs_paths
 
+    if ctx.delegated:
+        progress = lambda pb: ctx.set_progress(progress=pb.progress)
+        kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
     target_view.export(
         export_dir=export_dir,
         dataset_type=dataset_type,
@@ -2534,10 +2552,17 @@ def execute(self, ctx):
 
         target_view = _get_target_view(ctx, target)
 
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         target_view.draw_labels(
             output_dir,
             label_fields=label_fields,
             overwrite=overwrite,
+            **kwargs,
         )
 
 

plugins/io/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/io"
 description: A collection of import/export utilities
 version: 1.0.1
 fiftyone:
-  version: ">=0.22.2"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/io
 license: Apache 2.0
 operators:

plugins/utils/__init__.py

@@ -6,6 +6,7 @@
 |
 """
 import contextlib
+import inspect
 import json
 import multiprocessing.dummy
 
@@ -1479,7 +1480,14 @@ def execute(self, ctx):
         view = _get_target_view(ctx, target)
 
         if ctx.delegated:
-            view.compute_metadata(overwrite=overwrite, num_workers=num_workers)
+            kwargs = {}
+
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
+            view.compute_metadata(
+                overwrite=overwrite, num_workers=num_workers, **kwargs
+            )
         else:
             for update in _compute_metadata_generator(
                 ctx, view, overwrite=overwrite, num_workers=num_workers
@@ -1785,13 +1793,20 @@ def execute(self, ctx):
         if not ctx.delegated:
             num_workers = 0
 
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         foui.transform_images(
             view,
             size=size,
             output_field=thumbnail_path,
             output_dir=output_dir,
             num_workers=num_workers,
             skip_failures=True,
+            **kwargs,
         )
 
         if thumbnail_path not in ctx.dataset.app_config.media_fields:
@@ -2118,6 +2133,17 @@ def execute(self, ctx):
         args = ctx.params["args"]
         kwargs = ctx.params["kwargs"]
 
+        # Special handling if we find a `progress` kwarg that is float/int
+        progress = kwargs.get("progress", None)
+        if isinstance(progress, float):
+            # Report progress every `progress` seconds
+            set_progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(set_progress, dt=progress)
+        elif isinstance(progress, int):
+            # Report progress every in `progress` equally-spaced increments
+            set_progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(set_progress, n=progress)
+
         if has_view:
             sample_collection = ctx.view
         elif has_dataset:

plugins/utils/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/utils"
 description: A collection of utility operators
 version: 1.1.1
 fiftyone:
-  version: ">=0.22"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/utils
 license: Apache 2.0
 operators:

plugins/zoo/__init__.py

@@ -6,6 +6,7 @@
 |
 """
 from collections import defaultdict
+import inspect
 
 import fiftyone as fo
 import fiftyone.operators as foo
@@ -48,6 +49,10 @@ def execute(self, ctx):
 
         dataset_name = _get_zoo_dataset_name(ctx)
 
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         dataset = foz.load_zoo_dataset(
             name,
             splits=splits,
@@ -547,6 +552,12 @@ def execute(self, ctx):
         if not ctx.delegated:
             num_workers = 0
 
+        kwargs = {}
+
+        if ctx.delegated:
+            progress = lambda pb: ctx.set_progress(progress=pb.progress)
+            kwargs["progress"] = fo.report_progress(progress, dt=5.0)
+
         if embeddings and patches_field is not None:
             target_view.compute_patch_embeddings(
                 model,
@@ -555,6 +566,7 @@ def execute(self, ctx):
                 batch_size=batch_size,
                 num_workers=num_workers,
                 skip_failures=skip_failures,
+                **kwargs,
             )
         elif embeddings:
             target_view.compute_embeddings(
@@ -563,6 +575,7 @@ def execute(self, ctx):
                 batch_size=batch_size,
                 num_workers=num_workers,
                 skip_failures=skip_failures,
+                **kwargs,
             )
         else:
             target_view.apply_model(
@@ -575,6 +588,7 @@ def execute(self, ctx):
                 skip_failures=skip_failures,
                 output_dir=output_dir,
                 rel_dir=rel_dir,
+                **kwargs,
             )
 
         if not ctx.delegated:

plugins/zoo/fiftyone.yml

@@ -2,7 +2,7 @@ name: "@voxel51/zoo"
 description: A collection of FiftyOne Zoo utilities
 version: 1.1.1
 fiftyone:
-  version: ">=0.22"
+  version: ">=0.23.3"
 url: https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/zoo
 license: Apache 2.0
 operators: