RSDK-7469: Add CaptureAllFromCamera()

rimage/image_file.go

@@ -256,10 +256,13 @@ func DecodeImage(ctx context.Context, imgBytes []byte, mimeType string) (image.I
 		return NewLazyEncodedImage(imgBytes, mimeType), nil
 	}
 	switch mimeType {
-	case "", ut.MimeTypeJPEG:
+	case "":
 		img, err := DecodeJPEG(bytes.NewReader(imgBytes))
 		if err != nil {
-			return nil, err
+			img, _, err = image.Decode(bytes.NewReader(imgBytes))
+			if err != nil {
+				return nil, err
+			}
 		}
 		return img, nil
 	default:

services/vision/client.go

@@ -23,6 +23,7 @@ import (
 	"go.viam.com/rdk/vision"
 	"go.viam.com/rdk/vision/classification"
 	objdet "go.viam.com/rdk/vision/objectdetection"
+	"go.viam.com/rdk/vision/viscapture"
 )
 
 // client implements VisionServiceClient.
@@ -72,16 +73,7 @@ func (c *client) DetectionsFromCamera(
 	if err != nil {
 		return nil, err
 	}
-	detections := make([]objdet.Detection, 0, len(resp.Detections))
-	for _, d := range resp.Detections {
-		if d.XMin == nil || d.XMax == nil || d.YMin == nil || d.YMax == nil {
-			return nil, fmt.Errorf("invalid detection %+v", d)
-		}
-		box := image.Rect(int(*d.XMin), int(*d.YMin), int(*d.XMax), int(*d.YMax))
-		det := objdet.NewDetection(box, d.Confidence, d.ClassName)
-		detections = append(detections, det)
-	}
-	return detections, nil
+	return protoToDets(resp.Detections)
 }
 
 func (c *client) Detections(ctx context.Context, img image.Image, extra map[string]interface{},
@@ -111,8 +103,12 @@ func (c *client) Detections(ctx context.Context, img image.Image, extra map[stri
 	if err != nil {
 		return nil, err
 	}
-	detections := make([]objdet.Detection, 0, len(resp.Detections))
-	for _, d := range resp.Detections {
+	return protoToDets(resp.Detections)
+}
+
+func protoToDets(protoDets []*pb.Detection) ([]objdet.Detection, error) {
+	detections := make([]objdet.Detection, 0, len(protoDets))
+	for _, d := range protoDets {
 		if d.XMin == nil || d.XMax == nil || d.YMin == nil || d.YMax == nil {
 			return nil, fmt.Errorf("invalid detection %+v", d)
 		}
@@ -144,12 +140,7 @@ func (c *client) ClassificationsFromCamera(
 	if err != nil {
 		return nil, err
 	}
-	classifications := make([]classification.Classification, 0, len(resp.Classifications))
-	for _, c := range resp.Classifications {
-		classif := classification.NewClassification(c.Confidence, c.ClassName)
-		classifications = append(classifications, classif)
-	}
-	return classifications, nil
+	return protoToClas(resp.Classifications), nil
 }
 
 func (c *client) Classifications(ctx context.Context, img image.Image,
@@ -181,12 +172,16 @@ func (c *client) Classifications(ctx context.Context, img image.Image,
 	if err != nil {
 		return nil, err
 	}
-	classifications := make([]classification.Classification, 0, len(resp.Classifications))
-	for _, c := range resp.Classifications {
+	return protoToClas(resp.Classifications), nil
+}
+
+func protoToClas(protoClass []*pb.Classification) classification.Classifications {
+	classifications := make([]classification.Classification, 0, len(protoClass))
+	for _, c := range protoClass {
 		classif := classification.NewClassification(c.Confidence, c.ClassName)
 		classifications = append(classifications, classif)
 	}
-	return classifications, nil
+	return classifications
 }
 
 func (c *client) GetObjectPointClouds(
@@ -262,6 +257,61 @@ func (c *client) GetProperties(ctx context.Context, extra map[string]interface{}
 	return &Properties{resp.ClassificationsSupported, resp.DetectionsSupported, resp.ObjectPointCloudsSupported}, nil
 }
 
+func (c *client) CaptureAllFromCamera(
+	ctx context.Context,
+	cameraName string,
+	captureOptions viscapture.CaptureOptions,
+	extra map[string]interface{},
+) (viscapture.VisCapture, error) {
+	ctx, span := trace.StartSpan(ctx, "service::vision::client::ClassificationsFromCamera")
+	defer span.End()
+	ext, err := protoutils.StructToStructPb(extra)
+	if err != nil {
+		return viscapture.VisCapture{}, err
+	}
+	resp, err := c.client.CaptureAllFromCamera(ctx, &pb.CaptureAllFromCameraRequest{
+		Name:                    c.name,
+		CameraName:              cameraName,
+		ReturnImage:             captureOptions.ReturnImage,
+		ReturnDetections:        captureOptions.ReturnDetections,
+		ReturnClassifications:   captureOptions.ReturnClassifications,
+		ReturnObjectPointClouds: captureOptions.ReturnObject,
+		Extra:                   ext,
+	})
+	if err != nil {
+		return viscapture.VisCapture{}, err
+	}
+
+	dets, err := protoToDets(resp.Detections)
+	if err != nil {
+		return viscapture.VisCapture{}, err
+	}
+
+	class := protoToClas(resp.Classifications)
+
+	objPCD, err := protoToObjects(resp.Objects)
+	if err != nil {
+		return viscapture.VisCapture{}, err
+	}
+	var img image.Image
+	if resp.Image != nil {
+		mimeType := utils.FormatToMimeType[resp.Image.GetFormat()]
+		img, err = rimage.DecodeImage(ctx, resp.Image.Image, mimeType)
+		if err != nil {
+			return viscapture.VisCapture{}, err
+		}
+	}
+
+	capt := viscapture.VisCapture{
+		Image:           img,
+		Detections:      dets,
+		Classifications: class,
+		Objects:         objPCD,
+	}
+
+	return capt, nil
+}
+
 func (c *client) DoCommand(ctx context.Context, cmd map[string]interface{}) (map[string]interface{}, error) {
 	ctx, span := trace.StartSpan(ctx, "service::vision::client::DoCommand")
 	defer span.End()

services/vision/client_test.go

@@ -17,6 +17,7 @@ import (
 	"go.viam.com/rdk/testutils"
 	"go.viam.com/rdk/testutils/inject"
 	"go.viam.com/rdk/vision/objectdetection"
+	"go.viam.com/rdk/vision/viscapture"
 )
 
 var visName1 = vision.Named("vision1")
@@ -44,7 +45,19 @@ func TestClient(t *testing.T) {
 	srv.GetPropertiesFunc = func(ctx context.Context, extra map[string]interface{}) (*vision.Properties, error) {
 		return &vision.Properties{ClassificationSupported: false, DetectionSupported: true, ObjectPCDsSupported: false}, nil
 	}
+
 	test.That(t, err, test.ShouldBeNil)
+
+	srv.CaptureAllFromCameraFunc = func(ctx context.Context,
+		cameraName string,
+		opts viscapture.CaptureOptions,
+		extra map[string]interface{},
+	) (viscapture.VisCapture, error) {
+		det1 := objectdetection.NewDetection(image.Rectangle{}, 0.5, "yes")
+		return viscapture.VisCapture{
+			Detections: []objectdetection.Detection{det1},
+		}, nil
+	}
 	m := map[resource.Name]vision.Service{
 		vision.Named(testVisionServiceName): srv,
 	}
@@ -121,6 +134,25 @@ func TestClient(t *testing.T) {
 		test.That(t, client.Close(context.Background()), test.ShouldBeNil)
 		test.That(t, conn.Close(), test.ShouldBeNil)
 	})
+
+	t.Run("capture all from camera", func(t *testing.T) {
+		conn, err := viamgrpc.Dial(context.Background(), listener1.Addr().String(), logger)
+		test.That(t, err, test.ShouldBeNil)
+		client, err := vision.NewClientFromConn(context.Background(), conn, "", vision.Named(testVisionServiceName), logger)
+		test.That(t, err, test.ShouldBeNil)
+		opts := viscapture.CaptureOptions{
+			true,
+			true, true, true,
+		}
+		capt, err := client.CaptureAllFromCamera(context.Background(), "", opts, map[string]interface{}{})
+		test.That(t, err, test.ShouldBeNil)
+
+		test.That(t, capt.Detections, test.ShouldHaveLength, 1)
+		test.That(t, capt.Detections[0].Label(), test.ShouldEqual, "yes")
+		test.That(t, capt.Detections[0].Score(), test.ShouldEqual, 0.5)
+		test.That(t, client.Close(context.Background()), test.ShouldBeNil)
+		test.That(t, conn.Close(), test.ShouldBeNil)
+	})
 }
 
 func TestInjectedServiceClient(t *testing.T) {

services/vision/obstaclesdepth/obstacles_depth_test.go

@@ -17,6 +17,7 @@ import (
 	"go.viam.com/rdk/services/vision"
 	"go.viam.com/rdk/spatialmath"
 	"go.viam.com/rdk/testutils/inject"
+	"go.viam.com/rdk/vision/viscapture"
 )
 
 // testReader creates and serves a fake depth image for testing.
@@ -72,6 +73,10 @@ func TestObstacleDepth(t *testing.T) {
 	r := &inject.Robot{ResourceNamesFunc: func() []resource.Name {
 		return []resource.Name{camera.Named("testCam"), camera.Named("noIntrinsicsCam")}
 	}}
+	r.LoggerFunc = func() logging.Logger {
+		return logging.NewLogger("test")
+	}
+
 	// camera with intrinsics
 	fr := fullReader{}
 	syst := transform.PinholeCameraModel{&someIntrinsics, nil}
@@ -143,6 +148,33 @@ func TestObstacleDepth(t *testing.T) {
 			test.That(t, o.Geometry, test.ShouldNotBeNil)
 		}
 	})
+
+	t.Run("capture all from camera", func(t *testing.T) {
+		srv2, err := registerObstaclesDepth(ctx, name, &withIntrinsicsCfg, r, testLogger)
+		test.That(t, err, test.ShouldBeNil)
+		test.That(t, srv2, test.ShouldNotBeNil)
+
+		obs, err := srv2.GetObjectPointClouds(ctx, "testCam", nil)
+		test.That(t, err, test.ShouldBeNil)
+		test.That(t, obs, test.ShouldNotBeNil)
+		test.That(t, len(obs), test.ShouldEqual, 2)
+
+		captOpts := viscapture.CaptureOptions{
+			ReturnImage:           true,
+			ReturnClassifications: true,
+			ReturnDetections:      true,
+			ReturnObject:          true,
+		}
+
+		capt, err := srv2.CaptureAllFromCamera(ctx, "testCam", captOpts, nil)
+		test.That(t, err, test.ShouldBeNil)
+		test.That(t, capt, test.ShouldNotBeNil)
+		test.That(t, len(capt.Objects), test.ShouldEqual, len(obs))
+
+		test.That(t, capt.Image, test.ShouldNotBeNil)
+		test.That(t, capt.Classifications, test.ShouldBeNil)
+		test.That(t, capt.Detections, test.ShouldBeNil)
+	})
 }
 
 func BenchmarkObstacleDepthIntrinsics(b *testing.B) {