Skip to content

Commit 3d04939

Browse files
hexbabenicksanford
hexbabe
and
nicksanford
authored
[RSDK-9132] Add (Get)Image to the camera interface (#4487)
Co-authored-by: nicksanford <[email protected]> Co-authored-by: nicksanford <[email protected]>
1 parent ac9e37e commit 3d04939

35 files changed

+703
-608
lines changed

‎components/camera/camera.go

+38-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package camera
66

77
import (
88
"context"
9+
"fmt"
910
"image"
1011

1112
"github.com/pkg/errors"
@@ -15,6 +16,7 @@ import (
1516
"go.viam.com/rdk/gostream"
1617
"go.viam.com/rdk/pointcloud"
1718
"go.viam.com/rdk/resource"
19+
"go.viam.com/rdk/rimage"
1820
"go.viam.com/rdk/rimage/transform"
1921
"go.viam.com/rdk/robot"
2022
)
@@ -70,15 +72,30 @@ type NamedImage struct {
7072
SourceName string
7173
}
7274

75+
// ImageMetadata contains useful information about returned image bytes such as its mimetype.
76+
type ImageMetadata struct {
77+
MimeType string
78+
}
79+
7380
// A Camera is a resource that can capture frames.
7481
type Camera interface {
7582
resource.Resource
7683
VideoSource
7784
}
7885

79-
// A VideoSource represents anything that can capture frames.
86+
// VideoSource represents anything that can capture frames.
8087
// For more information, see the [camera component docs].
8188
//
89+
// Image example:
90+
//
91+
// myCamera, err := camera.FromRobot(machine, "my_camera")
92+
// imageBytes, mimeType, err := myCamera.Image(context.Background(), utils.MimeTypeJPEG, nil)
93+
//
94+
// Or try to directly decode as an image.Image:
95+
//
96+
// myCamera, err := camera.FromRobot(machine, "my_camera")
97+
// img, err = camera.DecodeImageFromCamera(context.Background(), utils.MimeTypeJPEG, nil, myCamera)
98+
//
8299
// Images example:
83100
//
84101
// myCamera, err := camera.FromRobot(machine, "my_camera")
@@ -111,6 +128,10 @@ type Camera interface {
111128
//
112129
// [camera component docs]: https://docs.viam.com/components/camera/
113130
type VideoSource interface {
131+
// Image returns a byte slice representing an image that tries to adhere to the MIME type hint.
132+
// Image also may return metadata about the frame.
133+
Image(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, ImageMetadata, error)
134+
114135
// Images is used for getting simultaneous images from different imagers,
115136
// along with associated metadata (just timestamp for now). It's not for getting a time series of images from the same imager.
116137
Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error)
@@ -136,6 +157,22 @@ func ReadImage(ctx context.Context, src gostream.VideoSource) (image.Image, func
136157
return gostream.ReadImage(ctx, src)
137158
}
138159

160+
// DecodeImageFromCamera retrieves image bytes from a camera resource and serializes it as an image.Image.
161+
func DecodeImageFromCamera(ctx context.Context, mimeType string, extra map[string]interface{}, cam Camera) (image.Image, error) {
162+
resBytes, resMetadata, err := cam.Image(ctx, mimeType, extra)
163+
if err != nil {
164+
return nil, fmt.Errorf("could not get image bytes from camera: %w", err)
165+
}
166+
if len(resBytes) == 0 {
167+
return nil, errors.New("received empty bytes from camera")
168+
}
169+
img, err := rimage.DecodeImage(ctx, resBytes, resMetadata.MimeType)
170+
if err != nil {
171+
return nil, fmt.Errorf("could not decode into image.Image: %w", err)
172+
}
173+
return img, nil
174+
}
175+
139176
// A PointCloudSource is a source that can generate pointclouds.
140177
type PointCloudSource interface {
141178
NextPointCloud(ctx context.Context) (pointcloud.PointCloud, error)

‎components/camera/camera_test.go

+50-20
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"go.viam.com/rdk/components/camera"
1313
"go.viam.com/rdk/gostream"
14+
"go.viam.com/rdk/logging"
1415
"go.viam.com/rdk/pointcloud"
1516
"go.viam.com/rdk/resource"
1617
"go.viam.com/rdk/rimage"
@@ -168,35 +169,46 @@ func (cs *cloudSource) NextPointCloud(ctx context.Context) (pointcloud.PointClou
168169
}
169170

170171
func TestCameraWithNoProjector(t *testing.T) {
172+
logger := logging.NewTestLogger(t)
171173
videoSrc := &simpleSource{"rimage/board1"}
172174
noProj, err := camera.NewVideoSourceFromReader(context.Background(), videoSrc, nil, camera.DepthStream)
173175
test.That(t, err, test.ShouldBeNil)
174176
_, err = noProj.NextPointCloud(context.Background())
175177
test.That(t, errors.Is(err, transform.ErrNoIntrinsics), test.ShouldBeTrue)
176178

177179
// make a camera with a NextPointCloudFunction
178-
videoSrc2 := &cloudSource{Named: camera.Named("foo").AsNamed(), simpleSource: videoSrc}
179-
noProj2, err := camera.NewVideoSourceFromReader(context.Background(), videoSrc2, nil, camera.DepthStream)
180+
cloudSrc2 := &cloudSource{Named: camera.Named("foo").AsNamed(), simpleSource: videoSrc}
181+
videoSrc2, err := camera.NewVideoSourceFromReader(context.Background(), cloudSrc2, nil, camera.DepthStream)
182+
noProj2 := camera.FromVideoSource(resource.NewName(camera.API, "bar"), videoSrc2, logger)
180183
test.That(t, err, test.ShouldBeNil)
181184
pc, err := noProj2.NextPointCloud(context.Background())
182185
test.That(t, err, test.ShouldBeNil)
183186
_, got := pc.At(0, 0, 0)
184187
test.That(t, got, test.ShouldBeTrue)
185188

186-
img, _, err := camera.ReadImage(
187-
gostream.WithMIMETypeHint(context.Background(), rutils.WithLazyMIMEType(rutils.MimeTypePNG)),
188-
noProj2)
189+
// TODO(hexbabe): remove below test when Stream is refactored
190+
t.Run("ReadImage depth map without projector", func(t *testing.T) {
191+
img, _, err := camera.ReadImage(
192+
gostream.WithMIMETypeHint(context.Background(), rutils.WithLazyMIMEType(rutils.MimeTypePNG)),
193+
noProj2)
194+
test.That(t, err, test.ShouldBeNil)
195+
depthImg := img.(*rimage.DepthMap)
196+
test.That(t, err, test.ShouldBeNil)
197+
test.That(t, depthImg.Bounds().Dx(), test.ShouldEqual, 1280)
198+
test.That(t, depthImg.Bounds().Dy(), test.ShouldEqual, 720)
199+
})
200+
201+
img, err := camera.DecodeImageFromCamera(context.Background(), rutils.WithLazyMIMEType(rutils.MimeTypePNG), nil, noProj2)
189202
test.That(t, err, test.ShouldBeNil)
190203

191-
depthImg := img.(*rimage.DepthMap)
192-
test.That(t, err, test.ShouldBeNil)
193-
test.That(t, depthImg.Bounds().Dx(), test.ShouldEqual, 1280)
194-
test.That(t, depthImg.Bounds().Dy(), test.ShouldEqual, 720)
204+
test.That(t, img.Bounds().Dx(), test.ShouldEqual, 1280)
205+
test.That(t, img.Bounds().Dy(), test.ShouldEqual, 720)
195206

196207
test.That(t, noProj2.Close(context.Background()), test.ShouldBeNil)
197208
}
198209

199210
func TestCameraWithProjector(t *testing.T) {
211+
logger := logging.NewTestLogger(t)
200212
videoSrc := &simpleSource{"rimage/board1"}
201213
params1 := &transform.PinholeCameraIntrinsics{ // not the real camera parameters -- fake for test
202214
Width: 1280,
@@ -219,32 +231,50 @@ func TestCameraWithProjector(t *testing.T) {
219231
test.That(t, src.Close(context.Background()), test.ShouldBeNil)
220232

221233
// camera with a point cloud function
222-
videoSrc2 := &cloudSource{Named: camera.Named("foo").AsNamed(), simpleSource: videoSrc}
234+
cloudSrc2 := &cloudSource{Named: camera.Named("foo").AsNamed(), simpleSource: videoSrc}
223235
props, err := src.Properties(context.Background())
224236
test.That(t, err, test.ShouldBeNil)
225-
cam2, err := camera.NewVideoSourceFromReader(
237+
videoSrc2, err := camera.NewVideoSourceFromReader(
226238
context.Background(),
227-
videoSrc2,
239+
cloudSrc2,
228240
&transform.PinholeCameraModel{PinholeCameraIntrinsics: props.IntrinsicParams},
229241
camera.DepthStream,
230242
)
243+
cam2 := camera.FromVideoSource(resource.NewName(camera.API, "bar"), videoSrc2, logger)
231244
test.That(t, err, test.ShouldBeNil)
232-
pc, err = cam2.NextPointCloud(context.Background())
245+
pc, err = videoSrc2.NextPointCloud(context.Background())
233246
test.That(t, err, test.ShouldBeNil)
234247
_, got := pc.At(0, 0, 0)
235248
test.That(t, got, test.ShouldBeTrue)
236249

237-
img, _, err := camera.ReadImage(
238-
gostream.WithMIMETypeHint(context.Background(), rutils.MimeTypePNG),
239-
cam2)
250+
// TODO(hexbabe): remove below test when Stream/ReadImage pattern is refactored
251+
t.Run("ReadImage depth map with projector", func(t *testing.T) {
252+
img, _, err := camera.ReadImage(
253+
gostream.WithMIMETypeHint(context.Background(), rutils.MimeTypePNG),
254+
cam2)
255+
test.That(t, err, test.ShouldBeNil)
256+
257+
depthImg := img.(*rimage.DepthMap)
258+
test.That(t, err, test.ShouldBeNil)
259+
test.That(t, depthImg.Bounds().Dx(), test.ShouldEqual, 1280)
260+
test.That(t, depthImg.Bounds().Dy(), test.ShouldEqual, 720)
261+
// cam2 should implement a default GetImages, that just returns the one image
262+
images, _, err := cam2.Images(context.Background())
263+
test.That(t, err, test.ShouldBeNil)
264+
test.That(t, len(images), test.ShouldEqual, 1)
265+
test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.DepthMap{})
266+
test.That(t, images[0].Image.Bounds().Dx(), test.ShouldEqual, 1280)
267+
test.That(t, images[0].Image.Bounds().Dy(), test.ShouldEqual, 720)
268+
})
269+
270+
img, err := camera.DecodeImageFromCamera(context.Background(), rutils.MimeTypePNG, nil, cam2)
240271
test.That(t, err, test.ShouldBeNil)
241272

242-
depthImg := img.(*rimage.DepthMap)
243273
test.That(t, err, test.ShouldBeNil)
244-
test.That(t, depthImg.Bounds().Dx(), test.ShouldEqual, 1280)
245-
test.That(t, depthImg.Bounds().Dy(), test.ShouldEqual, 720)
274+
test.That(t, img.Bounds().Dx(), test.ShouldEqual, 1280)
275+
test.That(t, img.Bounds().Dy(), test.ShouldEqual, 720)
246276
// cam2 should implement a default GetImages, that just returns the one image
247-
images, _, err := cam2.Images(context.Background())
277+
images, _, err := videoSrc2.Images(context.Background())
248278
test.That(t, err, test.ShouldBeNil)
249279
test.That(t, len(images), test.ShouldEqual, 1)
250280
test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.DepthMap{})

‎components/camera/client.go

+35-60
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ import (
2121
goprotoutils "go.viam.com/utils/protoutils"
2222
"go.viam.com/utils/rpc"
2323
"golang.org/x/exp/slices"
24-
"google.golang.org/protobuf/proto"
25-
"google.golang.org/protobuf/types/known/structpb"
2624

2725
"go.viam.com/rdk/components/camera/rtppassthrough"
2826
"go.viam.com/rdk/data"
@@ -100,60 +98,6 @@ func NewClientFromConn(
10098
}, nil
10199
}
102100

103-
func getExtra(ctx context.Context) (*structpb.Struct, error) {
104-
ext := &structpb.Struct{}
105-
if extra, ok := FromContext(ctx); ok {
106-
var err error
107-
if ext, err = goprotoutils.StructToStructPb(extra); err != nil {
108-
return nil, err
109-
}
110-
}
111-
112-
dataExt, err := data.GetExtraFromContext(ctx)
113-
if err != nil {
114-
return nil, err
115-
}
116-
117-
proto.Merge(ext, dataExt)
118-
return ext, nil
119-
}
120-
121-
// RSDK-8663: This method signature is depended on by the `camera.serviceServer` optimization that
122-
// avoids using an image stream just to get a single image.
123-
func (c *client) Read(ctx context.Context) (image.Image, func(), error) {
124-
ctx, span := trace.StartSpan(ctx, "camera::client::Read")
125-
defer span.End()
126-
mimeType := gostream.MIMETypeHint(ctx, "")
127-
expectedType, _ := utils.CheckLazyMIMEType(mimeType)
128-
129-
ext, err := getExtra(ctx)
130-
if err != nil {
131-
return nil, nil, err
132-
}
133-
134-
resp, err := c.client.GetImage(ctx, &pb.GetImageRequest{
135-
Name: c.name,
136-
MimeType: expectedType,
137-
Extra: ext,
138-
})
139-
if err != nil {
140-
return nil, nil, err
141-
}
142-
143-
if expectedType != "" && resp.MimeType != expectedType {
144-
c.logger.CDebugw(ctx, "got different MIME type than what was asked for", "sent", expectedType, "received", resp.MimeType)
145-
} else {
146-
resp.MimeType = mimeType
147-
}
148-
149-
resp.MimeType = utils.WithLazyMIMEType(resp.MimeType)
150-
img, err := rimage.DecodeImage(ctx, resp.Image, resp.MimeType)
151-
if err != nil {
152-
return nil, nil, err
153-
}
154-
return img, func() {}, nil
155-
}
156-
157101
func (c *client) Stream(
158102
ctx context.Context,
159103
errHandlers ...gostream.ErrorHandler,
@@ -184,7 +128,8 @@ func (c *client) Stream(
184128
// with those from the new "generation".
185129
healthyClientCh := c.maybeResetHealthyClientCh()
186130

187-
ctxWithMIME := gostream.WithMIMETypeHint(context.Background(), gostream.MIMETypeHint(ctx, ""))
131+
mimeTypeFromCtx := gostream.MIMETypeHint(ctx, "")
132+
ctxWithMIME := gostream.WithMIMETypeHint(context.Background(), mimeTypeFromCtx)
188133
streamCtx, stream, frameCh := gostream.NewMediaStreamForChannel[image.Image](ctxWithMIME)
189134

190135
c.activeBackgroundWorkers.Add(1)
@@ -201,7 +146,7 @@ func (c *client) Stream(
201146
return
202147
}
203148

204-
frame, release, err := c.Read(streamCtx)
149+
img, err := DecodeImageFromCamera(streamCtx, mimeTypeFromCtx, nil, c)
205150
if err != nil {
206151
for _, handler := range errHandlers {
207152
handler(streamCtx, err)
@@ -217,8 +162,8 @@ func (c *client) Stream(
217162
}
218163
return
219164
case frameCh <- gostream.MediaReleasePairWithError[image.Image]{
220-
Media: frame,
221-
Release: release,
165+
Media: img,
166+
Release: func() {},
222167
Err: err,
223168
}:
224169
}
@@ -228,6 +173,36 @@ func (c *client) Stream(
228173
return stream, nil
229174
}
230175

176+
func (c *client) Image(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, ImageMetadata, error) {
177+
ctx, span := trace.StartSpan(ctx, "camera::client::Image")
178+
defer span.End()
179+
expectedType, _ := utils.CheckLazyMIMEType(mimeType)
180+
181+
convertedExtra, err := goprotoutils.StructToStructPb(extra)
182+
if err != nil {
183+
return nil, ImageMetadata{}, err
184+
}
185+
resp, err := c.client.GetImage(ctx, &pb.GetImageRequest{
186+
Name: c.name,
187+
MimeType: expectedType,
188+
Extra: convertedExtra,
189+
})
190+
if err != nil {
191+
return nil, ImageMetadata{}, err
192+
}
193+
if len(resp.Image) == 0 {
194+
return nil, ImageMetadata{}, errors.New("received empty bytes from client GetImage")
195+
}
196+
197+
if expectedType != "" && resp.MimeType != expectedType {
198+
c.logger.CDebugw(ctx, "got different MIME type than what was asked for", "sent", expectedType, "received", resp.MimeType)
199+
} else {
200+
resp.MimeType = mimeType
201+
}
202+
203+
return resp.Image, ImageMetadata{MimeType: utils.WithLazyMIMEType(resp.MimeType)}, nil
204+
}
205+
231206
func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {
232207
ctx, span := trace.StartSpan(ctx, "camera::client::Images")
233208
defer span.End()

0 commit comments

Comments
 (0)