[RSDK-9660] - fix-camera-image-returning-no-mime-type (#4674)

components/camera/camera.go

@@ -19,6 +19,7 @@ import (
 	"go.viam.com/rdk/rimage"
 	"go.viam.com/rdk/rimage/transform"
 	"go.viam.com/rdk/robot"
+	"go.viam.com/rdk/utils"
 )
 
 func init() {
@@ -166,7 +167,7 @@ func DecodeImageFromCamera(ctx context.Context, mimeType string, extra map[strin
 	if len(resBytes) == 0 {
 		return nil, errors.New("received empty bytes from camera")
 	}
-	img, err := rimage.DecodeImage(ctx, resBytes, resMetadata.MimeType)
+	img, err := rimage.DecodeImage(ctx, resBytes, utils.WithLazyMIMEType(resMetadata.MimeType))
 	if err != nil {
 		return nil, fmt.Errorf("could not decode into image.Image: %w", err)
 	}

components/camera/client.go

@@ -196,11 +196,14 @@ func (c *client) Image(ctx context.Context, mimeType string, extra map[string]in
 
 	if expectedType != "" && resp.MimeType != expectedType {
 		c.logger.CDebugw(ctx, "got different MIME type than what was asked for", "sent", expectedType, "received", resp.MimeType)
-	} else {
-		resp.MimeType = mimeType
+		if resp.MimeType == "" {
+			// if the user expected a mime_type and the successful response didn't have a mime type, assume the
+			// response's mime_type was what the user requested
+			resp.MimeType = mimeType
+		}
 	}
 
-	return resp.Image, ImageMetadata{MimeType: utils.WithLazyMIMEType(resp.MimeType)}, nil
+	return resp.Image, ImageMetadata{MimeType: resp.MimeType}, nil
 }
 
 func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {

components/camera/client_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"image"
 	"image/color"
+	"image/jpeg"
 	"image/png"
 	"net"
 	"testing"
@@ -428,7 +429,14 @@ func TestClientLazyImage(t *testing.T) {
 	test.That(t, png.Encode(&imgBuf, img), test.ShouldBeNil)
 	imgPng, err := png.Decode(bytes.NewReader(imgBuf.Bytes()))
 	test.That(t, err, test.ShouldBeNil)
+	var jpegBuf bytes.Buffer
+	test.That(t, jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 100}), test.ShouldBeNil)
+	imgJpeg, err := jpeg.Decode(bytes.NewBuffer(jpegBuf.Bytes()))
+	test.That(t, err, test.ShouldBeNil)
 
+	injectCamera.PropertiesFunc = func(ctx context.Context) (camera.Properties, error) {
+		return camera.Properties{}, nil
+	}
 	injectCamera.ImageFunc = func(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, camera.ImageMetadata, error) {
 		if mimeType == "" {
 			mimeType = rutils.MimeTypeRawRGBA
@@ -470,7 +478,6 @@ func TestClientLazyImage(t *testing.T) {
 	frameLazy := frame.(*rimage.LazyEncodedImage)
 	test.That(t, frameLazy.RawData(), test.ShouldResemble, imgBuf.Bytes())
 
-	ctx = context.Background()
 	frame, err = camera.DecodeImageFromCamera(ctx, rutils.WithLazyMIMEType(rutils.MimeTypePNG), nil, camera1Client)
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, frame, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
@@ -482,6 +489,24 @@ func TestClientLazyImage(t *testing.T) {
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, compVal, test.ShouldEqual, 0) // exact copy, no color conversion
 
+	// when DecodeImageFromCamera is called without a mime type, defaults to JPEG
+	var called bool
+	injectCamera.ImageFunc = func(ctx context.Context, mimeType string, extra map[string]interface{}) ([]byte, camera.ImageMetadata, error) {
+		called = true
+		test.That(t, mimeType, test.ShouldResemble, rutils.WithLazyMIMEType(rutils.MimeTypeJPEG))
+		resBytes, err := rimage.EncodeImage(ctx, imgPng, mimeType)
+		test.That(t, err, test.ShouldBeNil)
+		return resBytes, camera.ImageMetadata{MimeType: mimeType}, nil
+	}
+	frame, err = camera.DecodeImageFromCamera(ctx, "", nil, camera1Client)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, frame, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
+	frameLazy = frame.(*rimage.LazyEncodedImage)
+	test.That(t, frameLazy.MIMEType(), test.ShouldEqual, rutils.MimeTypeJPEG)
+	compVal, _, err = rimage.CompareImages(imgJpeg, frame)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, compVal, test.ShouldEqual, 0) // exact copy, no color conversion
+	test.That(t, called, test.ShouldBeTrue)
 	test.That(t, conn.Close(), test.ShouldBeNil)
 }
 

rimage/lazy_encoded.go

@@ -5,7 +5,11 @@ import (
 	"context"
 	"image"
 	"image/color"
+	"net/http"
+	"strings"
 	"sync"
+
+	"go.viam.com/rdk/logging"
 )
 
 // LazyEncodedImage defers the decoding of an image until necessary.
@@ -29,6 +33,16 @@ type LazyEncodedImage struct {
 // away with reading all metadata from the header of the image bytes.
 // NOTE: Usage of an image that would fail to decode causes a lazy panic.
 func NewLazyEncodedImage(imgBytes []byte, mimeType string) image.Image {
+	if mimeType == "" {
+		logging.Global().Warn("NewLazyEncodedImage called without a mime_type. " +
+			"Sniffing bytes to detect mime_type. Specify mime_type to reduce CPU utilization")
+		mimeType = http.DetectContentType(imgBytes)
+	}
+
+	if !strings.HasPrefix(mimeType, "image/") {
+		logging.Global().Warnf("NewLazyEncodedImage resolving to non image mime_type: %s", mimeType)
+	}
+
 	return &LazyEncodedImage{
 		imgBytes: imgBytes,
 		mimeType: mimeType,

rimage/lazy_encoded_test.go

@@ -3,6 +3,7 @@ package rimage
 import (
 	"bytes"
 	"image"
+	"image/jpeg"
 	"image/png"
 	"testing"
 
@@ -15,10 +16,14 @@ func TestLazyEncodedImage(t *testing.T) {
 	img := image.NewNRGBA(image.Rect(0, 0, 4, 8))
 	img.Set(3, 3, Red)
 
-	var buf bytes.Buffer
-	test.That(t, png.Encode(&buf, img), test.ShouldBeNil)
+	var pngBuf bytes.Buffer
+	test.That(t, png.Encode(&pngBuf, img), test.ShouldBeNil)
+	var jpegBuf bytes.Buffer
+	test.That(t, jpeg.Encode(&jpegBuf, img, &jpeg.Options{Quality: 100}), test.ShouldBeNil)
+	jpegImg, err := jpeg.Decode(bytes.NewBuffer(jpegBuf.Bytes()))
+	test.That(t, err, test.ShouldBeNil)
 
-	imgLazy := NewLazyEncodedImage(buf.Bytes(), utils.MimeTypePNG)
+	imgLazy := NewLazyEncodedImage(pngBuf.Bytes(), utils.MimeTypePNG)
 
 	test.That(t, imgLazy.(*LazyEncodedImage).MIMEType(), test.ShouldEqual, utils.MimeTypePNG)
 	test.That(t, NewColorFromColor(imgLazy.At(0, 0)), test.ShouldEqual, Black)
@@ -46,4 +51,26 @@ func TestLazyEncodedImage(t *testing.T) {
 	test.That(t, func() { imgLazy.ColorModel() }, test.ShouldPanic)
 	test.That(t, func() { NewColorFromColor(imgLazy.At(0, 0)) }, test.ShouldPanic)
 	test.That(t, func() { NewColorFromColor(imgLazy.At(4, 4)) }, test.ShouldPanic)
+
+	// png without a mime type
+	imgLazy = NewLazyEncodedImage(pngBuf.Bytes(), "")
+	test.That(t, imgLazy.(*LazyEncodedImage).MIMEType(), test.ShouldEqual, utils.MimeTypePNG)
+	test.That(t, NewColorFromColor(imgLazy.At(0, 0)), test.ShouldEqual, Black)
+	test.That(t, NewColorFromColor(imgLazy.At(3, 3)), test.ShouldEqual, Red)
+	test.That(t, imgLazy.Bounds(), test.ShouldResemble, img.Bounds())
+	test.That(t, imgLazy.ColorModel(), test.ShouldResemble, img.ColorModel())
+
+	img2, err = png.Decode(bytes.NewBuffer(imgLazy.(*LazyEncodedImage).RawData()))
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, img2, test.ShouldResemble, img)
+
+	// jpeg without a mime type
+	imgLazy = NewLazyEncodedImage(jpegBuf.Bytes(), "")
+	test.That(t, imgLazy.(*LazyEncodedImage).MIMEType(), test.ShouldEqual, utils.MimeTypeJPEG)
+	test.That(t, imgLazy.Bounds(), test.ShouldResemble, jpegImg.Bounds())
+	test.That(t, imgLazy.ColorModel(), test.ShouldResemble, jpegImg.ColorModel())
+
+	img2, err = jpeg.Decode(bytes.NewBuffer(imgLazy.(*LazyEncodedImage).RawData()))
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, img2, test.ShouldResemble, jpegImg)
 }