[RSDK-9928] - Add preloaded images to image_file camera (viamrobotics#4830)

Author: hexbabe

components/camera/fake/fakedata.go

@@ -3,10 +3,13 @@
 package fake
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"image"
+	"image/jpeg"
 	"time"
 
 	"go.viam.com/rdk/components/camera"
@@ -36,9 +39,16 @@ func init() {
 }
 
 func newCamera(ctx context.Context, name resource.Name, newConf *fileSourceConfig, logger logging.Logger) (camera.Camera, error) {
-	videoSrc := &fileSource{newConf.Color, newConf.Depth, newConf.PointCloud, newConf.CameraParameters}
+	videoSrc := &fileSource{
+		ColorFN:        newConf.Color,
+		DepthFN:        newConf.Depth,
+		PointCloudFN:   newConf.PointCloud,
+		Intrinsics:     newConf.CameraParameters,
+		PreloadedImage: newConf.PreloadedImage,
+	}
+
 	imgType := camera.ColorStream
-	if newConf.Color == "" {
+	if newConf.Color == "" && newConf.PreloadedImage == "" {
 		imgType = camera.DepthStream
 	}
 	cameraModel := camera.NewPinholeModelWithBrownConradyDistortion(newConf.CameraParameters, newConf.DistortionParameters)
@@ -56,10 +66,11 @@ func newCamera(ctx context.Context, name resource.Name, newConf *fileSourceConfi
 
 // fileSource stores the paths to a color and depth image and a pointcloud.
 type fileSource struct {
-	ColorFN      string
-	DepthFN      string
-	PointCloudFN string
-	Intrinsics   *transform.PinholeCameraIntrinsics
+	ColorFN        string
+	DepthFN        string
+	PointCloudFN   string
+	Intrinsics     *transform.PinholeCameraIntrinsics
+	PreloadedImage string
 }
 
 // fileSourceConfig is the attribute struct for fileSource.
@@ -69,6 +80,7 @@ type fileSourceConfig struct {
 	Color                string                             `json:"color_image_file_path,omitempty"`
 	Depth                string                             `json:"depth_image_file_path,omitempty"`
 	PointCloud           string                             `json:"pointcloud_file_path,omitempty"`
+	PreloadedImage       string                             `json:"preloaded_image,omitempty"` // can be "pizza", "dog", or "crowd"
 }
 
 // Validate ensures all parts of the config are valid.
@@ -81,23 +93,41 @@ func (c fileSourceConfig) Validate(path string) ([]string, error) {
 		}
 	}
 
+	if c.PreloadedImage != "" {
+		switch c.PreloadedImage {
+		case "pizza", "dog", "crowd":
+			// valid options
+		default:
+			return nil, fmt.Errorf("preloaded_image must be one of: pizza, dog, crowd. Got: %s", c.PreloadedImage)
+		}
+	}
+
 	return []string{}, nil
 }
 
 // Read returns just the RGB image if it is present, or the depth map if the RGB image is not present.
 func (fs *fileSource) Read(ctx context.Context) (image.Image, func(), error) {
-	if fs.ColorFN == "" && fs.DepthFN == "" {
+	if fs.ColorFN == "" && fs.DepthFN == "" && fs.PreloadedImage == "" {
 		return nil, nil, errors.New("no image file to read, so not implemented")
 	}
-	if fs.ColorFN == "" { // only depth info
+	if fs.ColorFN == "" && fs.PreloadedImage == "" { // only depth info
 		img, err := rimage.NewDepthMapFromFile(context.Background(), fs.DepthFN)
 		if err != nil {
 			return nil, nil, err
 		}
 		return img, func() {}, err
 	}
 
-	img, err := rimage.NewImageFromFile(fs.ColorFN)
+	var img *rimage.Image
+	var err error
+
+	// Get image from preloaded image or file
+	if fs.PreloadedImage != "" {
+		img, err = getPreloadedImage(fs.PreloadedImage)
+	} else {
+		img, err = rimage.NewImageFromFile(fs.ColorFN)
+	}
+
 	if err != nil {
 		return nil, nil, err
 	}
@@ -123,24 +153,35 @@ func (fs *fileSource) Read(ctx context.Context) (image.Image, func(), error) {
 
 // Images returns the saved color and depth image if they are present.
 func (fs *fileSource) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
-	if fs.ColorFN == "" && fs.DepthFN == "" {
-		return nil, resource.ResponseMetadata{}, errors.New("no image file to read, so not implemented")
+	if fs.ColorFN == "" && fs.DepthFN == "" && fs.PreloadedImage == "" {
+		return nil, resource.ResponseMetadata{}, errors.New("no image files to read, so not implemented")
 	}
 	imgs := []camera.NamedImage{}
+
+	if fs.PreloadedImage != "" {
+		img, err := getPreloadedImage(fs.PreloadedImage)
+		if err != nil {
+			return nil, resource.ResponseMetadata{}, err
+		}
+		imgs = append(imgs, camera.NamedImage{img, "preloaded"})
+	}
+
 	if fs.ColorFN != "" {
 		img, err := rimage.NewImageFromFile(fs.ColorFN)
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
 		}
 		imgs = append(imgs, camera.NamedImage{img, "color"})
 	}
+
 	if fs.DepthFN != "" {
 		dm, err := rimage.NewDepthMapFromFile(context.Background(), fs.DepthFN)
 		if err != nil {
 			return nil, resource.ResponseMetadata{}, err
 		}
 		imgs = append(imgs, camera.NamedImage{dm, "depth"})
 	}
+
 	ts := time.Now()
 	return imgs, resource.ResponseMetadata{CapturedAt: ts}, nil
 }
@@ -227,3 +268,25 @@ func (ss *StaticSource) NextPointCloud(ctx context.Context) (pointcloud.PointClo
 func (ss *StaticSource) Close(ctx context.Context) error {
 	return nil
 }
+
+// getPreloadedImage returns one of the preloaded images based on the name.
+func getPreloadedImage(name string) (*rimage.Image, error) {
+	var imageBase64 []byte
+	switch name {
+	case "pizza":
+		imageBase64 = pizzaBase64
+	case "dog":
+		imageBase64 = dogBase64
+	case "crowd":
+		imageBase64 = crowdBase64
+	default:
+		return nil, fmt.Errorf("unknown preloaded image: %s", name)
+	}
+
+	d := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(imageBase64))
+	img, err := jpeg.Decode(d)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+	return rimage.ConvertImage(img), nil
+}

components/camera/fake/image_file.go

@@ -114,3 +114,65 @@ func TestColorOddResolution(t *testing.T) {
 	err = cam.Close(ctx)
 	test.That(t, err, test.ShouldBeNil)
 }
+
+func TestPreloadedImages(t *testing.T) {
+	ctx := context.Background()
+	logger := logging.NewTestLogger(t)
+	preloadedImages := []string{"pizza", "dog", "crowd"}
+
+	for _, imgName := range preloadedImages {
+		t.Run(imgName, func(t *testing.T) {
+			cfg := &fileSourceConfig{PreloadedImage: imgName}
+			cam, err := newCamera(ctx, resource.Name{API: camera.API}, cfg, logger)
+			test.That(t, err, test.ShouldBeNil)
+
+			img, err := camera.DecodeImageFromCamera(ctx, utils.MimeTypeRawRGBA, nil, cam)
+			test.That(t, err, test.ShouldBeNil)
+			test.That(t, img, test.ShouldNotBeNil)
+
+			bounds := img.Bounds()
+			test.That(t, bounds.Dx() > 0, test.ShouldBeTrue)
+			test.That(t, bounds.Dy() > 0, test.ShouldBeTrue)
+
+			namedImages, metadata, err := cam.Images(ctx)
+			test.That(t, err, test.ShouldBeNil)
+			test.That(t, len(namedImages), test.ShouldEqual, 1)
+			test.That(t, namedImages[0].SourceName, test.ShouldEqual, "preloaded")
+			test.That(t, metadata.CapturedAt.IsZero(), test.ShouldBeFalse)
+
+			jpegBytes, mime, err := cam.Image(ctx, utils.MimeTypeJPEG, nil)
+			test.That(t, err, test.ShouldBeNil)
+			test.That(t, mime.MimeType, test.ShouldEqual, utils.MimeTypeJPEG)
+			test.That(t, len(jpegBytes) > 0, test.ShouldBeTrue)
+
+			err = cam.Close(ctx)
+			test.That(t, err, test.ShouldBeNil)
+		})
+	}
+
+	colorImgPath := artifact.MustPath("vision/objectdetection/detection_test.jpg")
+	cfg := &fileSourceConfig{
+		PreloadedImage: "pizza",
+		Color:          colorImgPath,
+	}
+	cam, err := newCamera(ctx, resource.Name{API: camera.API}, cfg, logger)
+	test.That(t, err, test.ShouldBeNil)
+
+	// Should return both images
+	namedImages, _, err := cam.Images(ctx)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, len(namedImages), test.ShouldEqual, 2)
+	test.That(t, namedImages[0].SourceName, test.ShouldEqual, "preloaded")
+	test.That(t, namedImages[1].SourceName, test.ShouldEqual, "color")
+
+	cameraImg, err := camera.DecodeImageFromCamera(ctx, utils.MimeTypeRawRGBA, nil, cam)
+	test.That(t, err, test.ShouldBeNil)
+	preloadedImg, err := getPreloadedImage("pizza")
+	test.That(t, err, test.ShouldBeNil)
+	diff, _, err := rimage.CompareImages(cameraImg, preloadedImg)
+	test.That(t, err, test.ShouldBeNil)
+	test.That(t, diff, test.ShouldEqual, 0)
+
+	err = cam.Close(ctx)
+	test.That(t, err, test.ShouldBeNil)
+}