Rsdk 4095: Edit the camera interface to match new get_images proto (viamrobotics#2766)

Author: bhaney

components/camera/camera.go

@@ -63,6 +63,12 @@ type Properties struct {
 	DistortionParams transform.Distorter
 }
 
+// NamedImage is a struct that associates the source from where the image came from to the Image.
+type NamedImage struct {
+	Image      image.Image
+	SourceName string
+}
+
 // A Camera is a resource that can capture frames.
 type Camera interface {
 	resource.Resource
@@ -75,7 +81,7 @@ type VideoSource interface {
 
 	// Images is used for getting simultaneous images from different sensors,
 	// along with associated metadata (just timestamp for now). It's not for getting a time series of images from the same sensor.
-	Images(ctx context.Context) ([]image.Image, time.Time, error)
+	Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error)
 	// Stream returns a stream that makes a best effort to return consecutive images
 	// that may have a MIME type hint dictated in the context via gostream.WithMIMETypeHint.
 	Stream(ctx context.Context, errHandlers ...gostream.ErrorHandler) (gostream.VideoStream, error)
@@ -105,7 +111,7 @@ type PointCloudSource interface {
 
 // A ImagesSource is a source that can return a list of images with timestamp.
 type ImagesSource interface {
-	Images(ctx context.Context) ([]image.Image, time.Time, error)
+	Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error)
 }
 
 // FromVideoSource creates a Camera resource from a VideoSource.
@@ -242,23 +248,23 @@ func (vs *videoSource) Stream(ctx context.Context, errHandlers ...gostream.Error
 // Images is for getting simultaneous images from different sensors
 // If the underlying source did not specify an Images function, a default is applied.
 // The default returns a list of 1 image from ReadImage, and the current time.
-func (vs *videoSource) Images(ctx context.Context) ([]image.Image, time.Time, error) {
+func (vs *videoSource) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {
 	ctx, span := trace.StartSpan(ctx, "camera::videoSource::Images")
 	defer span.End()
 	if c, ok := vs.actualSource.(ImagesSource); ok {
 		return c.Images(ctx)
 	}
 	img, release, err := ReadImage(ctx, vs.videoSource)
 	if err != nil {
-		return nil, time.Time{}, errors.Wrap(err, "videoSource: call to get Images failed")
+		return nil, resource.ResponseMetadata{}, errors.Wrap(err, "videoSource: call to get Images failed")
 	}
 	defer func() {
 		if release != nil {
 			release()
 		}
 	}()
 	ts := time.Now()
-	return []image.Image{img}, ts, nil
+	return []NamedImage{{img, ""}}, resource.ResponseMetadata{CapturedAt: ts}, nil
 }
 
 // NextPointCloud returns the next PointCloud from the camera, or will error if not supported.

components/camera/camera_test.go

@@ -247,9 +247,9 @@ func TestCameraWithProjector(t *testing.T) {
 	images, _, err := cam2.Images(context.Background())
 	test.That(t, err, test.ShouldBeNil)
 	test.That(t, len(images), test.ShouldEqual, 1)
-	test.That(t, images[0], test.ShouldHaveSameTypeAs, &rimage.DepthMap{})
-	test.That(t, images[0].Bounds().Dx(), test.ShouldEqual, 1280)
-	test.That(t, images[0].Bounds().Dy(), test.ShouldEqual, 720)
+	test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.DepthMap{})
+	test.That(t, images[0].Image.Bounds().Dx(), test.ShouldEqual, 1280)
+	test.That(t, images[0].Image.Bounds().Dy(), test.ShouldEqual, 720)
 
 	test.That(t, cam2.Close(context.Background()), test.ShouldBeNil)
 }

components/camera/client.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"image"
 	"sync"
-	"time"
 
 	"github.com/edaniels/golog"
 	"github.com/pkg/errors"
@@ -138,18 +137,18 @@ func (c *client) Stream(
 	return stream, nil
 }
 
-func (c *client) Images(ctx context.Context) ([]image.Image, time.Time, error) {
+func (c *client) Images(ctx context.Context) ([]NamedImage, resource.ResponseMetadata, error) {
 	ctx, span := trace.StartSpan(ctx, "camera::client::Images")
 	defer span.End()
 
 	resp, err := c.client.GetImages(ctx, &pb.GetImagesRequest{
 		Name: c.name,
 	})
 	if err != nil {
-		return nil, time.Time{}, errors.Wrap(err, "camera client: could not gets images from the camera")
+		return nil, resource.ResponseMetadata{}, errors.Wrap(err, "camera client: could not gets images from the camera")
 	}
 
-	images := make([]image.Image, 0, len(resp.Images))
+	images := make([]NamedImage, 0, len(resp.Images))
 	// keep everything lazy encoded by default, if type is unknown, attempt to decode it
 	for _, img := range resp.Images {
 		var rdkImage image.Image
@@ -165,12 +164,12 @@ func (c *client) Images(ctx context.Context) ([]image.Image, time.Time, error) {
 		case pb.Format_FORMAT_UNSPECIFIED:
 			rdkImage, _, err = image.Decode(bytes.NewReader(img.Image))
 			if err != nil {
-				return nil, time.Time{}, err
+				return nil, resource.ResponseMetadata{}, err
 			}
 		}
-		images = append(images, rdkImage)
+		images = append(images, NamedImage{rdkImage, img.SourceName})
 	}
-	return images, resp.ResponseMetadata.CapturedAt.AsTime(), nil
+	return images, resource.ResponseMetadataFromProto(resp.ResponseMetadata), nil
 }
 
 func (c *client) NextPointCloud(ctx context.Context) (pointcloud.PointCloud, error) {

components/camera/client_test.go

@@ -75,17 +75,17 @@ func TestClient(t *testing.T) {
 	injectCamera.ProjectorFunc = func(ctx context.Context) (transform.Projector, error) {
 		return projA, nil
 	}
-	injectCamera.ImagesFunc = func(ctx context.Context) ([]image.Image, time.Time, error) {
-		images := []image.Image{}
+	injectCamera.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+		images := []camera.NamedImage{}
 		// one color image
 		color := rimage.NewImage(40, 50)
-		images = append(images, color)
+		images = append(images, camera.NamedImage{color, "color"})
 		// one depth image
 		depth := rimage.NewEmptyDepthMap(10, 20)
-		images = append(images, depth)
+		images = append(images, camera.NamedImage{depth, "depth"})
 		// a timestamp of 12345
 		ts := time.UnixMilli(12345)
-		return images, ts, nil
+		return images, resource.ResponseMetadata{ts}, nil
 	}
 	injectCamera.StreamFunc = func(ctx context.Context, errHandlers ...gostream.ErrorHandler) (gostream.VideoStream, error) {
 		return gostream.NewEmbeddedVideoStreamFromReader(gostream.VideoReaderFunc(func(ctx context.Context) (image.Image, func(), error) {
@@ -192,18 +192,20 @@ func TestClient(t *testing.T) {
 		test.That(t, propsB.SupportsPCD, test.ShouldBeTrue)
 		test.That(t, propsB.IntrinsicParams, test.ShouldResemble, intrinsics)
 
-		images, ts, err := camera1Client.Images(context.Background())
+		images, meta, err := camera1Client.Images(context.Background())
 		test.That(t, err, test.ShouldBeNil)
-		test.That(t, ts, test.ShouldEqual, time.UnixMilli(12345))
+		test.That(t, meta.CapturedAt, test.ShouldEqual, time.UnixMilli(12345))
 		test.That(t, len(images), test.ShouldEqual, 2)
-		test.That(t, images[0].Bounds().Dx(), test.ShouldEqual, 40)
-		test.That(t, images[0].Bounds().Dy(), test.ShouldEqual, 50)
-		test.That(t, images[0], test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
-		test.That(t, images[0].ColorModel(), test.ShouldHaveSameTypeAs, color.RGBAModel)
-		test.That(t, images[1].Bounds().Dx(), test.ShouldEqual, 10)
-		test.That(t, images[1].Bounds().Dy(), test.ShouldEqual, 20)
-		test.That(t, images[1], test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
-		test.That(t, images[1].ColorModel(), test.ShouldHaveSameTypeAs, color.Gray16Model)
+		test.That(t, images[0].SourceName, test.ShouldEqual, "color")
+		test.That(t, images[0].Image.Bounds().Dx(), test.ShouldEqual, 40)
+		test.That(t, images[0].Image.Bounds().Dy(), test.ShouldEqual, 50)
+		test.That(t, images[0].Image, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
+		test.That(t, images[0].Image.ColorModel(), test.ShouldHaveSameTypeAs, color.RGBAModel)
+		test.That(t, images[1].SourceName, test.ShouldEqual, "depth")
+		test.That(t, images[1].Image.Bounds().Dx(), test.ShouldEqual, 10)
+		test.That(t, images[1].Image.Bounds().Dy(), test.ShouldEqual, 20)
+		test.That(t, images[1].Image, test.ShouldHaveSameTypeAs, &rimage.LazyEncodedImage{})
+		test.That(t, images[1].Image.ColorModel(), test.ShouldHaveSameTypeAs, color.Gray16Model)
 
 		// Do
 		resp, err := camera1Client.DoCommand(context.Background(), testutils.TestCommand)

components/camera/replaypcd/replaypcd.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"image"
 	"sync"
 	"time"
 
@@ -291,8 +290,8 @@ func addGRPCMetadata(ctx context.Context, timeRequested, timeReceived *timestamp
 }
 
 // Images is a part of the camera interface but is not implemented for replay.
-func (replay *pcdCamera) Images(ctx context.Context) ([]image.Image, time.Time, error) {
-	return nil, time.Time{}, errors.New("Images is unimplemented")
+func (replay *pcdCamera) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+	return nil, resource.ResponseMetadata{}, errors.New("Images is unimplemented")
 }
 
 // Properties is a part of the camera interface but is not implemented for replay.

components/camera/server.go

@@ -12,7 +12,6 @@ import (
 	commonpb "go.viam.com/api/common/v1"
 	pb "go.viam.com/api/component/camera/v1"
 	"google.golang.org/genproto/googleapis/api/httpbody"
-	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"go.viam.com/rdk/pointcloud"
 	"go.viam.com/rdk/protoutils"
@@ -106,30 +105,27 @@ func (s *serviceServer) GetImages(
 	}
 	// request the images, and then check to see what the underlying type is to determine
 	// what to encode as. If it's color, just encode as JPEG.
-	imgs, ts, err := cam.Images(ctx)
+	imgs, metadata, err := cam.Images(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "camera server GetImages could not call Images on the camera")
 	}
 	imagesMessage := make([]*pb.Image, 0, len(imgs))
 	for _, img := range imgs {
-		format, outBytes, err := encodeImageFromUnderlyingType(ctx, img)
+		format, outBytes, err := encodeImageFromUnderlyingType(ctx, img.Image)
 		if err != nil {
 			return nil, errors.Wrap(err, "camera server GetImages could not encode the images")
 		}
 		imgMes := &pb.Image{
-			SourceName: req.Name, // same as the camera name
+			SourceName: img.SourceName,
 			Format:     format,
 			Image:      outBytes,
 		}
 		imagesMessage = append(imagesMessage, imgMes)
 	}
 	// right now the only metadata is timestamp
-	metadata := &commonpb.ResponseMetadata{
-		CapturedAt: timestamppb.New(ts),
-	}
 	resp := &pb.GetImagesResponse{
 		Images:           imagesMessage,
-		ResponseMetadata: metadata,
+		ResponseMetadata: metadata.AsProto(),
 	}
 
 	return resp, nil

components/camera/server_test.go

@@ -90,17 +90,17 @@ func TestServer(t *testing.T) {
 			IntrinsicParams: intrinsics,
 		}, nil
 	}
-	injectCamera.ImagesFunc = func(ctx context.Context) ([]image.Image, time.Time, error) {
-		images := []image.Image{}
+	injectCamera.ImagesFunc = func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
+		images := []camera.NamedImage{}
 		// one color image
 		color := rimage.NewImage(40, 50)
-		images = append(images, color)
+		images = append(images, camera.NamedImage{color, "color"})
 		// one depth image
 		depth := rimage.NewEmptyDepthMap(10, 20)
-		images = append(images, depth)
+		images = append(images, camera.NamedImage{depth, "depth"})
 		// a timestamp of 12345
 		ts := time.UnixMilli(12345)
-		return images, ts, nil
+		return images, resource.ResponseMetadata{ts}, nil
 	}
 	injectCamera.ProjectorFunc = func(ctx context.Context) (transform.Projector, error) {
 		return projA, nil
@@ -395,7 +395,9 @@ func TestServer(t *testing.T) {
 		test.That(t, resp.ResponseMetadata.CapturedAt.AsTime(), test.ShouldEqual, time.UnixMilli(12345))
 		test.That(t, len(resp.Images), test.ShouldEqual, 2)
 		test.That(t, resp.Images[0].Format, test.ShouldEqual, pb.Format_FORMAT_JPEG)
+		test.That(t, resp.Images[0].SourceName, test.ShouldEqual, "color")
 		test.That(t, resp.Images[1].Format, test.ShouldEqual, pb.Format_FORMAT_RAW_DEPTH)
+		test.That(t, resp.Images[1].SourceName, test.ShouldEqual, "depth")
 	})
 
 	t.Run("GetProperties", func(t *testing.T) {

components/camera/videosource/webcam.go

@@ -514,20 +514,20 @@ func (c *monitoredWebcam) Projector(ctx context.Context) (transform.Projector, e
 	return c.exposedProjector.Projector(ctx)
 }
 
-func (c *monitoredWebcam) Images(ctx context.Context) ([]image.Image, time.Time, error) {
+func (c *monitoredWebcam) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 	if c, ok := c.underlyingSource.(camera.ImagesSource); ok {
 		return c.Images(ctx)
 	}
 	img, release, err := camera.ReadImage(ctx, c.underlyingSource)
 	if err != nil {
-		return nil, time.Time{}, errors.Wrap(err, "monitoredWebcam: call to get Images failed")
+		return nil, resource.ResponseMetadata{}, errors.Wrap(err, "monitoredWebcam: call to get Images failed")
 	}
 	defer func() {
 		if release != nil {
 			release()
 		}
 	}()
-	return []image.Image{img}, time.Now(), nil
+	return []camera.NamedImage{{img, c.Name().Name}}, resource.ResponseMetadata{time.Now()}, nil
 }
 
 func (c *monitoredWebcam) Stream(ctx context.Context, errHandlers ...gostream.ErrorHandler) (gostream.VideoStream, error) {

resource/response_metadata.go

@@ -0,0 +1,27 @@
+package resource
+
+import (
+	"time"
+
+	commonpb "go.viam.com/api/common/v1"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+// ResponseMetadata contains extra info associated with a Resource's standard response.
+type ResponseMetadata struct {
+	CapturedAt time.Time
+}
+
+// AsProto turns the ResponseMetadata struct into a protobuf message.
+func (rm ResponseMetadata) AsProto() *commonpb.ResponseMetadata {
+	metadata := &commonpb.ResponseMetadata{}
+	metadata.CapturedAt = timestamppb.New(rm.CapturedAt)
+	return metadata
+}
+
+// ResponseMetadataFromProto turns the protobuf message into a ResponseMetadata struct.
+func ResponseMetadataFromProto(proto *commonpb.ResponseMetadata) ResponseMetadata {
+	metadata := ResponseMetadata{}
+	metadata.CapturedAt = proto.CapturedAt.AsTime()
+	return metadata
+}

resource/response_metadata_test.go

@@ -0,0 +1,24 @@
+package resource
+
+import (
+	"testing"
+	"time"
+
+	commonpb "go.viam.com/api/common/v1"
+	"go.viam.com/test"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestResponseToProto(t *testing.T) {
+	ts := time.UnixMilli(12345)
+	metadata := ResponseMetadata{CapturedAt: ts}
+	proto := metadata.AsProto()
+	test.That(t, proto.CapturedAt.AsTime(), test.ShouldEqual, ts)
+}
+
+func TestResponseFromProto(t *testing.T) {
+	ts := &timestamppb.Timestamp{Seconds: 12, Nanos: 345000000}
+	proto := &commonpb.ResponseMetadata{CapturedAt: ts}
+	metadata := ResponseMetadataFromProto(proto)
+	test.That(t, metadata.CapturedAt, test.ShouldEqual, time.UnixMilli(12345))
+}

testutils/inject/camera.go

@@ -2,8 +2,6 @@ package inject
 
 import (
 	"context"
-	"image"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/viamrobotics/gostream"
@@ -19,7 +17,7 @@ type Camera struct {
 	camera.Camera
 	name       resource.Name
 	DoFunc     func(ctx context.Context, cmd map[string]interface{}) (map[string]interface{}, error)
-	ImagesFunc func(ctx context.Context) ([]image.Image, time.Time, error)
+	ImagesFunc func(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error)
 	StreamFunc func(
 		ctx context.Context,
 		errHandlers ...gostream.ErrorHandler,
@@ -79,7 +77,7 @@ func (c *Camera) Properties(ctx context.Context) (camera.Properties, error) {
 }
 
 // Images calls the injected Images or the real version.
-func (c *Camera) Images(ctx context.Context) ([]image.Image, time.Time, error) {
+func (c *Camera) Images(ctx context.Context) ([]camera.NamedImage, resource.ResponseMetadata, error) {
 	if c.ImagesFunc == nil {
 		return c.Camera.Images(ctx)
 	}