feat(backup): extends backup manifest with info needed for 1-to-1 restore.

go.mod

@@ -3,6 +3,7 @@ module github.com/scylladb/scylla-manager/v3
 go 1.23.2
 
 require (
+	cloud.google.com/go/compute/metadata v0.3.0
 	github.com/aws/aws-sdk-go v1.35.17
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cespare/xxhash/v2 v2.3.0
@@ -32,7 +33,7 @@ require (
 	github.com/scylladb/gocqlx/v2 v2.8.0
 	github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241104134613-aba35605c28b
 	github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241104134613-aba35605c28b
-	github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241112131737-4fc93b5355fd
+	github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250103082619-ef3b968167d0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stoewer/go-strcase v1.3.0
@@ -48,7 +49,6 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/Azure/azure-storage-blob-go v0.13.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -112,12 +112,12 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/config v1.4.0 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
-	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/term v0.27.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.27.0 // indirect
+	golang.org/x/tools v0.28.0 // indirect
 	google.golang.org/api v0.114.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

go.sum

@@ -1066,8 +1066,8 @@ github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241104134613-ab
 github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241104134613-aba35605c28b/go.mod h1:Tss7a99vrgds+B70w8ZFG3Skxfr9Br3kAzrKP2b3CmQ=
 github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241104134613-aba35605c28b h1:7CHNmPrQqSdApaEh5nkRL+D52KFHaOHVBBVDvytHEOY=
 github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241104134613-aba35605c28b/go.mod h1:+sPCx2oaOXmMpy/ODNNEDGJ7vCghBeKP4S7xEfMI+eA=
-github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241112131737-4fc93b5355fd h1:NNkXlN5SnutcpictWx3sc4jaOz2QMeGmzxES2XdC9RQ=
-github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241112131737-4fc93b5355fd/go.mod h1:Oxfuz1XcXi9iV4ggSGfQdn+p6gPz6djPOegRMMe/6/s=
+github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250103082619-ef3b968167d0 h1:P2RcsgtiMsaKrDUWRI7+eBd7Io0YCctaBFPyqqlsH0I=
+github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20250103082619-ef3b968167d0/go.mod h1:nCN5P0jiWL0W7jbcZ9p0ndtZAPoyEWXefddx/nbyFes=
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4 h1:8qmTC5ByIXO3GP/IzBkxcZ/99VITvnIETDhdFz/om7A=
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
@@ -1296,8 +1296,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
+golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1528,8 +1528,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
-golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o=
-golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q=
+golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
+golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/cloudmeta/metadata.go

@@ -87,23 +87,31 @@ func (cloud *CloudMeta) GetInstanceMetadata(ctx context.Context) (InstanceMetada
 		go func(provider CloudMetadataProvider) {
 			meta, err := cloud.runWithTimeout(ctx, provider)
 
-			select {
-			case <-ctx.Done():
+			if ctx.Err() != nil {
 				return
-			case results <- msg{meta: meta, err: err}:
 			}
+
+			results <- msg{meta: meta, err: err}
 		}(provider)
 	}
 
 	// Return the first non error result or wait until all providers return err.
 	var mErr error
 	for range len(cloud.providers) {
-		res := <-results
-		if res.err != nil {
-			mErr = multierr.Append(mErr, res.err)
-			continue
+		select {
+		case <-ctx.Done():
+			return InstanceMetadata{}, ctx.Err()
+		case res := <-results:
+			// Additional context check just in case messages in results and in ctx.Done channels were available at the same time.
+			if err := ctx.Err(); err != nil {
+				return InstanceMetadata{}, err
+			}
+			if res.err != nil {
+				mErr = multierr.Append(mErr, res.err)
+				continue
+			}
+			return res.meta, nil
 		}
-		return res.meta, nil
 	}
 	return InstanceMetadata{}, mErr
 }

pkg/cloudmeta/metadata_test.go

@@ -77,7 +77,8 @@ func TestGetInstanceMetadata(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			cloudmeta := &CloudMeta{
-				providers: tc.providers,
+				providers:       tc.providers,
+				providerTimeout: 1 * time.Second,
 			}
 
 			meta, err := cloudmeta.GetInstanceMetadata(context.Background())
@@ -101,6 +102,31 @@ func TestGetInstanceMetadata(t *testing.T) {
 	}
 }
 
+func TestGetInstanceMetadataWithCancelledContext(t *testing.T) {
+	cloudmeta := &CloudMeta{
+		providers: []CloudMetadataProvider{
+			newTestProvider(t, "test_provider_1", "x-test-1", 1*time.Second, nil),
+		},
+		providerTimeout: 100 * time.Millisecond,
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	_ = time.AfterFunc(50*time.Millisecond, cancel)
+
+	meta, err := cloudmeta.GetInstanceMetadata(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+
+	if meta.CloudProvider != "" {
+		t.Fatalf("meta.CloudProvider should be empty, got %s", meta.CloudProvider)
+	}
+
+	if meta.InstanceType != "" {
+		t.Fatalf("meta.InstanceType should be empty, got %s", meta.InstanceType)
+	}
+}
+
 func newTestProvider(t *testing.T, providerName, instanceType string, latency time.Duration, err error) *testProvider {
 	t.Helper()
 
@@ -120,13 +146,17 @@ type testProvider struct {
 }
 
 func (tp testProvider) Metadata(ctx context.Context) (InstanceMetadata, error) {
-	time.Sleep(tp.latency)
+	select {
+	case <-time.After(tp.latency):
+	case <-ctx.Done():
+		return InstanceMetadata{}, ctx.Err()
+	}
 
 	if tp.err != nil {
 		return InstanceMetadata{}, tp.err
 	}
 	return InstanceMetadata{
 		CloudProvider: tp.name,
 		InstanceType:  tp.instanceType,
-	}, nil
+	}, ctx.Err()
 }

pkg/cmd/agent/agent.go

@@ -16,7 +16,7 @@ import (
 	"github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models"
 )
 
-func newAgentHandler(c agent.Config, rclone http.Handler, logger log.Logger) *chi.Mux {
+func newAgentHandler(c agent.Config, rclone http.Handler, cloudMeta http.HandlerFunc, logger log.Logger) *chi.Mux {
 	m := chi.NewMux()
 
 	m.Get("/node_info", newNodeInfoHandler(c).getNodeInfo)
@@ -49,6 +49,8 @@ func newAgentHandler(c agent.Config, rclone http.Handler, logger log.Logger) *ch
 		debug.FreeOSMemory()
 	})
 
+	m.Get("/cloud/metadata", cloudMeta)
+
 	// Rclone server
 	m.Mount("/rclone", http.StripPrefix("/agent/rclone", rclone))
 

pkg/cmd/agent/metadata.go

@@ -0,0 +1,62 @@
+// Copyright (C) 2024 ScyllaDB
+
+package main
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	"github.com/go-chi/render"
+	"github.com/pkg/errors"
+	"github.com/scylladb/go-log"
+	"github.com/scylladb/scylla-manager/v3/pkg/cloudmeta"
+	"github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models"
+)
+
+func newMetadataHandler(logger log.Logger) http.HandlerFunc {
+	var (
+		m        sync.Mutex
+		loaded   bool
+		metadata cloudmeta.InstanceMetadata
+	)
+
+	// Caches only successful result of GetInstanceMetadata.
+	lazyGetMetadata := func(ctx context.Context) (cloudmeta.InstanceMetadata, error) {
+		m.Lock()
+		defer m.Unlock()
+		if loaded {
+			return metadata, nil
+		}
+
+		metaSvc, err := cloudmeta.NewCloudMeta(logger)
+		if err != nil {
+			return cloudmeta.InstanceMetadata{}, errors.Wrap(err, "NewCloudMeta")
+		}
+
+		metadata, err = metaSvc.GetInstanceMetadata(ctx)
+		if err != nil {
+			return cloudmeta.InstanceMetadata{}, err
+		}
+
+		loaded = true
+		return metadata, nil
+	}
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		instanceMeta, err := lazyGetMetadata(ctx)
+		if err != nil {
+			// Metadata may not be available for several reasons:
+			// 1. running on-premise 2. disabled 3. smth went wrong with metadata server.
+			// As we cannot distinguish between these cases, we can only log err.
+			logger.Error(ctx, "GetInstanceMetadata", "err", err)
+			render.Respond(w, r, models.InstanceMetadata{})
+			return
+		}
+		render.Respond(w, r, models.InstanceMetadata{
+			CloudProvider: string(instanceMeta.CloudProvider),
+			InstanceType:  instanceMeta.InstanceType,
+		})
+	}
+}

pkg/cmd/agent/nodeinfo_linux.go

@@ -7,6 +7,7 @@ package main
 import (
 	"runtime"
 
+	"github.com/pkg/errors"
 	"github.com/scylladb/scylla-manager/v3/pkg/scyllaclient"
 	"golang.org/x/sys/unix"
 )
@@ -17,9 +18,16 @@ func (h *nodeInfoHandler) sysInfo(info *scyllaclient.NodeInfo) error {
 		return err
 	}
 
+	var statfs unix.Statfs_t
+	if err := unix.Statfs(info.DataDirectory, &statfs); err != nil {
+		return errors.Wrap(err, "statfs")
+	}
+	total := statfs.Blocks * uint64(statfs.Bsize)
+
 	info.MemoryTotal = int64(si.Totalram)
 	info.CPUCount = int64(runtime.NumCPU())
 	info.Uptime = si.Uptime
+	info.StorageSize = total
 
 	return nil
 }

pkg/cmd/agent/nodeinfo_others.go

@@ -15,5 +15,6 @@ func (h *nodeInfoHandler) sysInfo(info *scyllaclient.NodeInfo) error {
 	info.MemoryTotal = 0
 	info.CPUCount = 0
 	info.Uptime = 0
+	info.StorageSize = 0
 	return nil
 }

pkg/cmd/agent/router.go

@@ -18,7 +18,7 @@ import (
 
 var unauthorizedErrorBody = json.RawMessage(`{"message":"unauthorized","code":401}`)
 
-func newRouter(c agent.Config, metrics AgentMetrics, rclone http.Handler, logger log.Logger) http.Handler {
+func newRouter(c agent.Config, metrics AgentMetrics, rclone http.Handler, cloudMeta http.HandlerFunc, logger log.Logger) http.Handler {
 	r := chi.NewRouter()
 
 	// Common middleware
@@ -34,7 +34,7 @@ func newRouter(c agent.Config, metrics AgentMetrics, rclone http.Handler, logger
 		auth.ValidateToken(c.AuthToken, time.Second, unauthorizedErrorBody),
 	)
 	// Agent specific endpoints
-	priv.Mount("/agent", newAgentHandler(c, rclone, logger.Named("agent")))
+	priv.Mount("/agent", newAgentHandler(c, rclone, cloudMeta, logger.Named("agent")))
 	// Scylla prometheus proxy
 	priv.Mount("/metrics", promProxy(c))
 	// Fallback to Scylla API proxy

pkg/cmd/agent/router_test.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"encoding/json"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -25,7 +26,7 @@ func TestRcloneRouting(t *testing.T) {
 	c := agent.Config{}
 	rclone := assertURLPath(t, "/foo")
 
-	h := newRouter(c, NewAgentMetrics(), rclone, log.NewDevelopment())
+	h := newRouter(c, NewAgentMetrics(), rclone, nil, log.NewDevelopment())
 	r := httptest.NewRequest(http.MethodGet, "/agent/rclone/foo", nil)
 	w := httptest.NewRecorder()
 	h.ServeHTTP(w, r)
@@ -52,7 +53,7 @@ func TestProxyRouting(t *testing.T) {
 		},
 	}
 
-	h := newRouter(c, NewAgentMetrics(), nil, log.NewDevelopment())
+	h := newRouter(c, NewAgentMetrics(), nil, nil, log.NewDevelopment())
 
 	r := httptest.NewRequest(http.MethodGet, "/metrics", nil)
 	w := httptest.NewRecorder()
@@ -70,3 +71,41 @@ func TestProxyRouting(t *testing.T) {
 		t.Errorf("Response Code=%d expected %d", w.Code, http.StatusOK)
 	}
 }
+
+func TestCloudMetadataRouting(t *testing.T) {
+	c := agent.Config{}
+	rclone := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
+	cloudMeta := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`{"cloud_provider":"","instance_type":""}`))
+	})
+
+	h := newRouter(c, NewAgentMetrics(), rclone, cloudMeta, log.NewDevelopment())
+	r := httptest.NewRequest(http.MethodGet, "/agent/cloud/metadata", nil)
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, r)
+
+	if w.Code != http.StatusOK {
+		t.Errorf("Response Code=%d expected %d", w.Code, http.StatusOK)
+	}
+
+	responseBody := map[string]string{}
+	if err := json.NewDecoder(w.Result().Body).Decode(&responseBody); err != nil {
+		t.Fatalf("decode body, unexpected err: %v", err)
+	}
+
+	cloudProvider, ok := responseBody["cloud_provider"]
+	if !ok {
+		t.Fatalf("`cloud_provider` field is expected")
+	}
+	if cloudProvider != "" {
+		t.Fatalf("expects `cloud_provider` to be empty, got %s", cloudProvider)
+	}
+
+	instanceType, ok := responseBody["instance_type"]
+	if !ok {
+		t.Fatalf("`instance_type` field is expected")
+	}
+	if instanceType != "" {
+		t.Fatalf("expects `instance_type` to be empty, got %s", instanceType)
+	}
+}

pkg/cmd/agent/server.go

@@ -160,10 +160,11 @@ func (s *server) makeServers(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrapf(err, "tls")
 	}
+	cloudMeta := newMetadataHandler(s.logger.Named("metadata"))
 	s.httpsServer = &http.Server{
 		Addr:      s.config.HTTPS,
 		TLSConfig: tlsConfig,
-		Handler:   newRouter(s.config, s.metrics, rcserver.New(), s.logger.Named("http")),
+		Handler:   newRouter(s.config, s.metrics, rcserver.New(), cloudMeta, s.logger.Named("http")),
 	}
 	if s.config.Prometheus != "" {
 		s.prometheusServer = &http.Server{