Merge pull request #32841 from vespa-engine/mpolden/snapshot-encryption

Generate and store a sealed encryption key for snapshots

config-provisioning/src/main/java/com/yahoo/config/provision/Cloud.java

@@ -2,6 +2,7 @@
 package com.yahoo.config.provision;
 
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Properties of the cloud service where the zone is deployed.
@@ -17,15 +18,17 @@ public class Cloud {
     private final boolean allowEnclave;
     private final boolean requireAccessControl;
     private final CloudAccount account;
+    private final Optional<String> snapshotPrivateKeySecretName;
 
     private Cloud(CloudName name, boolean dynamicProvisioning, boolean allowHostSharing, boolean allowEnclave,
-                  boolean requireAccessControl, CloudAccount account) {
+                  boolean requireAccessControl, CloudAccount account, Optional<String> snapshotPrivateKeySecretName) {
         this.name = Objects.requireNonNull(name);
         this.dynamicProvisioning = dynamicProvisioning;
         this.allowHostSharing = allowHostSharing;
         this.allowEnclave = allowEnclave;
         this.requireAccessControl = requireAccessControl;
         this.account = Objects.requireNonNull(account);
+        this.snapshotPrivateKeySecretName = Objects.requireNonNull(snapshotPrivateKeySecretName);
         if (allowEnclave && account.isUnspecified()) {
             throw new IllegalArgumentException("Account must be non-empty in '" + name + "'");
         }
@@ -62,6 +65,11 @@ public boolean useProxyProtocol() {
         return !name.equals(CloudName.AZURE);
     }
 
+    /** Name of private key secret used for sealing snapshot encryption keys */
+    public Optional<String> snapshotPrivateKeySecretName() {
+        return snapshotPrivateKeySecretName;
+    }
+
     /** For testing purposes only */
     public static Cloud defaultCloud() {
         return new Builder().build();
@@ -79,6 +87,7 @@ public static class Builder {
         private boolean allowEnclave = false;
         private boolean requireAccessControl = false;
         private CloudAccount account = CloudAccount.empty;
+        private Optional<String> snapshotPrivateKeySecretName = Optional.empty();
 
         public Builder() {}
 
@@ -112,8 +121,14 @@ public Builder account(CloudAccount account) {
             return this;
         }
 
+        public Builder snapshotPrivateKeySecretName(String snapshotPrivateKeySecretName) {
+            this.snapshotPrivateKeySecretName = Optional.of(snapshotPrivateKeySecretName)
+                                                        .filter(s -> !s.isEmpty());
+            return this;
+        }
+
         public Cloud build() {
-            return new Cloud(name, dynamicProvisioning, allowHostSharing, allowEnclave, requireAccessControl, account);
+            return new Cloud(name, dynamicProvisioning, allowHostSharing, allowEnclave, requireAccessControl, account, snapshotPrivateKeySecretName);
         }
 
     }

config-provisioning/src/main/java/com/yahoo/config/provision/Zone.java

@@ -30,6 +30,7 @@ public Zone(ConfigserverConfig configserverConfig, CloudConfig cloudConfig) {
                   .allowEnclave(configserverConfig.cloud().equals("aws") || configserverConfig.cloud().equals("gcp"))
                   .requireAccessControl(cloudConfig.requireAccessControl())
                   .account(CloudAccount.from(cloudConfig.account()))
+                  .snapshotPrivateKeySecretName(cloudConfig.snapshotPrivateKeySecretName())
                   .build(),
              SystemName.from(configserverConfig.system()),
              Environment.from(configserverConfig.environment()),

config-provisioning/src/main/resources/configdefinitions/config.provisioning.cloud.def

@@ -17,3 +17,6 @@ account string default=""
 
 # The cloud-specific region for this zone (as opposed to the Vespa region).
 region string default=""
+
+# Name of private key secret used for sealing snapshot encryption keys.
+snapshotPrivateKeySecretName string default=""

node-repository/pom.xml

@@ -76,6 +76,12 @@
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
+      <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>security-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+      </dependency>
 
         <!-- compile -->
         <dependency>

node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java

@@ -1,6 +1,7 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
+import ai.vespa.secret.internal.TypedSecretStore;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.annotation.Inject;
 import com.yahoo.concurrent.maintenance.JobControl;
@@ -90,7 +91,8 @@ public NodeRepository(NodeRepositoryConfig config,
                           Zone zone,
                           FlagSource flagSource,
                           MetricsDb metricsDb,
-                          Orchestrator orchestrator) {
+                          Orchestrator orchestrator,
+                          TypedSecretStore secretStore) {
         this(flavors,
              provisionServiceProvider,
              curator,
@@ -104,7 +106,8 @@ public NodeRepository(NodeRepositoryConfig config,
              metricsDb,
              orchestrator,
              config.useCuratorClientCache(),
-             zone.environment().isProduction() && !zone.cloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0);
+             zone.environment().isProduction() && !zone.cloud().dynamicProvisioning() && !zone.system().isCd() ? 1 : 0,
+             secretStore);
     }
 
     /**
@@ -124,7 +127,8 @@ public NodeRepository(NodeFlavors flavors,
                           MetricsDb metricsDb,
                           Orchestrator orchestrator,
                           boolean useCuratorClientCache,
-                          int spareCount) {
+                          int spareCount,
+                          TypedSecretStore secretStore) {
         if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.cloud().dynamicProvisioning())
             throw new IllegalArgumentException(String.format(
                     "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s",
@@ -135,7 +139,7 @@ public NodeRepository(NodeFlavors flavors,
         this.clock = clock;
         this.zone = zone;
         this.applications = new Applications(db);
-        this.snapshots = new Snapshots(this, provisionServiceProvider.getSnapshotStore());
+        this.snapshots = new Snapshots(this, secretStore, provisionServiceProvider.getSnapshotStore(), zone.cloud().snapshotPrivateKeySecretName());
         this.nodes = new Nodes(db, zone, clock, orchestrator, applications, snapshots, flagSource);
         this.flavors = flavors;
         this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator();

node-repository/src/main/java/com/yahoo/vespa/hosted/provision/backup/Snapshot.java

@@ -21,14 +21,15 @@
  * @author mpolden
  */
 public record Snapshot(SnapshotId id, HostName hostname, State state, History history, ClusterId cluster,
-                       int clusterIndex, CloudAccount cloudAccount) {
+                       int clusterIndex, CloudAccount cloudAccount, Optional<SnapshotKey> key) {
 
     public Snapshot {
         Objects.requireNonNull(id);
         Objects.requireNonNull(state);
         Objects.requireNonNull(hostname);
         Objects.requireNonNull(history);
         Objects.requireNonNull(cluster);
+        Objects.requireNonNull(key);
         if (clusterIndex < 0) {
             throw new IllegalArgumentException("clusterIndex cannot be negative, got " + cluster);
         }
@@ -47,7 +48,7 @@ public Snapshot with(State state, Instant at) {
         if (!canChangeTo(state)) {
             throw new IllegalArgumentException("Cannot change state of " + this + " to " + state);
         }
-        return new Snapshot(id, hostname, state, history.with(state, at), cluster, clusterIndex, cloudAccount);
+        return new Snapshot(id, hostname, state, history.with(state, at), cluster, clusterIndex, cloudAccount, key);
     }
 
     private boolean canChangeTo(State state) {
@@ -122,8 +123,10 @@ public static SnapshotId generateId() {
         return new SnapshotId(UUID.randomUUID());
     }
 
-    public static Snapshot create(SnapshotId id, HostName hostname, CloudAccount cloudAccount, Instant at, ClusterId cluster, int clusterIndex) {
-        return new Snapshot(id, hostname, State.creating, History.of(State.creating, at), cluster, clusterIndex, cloudAccount);
+    public static Snapshot create(SnapshotId id, HostName hostname, CloudAccount cloudAccount, Instant at,
+                                  ClusterId cluster, int clusterIndex, SnapshotKey encryptionKey) {
+        return new Snapshot(id, hostname, State.creating, History.of(State.creating, at), cluster, clusterIndex,
+                            cloudAccount, Optional.of(encryptionKey));
     }
 
 }

node-repository/src/main/java/com/yahoo/vespa/hosted/provision/backup/SnapshotKey.java

@@ -0,0 +1,20 @@
+package com.yahoo.vespa.hosted.provision.backup;
+
+import ai.vespa.secret.model.SecretVersionId;
+import com.yahoo.security.SealedSharedKey;
+
+import java.util.Objects;
+
+/**
+ * The sealed encryption key for a {@link Snapshot}.
+ *
+ * @author mpolden
+ */
+public record SnapshotKey(SealedSharedKey sharedKey, SecretVersionId sealingKeyVersion) {
+
+    public SnapshotKey {
+        Objects.requireNonNull(sharedKey);
+        Objects.requireNonNull(sealingKeyVersion);
+    }
+
+}

node-repository/src/main/java/com/yahoo/vespa/hosted/provision/backup/Snapshots.java

@@ -1,11 +1,20 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.backup;
 
+import ai.vespa.secret.internal.TypedSecretStore;
+import ai.vespa.secret.model.Key;
+import ai.vespa.secret.model.Secret;
+import ai.vespa.secret.model.SecretVersionId;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SnapshotId;
+import com.yahoo.security.KeyId;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SealedSharedKey;
+import com.yahoo.security.SecretSharedKey;
+import com.yahoo.security.SharedKeyGenerator;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Lock;
@@ -18,6 +27,10 @@
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.provision.provisioning.SnapshotStore;
 
+import java.security.KeyPair;
+import java.security.PublicKey;
+import java.security.interfaces.XECPrivateKey;
+import java.security.interfaces.XECPublicKey;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,11 +48,17 @@ public class Snapshots {
     private final NodeRepository nodeRepository;
     private final CuratorDb db;
     private final Optional<SnapshotStore> snapshotStore;
+    private final TypedSecretStore secretStore;
+    private final Optional<String> sealingPrivateKeySecretName;
 
-    public Snapshots(NodeRepository nodeRepository, Optional<SnapshotStore> snapshotStore) {
+    public Snapshots(NodeRepository nodeRepository, TypedSecretStore secretStore,
+                     Optional<SnapshotStore> snapshotStore,
+                     Optional<String> sealingPrivateKeySecretName) {
         this.nodeRepository = Objects.requireNonNull(nodeRepository);
         this.db = nodeRepository.database();
         this.snapshotStore = Objects.requireNonNull(snapshotStore);
+        this.secretStore = Objects.requireNonNull(secretStore);
+        this.sealingPrivateKeySecretName = Objects.requireNonNull(sealingPrivateKeySecretName);
     }
 
     /** Read known backup snapshots, for all hosts */
@@ -67,36 +86,45 @@ public Snapshot require(SnapshotId id, String hostname) {
 
     /** Trigger a new snapshot for node of given hostname */
     public Snapshot create(String hostname, Instant instant) {
+        VersionedKeyPair keyPair = sealingKeyPair();
         try (var lock = lock(hostname)) {
             SnapshotId id = Snapshot.generateId();
+            SecretSharedKey encryptionKey = generateEncryptionKey(keyPair.keyPair(), id);
             return write(id, hostname, (node) -> {
                 if (busy(node)) {
                     throw new IllegalArgumentException("Cannot trigger new snapshot: Node " + hostname +
                                                        " is busy with snapshot " + node.status().snapshot().get().id());
                 }
                 ClusterId cluster = new ClusterId(node.allocation().get().owner(), node.allocation().get().membership().cluster().id());
                 return Snapshot.create(id, com.yahoo.config.provision.HostName.of(hostname), node.cloudAccount(),
-                                       instant, cluster, node.allocation().get().membership().index());
+                                       instant, cluster, node.allocation().get().membership().index(),
+                                       new SnapshotKey(encryptionKey.sealedSharedKey(), keyPair.version()));
             }, lock);
         }
     }
 
     /** Restore a node to given snapshot */
     public Snapshot restore(SnapshotId id, String hostname) {
         try (var lock = lock(hostname)) {
-            Snapshot snapshot = require(id, hostname);
+            Snapshot original = require(id, hostname);
             Instant now = nodeRepository.clock().instant();
             return write(id, hostname, (node) -> {
                 if (busy(node)) {
                     throw new IllegalArgumentException("Cannot restore snapshot: Node " + hostname +
-                                                       " is busy with snapshot " + node.status().snapshot().get().id() + " in "+
+                                                       " is busy with snapshot " + node.status().snapshot().get().id() + " in " +
                                                        node.status().snapshot().get().state() + " state");
                 }
-                return snapshot.with(Snapshot.State.restoring, now);
+                return original.with(Snapshot.State.restoring, now);
             }, lock);
         }
     }
 
+    /** Returns the encryption key for snapshot, sealed with given public key */
+    public SealedSharedKey keyOf(SnapshotId id, String hostname, PublicKey sealingKey) {
+        Snapshot snapshot = require(id, hostname);
+        return resealKeyOf(snapshot, sealingKey);
+    }
+
     /** Remove given snapshot */
     public void remove(SnapshotId id, String hostname, boolean force) {
         try (var lock = lock(hostname)) {
@@ -147,6 +175,39 @@ public Snapshot move(SnapshotId id, String hostname, Snapshot.State newState) {
         }
     }
 
+    private SecretSharedKey generateEncryptionKey(KeyPair keyPair, SnapshotId id) {
+        return SharedKeyGenerator.generateForReceiverPublicKey(keyPair.getPublic(),
+                                                               KeyId.ofString(id.toString()));
+    }
+
+    /** Reseal the encryption key for snapshot using given public key */
+    private SealedSharedKey resealKeyOf(Snapshot snapshot, PublicKey receiverPublicKey) {
+        if (snapshot.key().isEmpty()) {
+            throw new IllegalArgumentException("Snapshot " + snapshot.id() + " has no encryption key");
+        }
+        VersionedKeyPair sealingKeyPair = sealingKeyPair(snapshot.key().get().sealingKeyVersion());
+        SecretSharedKey unsealedKey = SharedKeyGenerator.fromSealedKey(snapshot.key().get().sharedKey(),
+                                                                       sealingKeyPair.keyPair().getPrivate());
+        return SharedKeyGenerator.reseal(unsealedKey, receiverPublicKey, KeyId.ofString(snapshot.id().toString()))
+                                 .sealedSharedKey();
+    }
+
+    /** Returns the key pair to use when sealing the snapshot encryption key */
+    private VersionedKeyPair sealingKeyPair(SecretVersionId version) {
+        if (sealingPrivateKeySecretName.isEmpty()) {
+            throw new IllegalArgumentException("Cannot retrieve sealing key because its secret name is unset");
+        }
+        Key key = Key.fromString(sealingPrivateKeySecretName.get());
+        Secret sealingPrivateKey = version == null ? secretStore.getSecret(key) : secretStore.getSecret(key, version);
+        XECPrivateKey privateKey = KeyUtils.fromBase64EncodedX25519PrivateKey(sealingPrivateKey.secretValue().value());
+        XECPublicKey publicKey = KeyUtils.extractX25519PublicKey(privateKey);
+        return new VersionedKeyPair(new KeyPair(publicKey, privateKey), sealingPrivateKey.version());
+    }
+
+    private VersionedKeyPair sealingKeyPair() {
+        return sealingKeyPair(null);
+    }
+
     private boolean active(SnapshotId id, Node node) {
         return node.status().snapshot().isPresent() && node.status().snapshot().get().id().equals(id);
     }
@@ -192,4 +253,6 @@ private static Node requireNode(NodeMutex nodeMutex) {
         return node;
     }
 
+    private record VersionedKeyPair(KeyPair keyPair, SecretVersionId version) {}
+
 }